diff --git a/.claude/skills/exec-local-compile/SKILL.md b/.claude/skills/exec-local-compile/SKILL.md index 9a45283174fe..6a2f3184895d 100644 --- a/.claude/skills/exec-local-compile/SKILL.md +++ b/.claude/skills/exec-local-compile/SKILL.md @@ -45,7 +45,7 @@ git checkout main && git pull Run the build command (**incremental by default** — omit `-c`/`--clean` unless explicitly requested or the incremental build fails): ```bash -./scripts/build_wheel.py --use_ccache -a "" -f --nvtx +./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache -a "" -f --nvtx ``` Replace `` with the target GPU architecture (see Architecture Reference below). If not specified by the user, auto-detect from `nvidia-smi`. @@ -66,6 +66,8 @@ python3 -c "import tensorrt_llm; print(tensorrt_llm.__version__)" | Flag | Description | |------|-------------| +| `--trt_root /usr/local/tensorrt` | TensorRT installation path (standard in NVIDIA containers) | +| `--benchmarks` | Build the C++ benchmarks | | `-a ""` | Target GPU architecture(s) | | `--nvtx` | Enable NVTX markers for profiling | | `--use_ccache` | Use ccache for faster recompilation | diff --git a/.claude/skills/exec-slurm-compile/SKILL.md b/.claude/skills/exec-slurm-compile/SKILL.md index 8a8005d06d7a..44c60ed374bd 100644 --- a/.claude/skills/exec-slurm-compile/SKILL.md +++ b/.claude/skills/exec-slurm-compile/SKILL.md @@ -204,6 +204,8 @@ A successful build ends with a message like `Successfully built tensorrt_llm` or | Flag | Description | |------|-------------| +| `--trt_root /usr/local/tensorrt` | TensorRT installation path (standard in NVIDIA containers) | +| `--benchmarks` | Build the C++ benchmarks | | `-a "100-real"` | Target architecture — `100` for Blackwell, `90` for Hopper, etc. | | `--nvtx` | Enable NVTX markers for profiling | | `--no-venv` | Skip virtual environment creation | @@ -226,6 +228,7 @@ Common architecture values: | `sbatch: error: invalid partition` | Verify partition name with `sinfo -s` | | `sbatch: error: invalid account` | Check available accounts with `sacctmgr show assoc user=$USER` | | Container image not found | Verify the `.sqsh` path exists and is readable | +| Build fails with missing TensorRT | Ensure `--trt_root` points to the correct path inside the container | | Build OOM (out of memory) | Reduce parallelism with `-j ` flag to `build_wheel.py` | | `srun: error: Unable to create step` | The node may lack enroot/pyxis — check with cluster admin | | Job stuck in `PD` state | Check `squeue -j -o %R` for the reason (e.g., resource limits, priority) | diff --git a/.claude/skills/exec-slurm-compile/scripts/compile.sh b/.claude/skills/exec-slurm-compile/scripts/compile.sh index 4822dbceb677..22b7882d55ff 100755 --- a/.claude/skills/exec-slurm-compile/scripts/compile.sh +++ b/.claude/skills/exec-slurm-compile/scripts/compile.sh @@ -19,7 +19,7 @@ # Usage: compile.sh [build_wheel_args...] # # Default build_wheel.py flags: -# -a "100-real" --nvtx --no-venv +# --trt_root /usr/local/tensorrt --benchmarks -a "100-real" --nvtx --no-venv # Any extra arguments after repo_dir are forwarded to build_wheel.py, # overriding the defaults above. @@ -36,6 +36,8 @@ if [[ $# -gt 0 ]]; then else echo "[compile.sh] Running default build command" python3 ./scripts/build_wheel.py \ + --trt_root /usr/local/tensorrt \ + --benchmarks \ -a "100-real" \ --nvtx fi diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 2caf9db60128..dc89ff60b2b8 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -22,26 +22,7 @@ reviews: auto_title_placeholder: '@coderabbitai title' auto_title_instructions: 'Format: "[] ". Category must be one of: fix, feat, doc, infra, style, refactor, perf, test, chore, revert. Enclose the category in square brackets. Title should be concise (<= 60 chars). Example: "[feat] Add logit_bias support".' commit_status: false - high_level_summary_instructions: | - Always produce two review sections in the summary: - - **Dev Engineer Review** - Review all changes for correctness and consistency, including: - - Code changes: correctness, performance, API consistency (CODING_GUIDELINES.md), error handling, regressions. - - Config files: valid values, no typos, consistency with related configs, no unintended scope changes. - - Test list files (test-db/, qa/, waives.txt): correct format, valid test paths, appropriate bug references, no duplicates. - - **QA Engineer Review** - Always include this section when any files under tests/ are touched. - For test-list-only changes (only tests/integration/test_lists/ files): - - List which test-db/ or qa/ files were modified and what entries were added or removed. - - Verdict: "needs follow-up" if CBTS coverage data is unavailable, otherwise "sufficient" or "insufficient". - For test-code changes (files outside tests/integration/test_lists/): - - List test functions added, modified, or removed. - - State whether each is covered in tests/integration/test_lists/ (test-db/ for CI, qa/ for manual QA). - - Verdict: sufficient, insufficient, or needs follow-up. - If no test files are touched, write "No test changes." - collapse_walkthrough: false + collapse_walkthrough: true assess_linked_issues: true related_issues: true related_prs: true @@ -50,27 +31,15 @@ reviews: poem: false review_status: false auto_review: - auto_incremental_review: true + auto_incremental_review: false drafts: false base_branches: ["main", "release/.+"] path_instructions: - path: "tests/**" instructions: | Act as a QA engineer reviewing test changes and coverage for TensorRT-LLM. - Always produce a test coverage summary, even if no issues are found. - - If the change touches ONLY files under tests/integration/test_lists/ (no test-code changes): - - Report which test-db/ or qa/ list files were modified and what entries were added or removed. - - Do NOT require changed test functions for this path. - - Use verdict "needs follow-up" when cbts_touchmap.sqlite or a CBTS coverage report is unavailable - to confirm the impacted test scope; otherwise use "sufficient" or "insufficient". - - If the change includes test-code files (outside tests/integration/test_lists/), the summary must include: - 1. Which test functions were added, modified, or removed. - 2. Whether each changed test is listed in the appropriate test list files under - tests/integration/test_lists/ (test-db/ for CI, qa/ for manual QA). - 3. A coverage verdict: sufficient, insufficient, or needs follow-up. - Keep feedback actionable: reference concrete list file names when suggesting additions. + Keep feedback actionable: suggest concrete list file names and whether + coverage is sufficient, insufficient, or needs follow-up outside the PR. - path: "tests/integration/test_lists/qa/**" instructions: | Files here are manually-triggered QA perf/regression lists, maintained diff --git a/.gitattributes b/.gitattributes index a8ca34bc5151..5f797908576a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,5 +16,4 @@ docs/source/blogs/media/tech_blog10_full_strategy_performance.png filter=lfs dif docs/source/blogs/media/tech_blog10_context_wait_performance.png filter=lfs diff=lfs merge=lfs -text cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/cubin/kernelMetaInfo_cubin.cpp filter=lfs diff=lfs merge=lfs -text cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/cubin/xqa_kernel_cubin.cpp filter=lfs diff=lfs merge=lfs -text -docs/source/blogs/media/tech_blog26_deepseek_v4_hybrid_attention.png filter=lfs diff=lfs merge=lfs -text -docs/source/blogs/media/tech_blog26_deepseek_v4_mhc_moe.png filter=lfs diff=lfs merge=lfs -text +tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/*/*/*.so filter=lfs diff=lfs merge=lfs -text diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4c72f3495a99..934c4a3c37c8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -37,6 +37,8 @@ /tests/unittest @NVIDIA/trt-llm-devs # ===== TensorRT backend (will be deprecated soon) — also on the trt-llm-devs fallback ===== +/cpp/include/tensorrt_llm/plugins @NVIDIA/trt-llm-devs +/cpp/tensorrt_llm/plugins @NVIDIA/trt-llm-devs /tensorrt_llm/builder.py @NVIDIA/trt-llm-devs /tensorrt_llm/commands/build.py @NVIDIA/trt-llm-devs /tensorrt_llm/commands/prune.py @NVIDIA/trt-llm-devs @@ -103,6 +105,7 @@ /cpp/tensorrt_llm/batch_manager @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/common @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/executor @NVIDIA/trt-llm-runtime-devs +/cpp/tensorrt_llm/executor_worker @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/layers @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/nanobind @NVIDIA/trt-llm-runtime-devs /cpp/tensorrt_llm/runtime @NVIDIA/trt-llm-runtime-devs @@ -166,6 +169,7 @@ /examples/llm-api/quickstart_multimodal.py @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners /examples/models @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners /examples/serve/*multimodal* @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners +/scripts/build_cpp_examples.py @NVIDIA/trt-llm-models-devs /scripts/generate_config_database_tests.py @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners /scripts/generate_config_table.py @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-doc-owners /tensorrt_llm/_torch/models @NVIDIA/trt-llm-models-devs @@ -217,11 +221,6 @@ /tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py @NVIDIA/trt-llm-kv-cache-manager-devs /tensorrt_llm/_torch/pyexecutor/resource_manager.py @NVIDIA/trt-llm-kv-cache-manager-devs /tensorrt_llm/runtime/kv_cache_manager_v2 @NVIDIA/trt-llm-kv-cache-manager-devs -/tests/unittest/_torch/executor/test_dual_pool_kv_cache.py @NVIDIA/trt-llm-kv-cache-manager-devs -/tests/unittest/_torch/executor/test_kv_cache* @NVIDIA/trt-llm-kv-cache-manager-devs -/tests/unittest/_torch/executor/test_kv_pool_rebalance.py @NVIDIA/trt-llm-kv-cache-manager-devs -/tests/unittest/_torch/executor/test_kvcache_aware_router.py @NVIDIA/trt-llm-kv-cache-manager-devs -/tests/unittest/_torch/executor/test_mamba_cache_manager.py @NVIDIA/trt-llm-kv-cache-manager-devs /tests/unittest/kv_cache_manager_v2_tests @NVIDIA/trt-llm-kv-cache-manager-devs # ===== DISAGGREGATED SERVING ===== @@ -241,17 +240,7 @@ /tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py @NVIDIA/trt-llm-disagg-devs /tensorrt_llm/disaggregated_params.py @NVIDIA/trt-llm-disagg-devs /tensorrt_llm/serve/openai_disagg_server.py @NVIDIA/trt-llm-disagg-devs -# Disagg tests: co-own with the owning team so disagg-devs review disagg-test changes. -/tests/integration/defs/accuracy/*disagg* @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa -/tests/integration/defs/disaggregated @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa -/tests/integration/defs/stress_test/disagg_cancel @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-qa -/tests/scripts/perf-sanity/disaggregated @NVIDIA/trt-llm-perf-devs @NVIDIA/trt-llm-disagg-devs -/tests/scripts/perf/disaggregated @NVIDIA/trt-llm-perf-devs @NVIDIA/trt-llm-disagg-devs -/tests/unittest/_torch/executor/*disagg* @NVIDIA/trt-llm-runtime-devs @NVIDIA/trt-llm-disagg-devs -/tests/unittest/_torch/multimodal/*disagg* @NVIDIA/trt-llm-models-devs @NVIDIA/trt-llm-disagg-devs /tests/unittest/disaggregated @NVIDIA/trt-llm-disagg-devs -/tests/unittest/llmapi/*disagg* @NVIDIA/trt-llm-runtime-devs @NVIDIA/trt-llm-disagg-devs -/tests/unittest/llmapi/apps/*disagg* @NVIDIA/trt-llm-runtime-devs @NVIDIA/trt-llm-disagg-devs # ===== ATTENTION ===== # Where a kernel is both a dir and sibling .cu/.h, keep the bare dir (subtree) AND a trailing-* (siblings). @@ -437,7 +426,6 @@ /cpp/tensorrt_llm/batch_manager/allocateKvCache.cpp @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp @NVIDIA/trt-llm-kv-cache-manager-devs -/tensorrt_llm/_torch/attention_backend/sparse/*/cache_manager.py @NVIDIA/trt-llm-kv-cache-manager-devs /tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py @NVIDIA/trt-llm-kv-cache-manager-devs /tensorrt_llm/_torch/pyexecutor/resource_manager.py @NVIDIA/trt-llm-kv-cache-manager-devs /cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.h @NVIDIA/trt-llm-kv-cache-manager-devs @@ -455,6 +443,7 @@ # ===== DISAGG BLAST-RADIUS CO-OWNS (Tier-1) ===== # Last-match cross-cutting: adds disagg-devs to shared files that can silently break disagg e2e. /tensorrt_llm/_torch/pyexecutor/resource_manager.py @NVIDIA/trt-llm-kv-cache-manager-devs @NVIDIA/trt-llm-disagg-devs +/tensorrt_llm/_torch/modules/mla.py @NVIDIA/trt-llm-torch-attention-devs @NVIDIA/trt-llm-disagg-devs /tensorrt_llm/_torch/pyexecutor/py_executor.py @NVIDIA/trt-llm-runtime-devs @NVIDIA/trt-llm-disagg-devs /tensorrt_llm/_torch/pyexecutor/model_engine.py @NVIDIA/trt-llm-runtime-devs @NVIDIA/trt-llm-disagg-devs diff --git a/.github/scripts/label_component.py b/.github/scripts/label_component.py deleted file mode 100644 index 1fa64cf7351b..000000000000 --- a/.github/scripts/label_component.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python3 -"""Label pull requests by component, driven by .github/CODEOWNERS. - -For each configured (CODEOWNERS team handle -> label) mapping, this resolves the -effective owners of every file a PR changes using CODEOWNERS last-match-wins -semantics, and applies the mapped label when any changed file is owned by that -team. Labels are only added, never removed. - -Usage: - # single PR (what the GitHub Action runs) - label_component.py --pr 16143 --codeowners .github/CODEOWNERS - # a few PRs - label_component.py --pr 16143 16142 - # sweep every open PR (preview first with --dry-run) - label_component.py --all-open --dry-run - -The repo defaults to the upstream NVIDIA/TensorRT-LLM. CODEOWNERS is read from ---codeowners when given, otherwise fetched from the repo's default branch. The -token comes from GITHUB_TOKEN / GH_TOKEN, falling back to `gh auth token`. -""" - -import argparse -import os -import re -import subprocess -import sys - -import requests - -GITHUB_API_URL = "https://api.github.com" -DEFAULT_REPO = "NVIDIA/TensorRT-LLM" - -# CODEOWNERS team handle (lower-cased) -> label to apply. -# Extend this dict to cover more components. -COMPONENT_LABELS = { - "@nvidia/trt-llm-torch-visual-gen-devs": "VisualGen", -} - - -# --- CODEOWNERS parsing / matching --------------------------------------- - - -def parse_codeowners(text): - """Parse CODEOWNERS into an ordered list of (compiled_regex, owners).""" - rules = [] - for raw in text.splitlines(): - line = raw.split("#", 1)[0].strip() - if not line: - continue - parts = line.split() - pattern, owners = parts[0], [o.lower() for o in parts[1:]] - rules.append((_pattern_to_regex(pattern), owners)) - return rules - - -def _pattern_to_regex(pattern): - """Translate a CODEOWNERS (gitignore-style) pattern to a regex. - - '*' matches within a path segment, '**' crosses segments, and a directory - pattern matches everything beneath it. All CODEOWNERS patterns here are - root-anchored. - """ - body = re.escape(pattern.strip("/")) - body = body.replace(r"\*\*", ".*").replace(r"\*", "[^/]*") - return re.compile(rf"(?:{body})(?:/.*)?$") - - -def owners_for_path(path, rules): - """Effective CODEOWNERS owners for a path (last matching rule wins).""" - owners = [] - for regex, rule_owners in rules: - if regex.match(path): - owners = rule_owners - return owners - - -def labels_for_files(files, rules, component_labels=COMPONENT_LABELS): - labels = set() - for path in files: - owners = owners_for_path(path, rules) - for team, label in component_labels.items(): - if team in owners: - labels.add(label) - return labels - - -# --- GitHub access ------------------------------------------------------- - - -def resolve_token(): - for var in ("GITHUB_TOKEN", "GH_TOKEN"): - if os.environ.get(var): - return os.environ[var] - try: - return subprocess.check_output(["gh", "auth", "token"], text=True).strip() - except (OSError, subprocess.CalledProcessError): - raise SystemExit("No token found: set GITHUB_TOKEN / GH_TOKEN, or run `gh auth login`.") - - -def make_session(token): - session = requests.Session() - session.headers.update( - { - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "trtllm-label-component/1.0", - "Authorization": f"token {token}", - } - ) - return session - - -def load_codeowners(session, repo, path): - if path: - with open(path, encoding="utf-8") as fh: - return parse_codeowners(fh.read()) - r = session.get( - f"{GITHUB_API_URL}/repos/{repo}/contents/.github/CODEOWNERS", - headers={"Accept": "application/vnd.github.raw"}, - timeout=30, - ) - r.raise_for_status() - return parse_codeowners(r.text) - - -def iter_open_prs(session, repo, limit=None): - """Yield (number, existing_labels) for open PRs; labels come free here.""" - page, seen = 1, 0 - while True: - r = session.get( - f"{GITHUB_API_URL}/repos/{repo}/pulls", - params={ - "state": "open", - "per_page": 100, - "page": page, - "sort": "created", - "direction": "desc", - }, - timeout=30, - ) - r.raise_for_status() - batch = r.json() - if not batch: - return - for pr in batch: - yield pr["number"], {lbl["name"] for lbl in pr.get("labels", [])} - seen += 1 - if limit and seen >= limit: - return - page += 1 - - -def get_changed_files(session, repo, pr_number): - files, page = [], 1 - while True: - r = session.get( - f"{GITHUB_API_URL}/repos/{repo}/pulls/{pr_number}/files", - params={"per_page": 100, "page": page}, - timeout=30, - ) - r.raise_for_status() - batch = r.json() - if not batch: - break - files.extend(f["filename"] for f in batch) - page += 1 - return files - - -def get_pr_labels(session, repo, pr_number): - r = session.get(f"{GITHUB_API_URL}/repos/{repo}/issues/{pr_number}", timeout=30) - r.raise_for_status() - return {lbl["name"] for lbl in r.json().get("labels", [])} - - -def add_labels(session, repo, pr_number, labels): - r = session.post( - f"{GITHUB_API_URL}/repos/{repo}/issues/{pr_number}/labels", - json={"labels": labels}, - timeout=30, - ) - r.raise_for_status() - - -def process_pr(session, repo, pr_number, rules, existing_labels, dry_run): - """Return the labels added (or that would be added). Empty if none.""" - files = get_changed_files(session, repo, pr_number) - wanted = labels_for_files(files, rules) - to_add = sorted(wanted - existing_labels) - if not to_add: - return [] - if dry_run: - print(f"PR #{pr_number}: would add {to_add}") - else: - add_labels(session, repo, pr_number, to_add) - print(f"PR #{pr_number}: added {to_add}") - return to_add - - -# --- CLI ----------------------------------------------------------------- - - -def parse_args(argv): - ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - ap.add_argument("--repo", default=DEFAULT_REPO, help=f"owner/name (default: {DEFAULT_REPO})") - target = ap.add_mutually_exclusive_group(required=True) - target.add_argument("--pr", type=int, nargs="+", metavar="N", help="label these PR number(s)") - target.add_argument("--all-open", action="store_true", help="label every open PR in the repo") - ap.add_argument( - "--codeowners", - metavar="PATH", - help="local CODEOWNERS file; if omitted, fetched from the repo's default branch", - ) - ap.add_argument( - "--limit", type=int, metavar="N", help="with --all-open, cap the number of PRs scanned" - ) - ap.add_argument( - "--dry-run", action="store_true", help="report what would change without adding labels" - ) - return ap.parse_args(argv) - - -def main(argv=None): - args = parse_args(argv) - session = make_session(resolve_token()) - rules = load_codeowners(session, args.repo, args.codeowners) - - if args.pr: - targets = [(n, get_pr_labels(session, args.repo, n)) for n in args.pr] - else: - targets = iter_open_prs(session, args.repo, args.limit) - - scanned, labeled, failed = 0, 0, 0 - for number, existing in targets: - scanned += 1 - # Isolate per-PR failures so one bad PR (transient 5xx, missing label) - # does not abort an --all-open sweep. - try: - if process_pr(session, args.repo, number, rules, existing, args.dry_run): - labeled += 1 - except requests.HTTPError as exc: - failed += 1 - print(f"PR #{number}: failed ({exc})", file=sys.stderr) - - verb = "would be labeled" if args.dry_run else "labeled" - print(f"Scanned {scanned} PR(s); {labeled} {verb}; {failed} failed.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml index f5a60658d8dd..d67685f216f2 100644 --- a/.github/workflows/blossom-ci.yml +++ b/.github/workflows/blossom-ci.yml @@ -187,7 +187,6 @@ jobs: "JunyiXu-nv", "JyChang012", "kaiyux", - "Kambili", "kanghui0204", "karljang", "karthikvetrivel", @@ -310,7 +309,6 @@ jobs: "shuyixiong", "shyeh25", "SimengLiu-nv", - "siyidNV", "sklevtsov-nvidia", "StanleySun639", "stnie", diff --git a/.github/workflows/bot-command.yml b/.github/workflows/bot-command.yml index 30b5de652f55..0a112dbf5435 100644 --- a/.github/workflows/bot-command.yml +++ b/.github/workflows/bot-command.yml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -52,14 +52,14 @@ jobs: "`--disable-reuse-test ` *(OPTIONAL)* : Explicitly prevent the pipeline from reusing build artifacts and skipping successful test stages from a previous pipeline. Ensure that all builds and tests are run regardless of previous successes.\n\n" + "`--disable-fail-fast ` *(OPTIONAL)* : Disable fail fast on build/tests/infra failures.\n\n" + "`--skip-test ` *(OPTIONAL)* : Skip all test stages, but still run build stages, package stages and sanity check stages. Note: Does **NOT** update GitHub check status.\n\n" + - "`--stage-list \"A10-PyTorch-1, xxx\"` *(OPTIONAL)* : Only run the specified test stages. Supports wildcard `*` for pattern matching (e.g., `\"*PerfSanity*\"` matches all stages containing PerfSanity). Examples: \"A10-PyTorch-1, xxx\", \"*PerfSanity*\". The patterns `\"*\"`, `\"*Post-Merge*\"`, and `\"*PerfSanity*\"`, including equivalent escaped or repeated-star forms and their use in comma-separated lists, require the `ci: post-merge approved` PR label. Note: Does **NOT** update GitHub check status.\n\n" + + "`--stage-list \"A10-PyTorch-1, xxx\"` *(OPTIONAL)* : Only run the specified test stages. Supports wildcard `*` for pattern matching (e.g., `\"*PerfSanity*\"` matches all stages containing PerfSanity). Examples: \"A10-PyTorch-1, xxx\", \"*PerfSanity*\". Note: Does **NOT** update GitHub check status.\n\n" + "`--gpu-type \"A30, H100_PCIe\"` *(OPTIONAL)* : Only run the test stages on the specified GPU types. Examples: \"A30, H100_PCIe\". Note: Does **NOT** update GitHub check status.\n\n" + "`--test-backend \"pytorch, cpp\"` *(OPTIONAL)* : Skip test stages which don't match the specified backends. Only support [pytorch, cpp, tensorrt, triton]. Examples: \"pytorch, cpp\" (does not run test stages with tensorrt or triton backend). Note: Does **NOT** update GitHub pipeline status.\n\n" + "`--only-multi-gpu-test ` *(OPTIONAL)* : Only run the multi-GPU tests. Note: Does **NOT** update GitHub check status.\n\n" + "`--disable-multi-gpu-test ` *(OPTIONAL)* : Disable the multi-GPU tests. Note: Does **NOT** update GitHub check status.\n\n" + "`--add-multi-gpu-test ` *(OPTIONAL)* : Force run the multi-GPU tests in addition to running L0 pre-merge pipeline.\n\n" + - "`--post-merge ` *(OPTIONAL)* : Run the L0 post-merge pipeline instead of the ordinary L0 pre-merge pipeline. Requires the `ci: post-merge approved` PR label applied by an active member of `NVIDIA/trt-llm-ci-approvers`. The approval label remains in place when new commits are pushed.\n\n" + - "`--extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\"` *(OPTIONAL)* : Run the ordinary L0 pre-merge pipeline and specified test stages. Supports wildcard `*` for pattern matching. Examples: --extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\", --extra-stage \"*Post-Merge*\". The patterns `\"*\"`, `\"*Post-Merge*\"`, and `\"*PerfSanity*\"`, including equivalent escaped or repeated-star forms and their use in comma-separated lists, require the `ci: post-merge approved` PR label.\n\n" + + "`--post-merge ` *(OPTIONAL)* : Run the L0 post-merge pipeline instead of the ordinary L0 pre-merge pipeline.\n\n" + + "`--extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\"` *(OPTIONAL)* : Run the ordinary L0 pre-merge pipeline and specified test stages. Supports wildcard `*` for pattern matching. Examples: --extra-stage \"H100_PCIe-TensorRT-Post-Merge-1, xxx\", --extra-stage \"*Post-Merge*\".\n\n" + "`--detailed-log ` *(OPTIONAL)* : Enable flushing out all logs to the Jenkins console. This will significantly increase the log volume and may slow down the job.\n\n" + "`--debug ` *(OPTIONAL)* : **Experimental feature**. Enable access to the CI container for debugging purpose. Note: Specify exactly one stage in the `stage-list` parameter to access the appropriate container environment. Note: Does **NOT** update GitHub check status.\n\n" + "`--high-priority ` *(OPTIONAL)* : Run the pipeline with high priority. This option is restricted to authorized users only and will route the job to a high-priority queue.\n\n" + diff --git a/.github/workflows/label_component_pr.yml b/.github/workflows/label_component_pr.yml deleted file mode 100644 index 5d584eb237cd..000000000000 --- a/.github/workflows/label_component_pr.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Label Component for PR - -on: - pull_request_target: - types: [opened, reopened] - -permissions: - contents: read - pull-requests: write - -jobs: - label-component: - runs-on: ubuntu-latest - if: github.repository == 'NVIDIA/TensorRT-LLM' - # This workflow is advisory: it must never turn a PR check red. Every step - # is continue-on-error, so the check is always green even if setup or - # labeling fails. - steps: - - name: Checkout base repository - continue-on-error: true - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Set up Python - continue-on-error: true - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Install dependencies - continue-on-error: true - run: pip install requests - - - name: Label PR by component - continue-on-error: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: >- - python .github/scripts/label_component.py - --pr ${{ github.event.pull_request.number }} - --codeowners .github/CODEOWNERS diff --git a/.github/workflows/lfs-sync.yml b/.github/workflows/lfs-sync.yml index 3143afedbd90..bccdedc858e1 100644 --- a/.github/workflows/lfs-sync.yml +++ b/.github/workflows/lfs-sync.yml @@ -44,12 +44,9 @@ jobs: steps: - name: Checkout merge commit without LFS smudge - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.merge_commit_sha }} - # This job only handles merged PRs and does not execute code from the - # checked-out tree, so explicitly allow the required merge checkout. - allow-unsafe-pr-checkout: true lfs: false fetch-depth: 2 token: ${{ secrets.GITHUB_TOKEN }} @@ -203,7 +200,7 @@ jobs: - name: Comment on PR — already in storage if: steps.detect-lfs.outputs.lfs_status == 'present' - uses: actions/github-script@v8 + uses: actions/github-script@v7 with: script: | const files = require('fs').readFileSync('/tmp/lfs_files.txt', 'utf8').trim().split('\n'); @@ -224,7 +221,7 @@ jobs: - name: Comment on PR — sync succeeded if: steps.detect-lfs.outputs.lfs_status == 'pending' && steps.verify.outcome == 'success' - uses: actions/github-script@v8 + uses: actions/github-script@v7 with: script: | const files = require('fs').readFileSync('/tmp/lfs_files.txt', 'utf8').trim().split('\n'); @@ -245,7 +242,7 @@ jobs: - name: Comment on PR — sync failed if: always() && steps.detect-lfs.outputs.lfs_status == 'pending' && (steps.fetch-lfs.outcome != 'success' || steps.push-lfs.outcome != 'success' || steps.verify.outcome != 'success') - uses: actions/github-script@v8 + uses: actions/github-script@v7 with: script: | const forkRepo = context.payload.pull_request.head.repo.full_name; diff --git a/.github/workflows/post-merge-approval.yml b/.github/workflows/post-merge-approval.yml deleted file mode 100644 index 66b3490843dd..000000000000 --- a/.github/workflows/post-merge-approval.yml +++ /dev/null @@ -1,222 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Guard Post-Merge Approval Label - -on: - # Intentional: this runs static default-branch API logic only. It never checks - # out or executes PR code; it uses GitHub APIs to validate membership and - # manage this PR's approval label/comment. - pull_request_target: - types: [labeled] - -permissions: - contents: read - pull-requests: write - -jobs: - guard-post-merge-approval: - concurrency: - group: post-merge-approval-${{ github.event.pull_request.number }} - cancel-in-progress: false - # Keep a started job eligible to reach fail-closed cleanup after cancellation. - if: >- - always() && - github.repository == 'NVIDIA/TensorRT-LLM' && - github.event.action == 'labeled' && - github.event.label.name == 'ci: post-merge approved' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Validate post-merge approver - id: validate - if: github.event.action == 'labeled' - uses: actions/github-script@v8 - with: - github-token: ${{ secrets.TRTLLM_AGENT_SHARED_TOKEN }} - result-encoding: string - script: | - const approvalLabel = 'ci: post-merge approved'; - const owner = context.repo.owner; - const repo = context.repo.repo; - const issueNumber = context.payload.pull_request.number; - let actor = context.payload.sender?.login || context.actor; - let labelEventId = ''; - let timelineVerified = false; - - try { - const events = await github.paginate( - github.rest.issues.listEventsForTimeline, - { owner, repo, issue_number: issueNumber, per_page: 100 } - ); - const latestApprovalEvent = events - .filter( - (event) => - event.event === 'labeled' && - event.label?.name?.toLowerCase() === approvalLabel.toLowerCase() - ) - .at(-1); - if (latestApprovalEvent) { - actor = latestApprovalEvent.actor?.login || actor; - labelEventId = String(latestApprovalEvent.id); - timelineVerified = true; - } else { - core.warning('Could not identify the latest post-merge approval event.'); - } - } catch (error) { - core.warning( - 'Could not read the latest post-merge approval event: ' + error.message - ); - } - - core.setOutput('validated_actor', actor); - core.setOutput('label_event_id', labelEventId); - if (!timelineVerified) { - return 'false'; - } - try { - const response = await github.request( - 'GET /orgs/{org}/teams/{team_slug}/memberships/{username}', - { - org: 'NVIDIA', - team_slug: 'trt-llm-ci-approvers', - username: actor, - } - ); - const authorized = response.data.state === 'active'; - console.log( - actor + ' active membership in NVIDIA/trt-llm-ci-approvers: ' + authorized - ); - return authorized ? 'true' : 'false'; - } catch (error) { - if (error.status === 404) { - console.log( - actor + ' is not an active member of NVIDIA/trt-llm-ci-approvers.' - ); - } else { - core.warning( - 'Could not verify post-merge approver ' + actor + ': ' + error.message - ); - } - return 'false'; - } - - - name: Clear unauthorized post-merge approval - # Let a started job attempt fail-closed cleanup after normal cancellation. - if: always() && steps.validate.outputs.result != 'true' - uses: actions/github-script@v8 - env: - VALIDATED_ACTOR: ${{ steps.validate.outputs.validated_actor }} - VALIDATED_LABEL_EVENT_ID: ${{ steps.validate.outputs.label_event_id }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const approvalLabel = 'ci: post-merge approved'; - const actor = - process.env.VALIDATED_ACTOR || - context.payload.sender?.login || - context.actor; - const validatedEventId = process.env.VALIDATED_LABEL_EVENT_ID || ''; - const owner = context.repo.owner; - const repo = context.repo.repo; - const issueNumber = context.payload.pull_request.number; - - try { - const pullRequest = await github.rest.pulls.get({ - owner, - repo, - pull_number: issueNumber, - }); - const labelIsPresent = pullRequest.data.labels.some( - (label) => - label.name?.toLowerCase() === approvalLabel.toLowerCase() - ); - if (!labelIsPresent) { - console.log('Post-merge approval label is already absent; no cleanup needed.'); - return; - } - } catch (error) { - core.warning( - 'Could not read the current post-merge approval label state; continuing validation: ' + - error.message - ); - } - - try { - const events = await github.paginate( - github.rest.issues.listEventsForTimeline, - { owner, repo, issue_number: issueNumber, per_page: 100 } - ); - const latestApprovalEvent = events - .filter( - (event) => - event.event === 'labeled' && - event.label?.name?.toLowerCase() === approvalLabel.toLowerCase() - ) - .at(-1); - const latestEventId = latestApprovalEvent - ? String(latestApprovalEvent.id) - : ''; - const latestActor = latestApprovalEvent?.actor?.login || ''; - if (!latestApprovalEvent) { - core.warning( - 'Could not identify the latest post-merge approval event; removing the label to fail closed.' - ); - } else if (!latestActor) { - core.warning( - 'Could not identify the latest post-merge approval actor; removing the label to fail closed.' - ); - } else if (!validatedEventId) { - core.warning( - 'The validation run did not bind an approval event; removing the label to fail closed.' - ); - } else if ( - latestEventId !== validatedEventId || - latestActor !== actor - ) { - console.log( - 'A newer post-merge approval event was found; leaving it for its own validation run.' - ); - return; - } - } catch (error) { - core.warning( - 'Could not re-check the latest post-merge approval event; removing the label to fail closed: ' + - error.message - ); - } - - try { - await github.rest.issues.removeLabel({ - owner, - repo, - issue_number: issueNumber, - name: approvalLabel, - }); - } catch (error) { - if (error.status !== 404) { - throw error; - } - } - - await github.rest.issues.createComment({ - owner, - repo, - issue_number: issueNumber, - body: - 'Removed the "' + approvalLabel + '" label because @' + actor + - ' could not be verified as an active member of ' + - 'NVIDIA/trt-llm-ci-approvers. Ask a member of that team to apply it.', - }); diff --git a/.gitignore b/.gitignore index 9a9332335ca5..47e39a1b2d71 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ tensorrt_llm/flash_mla/ tensorrt_llm/flash_mla_cpp_tllm.*.so tensorrt_llm/flash_mla_cpp_tllm.pyi tensorrt_llm/runtime/kv_cache_manager_v2/**/*.so +!tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/**/*.so **/*__mypyc*.so tensorrt_llm/scripts *docs/cpp_docs* diff --git a/.gitmodules b/.gitmodules index 627760b34da2..e69de29bb2d1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "3rdparty/MSA"] - path = 3rdparty/MSA - url = https://gitlab.com/nvidia/tensorrt-llm/oss-components/msa.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37c51fbbb377..452c3be57ea1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,14 @@ common-files: &common_files | .devcontainer/make_env.py | .github/scripts/label_community_user.py | .github/scripts/pr_checklist_check.py | + benchmarks/cpp/__init__.py | + benchmarks/cpp/prepare_dataset.py | + benchmarks/cpp/utils/__init__.py | + benchmarks/cpp/utils/convert_nemo_dataset.py | + benchmarks/cpp/utils/generate_rand_loras.py | + benchmarks/cpp/utils/prepare_real_data.py | + benchmarks/cpp/utils/prepare_synthetic_data.py | + benchmarks/cpp/utils/utils.py | cpp/conanfile.py | cpp/kernels/fmha_v2/conftest.py | cpp/kernels/fmha_v2/fmha_test.py | @@ -33,17 +41,58 @@ common-files: &common_files | cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py | cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py | cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py | + cpp/tests/resources/scripts/build_chatglm_engines.py | + cpp/tests/resources/scripts/build_eagle_engines.py | + cpp/tests/resources/scripts/build_enc_dec_engines.py | + cpp/tests/resources/scripts/build_engines_utils.py | + cpp/tests/resources/scripts/build_gpt_engines.py | + cpp/tests/resources/scripts/build_gptj_engines.py | + cpp/tests/resources/scripts/build_llama_engines.py | + cpp/tests/resources/scripts/build_mamba_engines.py | + cpp/tests/resources/scripts/build_medusa_engines.py | + cpp/tests/resources/scripts/build_recurrentgemma_engines.py | + cpp/tests/resources/scripts/build_redrafter_engines.py | + cpp/tests/resources/scripts/generate_expected_chatglm_output.py | + cpp/tests/resources/scripts/generate_expected_eagle_output.py | + cpp/tests/resources/scripts/generate_expected_enc_dec_output.py | + cpp/tests/resources/scripts/generate_expected_gpt_output.py | + cpp/tests/resources/scripts/generate_expected_gptj_output.py | + cpp/tests/resources/scripts/generate_expected_llama_output.py | + cpp/tests/resources/scripts/generate_expected_mamba_output.py | + cpp/tests/resources/scripts/generate_expected_medusa_output.py | + cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py | + cpp/tests/resources/scripts/generate_expected_redrafter_output.py | + cpp/tests/resources/scripts/generate_hf_gpt_output.py | cpp/tests/resources/scripts/generate_test_lora_weights.py | + cpp/tests/resources/scripts/io_converter.py | docs/source/conf.py | docs/source/helper.py | examples/apps/chat.py | examples/apps/fastapi_server.py | + examples/bindings/executor/example_advanced.py | + examples/bindings/executor/example_basic.py | + examples/bindings/executor/example_debug.py | + examples/bindings/executor/example_logits_processor.py | examples/disaggregated/clients/disagg_client.py | examples/disaggregated/slurm/benchmark/submit.py | + examples/dora/normalize_weights.py | + examples/eagle/convert_checkpoint.py | + examples/eval_long_context.py | + examples/generate_checkpoint_config.py | + examples/generate_xgrammar_tokenizer_info.py | + examples/hf_lora_convert.py | examples/infinitebench/args.py | examples/infinitebench/compute_scores.py | examples/infinitebench/construct_synthetic_dataset.py | examples/infinitebench/eval_utils.py | + examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py | + examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py | + examples/llm-api/_tensorrt_engine/llm_inference_customize.py | + examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py | + examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py | + examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py | + examples/llm-api/_tensorrt_engine/llm_quantization.py | + examples/llm-api/_tensorrt_engine/quickstart_example.py | examples/llm-api/llm_guided_decoding.py | examples/llm-api/llm_inference.py | examples/llm-api/llm_inference_async.py | @@ -63,17 +112,122 @@ common-files: &common_files | examples/llm-api/quickstart_example.py | examples/llm-api/quickstart_multimodal.py | examples/llm-api/star_attention.py | + examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py | examples/longbench/eval_longbench_v1.py | + examples/medusa/convert_checkpoint.py | + examples/mmlu.py | + examples/models/contrib/baichuan/convert_checkpoint.py | + examples/models/contrib/bloom/convert_checkpoint.py | + examples/models/contrib/chatglm-6b/tokenization_chatglm.py | + examples/models/contrib/chatglm2-6b/tokenization_chatglm.py | + examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py | + examples/models/contrib/cogvlm/convert_checkpoint.py | + examples/models/contrib/dbrx/convert_checkpoint.py | + examples/models/contrib/deepseek_v1/__init__.py | + examples/models/contrib/deepseek_v1/convert_checkpoint.py | + examples/models/contrib/deepseek_v2/convert_checkpoint.py | + examples/models/contrib/dit/convert_checkpoint.py | + examples/models/contrib/dit/diffusion.py | + examples/models/contrib/dit/sample.py | + examples/models/contrib/dit/utils_modelopt.py | + examples/models/contrib/dit/vae_decoder_trt.py | + examples/models/contrib/falcon/convert_checkpoint.py | + examples/models/contrib/gptj/convert_checkpoint.py | + examples/models/contrib/gptneox/convert_checkpoint.py | + examples/models/contrib/grok/convert_checkpoint.py | + examples/models/contrib/mmdit/convert_checkpoint.py | + examples/models/contrib/mmdit/sample.py | + examples/models/contrib/mpt/convert_checkpoint.py | + examples/models/contrib/opt/convert_checkpoint.py | + examples/models/contrib/sdxl/build_sdxl_unet.py | + examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py | + examples/models/contrib/sdxl/run_sdxl.py | + examples/models/contrib/stdit/aspect.py | + examples/models/contrib/stdit/convert_checkpoint.py | + examples/models/contrib/stdit/pipeline_tllm.py | + examples/models/contrib/stdit/sample.py | + examples/models/contrib/stdit/scheduler.py | + examples/models/contrib/stdit/text_encoder.py | + examples/models/contrib/stdit/utils.py | + examples/models/contrib/stdit/vae.py | + examples/models/contrib/stdit/video_transforms.py | + examples/models/core/bert/__init__.py | + examples/models/core/bert/convert_checkpoint.py | + examples/models/core/bert/run.py | + examples/models/core/bert/utils.py | + examples/models/core/commandr/convert_checkpoint.py | + examples/models/core/enc_dec/__init__.py | + examples/models/core/enc_dec/convert_checkpoint.py | + examples/models/core/enc_dec/helper.py | + examples/models/core/enc_dec/run.py | + examples/models/core/gemma/convert_checkpoint.py | + examples/models/core/glm-4-9b/convert_checkpoint.py | + examples/models/core/glm-4-9b/tokenization_chatglm.py | + examples/models/core/gpt/convert_checkpoint.py | + examples/models/core/gpt/merge_ptuning_tables.py | + examples/models/core/gpt/nemo_lora_convert.py | + examples/models/core/gpt/nemo_prompt_convert.py | + examples/models/core/gpt/run_hf.py | examples/models/core/gpt_oss/openai_chat_client_function_calling.py | + examples/models/core/internlm2/convert_checkpoint.py | examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py | + examples/models/core/llama/convert_checkpoint.py | + examples/models/core/llama/summarize_long.py | + examples/models/core/mamba/convert_checkpoint.py | + examples/models/core/mllama/convert_checkpoint.py | + examples/models/core/multimodal/__init__.py | + examples/models/core/multimodal/build_multimodal_engine.py | + examples/models/core/multimodal/eval.py | + examples/models/core/multimodal/run.py | + examples/models/core/multimodal/utils.py | + examples/models/core/nemotron_nas/calibration_utils.py | + examples/models/core/nemotron_nas/convert_checkpoint.py | + examples/models/core/phi/convert_checkpoint.py | + examples/models/core/qwen/convert_checkpoint.py | + examples/models/core/qwen2audio/run.py | + examples/models/core/qwen2audio/run_chat.py | + examples/models/core/qwen2audio/utils.py | + examples/models/core/qwenvl/run.py | + examples/models/core/qwenvl/run_chat.py | + examples/models/core/qwenvl/show_pic.py | + examples/models/core/qwenvl/vit_onnx_trt.py | + examples/models/core/recurrentgemma/convert_checkpoint.py | + examples/models/core/vit/convert_checkpoint.py | + examples/models/core/whisper/convert_checkpoint.py | + examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py | + examples/models/core/whisper/run.py | + examples/models/core/whisper/tokenizer.py | + examples/models/core/whisper/whisper_utils.py | + examples/ngram/run_dtm_ngram.py | + examples/openai_triton/manual_plugin/build.py | + examples/openai_triton/manual_plugin/fmha_triton.py | + examples/openai_triton/manual_plugin/plugin.py | + examples/openai_triton/manual_plugin/run.py | + examples/openai_triton/plugin_autogen/build_engine.py | + examples/openai_triton/plugin_autogen/kernel_config.py | + examples/openai_triton/plugin_autogen/run_engine.py | + examples/python_plugin/build_lookup.py | + examples/python_plugin/plugin_lib/__init__.py | + examples/python_plugin/plugin_lib/lookup_kernel.py | + examples/python_plugin/plugin_lib/lookup_plugin.py | + examples/python_plugin/run_lookup.py | + examples/quantization/quantize.py | examples/quantization/quantize_mixed_precision_moe.py | examples/ray_orchestrator/llm_inference_async_ray.py | examples/ray_orchestrator/llm_inference_distributed_ray.py | + examples/redrafter/convert_checkpoint.py | + examples/run.py | examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py | examples/scaffolding/contrib/DeepConf/run_generation.py | examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py | examples/scaffolding/contrib/TreeInference/run_mcts_example.py | examples/scaffolding/contrib/TreeInference/run_tot_example.py | + examples/scaffolding/contrib/mcp/e2b/e2bserver.py | + examples/scaffolding/contrib/mcp/e2b/main.py | + examples/scaffolding/contrib/mcp/mcptest.py | + examples/scaffolding/contrib/mcp/weather/weather.py | + examples/scaffolding/contrib/mcp/websearch/main.py | + examples/scaffolding/contrib/mcp/websearch/websearch.py | examples/scaffolding/run_basic_generation.py | examples/scaffolding/run_best_of_n_with_reward.py | examples/scaffolding/run_majority_vote_aime24.py | @@ -83,6 +237,8 @@ common-files: &common_files | examples/serve/openai_completion_client.py | examples/serve/openai_completion_client_for_lora.py | examples/serve/openai_completion_client_json_schema.py | + examples/summarize.py | + examples/utils.py | examples/wide_ep/ep_load_balancer/generate_eplb_config.py | examples/wide_ep/ep_load_balancer/report_load_statistics.py | examples/wide_ep/ep_load_balancer/utils.py | @@ -90,10 +246,12 @@ common-files: &common_files | jenkins/scripts/mergeWaiveList.py | jenkins/scripts/open_search_db.py | jenkins/scripts/test_rerun.py | + scripts/build_cpp_examples.py | scripts/build_wheel.py | scripts/check_test_list.py | scripts/dco_check.py | scripts/format_test_list.py | + scripts/generate_duration.py | scripts/generate_lock_file.py | scripts/get_wheel_from_package.py | scripts/git_replace.py | @@ -104,6 +262,7 @@ common-files: &common_files | setup.py | tensorrt_llm/__init__.py | tensorrt_llm/_ray_utils.py | + tensorrt_llm/_tensorrt_engine/__init__.py | tensorrt_llm/_torch/__init__.py | tensorrt_llm/_torch/attention_backend/__init__.py | tensorrt_llm/_torch/attention_backend/flashinfer.py | @@ -307,6 +466,7 @@ common-files: &common_files | tensorrt_llm/_torch/pyexecutor/guided_decoder.py | tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py | tensorrt_llm/_torch/pyexecutor/handle_logits.py | + tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py | tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py | tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py | tensorrt_llm/_torch/pyexecutor/llm_request.py | @@ -344,6 +504,11 @@ common-files: &common_files | tensorrt_llm/bench/benchmark/utils/asynchronous.py | tensorrt_llm/bench/benchmark/utils/general.py | tensorrt_llm/bench/benchmark/utils/processes.py | + tensorrt_llm/bench/build/__init__.py | + tensorrt_llm/bench/build/build.py | + tensorrt_llm/bench/build/dataclasses.py | + tensorrt_llm/bench/build/tuning.py | + tensorrt_llm/bench/build/utils.py | tensorrt_llm/bench/dataclasses/__init__.py | tensorrt_llm/bench/dataclasses/configuration.py | tensorrt_llm/bench/dataclasses/engine.py | @@ -353,9 +518,13 @@ common-files: &common_files | tensorrt_llm/bench/dataclasses/statistics.py | tensorrt_llm/bench/utils/__init__.py | tensorrt_llm/bench/utils/data.py | + tensorrt_llm/builder.py | tensorrt_llm/commands/__init__.py | tensorrt_llm/commands/bench.py | + tensorrt_llm/commands/build.py | tensorrt_llm/commands/eval.py | + tensorrt_llm/commands/prune.py | + tensorrt_llm/commands/refit.py | tensorrt_llm/commands/serve.py | tensorrt_llm/evaluate/__init__.py | tensorrt_llm/evaluate/cnn_dailymail.py | @@ -391,7 +560,23 @@ common-files: &common_files | tensorrt_llm/inputs/multimodal.py | tensorrt_llm/inputs/registry.py | tensorrt_llm/inputs/utils.py | + tensorrt_llm/layers/__init__.py | + tensorrt_llm/layers/activation.py | + tensorrt_llm/layers/attention.py | + tensorrt_llm/layers/cast.py | + tensorrt_llm/layers/conv.py | + tensorrt_llm/layers/embedding.py | + tensorrt_llm/layers/language_adapter.py | + tensorrt_llm/layers/linear.py | + tensorrt_llm/layers/lora.py | + tensorrt_llm/layers/mlp.py | + tensorrt_llm/layers/moe.py | + tensorrt_llm/layers/normalization.py | + tensorrt_llm/layers/pooling.py | + tensorrt_llm/layers/recurrent.py | + tensorrt_llm/layers/ssm.py | tensorrt_llm/llmapi/__init__.py | + tensorrt_llm/llmapi/build_cache.py | tensorrt_llm/llmapi/disagg_utils.py | tensorrt_llm/llmapi/kv_cache_type.py | tensorrt_llm/llmapi/llm.py | @@ -414,17 +599,179 @@ common-files: &common_files | tensorrt_llm/metrics/enums.py | tensorrt_llm/models/__init__.py | tensorrt_llm/models/automodel.py | + tensorrt_llm/models/baichuan/__init__.py | + tensorrt_llm/models/baichuan/config.py | + tensorrt_llm/models/baichuan/convert.py | + tensorrt_llm/models/baichuan/model.py | + tensorrt_llm/models/bert/__init__.py | + tensorrt_llm/models/bert/config.py | + tensorrt_llm/models/bert/convert.py | + tensorrt_llm/models/bert/model.py | + tensorrt_llm/models/bloom/__init__.py | + tensorrt_llm/models/bloom/model.py | + tensorrt_llm/models/chatglm/__init__.py | + tensorrt_llm/models/chatglm/config.py | + tensorrt_llm/models/chatglm/convert.py | + tensorrt_llm/models/chatglm/model.py | + tensorrt_llm/models/clip/__init__.py | + tensorrt_llm/models/clip/model.py | + tensorrt_llm/models/cogvlm/__init__.py | + tensorrt_llm/models/cogvlm/config.py | + tensorrt_llm/models/cogvlm/convert.py | + tensorrt_llm/models/cogvlm/model.py | + tensorrt_llm/models/commandr/__init__.py | + tensorrt_llm/models/commandr/config.py | + tensorrt_llm/models/commandr/model.py | tensorrt_llm/models/convert_utils.py | + tensorrt_llm/models/dbrx/__init__.py | + tensorrt_llm/models/dbrx/config.py | + tensorrt_llm/models/dbrx/model.py | + tensorrt_llm/models/deepseek_v1/__init__.py | + tensorrt_llm/models/deepseek_v1/config.py | + tensorrt_llm/models/deepseek_v1/convert.py | + tensorrt_llm/models/deepseek_v1/model.py | + tensorrt_llm/models/deepseek_v2/__init__.py | + tensorrt_llm/models/deepseek_v2/config.py | + tensorrt_llm/models/deepseek_v2/convert.py | + tensorrt_llm/models/deepseek_v2/model.py | + tensorrt_llm/models/dit/__init__.py | + tensorrt_llm/models/dit/model.py | + tensorrt_llm/models/eagle/__init__.py | + tensorrt_llm/models/eagle/config.py | + tensorrt_llm/models/eagle/model.py | + tensorrt_llm/models/enc_dec/__init__.py | + tensorrt_llm/models/enc_dec/model.py | + tensorrt_llm/models/falcon/__init__.py | + tensorrt_llm/models/falcon/config.py | + tensorrt_llm/models/falcon/convert.py | + tensorrt_llm/models/falcon/model.py | + tensorrt_llm/models/gemma/__init__.py | + tensorrt_llm/models/gemma/config.py | + tensorrt_llm/models/gemma/convert.py | + tensorrt_llm/models/gemma/model.py | + tensorrt_llm/models/gemma/smoothquant.py | + tensorrt_llm/models/gemma/utils/__init__.py | + tensorrt_llm/models/gemma/utils/layers.py | + tensorrt_llm/models/gemma/utils/modules.py | + tensorrt_llm/models/gemma/utils/params.py | + tensorrt_llm/models/gemma/utils/positional_embeddings.py | + tensorrt_llm/models/gemma/utils/sampler.py | + tensorrt_llm/models/gemma/utils/transformer.py | + tensorrt_llm/models/gemma/weight.py | + tensorrt_llm/models/generation_mixin.py | + tensorrt_llm/models/gpt/__init__.py | + tensorrt_llm/models/gpt/config.py | + tensorrt_llm/models/gpt/convert.py | + tensorrt_llm/models/gpt/model.py | + tensorrt_llm/models/gptj/__init__.py | + tensorrt_llm/models/gptj/config.py | + tensorrt_llm/models/gptj/convert.py | + tensorrt_llm/models/gptj/model.py | + tensorrt_llm/models/gptneox/__init__.py | + tensorrt_llm/models/gptneox/model.py | + tensorrt_llm/models/grok/__init__.py | + tensorrt_llm/models/grok/convert.py | + tensorrt_llm/models/grok/model.py | + tensorrt_llm/models/grok/weight.py | + tensorrt_llm/models/llama/__init__.py | + tensorrt_llm/models/llama/config.py | + tensorrt_llm/models/llama/convert.py | + tensorrt_llm/models/llama/model.py | + tensorrt_llm/models/mamba/__init__.py | + tensorrt_llm/models/mamba/config.py | + tensorrt_llm/models/mamba/convert.py | + tensorrt_llm/models/mamba/model.py | + tensorrt_llm/models/medusa/__init__.py | + tensorrt_llm/models/medusa/config.py | + tensorrt_llm/models/medusa/model.py | + tensorrt_llm/models/medusa/weight.py | + tensorrt_llm/models/mllama/__init__.py | + tensorrt_llm/models/mllama/config.py | + tensorrt_llm/models/mllama/model.py | + tensorrt_llm/models/mmdit_sd3/__init__.py | + tensorrt_llm/models/mmdit_sd3/config.py | + tensorrt_llm/models/mmdit_sd3/model.py | + tensorrt_llm/models/model_weights_loader.py | tensorrt_llm/models/modeling_utils.py | + tensorrt_llm/models/mpt/__init__.py | + tensorrt_llm/models/mpt/model.py | + tensorrt_llm/models/multimodal_encoders/__init__.py | + tensorrt_llm/models/multimodal_encoders/config.py | + tensorrt_llm/models/multimodal_encoders/model.py | + tensorrt_llm/models/nemotron_nas/__init__.py | + tensorrt_llm/models/nemotron_nas/config.py | + tensorrt_llm/models/nemotron_nas/convert.py | + tensorrt_llm/models/nemotron_nas/layer_config.py | + tensorrt_llm/models/nemotron_nas/model.py | + tensorrt_llm/models/opt/__init__.py | + tensorrt_llm/models/opt/model.py | + tensorrt_llm/models/phi/__init__.py | + tensorrt_llm/models/phi/config.py | + tensorrt_llm/models/phi/convert.py | + tensorrt_llm/models/phi/model.py | + tensorrt_llm/models/phi3/__init__.py | + tensorrt_llm/models/phi3/config.py | + tensorrt_llm/models/phi3/convert.py | + tensorrt_llm/models/phi3/model.py | + tensorrt_llm/models/phi3/split_weights.py | + tensorrt_llm/models/qwen/__init__.py | + tensorrt_llm/models/qwen/config.py | + tensorrt_llm/models/qwen/convert.py | + tensorrt_llm/models/qwen/model.py | + tensorrt_llm/models/qwen/utils.py | + tensorrt_llm/models/recurrentgemma/__init__.py | + tensorrt_llm/models/recurrentgemma/model.py | + tensorrt_llm/models/redrafter/__init__.py | + tensorrt_llm/models/redrafter/drafter.py | + tensorrt_llm/models/redrafter/model.py | + tensorrt_llm/models/redrafter/redrafter_helper.py | + tensorrt_llm/models/stdit/__init__.py | + tensorrt_llm/models/stdit/config.py | + tensorrt_llm/models/stdit/model.py | + tensorrt_llm/models/unet/__init__.py | + tensorrt_llm/models/unet/attention.py | + tensorrt_llm/models/unet/embeddings.py | + tensorrt_llm/models/unet/pp/__init__.py | + tensorrt_llm/models/unet/pp/attention.py | + tensorrt_llm/models/unet/pp/conv2d.py | + tensorrt_llm/models/unet/pp/groupnorm.py | + tensorrt_llm/models/unet/pp/unet_pp.py | + tensorrt_llm/models/unet/resnet.py | + tensorrt_llm/models/unet/unet_2d_blocks.py | + tensorrt_llm/models/unet/unet_2d_condition.py | + tensorrt_llm/models/unet/weights.py | + tensorrt_llm/network.py | + tensorrt_llm/parameter.py | + tensorrt_llm/plugin/__init__.py | + tensorrt_llm/plugin/plugin.py | tensorrt_llm/quantization/__init__.py | tensorrt_llm/quantization/functional.py | + tensorrt_llm/quantization/image_processing.py | + tensorrt_llm/quantization/layers.py | tensorrt_llm/quantization/mode.py | + tensorrt_llm/quantization/quantize.py | + tensorrt_llm/quantization/quantize_by_modelopt.py | tensorrt_llm/quantization/utils/__init__.py | tensorrt_llm/quantization/utils/fp4_utils.py | tensorrt_llm/quantization/utils/fp8_utils.py | tensorrt_llm/ray_stub.py | tensorrt_llm/runtime/__init__.py | + tensorrt_llm/runtime/enc_dec_model_runner.py | + tensorrt_llm/runtime/generation.py | + tensorrt_llm/runtime/kv_cache_manager.py | + tensorrt_llm/runtime/medusa_utils.py | tensorrt_llm/runtime/memory_pools/__init__.py | + tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py | + tensorrt_llm/runtime/memory_pools/pool.py | + tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py | + tensorrt_llm/runtime/model_runner.py | + tensorrt_llm/runtime/model_runner_cpp.py | + tensorrt_llm/runtime/multimodal_model_runner.py | + tensorrt_llm/runtime/processor_wrapper/__init__.py | + tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py | + tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py | + tensorrt_llm/runtime/redrafter_utils.py | + tensorrt_llm/runtime/session.py | tensorrt_llm/scaffolding/__init__.py | tensorrt_llm/scaffolding/benchmark.py | tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py | @@ -479,7 +826,12 @@ common-files: &common_files | tensorrt_llm/tokenizer/tokenizer.py | tensorrt_llm/tools/__init__.py | tensorrt_llm/tools/importlib_utils.py | + tensorrt_llm/tools/multimodal_builder.py | + tensorrt_llm/tools/onnx_utils.py | tensorrt_llm/tools/plugin_gen/__init__.py | + tensorrt_llm/tools/plugin_gen/core.py | + tensorrt_llm/tools/plugin_gen/plugin_gen.py | + tensorrt_llm/tools/plugin_gen/shape_infer.py | tensorrt_llm/tools/ppl.py | tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py | tensorrt_llm/version.py | @@ -488,7 +840,9 @@ common-files: &common_files | tests/integration/defs/accuracy/accuracy_core.py | tests/integration/defs/accuracy/scripts/collect_evaluated_accuracies.py | tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py | + tests/integration/defs/accuracy/test_cli_flow.py | tests/integration/defs/accuracy/test_disaggregated_serving.py | + tests/integration/defs/accuracy/test_llm_api.py | tests/integration/defs/accuracy/test_llm_api_autodeploy.py | tests/integration/defs/accuracy/test_llm_api_pytorch.py | tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py | @@ -497,29 +851,63 @@ common-files: &common_files | tests/integration/defs/conftest.py | tests/integration/defs/cpp/conftest.py | tests/integration/defs/cpp/cpp_common.py | + tests/integration/defs/cpp/test_e2e.py | tests/integration/defs/cpp/test_multi_gpu.py | tests/integration/defs/cpp/test_unit_tests.py | + tests/integration/defs/deterministic/mixtral_deterministic.py | + tests/integration/defs/deterministic/test_mixtral_deterministic.py | tests/integration/defs/disaggregated/test_auto_scaling.py | tests/integration/defs/disaggregated/test_disaggregated.py | tests/integration/defs/disaggregated/test_disaggregated_etcd.py | tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py | tests/integration/defs/disaggregated/test_workers.py | + tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py | tests/integration/defs/examples/run_llm_quickstart_atexit.py | tests/integration/defs/examples/serve/test_serve.py | tests/integration/defs/examples/serve/test_serve_negative.py | tests/integration/defs/examples/test_ad_guided_decoding.py | + tests/integration/defs/examples/test_bert.py | + tests/integration/defs/examples/test_bindings.py | + tests/integration/defs/examples/test_chatglm.py | + tests/integration/defs/examples/test_commandr.py | + tests/integration/defs/examples/test_draft_target_model.py | + tests/integration/defs/examples/test_eagle.py | + tests/integration/defs/examples/test_enc_dec.py | + tests/integration/defs/examples/test_exaone.py | + tests/integration/defs/examples/test_gemma.py | tests/integration/defs/examples/test_gpt.py | + tests/integration/defs/examples/test_gptj.py | + tests/integration/defs/examples/test_granite.py | + tests/integration/defs/examples/test_internlm.py | + tests/integration/defs/examples/test_llama.py | tests/integration/defs/examples/test_llm_api_with_mpi.py | + tests/integration/defs/examples/test_mamba.py | + tests/integration/defs/examples/test_medusa.py | + tests/integration/defs/examples/test_mistral.py | + tests/integration/defs/examples/test_mixtral.py | + tests/integration/defs/examples/test_multimodal.py | + tests/integration/defs/examples/test_nemotron.py | + tests/integration/defs/examples/test_nemotron_nas.py | + tests/integration/defs/examples/test_ngram.py | + tests/integration/defs/examples/test_openai.py | tests/integration/defs/examples/test_phi.py | + tests/integration/defs/examples/test_qwen.py | + tests/integration/defs/examples/test_qwen2audio.py | + tests/integration/defs/examples/test_qwenvl.py | tests/integration/defs/examples/test_ray.py | + tests/integration/defs/examples/test_recurrentgemma.py | + tests/integration/defs/examples/test_redrafter.py | + tests/integration/defs/examples/test_whisper.py | tests/integration/defs/llmapi/__init__.py | tests/integration/defs/llmapi/_run_llmapi_llm.py | tests/integration/defs/llmapi/test_llm_api_connector.py | tests/integration/defs/llmapi/test_llm_api_qa.py | + tests/integration/defs/llmapi/test_llm_e2e.py | tests/integration/defs/llmapi/test_llm_examples.py | tests/integration/defs/local_venv.py | tests/integration/defs/perf/__init__.py | tests/integration/defs/perf/allowed_configs.py | + tests/integration/defs/perf/build.py | tests/integration/defs/perf/create_perf_comparison_report.py | tests/integration/defs/perf/data.py | tests/integration/defs/perf/data_export.py | @@ -540,21 +928,38 @@ common-files: &common_files | tests/integration/defs/test_fmha.py | tests/integration/defs/test_list_parser.py | tests/integration/defs/test_list_validation.py | + tests/integration/defs/test_mlpf_results.py | tests/integration/defs/test_sanity.py | tests/integration/defs/test_unittests.py | tests/integration/defs/triton_server/__init__.py | + tests/integration/defs/triton_server/build_engines.py | tests/integration/defs/triton_server/common.py | tests/integration/defs/triton_server/conftest.py | + tests/integration/defs/triton_server/local_venv.py | + tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py | + tests/integration/defs/triton_server/runner_interface.py | tests/integration/defs/triton_server/test_list_parser.py | + tests/integration/defs/triton_server/test_triton.py | + tests/integration/defs/triton_server/test_triton_llm.py | + tests/integration/defs/triton_server/test_triton_memleak.py | + tests/integration/defs/triton_server/test_triton_multi_node.py | + tests/integration/defs/triton_server/test_triton_rcca.py | tests/integration/defs/triton_server/trt_test_alternative.py | tests/integration/defs/trt_test_alternative.py | tests/integration/defs/utils/__init__.py | tests/integration/defs/utils/periodic_junit.py | tests/integration/defs/utils/timeout_manager.py | tests/microbenchmarks/all_reduce.py | + tests/microbenchmarks/build_time_benchmark.py | + tests/microbenchmarks/build_time_dashboard.py | tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py | tests/scripts/allreduce_perf/allreduce_perf_viz.py | tests/scripts/iteration_log_parser.py | + tests/scripts/perf-sanity/parse_benchmark_results.py | + tests/scripts/perf-sanity/run_benchmark_serve.py | + tests/unittest/_torch/attention/sparse/test_dsa_indexer.py | + tests/unittest/_torch/attention/sparse/test_flash_mla.py | + tests/unittest/_torch/attention/sparse/test_rocketkv.py | tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py | tests/unittest/_torch/attention/test_attention.py | tests/unittest/_torch/attention/test_attention_mla.py | @@ -575,6 +980,7 @@ common-files: &common_files | tests/unittest/_torch/misc/test_virtual_memory.py | tests/unittest/_torch/modeling/test_modeling_bert.py | tests/unittest/_torch/modeling/test_modeling_clip.py | + tests/unittest/_torch/modeling/test_modeling_exaone4.py | tests/unittest/_torch/modeling/test_modeling_gemma3.py | tests/unittest/_torch/modeling/test_modeling_gpt_oss.py | tests/unittest/_torch/modeling/test_modeling_llama.py | @@ -598,6 +1004,8 @@ common-files: &common_files | tests/unittest/_torch/modules/test_moe_routing.py | tests/unittest/_torch/modules/test_rotary_embedding.py | tests/unittest/_torch/modules/test_triton_linear.py | + tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py | + tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py | tests/unittest/_torch/multi_gpu/test_allreduce.py | tests/unittest/_torch/multi_gpu/test_alltoall.py | tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py | @@ -624,10 +1032,24 @@ common-files: &common_files | tests/unittest/_torch/sampler/test_beam_search.py | tests/unittest/_torch/sampler/test_best_of_n.py | tests/unittest/_torch/sampler/test_trtllm_sampler.py | + tests/unittest/_torch/speculative/test_draft_target.py | + tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py | + tests/unittest/_torch/speculative/test_draft_token_tree_verification.py | + tests/unittest/_torch/speculative/test_dynamic_spec_decode.py | tests/unittest/_torch/speculative/test_eagle3.py | + tests/unittest/_torch/speculative/test_kv_cache_reuse.py | + tests/unittest/_torch/speculative/test_mtp.py | + tests/unittest/_torch/speculative/test_ngram.py | + tests/unittest/_torch/speculative/test_save_state.py | + tests/unittest/_torch/speculative/test_spec_gate.py | + tests/unittest/_torch/speculative/test_torch_rejection_sampling.py | + tests/unittest/_torch/speculative/test_user_provided.py | tests/unittest/_torch/test_connector.py | tests/unittest/_torch/test_torch_multi_arange.py | tests/unittest/_torch/thop/parallel/deep_gemm_tests.py | + tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py | + tests/unittest/_torch/thop/parallel/test_cublas_mm.py | + tests/unittest/_torch/thop/parallel/test_custom_ops.py | tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py | tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py | tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py | @@ -641,6 +1063,11 @@ common-files: &common_files | tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py | tests/unittest/_torch/thop/parallel/test_fp8_quantize.py | tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py | + tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py | + tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py | + tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py | + tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py | + tests/unittest/_torch/thop/parallel/test_noaux_tc.py | tests/unittest/_torch/thop/parallel/test_scaled_mm.py | tests/unittest/_torch/thop/parallel/test_selective_scan_op.py | tests/unittest/_torch/thop/parallel/test_tinygemm2.py | @@ -654,6 +1081,7 @@ common-files: &common_files | tests/unittest/_torch/thop/serial/test_moe_alltoall.py | tests/unittest/api_stability/api_stability_core.py | tests/unittest/api_stability/test_llm_api.py | + tests/unittest/bindings/binding_test_utils.py | tests/unittest/bindings/test_bindings_moe.py | tests/unittest/bindings/test_bindings_ut.py | tests/unittest/bindings/test_executor_bindings.py | @@ -684,10 +1112,12 @@ common-files: &common_files | tests/unittest/llmapi/apps/_test_openai_chat_harmony.py | tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py | tests/unittest/llmapi/apps/_test_openai_completions.py | + tests/unittest/llmapi/apps/_test_openai_consistent_chat.py | tests/unittest/llmapi/apps/_test_openai_lora.py | tests/unittest/llmapi/apps/_test_openai_metrics.py | tests/unittest/llmapi/apps/_test_openai_misc.py | tests/unittest/llmapi/apps/_test_openai_mmencoder.py | + tests/unittest/llmapi/apps/_test_openai_multi_chat.py | tests/unittest/llmapi/apps/_test_openai_multi_gpu.py | tests/unittest/llmapi/apps/_test_openai_multi_nodes.py | tests/unittest/llmapi/apps/_test_openai_perf_metrics.py | @@ -710,12 +1140,15 @@ common-files: &common_files | tests/unittest/llmapi/run_llm_exit.py | tests/unittest/llmapi/run_llm_with_postproc.py | tests/unittest/llmapi/test_additional_model_outputs.py | + tests/unittest/llmapi/test_build_cache.py | tests/unittest/llmapi/test_executor.py | tests/unittest/llmapi/test_gc_utils.py | tests/unittest/llmapi/test_llm.py | tests/unittest/llmapi/test_llm_args.py | tests/unittest/llmapi/test_llm_download.py | tests/unittest/llmapi/test_llm_kv_cache_events.py | + tests/unittest/llmapi/test_llm_models.py | + tests/unittest/llmapi/test_llm_multi_gpu.py | tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py | tests/unittest/llmapi/test_llm_pytorch.py | tests/unittest/llmapi/test_llm_quant.py | @@ -726,14 +1159,25 @@ common-files: &common_files | tests/unittest/llmapi/test_serialization.py | tests/unittest/llmapi/test_utils.py | tests/unittest/others/__init__.py | + tests/unittest/others/test_builder.py | tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py | + tests/unittest/others/test_debugging_api.py | tests/unittest/others/test_exception.py | tests/unittest/others/test_export.py | + tests/unittest/others/test_graph_rewriter.py | + tests/unittest/others/test_kv_cache_manager.py | tests/unittest/others/test_kv_cache_transceiver.py | tests/unittest/others/test_kv_cache_update.py | + tests/unittest/others/test_layer.py | + tests/unittest/others/test_leak.py | tests/unittest/others/test_mapping.py | + tests/unittest/others/test_model_dtype.py | + tests/unittest/others/test_module.py | tests/unittest/others/test_multimodal_registry.py | + tests/unittest/others/test_plugins.py | + tests/unittest/others/test_precision_control.py | tests/unittest/others/test_pretrained_config.py | + tests/unittest/others/test_session.py | tests/unittest/others/test_time_breakdown.py | tests/unittest/profile_utils.py | tests/unittest/scaffolding/__init__.py | @@ -742,14 +1186,141 @@ common-files: &common_files | tests/unittest/scaffolding/test_scaffolding.py | tests/unittest/scaffolding/test_task_collection.py | tests/unittest/scaffolding/test_worker.py | + tests/unittest/test_model_runner_cpp.py | tests/unittest/test_pip_install.py | tests/unittest/tools/__init__.py | + tests/unittest/tools/plugin_gen/__init__.py | + tests/unittest/tools/plugin_gen/kernel_config.py | + tests/unittest/tools/plugin_gen/test_core.py | + tests/unittest/tools/plugin_gen/test_plugin_gen.py | + tests/unittest/tools/plugin_gen/test_shape_infer.py | tests/unittest/tools/test_prepare_dataset.py | tests/unittest/tools/test_test_to_stage_mapping.py | + tests/unittest/trt/__init__.py | + tests/unittest/trt/attention/test_bert_attention.py | + tests/unittest/trt/attention/test_gpt_attention.py | + tests/unittest/trt/attention/test_gpt_attention_IFB.py | + tests/unittest/trt/attention/test_gpt_attention_no_cache.py | + tests/unittest/trt/attention/test_sage_attention.py | + tests/unittest/trt/functional/__init__.py | + tests/unittest/trt/functional/test_alibi.py | + tests/unittest/trt/functional/test_allreduce_norm.py | + tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py | + tests/unittest/trt/functional/test_arange.py | + tests/unittest/trt/functional/test_argmax.py | + tests/unittest/trt/functional/test_assertion.py | + tests/unittest/trt/functional/test_avg_pool2d.py | + tests/unittest/trt/functional/test_cast.py | + tests/unittest/trt/functional/test_conv2d.py | + tests/unittest/trt/functional/test_conv3d.py | + tests/unittest/trt/functional/test_cos.py | + tests/unittest/trt/functional/test_cumsum.py | + tests/unittest/trt/functional/test_dora.py | + tests/unittest/trt/functional/test_einsum.py | + tests/unittest/trt/functional/test_embedding_single_gpu.py | + tests/unittest/trt/functional/test_exp.py | + tests/unittest/trt/functional/test_expand.py | + tests/unittest/trt/functional/test_flatten.py | + tests/unittest/trt/functional/test_flip.py | + tests/unittest/trt/functional/test_fp4_gemm.py | + tests/unittest/trt/functional/test_fp4_gemm_ootb.py | + tests/unittest/trt/functional/test_gather.py | + tests/unittest/trt/functional/test_gather_nd.py | + tests/unittest/trt/functional/test_geglu.py | + tests/unittest/trt/functional/test_gelu.py | + tests/unittest/trt/functional/test_gemm_swiglu.py | + tests/unittest/trt/functional/test_group_norm.py | + tests/unittest/trt/functional/test_identity.py | + tests/unittest/trt/functional/test_index_select.py | + tests/unittest/trt/functional/test_interpolate.py | + tests/unittest/trt/functional/test_logsoftmax.py | + tests/unittest/trt/functional/test_lora.py | + tests/unittest/trt/functional/test_low_latency_gemm.py | + tests/unittest/trt/functional/test_mamba_conv1d.py | + tests/unittest/trt/functional/test_masked_scatter.py | + tests/unittest/trt/functional/test_masked_select.py | + tests/unittest/trt/functional/test_matmul.py | + tests/unittest/trt/functional/test_meshgrid2d.py | + tests/unittest/trt/functional/test_moe.py | + tests/unittest/trt/functional/test_nccl.py | + tests/unittest/trt/functional/test_nonzero.py | + tests/unittest/trt/functional/test_outer.py | + tests/unittest/trt/functional/test_pad.py | + tests/unittest/trt/functional/test_permute.py | + tests/unittest/trt/functional/test_pp_reduce_scatter.py | + tests/unittest/trt/functional/test_quant.py | + tests/unittest/trt/functional/test_rearrange.py | + tests/unittest/trt/functional/test_repeat.py | + tests/unittest/trt/functional/test_repeat_interleave.py | + tests/unittest/trt/functional/test_rg_lru.py | + tests/unittest/trt/functional/test_sample.py | + tests/unittest/trt/functional/test_scatter.py | + tests/unittest/trt/functional/test_scatter_nd.py | + tests/unittest/trt/functional/test_select.py | + tests/unittest/trt/functional/test_selective_scan.py | + tests/unittest/trt/functional/test_sigmoid.py | + tests/unittest/trt/functional/test_silu.py | + tests/unittest/trt/functional/test_sin.py | + tests/unittest/trt/functional/test_slice.py | + tests/unittest/trt/functional/test_softplus.py | + tests/unittest/trt/functional/test_split.py | + tests/unittest/trt/functional/test_squeeze.py | + tests/unittest/trt/functional/test_swiglu.py | + tests/unittest/trt/functional/test_topk.py | + tests/unittest/trt/functional/test_transpose.py | + tests/unittest/trt/functional/test_unbind.py | + tests/unittest/trt/functional/test_unsqueeze.py | + tests/unittest/trt/functional/test_view.py | + tests/unittest/trt/functional/test_where.py | + tests/unittest/trt/model/__init__.py | + tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py | + tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py | + tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py | + tests/unittest/trt/model/redrafter/test_beams2tree.py | + tests/unittest/trt/model/redrafter/test_draft_token.py | + tests/unittest/trt/model/redrafter/test_draft_token_indices.py | + tests/unittest/trt/model/redrafter/test_gather_beams.py | + tests/unittest/trt/model/redrafter/test_mask.py | + tests/unittest/trt/model/redrafter/test_packed_position_ids.py | + tests/unittest/trt/model/redrafter/test_prefix_match_indices.py | + tests/unittest/trt/model/redrafter/test_prepare_input.py | + tests/unittest/trt/model/redrafter/test_process_logits.py | + tests/unittest/trt/model/redrafter/test_top1.py | + tests/unittest/trt/model/redrafter/test_unpack_gen_data.py | + tests/unittest/trt/model/redrafter/test_validate.py | + tests/unittest/trt/model/test_gpt.py | + tests/unittest/trt/model/test_gpt_e2e.py | + tests/unittest/trt/model/test_llama.py | + tests/unittest/trt/model/test_mamba.py | + tests/unittest/trt/model/test_mistral.py | + tests/unittest/trt/model/test_nemotron_nas.py | + tests/unittest/trt/model/test_phi.py | + tests/unittest/trt/model/test_unet.py | + tests/unittest/trt/model_api/test_model_api_multi_gpu.py | + tests/unittest/trt/model_api/test_model_level_api.py | + tests/unittest/trt/model_api/test_model_quantization.py | + tests/unittest/trt/python_plugin/plugin_wrapper_utils.py | + tests/unittest/trt/python_plugin/test_plugin_wrapper.py | + tests/unittest/trt/quantization/__init__.py | + tests/unittest/trt/quantization/_utils.py | + tests/unittest/trt/quantization/test_fp8_quantization.py | + tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py | + tests/unittest/trt/quantization/test_functional.py | + tests/unittest/trt/quantization/test_mode.py | + tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py | + tests/unittest/trt/quantization/test_qserve_gemm.py | + tests/unittest/trt/quantization/test_quant.py | + tests/unittest/trt/quantization/test_quant_layer.py | + tests/unittest/trt/quantization/test_smooth_quant_gemm.py | + tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py | + tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py | + tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py | + tests/unittest/trt/quantization/test_weight_only_quant_matmul.py | tests/unittest/utils/__init__.py | tests/unittest/utils/cpp_paths.py | tests/unittest/utils/llm_data.py | tests/unittest/utils/runtime_defaults.py | + tests/unittest/utils/test_medusa_utils.py | tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py | tests/unittest/utils/test_util.py | tests/unittest/utils/torch_ref.py | @@ -779,6 +1350,14 @@ legacy-files: &legacy_files | .devcontainer/make_env.py | .github/scripts/label_community_user.py | .github/scripts/pr_checklist_check.py | + benchmarks/cpp/__init__.py | + benchmarks/cpp/prepare_dataset.py | + benchmarks/cpp/utils/__init__.py | + benchmarks/cpp/utils/convert_nemo_dataset.py | + benchmarks/cpp/utils/generate_rand_loras.py | + benchmarks/cpp/utils/prepare_real_data.py | + benchmarks/cpp/utils/prepare_synthetic_data.py | + benchmarks/cpp/utils/utils.py | cpp/conanfile.py | cpp/kernels/fmha_v2/conftest.py | cpp/kernels/fmha_v2/fmha_test.py | @@ -803,17 +1382,58 @@ legacy-files: &legacy_files | cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py | cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py | cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py | + cpp/tests/resources/scripts/build_chatglm_engines.py | + cpp/tests/resources/scripts/build_eagle_engines.py | + cpp/tests/resources/scripts/build_enc_dec_engines.py | + cpp/tests/resources/scripts/build_engines_utils.py | + cpp/tests/resources/scripts/build_gpt_engines.py | + cpp/tests/resources/scripts/build_gptj_engines.py | + cpp/tests/resources/scripts/build_llama_engines.py | + cpp/tests/resources/scripts/build_mamba_engines.py | + cpp/tests/resources/scripts/build_medusa_engines.py | + cpp/tests/resources/scripts/build_recurrentgemma_engines.py | + cpp/tests/resources/scripts/build_redrafter_engines.py | + cpp/tests/resources/scripts/generate_expected_chatglm_output.py | + cpp/tests/resources/scripts/generate_expected_eagle_output.py | + cpp/tests/resources/scripts/generate_expected_enc_dec_output.py | + cpp/tests/resources/scripts/generate_expected_gpt_output.py | + cpp/tests/resources/scripts/generate_expected_gptj_output.py | + cpp/tests/resources/scripts/generate_expected_llama_output.py | + cpp/tests/resources/scripts/generate_expected_mamba_output.py | + cpp/tests/resources/scripts/generate_expected_medusa_output.py | + cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py | + cpp/tests/resources/scripts/generate_expected_redrafter_output.py | + cpp/tests/resources/scripts/generate_hf_gpt_output.py | cpp/tests/resources/scripts/generate_test_lora_weights.py | + cpp/tests/resources/scripts/io_converter.py | docs/source/conf.py | docs/source/helper.py | examples/apps/chat.py | examples/apps/fastapi_server.py | + examples/bindings/executor/example_advanced.py | + examples/bindings/executor/example_basic.py | + examples/bindings/executor/example_debug.py | + examples/bindings/executor/example_logits_processor.py | examples/disaggregated/clients/disagg_client.py | examples/disaggregated/slurm/benchmark/submit.py | + examples/dora/normalize_weights.py | + examples/eagle/convert_checkpoint.py | + examples/eval_long_context.py | + examples/generate_checkpoint_config.py | + examples/generate_xgrammar_tokenizer_info.py | + examples/hf_lora_convert.py | examples/infinitebench/args.py | examples/infinitebench/compute_scores.py | examples/infinitebench/construct_synthetic_dataset.py | examples/infinitebench/eval_utils.py | + examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py | + examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py | + examples/llm-api/_tensorrt_engine/llm_inference_customize.py | + examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py | + examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py | + examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py | + examples/llm-api/_tensorrt_engine/llm_quantization.py | + examples/llm-api/_tensorrt_engine/quickstart_example.py | examples/llm-api/llm_guided_decoding.py | examples/llm-api/llm_inference.py | examples/llm-api/llm_inference_async.py | @@ -833,17 +1453,122 @@ legacy-files: &legacy_files | examples/llm-api/quickstart_example.py | examples/llm-api/quickstart_multimodal.py | examples/llm-api/star_attention.py | + examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py | examples/longbench/eval_longbench_v1.py | + examples/medusa/convert_checkpoint.py | + examples/mmlu.py | + examples/models/contrib/baichuan/convert_checkpoint.py | + examples/models/contrib/bloom/convert_checkpoint.py | + examples/models/contrib/chatglm-6b/tokenization_chatglm.py | + examples/models/contrib/chatglm2-6b/tokenization_chatglm.py | + examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py | + examples/models/contrib/cogvlm/convert_checkpoint.py | + examples/models/contrib/dbrx/convert_checkpoint.py | + examples/models/contrib/deepseek_v1/__init__.py | + examples/models/contrib/deepseek_v1/convert_checkpoint.py | + examples/models/contrib/deepseek_v2/convert_checkpoint.py | + examples/models/contrib/dit/convert_checkpoint.py | + examples/models/contrib/dit/diffusion.py | + examples/models/contrib/dit/sample.py | + examples/models/contrib/dit/utils_modelopt.py | + examples/models/contrib/dit/vae_decoder_trt.py | + examples/models/contrib/falcon/convert_checkpoint.py | + examples/models/contrib/gptj/convert_checkpoint.py | + examples/models/contrib/gptneox/convert_checkpoint.py | + examples/models/contrib/grok/convert_checkpoint.py | + examples/models/contrib/mmdit/convert_checkpoint.py | + examples/models/contrib/mmdit/sample.py | + examples/models/contrib/mpt/convert_checkpoint.py | + examples/models/contrib/opt/convert_checkpoint.py | + examples/models/contrib/sdxl/build_sdxl_unet.py | + examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py | + examples/models/contrib/sdxl/run_sdxl.py | + examples/models/contrib/stdit/aspect.py | + examples/models/contrib/stdit/convert_checkpoint.py | + examples/models/contrib/stdit/pipeline_tllm.py | + examples/models/contrib/stdit/sample.py | + examples/models/contrib/stdit/scheduler.py | + examples/models/contrib/stdit/text_encoder.py | + examples/models/contrib/stdit/utils.py | + examples/models/contrib/stdit/vae.py | + examples/models/contrib/stdit/video_transforms.py | + examples/models/core/bert/__init__.py | + examples/models/core/bert/convert_checkpoint.py | + examples/models/core/bert/run.py | + examples/models/core/bert/utils.py | + examples/models/core/commandr/convert_checkpoint.py | + examples/models/core/enc_dec/__init__.py | + examples/models/core/enc_dec/convert_checkpoint.py | + examples/models/core/enc_dec/helper.py | + examples/models/core/enc_dec/run.py | + examples/models/core/gemma/convert_checkpoint.py | + examples/models/core/glm-4-9b/convert_checkpoint.py | + examples/models/core/glm-4-9b/tokenization_chatglm.py | + examples/models/core/gpt/convert_checkpoint.py | + examples/models/core/gpt/merge_ptuning_tables.py | + examples/models/core/gpt/nemo_lora_convert.py | + examples/models/core/gpt/nemo_prompt_convert.py | + examples/models/core/gpt/run_hf.py | examples/models/core/gpt_oss/openai_chat_client_function_calling.py | + examples/models/core/internlm2/convert_checkpoint.py | examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py | + examples/models/core/llama/convert_checkpoint.py | + examples/models/core/llama/summarize_long.py | + examples/models/core/mamba/convert_checkpoint.py | + examples/models/core/mllama/convert_checkpoint.py | + examples/models/core/multimodal/__init__.py | + examples/models/core/multimodal/build_multimodal_engine.py | + examples/models/core/multimodal/eval.py | + examples/models/core/multimodal/run.py | + examples/models/core/multimodal/utils.py | + examples/models/core/nemotron_nas/calibration_utils.py | + examples/models/core/nemotron_nas/convert_checkpoint.py | + examples/models/core/phi/convert_checkpoint.py | + examples/models/core/qwen/convert_checkpoint.py | + examples/models/core/qwen2audio/run.py | + examples/models/core/qwen2audio/run_chat.py | + examples/models/core/qwen2audio/utils.py | + examples/models/core/qwenvl/run.py | + examples/models/core/qwenvl/run_chat.py | + examples/models/core/qwenvl/show_pic.py | + examples/models/core/qwenvl/vit_onnx_trt.py | + examples/models/core/recurrentgemma/convert_checkpoint.py | + examples/models/core/vit/convert_checkpoint.py | + examples/models/core/whisper/convert_checkpoint.py | + examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py | + examples/models/core/whisper/run.py | + examples/models/core/whisper/tokenizer.py | + examples/models/core/whisper/whisper_utils.py | + examples/ngram/run_dtm_ngram.py | + examples/openai_triton/manual_plugin/build.py | + examples/openai_triton/manual_plugin/fmha_triton.py | + examples/openai_triton/manual_plugin/plugin.py | + examples/openai_triton/manual_plugin/run.py | + examples/openai_triton/plugin_autogen/build_engine.py | + examples/openai_triton/plugin_autogen/kernel_config.py | + examples/openai_triton/plugin_autogen/run_engine.py | + examples/python_plugin/build_lookup.py | + examples/python_plugin/plugin_lib/__init__.py | + examples/python_plugin/plugin_lib/lookup_kernel.py | + examples/python_plugin/plugin_lib/lookup_plugin.py | + examples/python_plugin/run_lookup.py | + examples/quantization/quantize.py | examples/quantization/quantize_mixed_precision_moe.py | examples/ray_orchestrator/llm_inference_async_ray.py | examples/ray_orchestrator/llm_inference_distributed_ray.py | + examples/redrafter/convert_checkpoint.py | + examples/run.py | examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py | examples/scaffolding/contrib/DeepConf/run_generation.py | examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py | examples/scaffolding/contrib/TreeInference/run_mcts_example.py | examples/scaffolding/contrib/TreeInference/run_tot_example.py | + examples/scaffolding/contrib/mcp/e2b/e2bserver.py | + examples/scaffolding/contrib/mcp/e2b/main.py | + examples/scaffolding/contrib/mcp/mcptest.py | + examples/scaffolding/contrib/mcp/weather/weather.py | + examples/scaffolding/contrib/mcp/websearch/main.py | + examples/scaffolding/contrib/mcp/websearch/websearch.py | examples/scaffolding/run_basic_generation.py | examples/scaffolding/run_best_of_n_with_reward.py | examples/scaffolding/run_majority_vote_aime24.py | @@ -853,6 +1578,8 @@ legacy-files: &legacy_files | examples/serve/openai_completion_client.py | examples/serve/openai_completion_client_for_lora.py | examples/serve/openai_completion_client_json_schema.py | + examples/summarize.py | + examples/utils.py | examples/wide_ep/ep_load_balancer/generate_eplb_config.py | examples/wide_ep/ep_load_balancer/report_load_statistics.py | examples/wide_ep/ep_load_balancer/utils.py | @@ -860,10 +1587,12 @@ legacy-files: &legacy_files | jenkins/scripts/mergeWaiveList.py | jenkins/scripts/open_search_db.py | jenkins/scripts/test_rerun.py | + scripts/build_cpp_examples.py | scripts/build_wheel.py | scripts/check_test_list.py | scripts/dco_check.py | scripts/format_test_list.py | + scripts/generate_duration.py | scripts/generate_lock_file.py | scripts/get_wheel_from_package.py | scripts/git_replace.py | @@ -874,6 +1603,7 @@ legacy-files: &legacy_files | setup.py | tensorrt_llm/__init__.py | tensorrt_llm/_ray_utils.py | + tensorrt_llm/_tensorrt_engine/__init__.py | tensorrt_llm/_torch/__init__.py | tensorrt_llm/_torch/attention_backend/__init__.py | tensorrt_llm/_torch/attention_backend/flashinfer.py | @@ -1077,6 +1807,7 @@ legacy-files: &legacy_files | tensorrt_llm/_torch/pyexecutor/guided_decoder.py | tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py | tensorrt_llm/_torch/pyexecutor/handle_logits.py | + tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py | tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py | tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py | tensorrt_llm/_torch/pyexecutor/llm_request.py | @@ -1114,6 +1845,11 @@ legacy-files: &legacy_files | tensorrt_llm/bench/benchmark/utils/asynchronous.py | tensorrt_llm/bench/benchmark/utils/general.py | tensorrt_llm/bench/benchmark/utils/processes.py | + tensorrt_llm/bench/build/__init__.py | + tensorrt_llm/bench/build/build.py | + tensorrt_llm/bench/build/dataclasses.py | + tensorrt_llm/bench/build/tuning.py | + tensorrt_llm/bench/build/utils.py | tensorrt_llm/bench/dataclasses/__init__.py | tensorrt_llm/bench/dataclasses/configuration.py | tensorrt_llm/bench/dataclasses/engine.py | @@ -1123,9 +1859,13 @@ legacy-files: &legacy_files | tensorrt_llm/bench/dataclasses/statistics.py | tensorrt_llm/bench/utils/__init__.py | tensorrt_llm/bench/utils/data.py | + tensorrt_llm/builder.py | tensorrt_llm/commands/__init__.py | tensorrt_llm/commands/bench.py | + tensorrt_llm/commands/build.py | tensorrt_llm/commands/eval.py | + tensorrt_llm/commands/prune.py | + tensorrt_llm/commands/refit.py | tensorrt_llm/commands/serve.py | tensorrt_llm/evaluate/__init__.py | tensorrt_llm/evaluate/cnn_dailymail.py | @@ -1161,7 +1901,23 @@ legacy-files: &legacy_files | tensorrt_llm/inputs/multimodal.py | tensorrt_llm/inputs/registry.py | tensorrt_llm/inputs/utils.py | + tensorrt_llm/layers/__init__.py | + tensorrt_llm/layers/activation.py | + tensorrt_llm/layers/attention.py | + tensorrt_llm/layers/cast.py | + tensorrt_llm/layers/conv.py | + tensorrt_llm/layers/embedding.py | + tensorrt_llm/layers/language_adapter.py | + tensorrt_llm/layers/linear.py | + tensorrt_llm/layers/lora.py | + tensorrt_llm/layers/mlp.py | + tensorrt_llm/layers/moe.py | + tensorrt_llm/layers/normalization.py | + tensorrt_llm/layers/pooling.py | + tensorrt_llm/layers/recurrent.py | + tensorrt_llm/layers/ssm.py | tensorrt_llm/llmapi/__init__.py | + tensorrt_llm/llmapi/build_cache.py | tensorrt_llm/llmapi/disagg_utils.py | tensorrt_llm/llmapi/kv_cache_type.py | tensorrt_llm/llmapi/llm.py | @@ -1184,17 +1940,179 @@ legacy-files: &legacy_files | tensorrt_llm/metrics/enums.py | tensorrt_llm/models/__init__.py | tensorrt_llm/models/automodel.py | + tensorrt_llm/models/baichuan/__init__.py | + tensorrt_llm/models/baichuan/config.py | + tensorrt_llm/models/baichuan/convert.py | + tensorrt_llm/models/baichuan/model.py | + tensorrt_llm/models/bert/__init__.py | + tensorrt_llm/models/bert/config.py | + tensorrt_llm/models/bert/convert.py | + tensorrt_llm/models/bert/model.py | + tensorrt_llm/models/bloom/__init__.py | + tensorrt_llm/models/bloom/model.py | + tensorrt_llm/models/chatglm/__init__.py | + tensorrt_llm/models/chatglm/config.py | + tensorrt_llm/models/chatglm/convert.py | + tensorrt_llm/models/chatglm/model.py | + tensorrt_llm/models/clip/__init__.py | + tensorrt_llm/models/clip/model.py | + tensorrt_llm/models/cogvlm/__init__.py | + tensorrt_llm/models/cogvlm/config.py | + tensorrt_llm/models/cogvlm/convert.py | + tensorrt_llm/models/cogvlm/model.py | + tensorrt_llm/models/commandr/__init__.py | + tensorrt_llm/models/commandr/config.py | + tensorrt_llm/models/commandr/model.py | tensorrt_llm/models/convert_utils.py | + tensorrt_llm/models/dbrx/__init__.py | + tensorrt_llm/models/dbrx/config.py | + tensorrt_llm/models/dbrx/model.py | + tensorrt_llm/models/deepseek_v1/__init__.py | + tensorrt_llm/models/deepseek_v1/config.py | + tensorrt_llm/models/deepseek_v1/convert.py | + tensorrt_llm/models/deepseek_v1/model.py | + tensorrt_llm/models/deepseek_v2/__init__.py | + tensorrt_llm/models/deepseek_v2/config.py | + tensorrt_llm/models/deepseek_v2/convert.py | + tensorrt_llm/models/deepseek_v2/model.py | + tensorrt_llm/models/dit/__init__.py | + tensorrt_llm/models/dit/model.py | + tensorrt_llm/models/eagle/__init__.py | + tensorrt_llm/models/eagle/config.py | + tensorrt_llm/models/eagle/model.py | + tensorrt_llm/models/enc_dec/__init__.py | + tensorrt_llm/models/enc_dec/model.py | + tensorrt_llm/models/falcon/__init__.py | + tensorrt_llm/models/falcon/config.py | + tensorrt_llm/models/falcon/convert.py | + tensorrt_llm/models/falcon/model.py | + tensorrt_llm/models/gemma/__init__.py | + tensorrt_llm/models/gemma/config.py | + tensorrt_llm/models/gemma/convert.py | + tensorrt_llm/models/gemma/model.py | + tensorrt_llm/models/gemma/smoothquant.py | + tensorrt_llm/models/gemma/utils/__init__.py | + tensorrt_llm/models/gemma/utils/layers.py | + tensorrt_llm/models/gemma/utils/modules.py | + tensorrt_llm/models/gemma/utils/params.py | + tensorrt_llm/models/gemma/utils/positional_embeddings.py | + tensorrt_llm/models/gemma/utils/sampler.py | + tensorrt_llm/models/gemma/utils/transformer.py | + tensorrt_llm/models/gemma/weight.py | + tensorrt_llm/models/generation_mixin.py | + tensorrt_llm/models/gpt/__init__.py | + tensorrt_llm/models/gpt/config.py | + tensorrt_llm/models/gpt/convert.py | + tensorrt_llm/models/gpt/model.py | + tensorrt_llm/models/gptj/__init__.py | + tensorrt_llm/models/gptj/config.py | + tensorrt_llm/models/gptj/convert.py | + tensorrt_llm/models/gptj/model.py | + tensorrt_llm/models/gptneox/__init__.py | + tensorrt_llm/models/gptneox/model.py | + tensorrt_llm/models/grok/__init__.py | + tensorrt_llm/models/grok/convert.py | + tensorrt_llm/models/grok/model.py | + tensorrt_llm/models/grok/weight.py | + tensorrt_llm/models/llama/__init__.py | + tensorrt_llm/models/llama/config.py | + tensorrt_llm/models/llama/convert.py | + tensorrt_llm/models/llama/model.py | + tensorrt_llm/models/mamba/__init__.py | + tensorrt_llm/models/mamba/config.py | + tensorrt_llm/models/mamba/convert.py | + tensorrt_llm/models/mamba/model.py | + tensorrt_llm/models/medusa/__init__.py | + tensorrt_llm/models/medusa/config.py | + tensorrt_llm/models/medusa/model.py | + tensorrt_llm/models/medusa/weight.py | + tensorrt_llm/models/mllama/__init__.py | + tensorrt_llm/models/mllama/config.py | + tensorrt_llm/models/mllama/model.py | + tensorrt_llm/models/mmdit_sd3/__init__.py | + tensorrt_llm/models/mmdit_sd3/config.py | + tensorrt_llm/models/mmdit_sd3/model.py | + tensorrt_llm/models/model_weights_loader.py | tensorrt_llm/models/modeling_utils.py | + tensorrt_llm/models/mpt/__init__.py | + tensorrt_llm/models/mpt/model.py | + tensorrt_llm/models/multimodal_encoders/__init__.py | + tensorrt_llm/models/multimodal_encoders/config.py | + tensorrt_llm/models/multimodal_encoders/model.py | + tensorrt_llm/models/nemotron_nas/__init__.py | + tensorrt_llm/models/nemotron_nas/config.py | + tensorrt_llm/models/nemotron_nas/convert.py | + tensorrt_llm/models/nemotron_nas/layer_config.py | + tensorrt_llm/models/nemotron_nas/model.py | + tensorrt_llm/models/opt/__init__.py | + tensorrt_llm/models/opt/model.py | + tensorrt_llm/models/phi/__init__.py | + tensorrt_llm/models/phi/config.py | + tensorrt_llm/models/phi/convert.py | + tensorrt_llm/models/phi/model.py | + tensorrt_llm/models/phi3/__init__.py | + tensorrt_llm/models/phi3/config.py | + tensorrt_llm/models/phi3/convert.py | + tensorrt_llm/models/phi3/model.py | + tensorrt_llm/models/phi3/split_weights.py | + tensorrt_llm/models/qwen/__init__.py | + tensorrt_llm/models/qwen/config.py | + tensorrt_llm/models/qwen/convert.py | + tensorrt_llm/models/qwen/model.py | + tensorrt_llm/models/qwen/utils.py | + tensorrt_llm/models/recurrentgemma/__init__.py | + tensorrt_llm/models/recurrentgemma/model.py | + tensorrt_llm/models/redrafter/__init__.py | + tensorrt_llm/models/redrafter/drafter.py | + tensorrt_llm/models/redrafter/model.py | + tensorrt_llm/models/redrafter/redrafter_helper.py | + tensorrt_llm/models/stdit/__init__.py | + tensorrt_llm/models/stdit/config.py | + tensorrt_llm/models/stdit/model.py | + tensorrt_llm/models/unet/__init__.py | + tensorrt_llm/models/unet/attention.py | + tensorrt_llm/models/unet/embeddings.py | + tensorrt_llm/models/unet/pp/__init__.py | + tensorrt_llm/models/unet/pp/attention.py | + tensorrt_llm/models/unet/pp/conv2d.py | + tensorrt_llm/models/unet/pp/groupnorm.py | + tensorrt_llm/models/unet/pp/unet_pp.py | + tensorrt_llm/models/unet/resnet.py | + tensorrt_llm/models/unet/unet_2d_blocks.py | + tensorrt_llm/models/unet/unet_2d_condition.py | + tensorrt_llm/models/unet/weights.py | + tensorrt_llm/network.py | + tensorrt_llm/parameter.py | + tensorrt_llm/plugin/__init__.py | + tensorrt_llm/plugin/plugin.py | tensorrt_llm/quantization/__init__.py | tensorrt_llm/quantization/functional.py | + tensorrt_llm/quantization/image_processing.py | + tensorrt_llm/quantization/layers.py | tensorrt_llm/quantization/mode.py | + tensorrt_llm/quantization/quantize.py | + tensorrt_llm/quantization/quantize_by_modelopt.py | tensorrt_llm/quantization/utils/__init__.py | tensorrt_llm/quantization/utils/fp4_utils.py | tensorrt_llm/quantization/utils/fp8_utils.py | tensorrt_llm/ray_stub.py | tensorrt_llm/runtime/__init__.py | + tensorrt_llm/runtime/enc_dec_model_runner.py | + tensorrt_llm/runtime/generation.py | + tensorrt_llm/runtime/kv_cache_manager.py | + tensorrt_llm/runtime/medusa_utils.py | tensorrt_llm/runtime/memory_pools/__init__.py | + tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py | + tensorrt_llm/runtime/memory_pools/pool.py | + tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py | + tensorrt_llm/runtime/model_runner.py | + tensorrt_llm/runtime/model_runner_cpp.py | + tensorrt_llm/runtime/multimodal_model_runner.py | + tensorrt_llm/runtime/processor_wrapper/__init__.py | + tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py | + tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py | + tensorrt_llm/runtime/redrafter_utils.py | + tensorrt_llm/runtime/session.py | tensorrt_llm/scaffolding/__init__.py | tensorrt_llm/scaffolding/benchmark.py | tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py | @@ -1249,7 +2167,12 @@ legacy-files: &legacy_files | tensorrt_llm/tokenizer/tokenizer.py | tensorrt_llm/tools/__init__.py | tensorrt_llm/tools/importlib_utils.py | + tensorrt_llm/tools/multimodal_builder.py | + tensorrt_llm/tools/onnx_utils.py | tensorrt_llm/tools/plugin_gen/__init__.py | + tensorrt_llm/tools/plugin_gen/core.py | + tensorrt_llm/tools/plugin_gen/plugin_gen.py | + tensorrt_llm/tools/plugin_gen/shape_infer.py | tensorrt_llm/tools/ppl.py | tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py | tensorrt_llm/version.py | @@ -1258,7 +2181,9 @@ legacy-files: &legacy_files | tests/integration/defs/accuracy/accuracy_core.py | tests/integration/defs/accuracy/scripts/collect_evaluated_accuracies.py | tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py | + tests/integration/defs/accuracy/test_cli_flow.py | tests/integration/defs/accuracy/test_disaggregated_serving.py | + tests/integration/defs/accuracy/test_llm_api.py | tests/integration/defs/accuracy/test_llm_api_autodeploy.py | tests/integration/defs/accuracy/test_llm_api_pytorch.py | tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py | @@ -1267,29 +2192,63 @@ legacy-files: &legacy_files | tests/integration/defs/conftest.py | tests/integration/defs/cpp/conftest.py | tests/integration/defs/cpp/cpp_common.py | + tests/integration/defs/cpp/test_e2e.py | tests/integration/defs/cpp/test_multi_gpu.py | tests/integration/defs/cpp/test_unit_tests.py | + tests/integration/defs/deterministic/mixtral_deterministic.py | + tests/integration/defs/deterministic/test_mixtral_deterministic.py | tests/integration/defs/disaggregated/test_auto_scaling.py | tests/integration/defs/disaggregated/test_disaggregated.py | tests/integration/defs/disaggregated/test_disaggregated_etcd.py | tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py | tests/integration/defs/disaggregated/test_workers.py | + tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py | tests/integration/defs/examples/run_llm_quickstart_atexit.py | tests/integration/defs/examples/serve/test_serve.py | tests/integration/defs/examples/serve/test_serve_negative.py | tests/integration/defs/examples/test_ad_guided_decoding.py | + tests/integration/defs/examples/test_bert.py | + tests/integration/defs/examples/test_bindings.py | + tests/integration/defs/examples/test_chatglm.py | + tests/integration/defs/examples/test_commandr.py | + tests/integration/defs/examples/test_draft_target_model.py | + tests/integration/defs/examples/test_eagle.py | + tests/integration/defs/examples/test_enc_dec.py | + tests/integration/defs/examples/test_exaone.py | + tests/integration/defs/examples/test_gemma.py | tests/integration/defs/examples/test_gpt.py | + tests/integration/defs/examples/test_gptj.py | + tests/integration/defs/examples/test_granite.py | + tests/integration/defs/examples/test_internlm.py | + tests/integration/defs/examples/test_llama.py | tests/integration/defs/examples/test_llm_api_with_mpi.py | + tests/integration/defs/examples/test_mamba.py | + tests/integration/defs/examples/test_medusa.py | + tests/integration/defs/examples/test_mistral.py | + tests/integration/defs/examples/test_mixtral.py | + tests/integration/defs/examples/test_multimodal.py | + tests/integration/defs/examples/test_nemotron.py | + tests/integration/defs/examples/test_nemotron_nas.py | + tests/integration/defs/examples/test_ngram.py | + tests/integration/defs/examples/test_openai.py | tests/integration/defs/examples/test_phi.py | + tests/integration/defs/examples/test_qwen.py | + tests/integration/defs/examples/test_qwen2audio.py | + tests/integration/defs/examples/test_qwenvl.py | tests/integration/defs/examples/test_ray.py | + tests/integration/defs/examples/test_recurrentgemma.py | + tests/integration/defs/examples/test_redrafter.py | + tests/integration/defs/examples/test_whisper.py | tests/integration/defs/llmapi/__init__.py | tests/integration/defs/llmapi/_run_llmapi_llm.py | tests/integration/defs/llmapi/test_llm_api_connector.py | tests/integration/defs/llmapi/test_llm_api_qa.py | + tests/integration/defs/llmapi/test_llm_e2e.py | tests/integration/defs/llmapi/test_llm_examples.py | tests/integration/defs/local_venv.py | tests/integration/defs/perf/__init__.py | tests/integration/defs/perf/allowed_configs.py | + tests/integration/defs/perf/build.py | tests/integration/defs/perf/create_perf_comparison_report.py | tests/integration/defs/perf/data.py | tests/integration/defs/perf/data_export.py | @@ -1310,21 +2269,38 @@ legacy-files: &legacy_files | tests/integration/defs/test_fmha.py | tests/integration/defs/test_list_parser.py | tests/integration/defs/test_list_validation.py | + tests/integration/defs/test_mlpf_results.py | tests/integration/defs/test_sanity.py | tests/integration/defs/test_unittests.py | tests/integration/defs/triton_server/__init__.py | + tests/integration/defs/triton_server/build_engines.py | tests/integration/defs/triton_server/common.py | tests/integration/defs/triton_server/conftest.py | + tests/integration/defs/triton_server/local_venv.py | + tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py | + tests/integration/defs/triton_server/runner_interface.py | tests/integration/defs/triton_server/test_list_parser.py | + tests/integration/defs/triton_server/test_triton.py | + tests/integration/defs/triton_server/test_triton_llm.py | + tests/integration/defs/triton_server/test_triton_memleak.py | + tests/integration/defs/triton_server/test_triton_multi_node.py | + tests/integration/defs/triton_server/test_triton_rcca.py | tests/integration/defs/triton_server/trt_test_alternative.py | tests/integration/defs/trt_test_alternative.py | tests/integration/defs/utils/__init__.py | tests/integration/defs/utils/periodic_junit.py | tests/integration/defs/utils/timeout_manager.py | tests/microbenchmarks/all_reduce.py | + tests/microbenchmarks/build_time_benchmark.py | + tests/microbenchmarks/build_time_dashboard.py | tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py | tests/scripts/allreduce_perf/allreduce_perf_viz.py | tests/scripts/iteration_log_parser.py | + tests/scripts/perf-sanity/parse_benchmark_results.py | + tests/scripts/perf-sanity/run_benchmark_serve.py | + tests/unittest/_torch/attention/sparse/test_dsa_indexer.py | + tests/unittest/_torch/attention/sparse/test_flash_mla.py | + tests/unittest/_torch/attention/sparse/test_rocketkv.py | tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py | tests/unittest/_torch/attention/test_attention.py | tests/unittest/_torch/attention/test_attention_mla.py | @@ -1345,6 +2321,7 @@ legacy-files: &legacy_files | tests/unittest/_torch/misc/test_virtual_memory.py | tests/unittest/_torch/modeling/test_modeling_bert.py | tests/unittest/_torch/modeling/test_modeling_clip.py | + tests/unittest/_torch/modeling/test_modeling_exaone4.py | tests/unittest/_torch/modeling/test_modeling_gemma3.py | tests/unittest/_torch/modeling/test_modeling_gpt_oss.py | tests/unittest/_torch/modeling/test_modeling_llama.py | @@ -1368,6 +2345,8 @@ legacy-files: &legacy_files | tests/unittest/_torch/modules/test_moe_routing.py | tests/unittest/_torch/modules/test_rotary_embedding.py | tests/unittest/_torch/modules/test_triton_linear.py | + tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py | + tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py | tests/unittest/_torch/multi_gpu/test_allreduce.py | tests/unittest/_torch/multi_gpu/test_alltoall.py | tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py | @@ -1394,10 +2373,24 @@ legacy-files: &legacy_files | tests/unittest/_torch/sampler/test_beam_search.py | tests/unittest/_torch/sampler/test_best_of_n.py | tests/unittest/_torch/sampler/test_trtllm_sampler.py | + tests/unittest/_torch/speculative/test_draft_target.py | + tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py | + tests/unittest/_torch/speculative/test_draft_token_tree_verification.py | + tests/unittest/_torch/speculative/test_dynamic_spec_decode.py | tests/unittest/_torch/speculative/test_eagle3.py | + tests/unittest/_torch/speculative/test_kv_cache_reuse.py | + tests/unittest/_torch/speculative/test_mtp.py | + tests/unittest/_torch/speculative/test_ngram.py | + tests/unittest/_torch/speculative/test_save_state.py | + tests/unittest/_torch/speculative/test_spec_gate.py | + tests/unittest/_torch/speculative/test_torch_rejection_sampling.py | + tests/unittest/_torch/speculative/test_user_provided.py | tests/unittest/_torch/test_connector.py | tests/unittest/_torch/test_torch_multi_arange.py | tests/unittest/_torch/thop/parallel/deep_gemm_tests.py | + tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py | + tests/unittest/_torch/thop/parallel/test_cublas_mm.py | + tests/unittest/_torch/thop/parallel/test_custom_ops.py | tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py | tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py | tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py | @@ -1411,6 +2404,11 @@ legacy-files: &legacy_files | tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py | tests/unittest/_torch/thop/parallel/test_fp8_quantize.py | tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py | + tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py | + tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py | + tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py | + tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py | + tests/unittest/_torch/thop/parallel/test_noaux_tc.py | tests/unittest/_torch/thop/parallel/test_scaled_mm.py | tests/unittest/_torch/thop/parallel/test_selective_scan_op.py | tests/unittest/_torch/thop/parallel/test_tinygemm2.py | @@ -1424,6 +2422,7 @@ legacy-files: &legacy_files | tests/unittest/_torch/thop/serial/test_moe_alltoall.py | tests/unittest/api_stability/api_stability_core.py | tests/unittest/api_stability/test_llm_api.py | + tests/unittest/bindings/binding_test_utils.py | tests/unittest/bindings/test_bindings_moe.py | tests/unittest/bindings/test_bindings_ut.py | tests/unittest/bindings/test_executor_bindings.py | @@ -1454,10 +2453,12 @@ legacy-files: &legacy_files | tests/unittest/llmapi/apps/_test_openai_chat_harmony.py | tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py | tests/unittest/llmapi/apps/_test_openai_completions.py | + tests/unittest/llmapi/apps/_test_openai_consistent_chat.py | tests/unittest/llmapi/apps/_test_openai_lora.py | tests/unittest/llmapi/apps/_test_openai_metrics.py | tests/unittest/llmapi/apps/_test_openai_misc.py | tests/unittest/llmapi/apps/_test_openai_mmencoder.py | + tests/unittest/llmapi/apps/_test_openai_multi_chat.py | tests/unittest/llmapi/apps/_test_openai_multi_gpu.py | tests/unittest/llmapi/apps/_test_openai_multi_nodes.py | tests/unittest/llmapi/apps/_test_openai_perf_metrics.py | @@ -1480,12 +2481,15 @@ legacy-files: &legacy_files | tests/unittest/llmapi/run_llm_exit.py | tests/unittest/llmapi/run_llm_with_postproc.py | tests/unittest/llmapi/test_additional_model_outputs.py | + tests/unittest/llmapi/test_build_cache.py | tests/unittest/llmapi/test_executor.py | tests/unittest/llmapi/test_gc_utils.py | tests/unittest/llmapi/test_llm.py | tests/unittest/llmapi/test_llm_args.py | tests/unittest/llmapi/test_llm_download.py | tests/unittest/llmapi/test_llm_kv_cache_events.py | + tests/unittest/llmapi/test_llm_models.py | + tests/unittest/llmapi/test_llm_multi_gpu.py | tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py | tests/unittest/llmapi/test_llm_pytorch.py | tests/unittest/llmapi/test_llm_quant.py | @@ -1496,14 +2500,25 @@ legacy-files: &legacy_files | tests/unittest/llmapi/test_serialization.py | tests/unittest/llmapi/test_utils.py | tests/unittest/others/__init__.py | + tests/unittest/others/test_builder.py | tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py | + tests/unittest/others/test_debugging_api.py | tests/unittest/others/test_exception.py | tests/unittest/others/test_export.py | + tests/unittest/others/test_graph_rewriter.py | + tests/unittest/others/test_kv_cache_manager.py | tests/unittest/others/test_kv_cache_transceiver.py | tests/unittest/others/test_kv_cache_update.py | + tests/unittest/others/test_layer.py | + tests/unittest/others/test_leak.py | tests/unittest/others/test_mapping.py | + tests/unittest/others/test_model_dtype.py | + tests/unittest/others/test_module.py | tests/unittest/others/test_multimodal_registry.py | + tests/unittest/others/test_plugins.py | + tests/unittest/others/test_precision_control.py | tests/unittest/others/test_pretrained_config.py | + tests/unittest/others/test_session.py | tests/unittest/others/test_time_breakdown.py | tests/unittest/profile_utils.py | tests/unittest/scaffolding/__init__.py | @@ -1512,14 +2527,141 @@ legacy-files: &legacy_files | tests/unittest/scaffolding/test_scaffolding.py | tests/unittest/scaffolding/test_task_collection.py | tests/unittest/scaffolding/test_worker.py | + tests/unittest/test_model_runner_cpp.py | tests/unittest/test_pip_install.py | tests/unittest/tools/__init__.py | + tests/unittest/tools/plugin_gen/__init__.py | + tests/unittest/tools/plugin_gen/kernel_config.py | + tests/unittest/tools/plugin_gen/test_core.py | + tests/unittest/tools/plugin_gen/test_plugin_gen.py | + tests/unittest/tools/plugin_gen/test_shape_infer.py | tests/unittest/tools/test_prepare_dataset.py | tests/unittest/tools/test_test_to_stage_mapping.py | + tests/unittest/trt/__init__.py | + tests/unittest/trt/attention/test_bert_attention.py | + tests/unittest/trt/attention/test_gpt_attention.py | + tests/unittest/trt/attention/test_gpt_attention_IFB.py | + tests/unittest/trt/attention/test_gpt_attention_no_cache.py | + tests/unittest/trt/attention/test_sage_attention.py | + tests/unittest/trt/functional/__init__.py | + tests/unittest/trt/functional/test_alibi.py | + tests/unittest/trt/functional/test_allreduce_norm.py | + tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py | + tests/unittest/trt/functional/test_arange.py | + tests/unittest/trt/functional/test_argmax.py | + tests/unittest/trt/functional/test_assertion.py | + tests/unittest/trt/functional/test_avg_pool2d.py | + tests/unittest/trt/functional/test_cast.py | + tests/unittest/trt/functional/test_conv2d.py | + tests/unittest/trt/functional/test_conv3d.py | + tests/unittest/trt/functional/test_cos.py | + tests/unittest/trt/functional/test_cumsum.py | + tests/unittest/trt/functional/test_dora.py | + tests/unittest/trt/functional/test_einsum.py | + tests/unittest/trt/functional/test_embedding_single_gpu.py | + tests/unittest/trt/functional/test_exp.py | + tests/unittest/trt/functional/test_expand.py | + tests/unittest/trt/functional/test_flatten.py | + tests/unittest/trt/functional/test_flip.py | + tests/unittest/trt/functional/test_fp4_gemm.py | + tests/unittest/trt/functional/test_fp4_gemm_ootb.py | + tests/unittest/trt/functional/test_gather.py | + tests/unittest/trt/functional/test_gather_nd.py | + tests/unittest/trt/functional/test_geglu.py | + tests/unittest/trt/functional/test_gelu.py | + tests/unittest/trt/functional/test_gemm_swiglu.py | + tests/unittest/trt/functional/test_group_norm.py | + tests/unittest/trt/functional/test_identity.py | + tests/unittest/trt/functional/test_index_select.py | + tests/unittest/trt/functional/test_interpolate.py | + tests/unittest/trt/functional/test_logsoftmax.py | + tests/unittest/trt/functional/test_lora.py | + tests/unittest/trt/functional/test_low_latency_gemm.py | + tests/unittest/trt/functional/test_mamba_conv1d.py | + tests/unittest/trt/functional/test_masked_scatter.py | + tests/unittest/trt/functional/test_masked_select.py | + tests/unittest/trt/functional/test_matmul.py | + tests/unittest/trt/functional/test_meshgrid2d.py | + tests/unittest/trt/functional/test_moe.py | + tests/unittest/trt/functional/test_nccl.py | + tests/unittest/trt/functional/test_nonzero.py | + tests/unittest/trt/functional/test_outer.py | + tests/unittest/trt/functional/test_pad.py | + tests/unittest/trt/functional/test_permute.py | + tests/unittest/trt/functional/test_pp_reduce_scatter.py | + tests/unittest/trt/functional/test_quant.py | + tests/unittest/trt/functional/test_rearrange.py | + tests/unittest/trt/functional/test_repeat.py | + tests/unittest/trt/functional/test_repeat_interleave.py | + tests/unittest/trt/functional/test_rg_lru.py | + tests/unittest/trt/functional/test_sample.py | + tests/unittest/trt/functional/test_scatter.py | + tests/unittest/trt/functional/test_scatter_nd.py | + tests/unittest/trt/functional/test_select.py | + tests/unittest/trt/functional/test_selective_scan.py | + tests/unittest/trt/functional/test_sigmoid.py | + tests/unittest/trt/functional/test_silu.py | + tests/unittest/trt/functional/test_sin.py | + tests/unittest/trt/functional/test_slice.py | + tests/unittest/trt/functional/test_softplus.py | + tests/unittest/trt/functional/test_split.py | + tests/unittest/trt/functional/test_squeeze.py | + tests/unittest/trt/functional/test_swiglu.py | + tests/unittest/trt/functional/test_topk.py | + tests/unittest/trt/functional/test_transpose.py | + tests/unittest/trt/functional/test_unbind.py | + tests/unittest/trt/functional/test_unsqueeze.py | + tests/unittest/trt/functional/test_view.py | + tests/unittest/trt/functional/test_where.py | + tests/unittest/trt/model/__init__.py | + tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py | + tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py | + tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py | + tests/unittest/trt/model/redrafter/test_beams2tree.py | + tests/unittest/trt/model/redrafter/test_draft_token.py | + tests/unittest/trt/model/redrafter/test_draft_token_indices.py | + tests/unittest/trt/model/redrafter/test_gather_beams.py | + tests/unittest/trt/model/redrafter/test_mask.py | + tests/unittest/trt/model/redrafter/test_packed_position_ids.py | + tests/unittest/trt/model/redrafter/test_prefix_match_indices.py | + tests/unittest/trt/model/redrafter/test_prepare_input.py | + tests/unittest/trt/model/redrafter/test_process_logits.py | + tests/unittest/trt/model/redrafter/test_top1.py | + tests/unittest/trt/model/redrafter/test_unpack_gen_data.py | + tests/unittest/trt/model/redrafter/test_validate.py | + tests/unittest/trt/model/test_gpt.py | + tests/unittest/trt/model/test_gpt_e2e.py | + tests/unittest/trt/model/test_llama.py | + tests/unittest/trt/model/test_mamba.py | + tests/unittest/trt/model/test_mistral.py | + tests/unittest/trt/model/test_nemotron_nas.py | + tests/unittest/trt/model/test_phi.py | + tests/unittest/trt/model/test_unet.py | + tests/unittest/trt/model_api/test_model_api_multi_gpu.py | + tests/unittest/trt/model_api/test_model_level_api.py | + tests/unittest/trt/model_api/test_model_quantization.py | + tests/unittest/trt/python_plugin/plugin_wrapper_utils.py | + tests/unittest/trt/python_plugin/test_plugin_wrapper.py | + tests/unittest/trt/quantization/__init__.py | + tests/unittest/trt/quantization/_utils.py | + tests/unittest/trt/quantization/test_fp8_quantization.py | + tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py | + tests/unittest/trt/quantization/test_functional.py | + tests/unittest/trt/quantization/test_mode.py | + tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py | + tests/unittest/trt/quantization/test_qserve_gemm.py | + tests/unittest/trt/quantization/test_quant.py | + tests/unittest/trt/quantization/test_quant_layer.py | + tests/unittest/trt/quantization/test_smooth_quant_gemm.py | + tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py | + tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py | + tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py | + tests/unittest/trt/quantization/test_weight_only_quant_matmul.py | tests/unittest/utils/__init__.py | tests/unittest/utils/cpp_paths.py | tests/unittest/utils/llm_data.py | tests/unittest/utils/runtime_defaults.py | + tests/unittest/utils/test_medusa_utils.py | tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py | tests/unittest/utils/test_util.py | tests/unittest/utils/torch_ref.py | diff --git a/3rdparty/MSA b/3rdparty/MSA deleted file mode 160000 index e2ebe7656649..000000000000 --- a/3rdparty/MSA +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e2ebe7656649f619af0ad1d457b534283034655e diff --git a/AGENTS.md b/AGENTS.md index cf6c717d0928..20fa71f70f7f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # AGENTS.md TensorRT-LLM: open-source library for optimized LLM inference on NVIDIA GPUs. -Python and C++ codebase with PyTorch and AutoDeploy execution paths. +Python and C++ codebase supporting TensorRT engine-based and PyTorch-based execution paths. > If a `CLAUDE.local.md` file exists alongside this file, read and respect it — it contains developer-specific overrides that supplement this shared guidance. @@ -55,16 +55,17 @@ See [architecture diagram](.github/tava_architecture_diagram.md) for the full Me |---------|--------|-------------|----------| | **PyTorch** | Default | `TorchLlmArgs` | `_torch/pyexecutor/` → `PyExecutor` → PyTorch Engine | | **AutoDeploy** | Beta | `_torch/auto_deploy/` shim | `_torch/auto_deploy/shim/ad_executor.py` → adapts `PyExecutor` → graph transforms + torch.export | +| **TensorRT** | Legacy | `TrtLlmArgs` | `builder.py` → `trtllm.Executor` → TensorRT Engine | ### Shared C++ Core (via Nanobind) -Both backends share these C++ components: +Both PyTorch and TensorRT backends share these C++ components: - **Scheduling pipeline**: Scheduler → BatchManager (in-flight batching) → KV Cache Manager - **Decoding pipeline**: Decoder (token generation orchestration) → Sampling ### Request Flow ```text -HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy) +HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy/TensorRT) → Scheduler → Model Forward → Decoder → Sampling → Generated Tokens ``` @@ -83,7 +84,7 @@ HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy) | `tensorrt_llm/models/modeling_utils.py` | Base classes for all models (`PretrainedConfig`, `PretrainedModel`) | | `tensorrt_llm/executor/executor.py` | Execution abstraction (`GenerationExecutor`) | | `tensorrt_llm/models/automodel.py` | Auto-discovery and model registry | -| `tensorrt_llm/_torch/models/` | PyTorch backend model implementations (distinct from the top-level `models/` package) | +| `tensorrt_llm/_torch/models/` | PyTorch backend model implementations (distinct from `models/` used by TensorRT backend) | | `tensorrt_llm/_torch/modules/ATTENTION_DEVELOPER_GUIDE.md` | Attention, MLA, backend families, sparse backends, metadata contracts, and KV-cache behavior - **read before modifying `tensorrt_llm/_torch/modules/attention.py`, `tensorrt_llm/_torch/modules/mla.py`, or `tensorrt_llm/_torch/attention_backend/`** | | `tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md` | MoE architecture, backends, communication, development patterns — **read before modifying MoE code** | | `CODING_GUIDELINES.md` | C++ and Python coding standards (referenced throughout, must read before contributing) | @@ -92,7 +93,7 @@ HuggingFace Model → LLM API → Executor (PyTorch/AutoDeploy) | Pattern | Key Points | |---------|------------| -| **Config hierarchy** | `BaseLlmArgs` → `TorchLlmArgs`, model-specific defaults override generics, Pydantic validation | +| **Config hierarchy** | `BaseLlmArgs` → `TrtLlmArgs` / `TorchLlmArgs`, model-specific defaults override generics, Pydantic validation | | **Model architecture** | Each model: `Config` (inherits `PretrainedConfig`) + `ForCausalLM` (inherits `PretrainedModel`) | | **Model defaults** | Architecture-specific overrides in `llm_utils.py` (attention kernels, quant, spec decoding, cache) | | **Attention backends** | `TorchLlmArgs.attn_backend` selects kernel: `TRTLLM` (default), `FlashInfer`, `FlashAttention` | @@ -124,6 +125,7 @@ Key files: - **Avoid broad exception handling** — catch specific exceptions, not bare `except:` (see `CODING_GUIDELINES.md`). - **One concern per PR** — avoid scope creep. If a PR touches unrelated areas, split it. - **User-facing configuration classes** - when editing or defining any user-facing configuration classes (particularly `BaseLlmArgs` or any class used in its fields), you **MUST** follow the Pydantic guidelines in `CODING_GUIDELINES.md`. +- **TensorRT backend is legacy** — `TrtLlmArgs` / `backend="tensorrt"` and all exclusive tooling (`trtllm-build`, `trtllm-refit`, `convert_checkpoint.py`, `ModelRunner*`) are legacy. Bug fixes OK; new features target PyTorch or AutoDeploy. ## Development Workflow diff --git a/ATTRIBUTIONS-Python.md b/ATTRIBUTIONS-Python.md index 7e74b846a85f..e43a78cafda0 100644 --- a/ATTRIBUTIONS-Python.md +++ b/ATTRIBUTIONS-Python.md @@ -5261,7 +5261,7 @@ For more information, please refer to <http://unlicense.org> - `Tracker`: https://github.com/tox-dev/py-filelock/issues -## flashinfer-python (0.6.15) +## flashinfer-python (0.6.14) ### Licenses License: `Apache-2.0` diff --git a/LICENSE b/LICENSE index ed5ca3b261ba..8ba867f30567 100644 --- a/LICENSE +++ b/LICENSE @@ -19,13 +19,6 @@ Original Source: https://github.com/Dao-AILab/causal-conv1d Copyright (c) 2024, Tri Dao. Licensed under the BSD 3-Clause License --------------------------------------------------------------------------------- -CUTLASS --------------------------------------------------------------------------------- -Original Source: https://github.com/NVIDIA/cutlass -Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -Licensed under the BSD 3-Clause License - -------------------------------------------------------------------------------- flash-attention -------------------------------------------------------------------------------- @@ -41,14 +34,6 @@ Original Source: https://github.com/fla-org/flash-linear-attention Copyright (c) 2023-2025 Songlin Yang Licensed under the MIT License --------------------------------------------------------------------------------- -FlashInfer --------------------------------------------------------------------------------- -Original Source: https://github.com/flashinfer-ai/flashinfer -Copyright 2025-2026 NVIDIA -Copyright 2023-2026 FlashInfer community (https://flashinfer.ai/) -Licensed under the Apache License 2.0 - -------------------------------------------------------------------------------- InstructEval -------------------------------------------------------------------------------- @@ -74,14 +59,6 @@ Original Source: https://github.com/state-spaces/mamba Copyright 2023 Tri Dao, Albert Gu Licensed under the Apache License 2.0 --------------------------------------------------------------------------------- -MSA (MiniMax Sparse Attention) --------------------------------------------------------------------------------- -Original Source: https://github.com/MiniMax-AI/MSA -Copyright (c) 2026 MiniMax -Licensed under the MIT License - - -------------------------------------------------------------------------------- Quack -------------------------------------------------------------------------------- diff --git a/README.md b/README.md index e1324129fd92..943083d6c4c6 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ TensorRT LLM [![python](https://img.shields.io/badge/python-3.10-green)](https://www.python.org/downloads/release/python-31012/) [![cuda](https://img.shields.io/badge/cuda-13.2.1-green)](https://developer.nvidia.com/cuda-downloads) [![torch](https://img.shields.io/badge/torch-2.11.0-green)](https://pytorch.org) -[![version](https://img.shields.io/badge/release-1.3.0rc23-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) +[![version](https://img.shields.io/badge/release-1.3.0rc21-green)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/version.py) [![license](https://img.shields.io/badge/license-Apache%202-blue)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/LICENSE) [Architecture](https://nvidia.github.io/TensorRT-LLM/developer-guide/overview.html)   |   [Performance](https://nvidia.github.io/TensorRT-LLM/developer-guide/perf-overview.html)   |   [Examples](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html)   |   [Documentation](https://nvidia.github.io/TensorRT-LLM/)   |   [Roadmap](https://github.com/NVIDIA/TensorRT-LLM/issues?q=is%3Aissue%20state%3Aopen%20label%3Aroadmap) @@ -22,9 +22,6 @@ TensorRT LLM <!-- Use github markdown link to link for the latest blog since the doc build has not happened yet. When the doc build is updated, it should be updated to the webpage link. --> -* [07/17] DeepSeek-V4 on NVIDIA Blackwell: Model-Specific and Agentic-Workload Optimizations in TensorRT LLM -✨ [➡️ link](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md) - * [07/01] Scaling Video Generation Across NVL72 Rack with TensorRT-LLM ✨ [➡️ link](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog25_Scaling_Video_Generation_Across_NVL72_Rack_with_TensorRT-LLM.md) diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000000..5d89f412ac5c --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,10 @@ +# TensorRT-LLM Benchmarks + +## Overview + +There are currently two workflows to benchmark TensorRT-LLM: +* [`trtllm-bench`](../docs/source/developer-guide/perf-benchmarking.md) + - `trtllm-bench` is native to TensorRT-LLM and is a Python benchmarker for reproducing and testing the performance of TensorRT-LLM. + - _NOTE_: This benchmarking suite is a current work in progress and is prone to large changes. +* [C++ benchmarks](./cpp) + - The recommended workflow that uses TensorRT-LLM C++ API and can take advantage of the latest features of TensorRT-LLM. diff --git a/benchmarks/cpp/CMakeLists.txt b/benchmarks/cpp/CMakeLists.txt new file mode 100644 index 000000000000..cb5ef1ee928b --- /dev/null +++ b/benchmarks/cpp/CMakeLists.txt @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +include_directories(${PROJECT_SOURCE_DIR}/include) + +set(TOP_LEVEL_DIR "${PROJECT_SOURCE_DIR}/..") + +add_custom_target(benchmarks) + +if(NOT TARGET cxxopts::cxxopts) + add_subdirectory(${CMAKE_BINARY_DIR}/_deps/cxxopts-src + ${CMAKE_CURRENT_BINARY_DIR}/cxxopts) +endif() + +function(add_benchmark test_name test_src) + add_executable(${test_name} ${test_src} utils/utils.cpp) + + target_link_libraries( + ${test_name} PUBLIC ${SHARED_TARGET} nvinfer_plugin_tensorrt_llm + cxxopts::cxxopts) + + target_compile_features(${test_name} PRIVATE cxx_std_17) + target_compile_definitions(${test_name} + PUBLIC TOP_LEVEL_DIR="${TOP_LEVEL_DIR}") + add_dependencies(benchmarks ${test_name}) +endfunction() + +add_benchmark(bertBenchmark bertBenchmark.cpp) +add_benchmark(gptManagerBenchmark gptManagerBenchmark.cpp) +add_benchmark(disaggServerBenchmark disaggServerBenchmark.cpp) diff --git a/benchmarks/cpp/README.md b/benchmarks/cpp/README.md new file mode 100644 index 000000000000..ae3287faf06c --- /dev/null +++ b/benchmarks/cpp/README.md @@ -0,0 +1,367 @@ +# Benchmark C++ Runtime + +This document explains how to benchmark the models supported by TensorRT-LLM on a single GPU, a single node with +multiple GPUs or multiple nodes with multiple GPUs using the C++ runtime. + +## Usage + +### 1. Build TensorRT-LLM and benchmarking source code + +Please follow the [`installation document`](../../README.md#installation) to build TensorRT-LLM. + +Note that the benchmarking source code for C++ runtime is not built by default, you can use the argument `--benchmarks` in [`build_wheel.py`](source:scripts/build_wheel.py) to build the corresponding executable. + +### 2. Launch C++ benchmarking (Inflight/V1 batching) + +#### Prepare dataset + +Run a preprocessing script to prepare/generate dataset into a json that `gptManagerBenchmark` can consume later. The processed output json has *input tokens length, input token ids and output tokens length*. + +For `tokenizer`, specifying the path to the local tokenizer that have already been downloaded, or simply the name of the tokenizer from HuggingFace like `meta-llama/Llama-2-7b` will both work. The tokenizer will be downloaded automatically for the latter case. + +This tool can be used in 3 different modes of traffic generation: `dataset`, `token-norm-dist` and `token-unif-dist`. + +##### 1 – Dataset + +The tool will tokenize the words and instruct the model to generate a specified number of output tokens for a request. + +``` +python3 prepare_dataset.py \ + --tokenizer <path/to/tokenizer> \ + --output preprocessed_dataset.json + dataset + --dataset-name <name of the dataset> \ + --dataset-split <split of the dataset to use> \ + --dataset-input-key <dataset dictionary key for input> \ + --dataset-prompt-key <dataset dictionary key for prompt> \ + --dataset-output-key <dataset dictionary key for output> \ + [--num-requests 100] \ + [--max-input-len 1000] \ + [--output-len-dist 100,10] +``` + +For datasets that don't have prompt key, set --dataset-prompt instead. +Take [cnn_dailymail dataset](https://huggingface.co/datasets/abisee/cnn_dailymail) for example: +``` +python3 prepare_dataset.py \ + --tokenizer <path/to/tokenizer> \ + --output cnn_dailymail.json + dataset + --dataset-name cnn_dailymail \ + --dataset-split validation \ + --dataset-config-name 3.0.0 \ + --dataset-input-key article \ + --dataset-prompt "Summarize the following article:" \ + --dataset-output-key "highlights" \ + [--num-requests 100] \ + [--max-input-len 1000] \ + [--output-len-dist 100,10] +``` + +##### 2 – Normal token length distribution + +This mode allows the user to generate normally distributed token lengths with a mean and std deviation specified. +For example, setting `mean=100` and `stdev=10` would generate requests where 95.4% of values are in <80,120> range following the normal probability distribution. Setting `stdev=0` will generate all requests with the same mean number of tokens. + +``` +python prepare_dataset.py \ + --output token-norm-dist.json \ + --tokenizer <path/to/tokenizer> \ + token-norm-dist \ + --num-requests 100 \ + --input-mean 100 --input-stdev 10 \ + --output-mean 15 --output-stdev 0 +``` + +##### 2 – Uniform token length distribution + +This mode allows the user to generate uniformly distributed token lengths with min and max lengths specified. +For example, setting `min=50` and `max=100` would generate requests where lengths are in the range `[50, 100]` following the uniform probability distribution. Setting `min=x` and `max=x` will generate all requests with the same mean number of tokens `x`. + +``` +python prepare_dataset.py \ + --output token-norm-dist.json \ + --tokenizer <path/to/tokenizer> \ + token-unif-dist \ + --num-requests 100 \ + --input-min 50 --input-max 100 \ + --output-min 10 --output-max 15 +``` + + +#### Prepare TensorRT-LLM engines + +Before you launch C++ benchmarking, please make sure that you have already built engine(s) using `trtllm-build` command. For more details on building engine(s), please refer to the [Quick Start Guide](../../docs/source/quick-start-guide.md). + +#### Launch benchmarking + +For detailed usage, you can do the following +``` +cd cpp/build + +# You can directly execute the binary for help information +./benchmarks/gptManagerBenchmark --help +``` + +`gptManagerBenchmark` now supports decoder-only models and encoder-decoder models. + +1. Decoder-only Models + + To benchmark decoder-only models, pass in the engine path with `--engine_dir` as executable input argument. + + Take GPT-350M as an example for 2-GPU inflight batching + ``` + mpirun -n 2 ./benchmarks/gptManagerBenchmark \ + --engine_dir ../../examples/models/core/gpt/trt_engine/gpt2-ib/fp16/2-gpu/ \ + --request_rate 10 \ + --dataset ../../benchmarks/cpp/preprocessed_dataset.json \ + --max_num_samples 500 + ``` + + `gptManagerBenchmark` by default uses the high-level C++ API defined by the `executor::Executor` class (see `cpp/include/tensorrt_llm/executor/executor.h`). + +2. Encoder-Decoder Models + To benchmark encoder-decoder models, pass in the encoder engine path with `--encoder_engine_dir` and the decoder engine path with `--decoder_engine_dir` as executable input arguments. `--decoder_engine_dir` is an alias of `--engine_dir`. + + Currently encoder-decoder engines only support `--api executor`, `--type IFB`, `--enable_kv_cache_reuse false`, which are all default values so no specific settings required. + + Prepare t5-small engine from [examples/models/core/enc_dec](/examples/models/core/enc_dec/README.md#convert-and-split-weights) for the encoder-decoder 4-GPU inflight batching example. + + Prepare the dataset suitable for engine input lengths. + ``` + python prepare_dataset.py \ + --tokenizer <path/to/tokenizer> \ + --output cnn_dailymail.json \ + dataset \ + --dataset-name cnn_dailymail \ + --dataset-split validation \ + --dataset-config-name 3.0.0 \ + --dataset-input-key article \ + --dataset-prompt "Summarize the following article:" \ + --dataset-output-key "highlights" \ + --num-requests 100 \ + --max-input-len 512 \ + --output-len-dist 128,20 + ``` + + Run the benchmark + ``` + mpirun --allow-run-as-root -np 4 ./benchmarks/gptManagerBenchmark \ + --encoder_engine_dir ../../examples/models/core/enc_dec/tmp/trt_engines/t5-small-4gpu/bfloat16/encoder \ + --decoder_engine_dir ../../examples/models/core/enc_dec/tmp/trt_engines/t5-small-4gpu/bfloat16/decoder \ + --dataset cnn_dailymail.json + ``` + + +#### Emulated static batching + +To emulate the deprecated `gptSessionBenchmark` static batching, you can use `gptManagerBenchmark` with the `--static_emulated_batch_size` and `--static_emulated-timeout` arguments. + +Given a `static_emulated_batch_size` of `n` the server will wait for `n` requests to arrive before submitting them to the batch manager at once. If the `static_emulated_timeout` (in ms) is reached before `n` requests are collected, the batch will be submitted prematurely with the current request count. New batches will only be submitted once the previous batch has been processed comepletely. + +Datasets with fixed input/output lengths for benchmarking can be generated with the preprocessing script, e.g. +``` + python prepare_dataset.py \ + --output tokens-fixed-lengths.json \ + --tokenizer <path/to/tokenizer> \ + token-norm-dist \ + --num-requests 128 \ + --input-mean 60 --input-stdev 0 \ + --output-mean 20 --output-stdev 0 +``` + +Take GPT-350M as an example for single GPU with static batching +``` +./benchmarks/gptManagerBenchmark \ + --engine_dir ../../examples/models/core/gpt/trt_engine/gpt2/fp16/1-gpu/ \ + --request_rate -1 \ + --static_emulated_batch_size 32 \ + --static_emulated_timeout 100 \ + --dataset ../../benchmarks/cpp/tokens-fixed-lengths.json +``` + +#### Benchmarking LoRA + +Using either of the `prepare_dataset.py` methods above, add `--rand-task-id <start-id> <end-id>` to the command. This will add a random `task_id` from `<start-id>` to `<end-id>` inclusive. +You can then use `utils/generate_rand_loras.py` to generate random LoRA weights for benchmarking purposes. `utils/generate_rand_loras.py` takes an example LoRA for the model you are benchmarking. +Then you can run `gptManagerBenchmark` with `--type IFB` and `--lora_dir /path/to/utils/generate_rand_loras/output` + +End-to-end LoRA benchmarking script + +``` +git-lfs clone https://huggingface.co/meta-llama/Llama-2-13b-hf +git-lfs clone https://huggingface.co/hfl/chinese-llama-2-lora-13b + +MODEL_CHECKPOINT=Llama-2-13b-hf +CONVERTED_CHECKPOINT=Llama-2-13b-hf-ckpt +TOKENIZER=Llama-2-13b-hf +LORA_ENGINE=Llama-2-13b-hf-engine + +DTYPE=float16 +TP=2 +PP=1 +MAX_LEN=1024 +MAX_BATCH=32 +NUM_LAYERS=40 +MAX_LORA_RANK=64 +NUM_LORA_MODS=7 +EOS_ID=2 + +SOURCE_LORA=chinese-llama-2-lora-13b +CPP_LORA=chinese-llama-2-lora-13b-cpp + +EG_DIR=/tmp/lora-eg + +# Build lora enabled engine +python examples/models/core/llama/convert_checkpoint.py --model_dir ${MODEL_CHECKPOINT} \ + --output_dir ${CONVERTED_CHECKPOINT} \ + --dtype ${DTYPE} \ + --tp_size ${TP} \ + --pp_size 1 + +${HOME}/.local/bin/trtllm-build \ + --checkpoint_dir ${CONVERTED_CHECKPOINT} \ + --output_dir ${LORA_ENGINE} \ + --max_batch_size ${MAX_BATCH} \ + --max_input_len $MAX_LEN \ + --max_seq_len $((2*${MAX_LEN})) \ + --gemm_plugin float16 \ + --lora_plugin float16 \ + --use_paged_context_fmha enable \ + --lora_target_modules attn_q attn_k attn_v attn_dense mlp_h_to_4h mlp_4h_to_h mlp_gate \ + --max_lora_rank ${MAX_LORA_RANK} + +NUM_LORAS=(8 16) +NUM_REQUESTS=1024 + +# Convert LoRA to cpp format +python examples/hf_lora_convert.py \ + -i $SOURCE_LORA \ + --storage-type $DTYPE \ + -o $CPP_LORA + +# Prepare datasets +mkdir -p $EG_DIR/data + +# Prepare dataset without lora_task_id +python benchmarks/cpp/prepare_dataset.py \ + --output "${EG_DIR}/data/token-norm-dist.json" \ + --tokenizer $TOKENIZER \ + token-norm-dist \ + --num-requests $NUM_REQUESTS \ + --input-mean 256 --input-stdev 16 --output-mean 128 --output-stdev 24 + +# Prepare dataset with lora_task_ids from 0 - $nloras +for nloras in ${NUM_LORAS[@]}; do + python benchmarks/cpp/prepare_dataset.py \ + --output "${EG_DIR}/data/token-norm-dist-lora-${nloras}.json" \ + --rand-task-id 0 $(( $nloras - 1 )) \ + --tokenizer $TOKENIZER \ + token-norm-dist \ + --num-requests $NUM_REQUESTS \ + --input-mean 256 --input-stdev 16 --output-mean 128 --output-stdev 24 +done + +# Generate random lora weights for 16 adapters +python benchmarks/cpp/utils/generate_rand_loras.py ${CPP_LORA} ${EG_DIR}/loras 16 + +# Perform benchmarking + +# First run inference without LoRAs +mkdir -p ${EG_DIR}/log-base-lora +mpirun -n ${TP} --output-filename ${EG_DIR}/log-base-lora \ + cpp/build/benchmarks/gptManagerBenchmark \ + --engine_dir $LORA_ENGINE \ + --type IFB \ + --dataset "${EG_DIR}/data/token-norm-dist.json" \ + --lora_host_cache_bytes 8589934592 \ + --lora_num_device_mod_layers $(( 32 * $NUM_LAYERS * $NUM_LORA_MODS * $MAX_LORA_RANK )) \ + --kv_cache_free_gpu_mem_fraction 0.70 \ + --log_level info \ + --eos_id ${EOS_ID} + +# Now run inference with various numbers or loras +# The host cache is set large enough to hold all the LoRAs in lora_dir +# GPU cache is set to hold 16 LoRAs +# This benchmark will preload all the LoRAs into the host cache +# We run inference on a range of active LoRAs exercising different cache miss rates. +for nloras in ${NUM_LORAS[@]}; do + mkdir -p ${EG_DIR}/log-lora-${nloras} + mpirun -n ${TP} --output-filename "${EG_DIR}/log-lora-${nloras}" \ + cpp/build/benchmarks/gptManagerBenchmark \ + --engine_dir $LORA_ENGINE \ + --type IFB \ + --dataset "${EG_DIR}/data/token-norm-dist-lora-${nloras}.json" \ + --lora_host_cache_bytes 8589934592 \ + --lora_num_device_mod_layers $(( 16 * $NUM_LAYERS * $NUM_LORA_MODS * $MAX_LORA_RANK )) \ + --kv_cache_free_gpu_mem_fraction 0.70 \ + --log_level info \ + --eos_id ${EOS_ID} \ + --lora_dir ${EG_DIR}/loras +done +``` + +### 3. [DEPRECATED] Launch C++ static batching benchmarking (Fixed BatchSize/InputLen/OutputLen) + +#### Prepare TensorRT-LLM engine(s) + +Before you launch C++ benchmarking, please make sure that you have already built engine(s) using TensorRT-LLM API, C++ benchmarking code cannot generate engine(s) for you. + +Use `trtllm-build` to build the TRT-LLM engine. Alternatively, if you have already benchmarked Python Runtime, you can reuse the engine(s) built previously, please see that [`document`](../python/README.md). + +#### Launch benchmarking + +For detailed usage, you can do the following +``` +cd cpp/build + +# You can directly execute the binary for help information +./benchmarks/bertBenchmark --help +``` + +*Please note that the expected outputs in that document are only for reference, specific performance numbers depend on the GPU you're using.* + + +### 4.launch C++ disaggServerBenchmark +Currently ,TensorRT-LLM has limited support for disaggregated inference, where context and generation phases of a request can run on different executors. `disaggServerBenchmark` is a tool to benchmark disaggregated inference. + +#### Usage +For detailed usage, you can do the following +``` +cd cpp/build + +# You can directly execute the binary for help information +./benchmarks/disaggServerBenchmark --help +``` +`disaggServerBenchmark` only supports `decoder-only` models. +Here is the basic usage: +``` +export TRTLLM_USE_UCX_KVCACHE=1 +mpirun -n ${proc} benchmarks/disaggServerBenchmark --context_engine_dirs ${context_engine_0},${context_engine_1}...,${context_engine_{m-1}} \ +--generation_engine_dirs ${generation_engine_0},${generation_engine_1}...,${generation_engine_{n-1}} --dataset ${dataset_path} +``` +This command will launch m context engines and n generation engines. You need to ensure `proc` is equal to the sum of the number of processes required for each engine plus 1. Since we use orchestrator mode for `disaggServerBenchmark` we need an additional process as the orchestrator. For example, if there are two context engines (one is TP2_PP1,another is TP1_PP1) and two generation engines(one is TP2_PP1,another is TP1_PP1), then the `proc` value should be set to 7. + +for example: +``` +export TRTLLM_USE_UCX_KVCACHE=1 +mpirun -n 7 benchmarks/disaggServerBenchmark --context_engine_dirs ${llama_7b_tp2_pp1_dir},${llama_7b_tp1_pp1_dir} --generation_engine_dirs ${llama_7b_tp1_pp1_dir},${llama_7b_tp2_pp1_dir} --dataset ${dataset_path} + +# need 6 gpus and 7 processes to launch the benchmark. +``` + +#### Known Issues + +##### 1. error `All available sequence slots are used` + +If generation_engine's pp_size >1, the error "All available sequence slots are used" may occur, setting and adjusting the parameter `--request_rate` may help alleviate the problem. + +##### 2.KVCache transfers are by default via PCIE on single node. +Currently, because of the dependency libraries,KVCache transfers are by default via PCIE on single node. + +If you want to use NVLink, please check the UCX version in the container by running: +``` +ucx_info -v +``` +If the UCX version is less than or equal to 1.17, set `UCX_RNDV_FRAG_MEM_TYPE=cuda` to enable KvCache transfers using NVLink. +If the UCX version is 1.18, please set `UCX_CUDA_COPY_ASYNC_MEM_TYPE=cuda` to enable KvCache transfers using NVLink. diff --git a/tensorrt_llm/_torch/kv_cache_compression/__init__.py b/benchmarks/cpp/__init__.py similarity index 100% rename from tensorrt_llm/_torch/kv_cache_compression/__init__.py rename to benchmarks/cpp/__init__.py diff --git a/benchmarks/cpp/bertBenchmark.cpp b/benchmarks/cpp/bertBenchmark.cpp new file mode 100644 index 000000000000..cc10a5b49eee --- /dev/null +++ b/benchmarks/cpp/bertBenchmark.cpp @@ -0,0 +1,260 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +#include <NvInfer.h> +#include <cxxopts.hpp> +#include <nlohmann/json.hpp> + +#include <chrono> +#include <filesystem> +#include <fstream> +#include <iostream> +#include <sstream> +#include <string> + +using namespace tensorrt_llm::runtime; + +namespace trt = nvinfer1; + +namespace +{ + +std::string engineFilename( + std::filesystem::path const& dataPath, WorldConfig const& worldConfig, std::string const& model) +{ + auto constexpr allowExceptions = true; + auto constexpr ignoreComments = true; + auto const jsonFilePath = dataPath / "config.json"; + TLLM_CHECK_WITH_INFO( + std::filesystem::exists(jsonFilePath), std::string("File does not exist: ") + jsonFilePath.string()); + std::ifstream jsonStream(jsonFilePath); + auto const json = nlohmann::json::parse(jsonStream, nullptr, allowExceptions, ignoreComments); + auto const& builderConfig = json.at("builder_config"); + auto const precision = builderConfig.at("precision").template get<std::string>(); + auto const worldSize = builderConfig.at("tensor_parallel").template get<SizeType32>(); + + TLLM_CHECK_WITH_INFO(worldSize == worldConfig.getSize(), "world size mismatch"); + return model + "_" + precision + "_tp" + std::to_string(worldConfig.getSize()) + "_rank" + + std::to_string(worldConfig.getRank()) + ".engine"; +} + +void benchmarkBert(std::string const& modelName, std::filesystem::path const& dataPath, + std::vector<int> const& batchSizes, std::vector<int> const& inLens, bool useGpuDirectStorage, + std::vector<float> const& gpuWeightsPercents, std::shared_ptr<nvinfer1::ILogger> const& logger, int warmUp, + int numRuns, int duration) +{ + auto const worldConfig = WorldConfig::mpi(); + auto const enginePath = dataPath / engineFilename(dataPath, worldConfig, modelName); + + for (float gpuWeightsPercent : gpuWeightsPercents) + { + auto rt = std::make_shared<TllmRuntime>( + RawEngine(enginePath), logger.get(), useGpuDirectStorage, gpuWeightsPercent); + rt->addContext(0); + for (auto inLen : inLens) + { + for (auto const batchSize : batchSizes) + { + auto& allocator = rt->getBufferManager(); + TllmRuntime::TensorMap tensorMap{}; + + // input_ids + std::vector<SizeType32> inputIdsHost(batchSize * inLen, inLen); + auto inputIdsBuffer = std::shared_ptr<ITensor>{ + allocator.copyFrom(inputIdsHost, ITensor::makeShape({batchSize, inLen}), MemoryType::kGPU)}; + allocator.setZero(*inputIdsBuffer); + tensorMap.insert(std::make_pair("input_ids", inputIdsBuffer)); + // input_lengths + std::vector<SizeType32> inputLengthsHost(batchSize); + auto inLensBuffer = std::shared_ptr<ITensor>{ + allocator.copyFrom(inputLengthsHost, ITensor::makeShape({batchSize}), MemoryType::kGPU)}; + allocator.setZero(*inLensBuffer); + tensorMap.insert(std::make_pair("input_lengths", inLensBuffer)); + + rt->setInputTensors(0, tensorMap); + rt->setOutputTensors(0, tensorMap); + cudaDeviceSynchronize(); + + for (auto r = 0; r < warmUp; ++r) + { + rt->executeContext(0); + rt->getStream().synchronize(); + } + cudaDeviceSynchronize(); + + int iterIdx = 0; + float curDuration = 0; + while (iterIdx < numRuns || curDuration / 1000 < duration) + { + auto const start = std::chrono::steady_clock::now(); + rt->executeContext(0); + rt->getStream().synchronize(); + auto const end = std::chrono::steady_clock::now(); + + iterIdx += 1; + curDuration += (static_cast<float>( + std::chrono::duration_cast<std::chrono::microseconds>(end - start).count()) + / 1000); + } + printf("Benchmarking done. Iteration: %d, duration: %.2f sec.\n", iterIdx, curDuration / 1000); + + auto averageLatency = curDuration / iterIdx; + + if (worldConfig.getRank() == 0) + { + printf("[BENCHMARK] batch_size %d input_length %d latency(ms) %.2f\n", batchSize, inLen, + averageLatency); + } + } + } + } +} + +} // namespace + +int main(int argc, char* argv[]) +{ + cxxopts::Options options("TensorRT LLM C++ Runtime Benchmark", "TensorRT LLM C++ Runtime Benchmark for BERT."); + options.add_options()("h,help", "Print usage"); + options.add_options()( + "m,model", "Model name specified for engines.", cxxopts::value<std::string>()->default_value("bert_base")); + options.add_options()("engine_dir", "Directory that store the engines.", cxxopts::value<std::string>()); + options.add_options()("batch_size", + "Specify batch size(s) you want to benchmark. Multiple batch sizes can be separated by \";\", example: " + "\"1;8;64\".", + cxxopts::value<std::string>()->default_value("8")); + options.add_options()("input_len", + "Specify input length(s) you want to benchmark. Multiple input lengths can be " + "separated by \";\", example: \"60;128\".", + cxxopts::value<std::string>()->default_value("128")); + + options.add_options()("log_level", "Choose log level between verbose/info/warning/error/internal_error.", + cxxopts::value<std::string>()->default_value("error")); + options.add_options()( + "warm_up", "Specify warm up iterations before benchmark starts.", cxxopts::value<int>()->default_value("2")); + options.add_options()("num_runs", "Minimal number of iterations to run during benchmarking.", + cxxopts::value<int>()->default_value("10")); + options.add_options()("duration", "Minimal duration of iterations to measure in seconds.", + cxxopts::value<int>()->default_value("60")); + options.add_options()("gpu_weights_percent", + "Specify the percentage of weights that reside on GPU (from 0.0 to 1.0). Multiple percentages can be separated " + "by \";\", " + "example: \"0.0;0.5;1.0\".", + cxxopts::value<std::string>()->default_value("1.0")); + options.add_options()("use_gpu_direct_storage", "Enable GPUDirect Storage (GDS) for loading engine.", + cxxopts::value<bool>()->default_value("false")); + + auto result = options.parse(argc, argv); + + if (result.count("help")) + { + std::cout << options.help() << std::endl; + exit(0); + } + + // Argument: Engine directory + if (!result.count("engine_dir")) + { + std::cout << options.help() << std::endl; + TLLM_LOG_ERROR("Please specify engine directory."); + return 1; + } + + // Argument: Batch sizes + std::istringstream ssBatchSizesArg; + ssBatchSizesArg.str(result["batch_size"].as<std::string>()); + std::vector<int> batchSizes; + for (std::string token; std::getline(ssBatchSizesArg, token, ';');) + { + batchSizes.push_back(std::stoi(token)); + } + + // Argument : Input lengths + std::istringstream ssInLenArg; + ssInLenArg.str(result["input_len"].as<std::string>()); + std::vector<int> inLens; + for (std::string token; std::getline(ssInLenArg, token, ';');) + { + inLens.push_back(std::stoi(token)); + } + + // Argument: GPU weights percentage + std::istringstream ssGpuPercentArg; + ssGpuPercentArg.str(result["gpu_weights_percent"].as<std::string>()); + std::vector<float> gpuWeightsPercents; + for (std::string token; std::getline(ssGpuPercentArg, token, ';');) + { + auto gpuWeightsPercent = std::stof(token); + if (gpuWeightsPercent < 0 || gpuWeightsPercent > 1) + { + TLLM_LOG_ERROR( + "--gpu_weights_percent must have percents between 0.0 and 1.0 but got: %f", gpuWeightsPercent); + return 1; + } + gpuWeightsPercents.push_back(gpuWeightsPercent); + } + + // Argument: Log level + auto logger = std::make_shared<TllmLogger>(); + auto const logLevel = result["log_level"].as<std::string>(); + if (logLevel == "verbose") + { + logger->setLevel(trt::ILogger::Severity::kVERBOSE); + } + else if (logLevel == "info") + { + logger->setLevel(trt::ILogger::Severity::kINFO); + } + else if (logLevel == "warning") + { + logger->setLevel(trt::ILogger::Severity::kWARNING); + } + else if (logLevel == "error") + { + logger->setLevel(trt::ILogger::Severity::kERROR); + } + else if (logLevel == "internal_error") + { + logger->setLevel(trt::ILogger::Severity::kINTERNAL_ERROR); + } + else + { + TLLM_LOG_ERROR("Unexpected log level: " + logLevel); + return 1; + } + initTrtLlmPlugins(logger.get()); + + try + { + benchmarkBert(result["model"].as<std::string>(), result["engine_dir"].as<std::string>(), batchSizes, inLens, + result["use_gpu_direct_storage"].as<bool>(), gpuWeightsPercents, logger, result["warm_up"].as<int>(), + result["num_runs"].as<int>(), result["duration"].as<int>()); + } + catch (std::exception const& e) + { + TLLM_LOG_ERROR(e.what()); + return 1; + } + return 0; +} diff --git a/benchmarks/cpp/disaggServerBenchmark.cpp b/benchmarks/cpp/disaggServerBenchmark.cpp new file mode 100644 index 000000000000..bc3a7a2659fd --- /dev/null +++ b/benchmarks/cpp/disaggServerBenchmark.cpp @@ -0,0 +1,1582 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/disaggServerUtil.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/gptJsonConfig.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "utils/utils.h" + +#include "cxxopts.hpp" +#include <nlohmann/json.hpp> + +#include <chrono> +#include <cstdint> +#include <cstdio> +#include <filesystem> +#include <memory> +#include <mutex> +#include <numeric> +#include <optional> +#include <string> +#include <thread> +#include <unordered_map> +#include <vector> + +using namespace tensorrt_llm::batch_manager; +using namespace tensorrt_llm::runtime; +using namespace tensorrt_llm::benchmark; +using namespace tensorrt_llm::executor::disagg_executor; +namespace texec = tensorrt_llm::executor; +namespace trt = nvinfer1; + +namespace +{ + +class Recorder +{ + +public: + explicit Recorder(std::string opCsvFile, bool streaming = false, int beamWidth = 1, + bool calculateKvCacheTransferTime = true, bool calculateQueueTime = true, std::string responsesJsonFile = "", + bool excludeInputInOutput = false) + : mOpCsvFile(std::move(opCsvFile)) + , mStreaming(streaming) + , mBeamWidth(beamWidth) + , mRespJsonFile(std::move(responsesJsonFile)) + , mOutputHasInput(!excludeInputInOutput) + , mCalculateKVCacheTransferTime(calculateKvCacheTransferTime) + , mCalculateQueueTime(calculateQueueTime) + { + } + + void initialize() + { + mStart = std::chrono::steady_clock::now(); + mSeqLatency.mDataTimes.clear(); + mFtLatency.mDataTimes.clear(); + mGenLatency.mDataTimes.clear(); + mGenFirstTokenLatency.mDataTimes.clear(); + mGenT2TLatency.mDataTimes.clear(); + mGenExcludeFirstIterT2TLatency.mDataTimes.clear(); + mContextReqQueuingLatency.mDataTimes.clear(); + mGenReqQueuingLatency.mDataTimes.clear(); + mGenReqKvCacheTransferLatency.mDataTimes.clear(); + mKvCacheThroughput.mDataTps.clear(); + } + + void finalize() + { + mEnd = std::chrono::steady_clock::now(); + } + + void recordContextQueueLatency(std::vector<float> const& latencies) + { + mContextReqQueuingLatency.mDataTimes.insert( + mContextReqQueuingLatency.mDataTimes.end(), latencies.begin(), latencies.end()); + } + + void recordGenQueueLatency(std::vector<float> const& latencies) + { + mGenReqQueuingLatency.mDataTimes.insert( + mGenReqQueuingLatency.mDataTimes.end(), latencies.begin(), latencies.end()); + } + + void recordKvCacheTransferLatency(std::vector<float> const& latencies) + { + mGenReqKvCacheTransferLatency.mDataTimes.insert( + mGenReqKvCacheTransferLatency.mDataTimes.end(), latencies.begin(), latencies.end()); + } + + void recordKvCacheThroughput(std::vector<float> const& throughputs) + { + mKvCacheThroughput.mDataTps.insert(mKvCacheThroughput.mDataTps.end(), throughputs.begin(), throughputs.end()); + } + + void recordContextStart(SizeType32 inputLength, SizeType32 maxNewTokens, uint64_t requestId, + std::chrono::time_point<std::chrono::steady_clock> const& start) + { + mRequestBenchInfos[requestId] = BenchInfo(inputLength, start); + } + + void recordContextEnd(tensorrt_llm::executor::IdType requestId, bool hasError) + { + TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); + mRequestBenchInfos.at(requestId).contextEnd = std::chrono::steady_clock::now(); + mRequestBenchInfos.at(requestId).contextHasError = hasError; + mRequestBenchInfos.at(requestId).decodingIter += 1; + } + + void recordToken(tensorrt_llm::executor::IdType requestId) + { + TLLM_CHECK(mStreaming); + TLLM_CHECK_WITH_INFO(mBeamWidth == 1, "gptManagerBenchmark streaming mode does not support beam > 1"); + TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); + + if (!mRequestBenchInfos.at(requestId).genFirstTokenSeen) + { + mRequestBenchInfos.at(requestId).genFirstTokenTs = std::chrono::steady_clock::now(); + mRequestBenchInfos.at(requestId).genFirstTokenSeen = true; + } + mRequestBenchInfos.at(requestId).decodingIter += 1; + } + + void recordToken(tensorrt_llm::executor::IdType requestId, texec::Response const& response) + { + + TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); + + auto outputTokenIds = response.getResult().outputTokenIds; + + int32_t outputLength = 1; + for (auto const& beam : outputTokenIds) + { + outputLength = std::max(static_cast<int32_t>(beam.size()), outputLength); + } + + mRequestBenchInfos[requestId].outputLength += outputLength; + this->recordToken(requestId); + } + + void recordGenStart( + tensorrt_llm::executor::IdType requestId, std::chrono::time_point<std::chrono::steady_clock> const& start) + { + + TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); + mRequestBenchInfos.at(requestId).genStart = start; + } + + void recordGenEnd(tensorrt_llm::executor::IdType requestId, bool hasError) + { + TLLM_CHECK(mRequestBenchInfos.find(requestId) != mRequestBenchInfos.end()); + mRequestBenchInfos.at(requestId).genEnd = std::chrono::steady_clock::now(); + mRequestBenchInfos.at(requestId).genHasError = hasError; + } + + void recordGenEnd(tensorrt_llm::executor::IdType requestId, texec::Response const& response) + { + recordGenEnd(requestId, response.hasError()); + if (!response.hasError()) + { + if (!mStreaming) + { + TLLM_LOG_DEBUG("response.getResult().outputTokenIds"); + auto outputTokenIds = response.getResult().outputTokenIds; + + int32_t outSeqLen = 0; + for (auto const& beam : outputTokenIds) + { + outSeqLen = std::max(static_cast<int32_t>(beam.size()), outSeqLen); + } + if (mOutputHasInput) + { + int inputSeqLen = mRequestBenchInfos[requestId].inputLength; + outSeqLen -= inputSeqLen; + } + mRequestBenchInfos[requestId].outputLength = outSeqLen; + mRequestBenchInfos[requestId].decodingIter = response.getResult().decodingIter; + } + else + { + recordToken(requestId, response); + } + } + } + + void reserve(size_t size) + { + mRequestBenchInfos.reserve(size); + } + + void calculateLatencies() + { + for (auto& reqInfo : mRequestBenchInfos) + { + + reqInfo.second.latency + = std::chrono::duration<float, std::milli>(reqInfo.second.genEnd - reqInfo.second.contextStart).count(); + reqInfo.second.firstTokenLatency + = std::chrono::duration<float, std::milli>(reqInfo.second.contextEnd - reqInfo.second.contextStart) + .count(); + reqInfo.second.genLatency + = std::chrono::duration<float, std::milli>(reqInfo.second.genEnd - reqInfo.second.genStart).count(); + if (mStreaming) + { + reqInfo.second.genFirstTokenLatency + = std::chrono::duration<float, std::milli>(reqInfo.second.genFirstTokenTs - reqInfo.second.genStart) + .count(); + // include the latency of the second token+ kv Cache transfer latency + + if (reqInfo.second.outputLength > 1) + { + reqInfo.second.avgGenT2TLatency + = std::chrono::duration<float, std::milli>(reqInfo.second.genEnd - reqInfo.second.genStart) + .count() + / static_cast<float>(reqInfo.second.outputLength - 1); + } + if (reqInfo.second.outputLength > 2) + { + reqInfo.second.avgGenExcludeFirstIterT2TLatency + = std::chrono::duration<float, std::milli>( + reqInfo.second.genEnd - reqInfo.second.genFirstTokenTs) + .count() + / static_cast<float>(reqInfo.second.outputLength - 2); + } + } + } + } + + void calculateMetrics() + { + + calculateLatencies(); + + int totalOutputTokens{0}; + int totalDecodingIter{0}; + mNumContextErrorSamples = 0; + mNumGenErrorSamples = 0; + mNumSamples = 0; + for (auto const& reqInfo : mRequestBenchInfos) + { + + if (!reqInfo.second.contextHasError && !reqInfo.second.genHasError) + { + mSeqLatency.mDataTimes.push_back(reqInfo.second.latency); + mNumSamples++; + } + if (!reqInfo.second.contextHasError) + { + mFtLatency.mDataTimes.push_back(reqInfo.second.firstTokenLatency); + } + else + { + mNumContextErrorSamples++; + } + if (!reqInfo.second.genHasError) + { + mGenLatency.mDataTimes.push_back(reqInfo.second.genLatency); + totalOutputTokens += reqInfo.second.outputLength; + totalDecodingIter += reqInfo.second.decodingIter; + if (mStreaming) + { + mGenFirstTokenLatency.mDataTimes.push_back(reqInfo.second.genFirstTokenLatency); + + if (reqInfo.second.avgGenT2TLatency.has_value()) + { + mGenT2TLatency.mDataTimes.push_back(reqInfo.second.avgGenT2TLatency.value()); + } + if (reqInfo.second.avgGenExcludeFirstIterT2TLatency.has_value()) + { + mGenExcludeFirstIterT2TLatency.mDataTimes.push_back( + reqInfo.second.avgGenExcludeFirstIterT2TLatency.value()); + } + } + } + else + { + mNumGenErrorSamples++; + } + } + mTotalLatency = std::chrono::duration<float, std::milli>(mEnd - mStart).count(); + mSeqThroughput = mNumSamples / (mTotalLatency / 1000); + mTokenThroughput = totalOutputTokens / (mTotalLatency / 1000); + mAcceptanceRate = totalDecodingIter + ? (static_cast<float>(totalOutputTokens) / static_cast<float>(totalDecodingIter)) + : 0.0F; + + mSeqLatency.calculate(); + mFtLatency.calculate(); + mGenLatency.calculate(); + if (mStreaming) + { + + mGenFirstTokenLatency.calculate(); + + if (!mGenT2TLatency.mDataTimes.empty()) + { + mGenT2TLatency.calculate(); + std::vector<float> userTokensPerSecond; + userTokensPerSecond.reserve(mGenT2TLatency.mDataTimes.size()); + for (auto const& latency : mGenT2TLatency.mDataTimes) + { + userTokensPerSecond.push_back(1000.F / latency); + } + mAvgUserTokensPerSecond = std::accumulate(userTokensPerSecond.begin(), userTokensPerSecond.end(), 0.F) + / userTokensPerSecond.size(); + } + if (!mGenExcludeFirstIterT2TLatency.mDataTimes.empty()) + { + + mGenExcludeFirstIterT2TLatency.calculate(); + } + } + if (mCalculateQueueTime) + { + + mContextReqQueuingLatency.calculate(); + mGenReqQueuingLatency.calculate(); + } + if (mCalculateKVCacheTransferTime) + { + mGenReqKvCacheTransferLatency.calculate(); + mKvCacheThroughput.calculate(); + } + } + + void report() + { + printf("[BENCHMARK] num_samples %d\n", mNumSamples); + printf("[BENCHMARK] num_context_error_samples %d\n", mNumContextErrorSamples); + printf("[BENCHMARK] num_gen_error_samples %d\n", mNumGenErrorSamples); + printf("\n[BENCHMARK] num_samples %d\n", mNumSamples); + printf("[BENCHMARK] total_latency(ms) %.2f\n", mTotalLatency); + printf("[BENCHMARK] seq_throughput(seq/sec) %.2f\n", mSeqThroughput); + printf("[BENCHMARK] token_throughput(token/sec) %.2f\n", mTokenThroughput); + if (mStreaming) + { + printf("[BENCHMARK] user_tokens_per_second(tokens/sec/user) %.2f\n", mAvgUserTokensPerSecond); + } + printf("[BENCHMARK] avg_acceptance_rate(tokens/decoding steps) %.2f\n\n", mAcceptanceRate); + + mSeqLatency.report(); + mFtLatency.report(); + mGenLatency.report(); + if (mStreaming) + { + mGenFirstTokenLatency.report(); + mGenT2TLatency.report(); + mGenExcludeFirstIterT2TLatency.report(); + } + if (mCalculateQueueTime) + { + mContextReqQueuingLatency.report(); + mGenReqQueuingLatency.report(); + } + if (mCalculateKVCacheTransferTime) + { + mGenReqKvCacheTransferLatency.report(); + mKvCacheThroughput.report(); + } + } + + void writeOpMetricsToCsv() + { + if (!mOpCsvFile.empty()) + { + std::vector<std::string> headers{"num_samples", "num_context_error_samples", "num_gen_error_samples", + "total_latency(ms)", "seq_throughput(seq/sec)", "token_throughput(token/sec)"}; + auto seqLatencyHeader = mSeqLatency.genHeaders(); + headers.insert(headers.end(), std::make_move_iterator(seqLatencyHeader.begin()), + std::make_move_iterator(seqLatencyHeader.end())); + auto contextLatencyHeader = mFtLatency.genHeaders(); + headers.insert(headers.end(), std::make_move_iterator(contextLatencyHeader.begin()), + std::make_move_iterator(contextLatencyHeader.end())); + auto genLatencyHeader = mGenLatency.genHeaders(); + headers.insert(headers.end(), std::make_move_iterator(genLatencyHeader.begin()), + std::make_move_iterator(genLatencyHeader.end())); + if (mStreaming) + { + auto genFirstTokenHeader = mGenFirstTokenLatency.genHeaders(); + headers.insert(headers.end(), std::make_move_iterator(genFirstTokenHeader.begin()), + std::make_move_iterator(genFirstTokenHeader.end())); + auto genIngterHeader = mGenT2TLatency.genHeaders(); + headers.insert(headers.end(), std::make_move_iterator(genIngterHeader.begin()), + std::make_move_iterator(genIngterHeader.end())); + auto excludeFirstIterIngterHeader = mGenExcludeFirstIterT2TLatency.genHeaders(); + headers.insert(headers.end(), std::make_move_iterator(excludeFirstIterIngterHeader.begin()), + std::make_move_iterator(excludeFirstIterIngterHeader.end())); + headers.push_back("avg_user_tokens_per_second(tokens/sec/user)"); + } + if (mCalculateKVCacheTransferTime) + { + auto genReqKVCacheTransferHeader = mGenReqKvCacheTransferLatency.genHeaders(); + headers.insert(headers.end(), std::make_move_iterator(genReqKVCacheTransferHeader.begin()), + std::make_move_iterator(genReqKVCacheTransferHeader.end())); + auto kvCacheTpHeader = mKvCacheThroughput.genHeaders(); + headers.insert(headers.end(), std::make_move_iterator(kvCacheTpHeader.begin()), + std::make_move_iterator(kvCacheTpHeader.end())); + } + + std::ofstream outputFile(mOpCsvFile); + + if (outputFile.is_open()) + { + for (auto const& header : headers) + { + outputFile << header << ","; + } + outputFile << "\n"; + + outputFile << mNumSamples << "," << mNumContextErrorSamples << "," << mNumGenErrorSamples << "," + << mTotalLatency << "," << mSeqThroughput << "," << mTokenThroughput << "," << mSeqLatency + << "," << mFtLatency << "," << mGenLatency; + if (mStreaming) + { + + outputFile << "," << mGenFirstTokenLatency << "," << mGenT2TLatency << "," + << mGenExcludeFirstIterT2TLatency << "," << mAvgUserTokensPerSecond; + } + if (mCalculateKVCacheTransferTime) + { + outputFile << "," << mGenReqKvCacheTransferLatency << "," << mKvCacheThroughput; + } + + outputFile << "\n"; + } + else + { + std::cerr << "Error opening file '" << mOpCsvFile << "' for writing.\n"; + } + } + } + +private: + struct BenchInfo + { + BenchInfo() = default; + + BenchInfo(int inputLength, std::chrono::time_point<std::chrono::steady_clock> start) + : inputLength(inputLength) + , contextStart(start) + { + } + + int inputLength{}; + int outputLength{}; + std::chrono::time_point<std::chrono::steady_clock> contextStart; + std::chrono::time_point<std::chrono::steady_clock> contextEnd; + std::chrono::time_point<std::chrono::steady_clock> genFirstTokenTs; + std::chrono::time_point<std::chrono::steady_clock> genStart; + std::chrono::time_point<std::chrono::steady_clock> genEnd; + float latency{}; // millisecond + float genLatency{}; + bool contextHasError{false}; + bool genHasError{false}; + float firstTokenLatency{}; + float genFirstTokenLatency{}; + std::optional<float> avgGenT2TLatency; + std::optional<float> avgGenExcludeFirstIterT2TLatency; + bool genFirstTokenSeen{false}; + SizeType32 decodingIter{0}; + }; + + std::unordered_map<uint64_t, BenchInfo> mRequestBenchInfos; + + std::chrono::time_point<std::chrono::steady_clock> mStart; + std::chrono::time_point<std::chrono::steady_clock> mEnd; + int mNumSamples{}; + int mNumContextErrorSamples{}; + int mNumGenErrorSamples{}; + float mTotalLatency{}; + float mSeqThroughput{}; + RecordTimeMetric mSeqLatency{"sequence_latency"}; + RecordTimeMetric mFtLatency{"context_latency"}; + RecordTimeMetric mGenLatency{"gen_latency"}; + + RecordTimeMetric mGenFirstTokenLatency{"time_to_gen_first_token"}; + RecordTimeMetric mGenT2TLatency{"inter_token_latency"}; + RecordTimeMetric mGenExcludeFirstIterT2TLatency{"exclude_first_iter_inter_token_latency"}; + RecordTimeMetric mContextReqQueuingLatency{"context_req_queueing_latency"}; + + RecordTimeMetric mGenReqQueuingLatency{"gen_req_queueing_latency"}; + RecordTimeMetric mGenReqKvCacheTransferLatency{"gen_req_kv_cache_transfer_latency"}; + + RecordBwMetric mKvCacheThroughput{"gen_req_kv_cache_transfer_throughput"}; + + float mTokenThroughput{}; + float mAcceptanceRate{}; + + std::string mOpCsvFile; + bool mStreaming; + int mBeamWidth; + std::string mRespJsonFile; + std::unordered_map<uint64_t, tensorrt_llm::executor::TensorPtr> mResponseTensors; + bool mOutputHasInput; + bool mCalculateKVCacheTransferTime; + bool mCalculateQueueTime; + float mAvgUserTokensPerSecond{}; +}; + +texec::Request makeExecutorContextRequest(Sample const& sample, SizeType32 const& beamWidth, + std::optional<SizeType32> const& eosId, std::optional<SizeType32> const& padId, bool streaming = false, + bool const& returnContextLogits = false, bool const& returnGenerationLogits = false, + std::optional<texec::LoraConfig> const& loraConfig = std::nullopt, + std::optional<texec::LookaheadDecodingConfig> const& lookaheadConfig = std::nullopt, + std::optional<texec::VecTokens> const& encoderInputTokenIds = std::nullopt) +{ + auto samplingConfig = texec::SamplingConfig{beamWidth}; + auto outputConfig = texec::OutputConfig{false, returnContextLogits, returnGenerationLogits, false}; + auto request + = texec::Request(sample.inputIds, sample.outputLen, streaming, samplingConfig, outputConfig, eosId, padId, + std::nullopt, // positionIds + std::nullopt, // badWords + std::nullopt, // stopWords + std::nullopt, // embeddingBias + std::nullopt, // speculativeDecoding + std::nullopt, // pTuning + std::nullopt, // multimodalInput + std::nullopt, // multimodalEmbedding + std::nullopt, // mRopeConfig + loraConfig, // loraConfig + lookaheadConfig, // lookaheadConfig + std::nullopt, // kvCacheRetentionConfig + std::nullopt, // logitsPostProcessorName + std::nullopt, // logitsPostProcessor + encoderInputTokenIds.has_value() ? encoderInputTokenIds : std::nullopt, + std::nullopt); // cacheSalt + request.setRequestType(tensorrt_llm::executor::RequestType::REQUEST_TYPE_CONTEXT_ONLY); + return request; +} + +class DisaggExecutorServer +{ + +public: + DisaggExecutorServer(std::vector<std::filesystem::path> const& contextEnginePaths, + std::vector<std::filesystem::path> const& genEnginePaths, + std::optional<std::vector<std::vector<SizeType32>>> const& deviceIdsForInstance, int32_t maxBeamWidth, + texec::CapacitySchedulerPolicy capacitySchedulerPolicy, BenchmarkParams const& benchmarkParams, + std::shared_ptr<Recorder> recorder, std::chrono::milliseconds waitSleep, bool logIterationData, + bool hasContextAwaitThreads, bool hasGenAwaitThreads) + : mRecorder(std::move(recorder)) + , mWaitSleep(waitSleep) + , mConcurrency(benchmarkParams.concurrency) + , mShutdown(false) + , mLogIterationData(logIterationData) + , mEnableCollectKvCacheTransferTime(benchmarkParams.enableCollectkvCacheTransferTime) + , mEnableCollectIterStats(benchmarkParams.enableCollectIterStats) + { + + int worldRank = tensorrt_llm::mpi::MpiComm::world().getRank(); + int worldSize = tensorrt_llm::mpi::MpiComm::world().getSize(); + mIsOrchestrator = (worldRank == 0); + auto contextNum = contextEnginePaths.size(); + auto genNum = genEnginePaths.size(); + int deviceCount = -1; + TLLM_CUDA_CHECK(cudaGetDeviceCount(&deviceCount)); + + std::vector<std::unique_ptr<tensorrt_llm::executor::Executor>> instances; + auto instanceNum = genNum + contextNum; + if (worldRank == 0) + { + TLLM_LOG_INFO("context enigne num :%d gen enigne num:%d", contextNum, genNum); + } + + int startRank = 0; + std::vector<texec::ExecutorConfig> ctxExecutorConfigs; + std::vector<texec::ExecutorConfig> genExecutorConfigs; + for (auto in = 0; in < instanceNum; in++) + { + auto&& enginePath = in < contextNum ? contextEnginePaths.at(in) : genEnginePaths.at(in - contextNum); + auto decoderJsonConfig = tensorrt_llm::runtime::GptJsonConfig::parse(enginePath / "config.json"); + size_t instanceRanks = decoderJsonConfig.getWorldSize(); + std::vector<SizeType32> participateRank(instanceRanks); + std::vector<SizeType32> deviceIds; + if (deviceIdsForInstance.has_value()) + { + deviceIds = deviceIdsForInstance.value().at(in); + } + for (int i = 0; i < instanceRanks; i++) + { + startRank++; + participateRank.at(i) = startRank; + if (!deviceIdsForInstance.has_value()) + { + deviceIds.push_back((startRank - 1) % deviceCount); + } + } + texec::DynamicBatchConfig dynamicBatchConfig(benchmarkParams.enableBatchSizeTuning); + texec::SchedulerConfig schedulerConfig(capacitySchedulerPolicy, std::nullopt, dynamicBatchConfig); + texec::KvCacheConfig kvCacheConfig(benchmarkParams.enableBlockReuse, + benchmarkParams.maxTokensInPagedKvCache, benchmarkParams.maxAttentionWindowVec, + benchmarkParams.sinkTokenLength, benchmarkParams.freeGpuMemoryFractions.at(in), + benchmarkParams.kvHostCacheSize); + texec::ExtendedRuntimePerfKnobConfig extendedRuntimePerfKnobConfig(benchmarkParams.multiBlockMode, + benchmarkParams.enableContextFMHAFP32Acc, benchmarkParams.cudaGraphMode, + benchmarkParams.cudaGraphCacheSize); + texec::ExecutorConfig executorConfig(maxBeamWidth, schedulerConfig, kvCacheConfig, + benchmarkParams.enableChunekedContextVec.at(in).value_or(false)); + executorConfig.setGpuWeightsPercent(benchmarkParams.gpuWeightsPercent); + texec::OrchestratorConfig orchestratorConfig{mIsOrchestrator, "", nullptr, false}; + texec::ParallelConfig parallelConfig{tensorrt_llm::executor::CommunicationType::kMPI, + tensorrt_llm::executor::CommunicationMode::kORCHESTRATOR, deviceIds, participateRank, + orchestratorConfig}; + executorConfig.setParallelConfig(parallelConfig); + if (benchmarkParams.maxBatchSizes.at(in)) + { + executorConfig.setMaxBatchSize(benchmarkParams.maxBatchSizes.at(in).value()); + } + if (benchmarkParams.maxNumTokensVec.at(in)) + { + executorConfig.setMaxNumTokens(benchmarkParams.maxNumTokensVec.at(in).value()); + } + + executorConfig.setDecodingConfig( + texec::DecodingConfig(benchmarkParams.medusaChoices.has_value() ? texec::DecodingMode::Medusa() + : benchmarkParams.executorLookaheadConfig.has_value() ? texec::DecodingMode::Lookahead() + : texec::DecodingMode::Auto(), + benchmarkParams.executorLookaheadConfig, benchmarkParams.medusaChoices)); + executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig); + executorConfig.setCacheTransceiverConfig( + texec::CacheTransceiverConfig(texec::CacheTransceiverConfig::BackendType::DEFAULT)); + constexpr int maxIterationsForRequestStats = 1000; + if (mEnableCollectKvCacheTransferTime) + { + executorConfig.setRequestStatsMaxIterations(maxIterationsForRequestStats); + } + if (!benchmarkParams.enableCollectIterStats) + { + executorConfig.setIterStatsMaxIterations(0); + } + + if (in < contextNum) + { + ctxExecutorConfigs.push_back(executorConfig); + } + else + { + genExecutorConfigs.push_back(executorConfig); + } + } + + mDisaggExecutor = std::make_unique<DisaggExecutorOrchestrator>(contextEnginePaths, genEnginePaths, + ctxExecutorConfigs, genExecutorConfigs, hasContextAwaitThreads, hasGenAwaitThreads); + + if (mIsOrchestrator) + { + + if (mEnableCollectIterStats || mEnableCollectKvCacheTransferTime) + { + mCollectStatsThread = std::thread(&DisaggExecutorServer::collectStats, this); + } + } + tensorrt_llm::mpi::MpiComm::world().barrier(); + } + + std::vector<tensorrt_llm::executor::IdType> enqueueContext(std::vector<texec::Request> const& requests, + std::optional<int> selectContextId = std::nullopt, bool warmup = false, bool batch = false) + { + std::vector<SizeType32> inputLengths; + std::vector<SizeType32> maxNewTokens; + if (!warmup) + { + for (auto const& request : requests) + { + inputLengths.push_back(static_cast<SizeType32>(request.getInputTokenIds().size())); + maxNewTokens.push_back(request.getMaxTokens()); + } + } + auto const start = std::chrono::steady_clock::now(); + std::vector<tensorrt_llm::executor::IdType> globalReqIds + = mDisaggExecutor->enqueueContext(requests, selectContextId, batch); + if (!warmup) + { + for (size_t i = 0; i < requests.size(); ++i) + { + mRecorder->recordContextStart(inputLengths.at(i), maxNewTokens.at(i), globalReqIds.at(i), start); + } + } + mNumContextActive += requests.size(); + return globalReqIds; + } + + void enqueueGeneration(std::vector<texec::Request> const& requests, + std::vector<tensorrt_llm::executor::IdType> const& globalRequestIds, + std::optional<int> selectGenIdx = std::nullopt, bool warmup = false, bool batch = false) + { + TLLM_CHECK(globalRequestIds.size() == requests.size()); + auto const start = std::chrono::steady_clock::now(); + mDisaggExecutor->enqueueGeneration(requests, globalRequestIds, selectGenIdx, batch); + if (!warmup) + { + for (int i = 0; i < requests.size(); i++) + { + + mRecorder->recordGenStart(globalRequestIds.at(i), start); + } + } + mNumGenActive += requests.size(); + } + + std::vector<ResponseWithId> waitForContextResponse(SizeType32 numRequests, bool warmup = false) + { + std::vector<ResponseWithId> ret; + ret.reserve(numRequests); + while ((mNumContextActive != 0) || (mNumContextFinished < numRequests)) + { + auto responses = mDisaggExecutor->awaitContextResponses(mWaitSleep); + for (auto&& response : responses) + { + TLLM_CHECK(response.response.getResult().isFinal); + if (response.response.getResult().isFinal) + { + mNumContextActive--; + mNumContextFinished++; + } + if (!warmup) + { + mRecorder->recordContextEnd(response.gid, response.response.hasError()); + } + ret.emplace_back(std::move(response)); + } + } + return ret; + } + + void waitForGenResponse(SizeType32 numRequests, bool warmup = false) + { + while (mNumGenActive > 0 || (mNumGenFinished < numRequests)) + { + auto responses = mDisaggExecutor->awaitGenerationResponses(mWaitSleep); + for (auto&& response : responses) + { + if (response.response.getResult().isFinal) + { + mNumGenActive--; + mNumGenFinished++; + + if (!warmup) + { + mRecorder->recordGenEnd(response.gid, response.response); + } + } + else + { + // streaming + if (!warmup && !response.response.hasError()) + { + mRecorder->recordToken(response.gid, response.response); + } + } + } + } + } + + bool canEnqueue(int numSentRequests) const + { + return mIsOrchestrator && (!mConcurrency || (numSentRequests - mNumGenFinished < mConcurrency)); + } + + ~DisaggExecutorServer() + { + mShutdown = true; + if (mCollectStatsThread.joinable()) + { + mCollectStatsThread.join(); + } + } + + void resetNumFinished() + { + mNumContextFinished = 0; + mNumGenFinished = 0; + } + + void resetNumActive() + { + mNumContextActive = 0; + mNumGenActive = 0; + } + + void collectStats() const + { + while (!mShutdown) + { + std::vector<std::deque<tensorrt_llm::executor::IterationStats>> contextStats; + std::vector<std::deque<tensorrt_llm::executor::IterationStats>> generationStats; + std::vector<std::deque<tensorrt_llm::executor::RequestStatsPerIteration>> + generationRequestStatsPerIteration; + contextStats.reserve(mDisaggExecutor->getContextExecutors().size()); + for (auto&& executor : mDisaggExecutor->getContextExecutors()) + { + if (executor->canEnqueueRequests()) + { + contextStats.emplace_back(executor->getLatestIterationStats()); + } + } + generationStats.reserve(mDisaggExecutor->getGenExecutors().size()); + for (auto&& executor : mDisaggExecutor->getGenExecutors()) + { + if (executor->canEnqueueRequests()) + { + if (mEnableCollectIterStats) + { + generationStats.emplace_back(executor->getLatestIterationStats()); + } + if (mEnableCollectKvCacheTransferTime) + { + + generationRequestStatsPerIteration.emplace_back(executor->getLatestRequestStats()); + } + } + } + if (mEnableCollectIterStats) + { + for (std::size_t i = 0; i < contextStats.size(); i++) + { + auto const& iterStats = contextStats.at(i); + for (auto const& stat : iterStats) + { + SizeType32 numNewActiveRequests = stat.numNewActiveRequests; + if (numNewActiveRequests > 0) + { + auto avgQueueingTime + = static_cast<float>(stat.newActiveRequestsQueueLatencyMS / numNewActiveRequests); + std::vector<float> requestsQueueLatencyMS(numNewActiveRequests, avgQueueingTime); + mRecorder->recordContextQueueLatency(requestsQueueLatencyMS); + } + if (mLogIterationData) + { + TLLM_LOG_INFO( + "ctx_id %d, ctx_stat: %s", i, texec::JsonSerialization::toJsonStr(stat).c_str()); + } + } + } + + for (std::size_t i = 0; i < generationStats.size(); i++) + { + auto const& iterStats = generationStats.at(i); + for (auto const& stat : iterStats) + { + SizeType32 numNewActiveRequests = stat.numNewActiveRequests; + if (numNewActiveRequests > 0) + { + float avgQueueingTime + = static_cast<float>(stat.newActiveRequestsQueueLatencyMS / numNewActiveRequests); + std::vector<float> requestsQueueLatencyMS(numNewActiveRequests, avgQueueingTime); + mRecorder->recordGenQueueLatency(requestsQueueLatencyMS); + } + if (mLogIterationData) + { + TLLM_LOG_INFO( + "gen_id %d, gen_stat: %s", i, texec::JsonSerialization::toJsonStr(stat).c_str()); + } + } + } + } + + if (mEnableCollectKvCacheTransferTime) + { + for (std::size_t i = 0; i < generationRequestStatsPerIteration.size(); i++) + { + auto const& stats = generationRequestStatsPerIteration.at(i); + for (auto const& stat : stats) + { + std::vector<float> kvCacheTransferMs; + std::vector<float> kvCacheThroughput; + for (auto const& requestStat : stat.requestStats) + { + if (requestStat.stage == tensorrt_llm::executor::RequestStage::kGENERATION_COMPLETE) + { + kvCacheTransferMs.push_back( + static_cast<float>(requestStat.disServingStats->kvCacheTransferMS)); + kvCacheThroughput.push_back(static_cast<float>(requestStat.disServingStats->kvCacheSize) + * 8 / (static_cast<float>(requestStat.disServingStats->kvCacheTransferMS) / 1000) + / 1e9f); + } + } + if (kvCacheTransferMs.size() > 0) + { + mRecorder->recordKvCacheTransferLatency(kvCacheTransferMs); + } + if (kvCacheThroughput.size() > 0) + { + mRecorder->recordKvCacheThroughput(kvCacheThroughput); + } + if (mLogIterationData) + { + TLLM_LOG_INFO( + "gen_id %d, gen_req_stat: %s", i, texec::JsonSerialization::toJsonStr(stat).c_str()); + } + } + } + } + auto const waitSleep = std::chrono::milliseconds(50); + std::this_thread::sleep_for(waitSleep); + } + } + + std::unique_ptr<DisaggExecutorOrchestrator> const& getDisaggExecutor() const noexcept + { + return mDisaggExecutor; + } + +private: + std::unique_ptr<DisaggExecutorOrchestrator> mDisaggExecutor; + + std::atomic<bool> mShutdown{false}; + bool mIsOrchestrator{false}; + + std::shared_ptr<Recorder> mRecorder; + std::chrono::milliseconds mWaitSleep; + std::optional<int> mConcurrency; + bool mLogIterationData{false}; + bool const mEnableCollectKvCacheTransferTime; + bool const mEnableCollectIterStats; + std::thread mCollectStatsThread; + std::atomic<uint64_t> mNumGenFinished{0}; + std::atomic<uint64_t> mNumContextFinished{0}; + std::atomic<uint64_t> mNumGenActive{0}; + std::atomic<uint64_t> mNumContextActive{0}; +}; + +} // namespace + +void benchmark(std::vector<std::filesystem::path> const& contextEngineDirs, + std::vector<std::filesystem::path> const& generationEngineDirs, + std::optional<std::vector<std::vector<int>>> const& deviceIdsForInstances, std::string const& datasetPath, + std::string const& opCsvFile, int maxNumSamples, int beamWidth, int warmUp, std::optional<int32_t> const& eosId, + std::optional<int32_t> const& padId, BenchmarkParams const& benchmarkParams, + texec::CapacitySchedulerPolicy capacitySchedulerPolicy, std::chrono::milliseconds waitSleep, + bool returnContextLogits, bool returnGenerationLogits, std::optional<int> const staticEmulatedBatchSize, + bool logIterationData, std::optional<SizeType32> const maxPromptLen, bool hasContextAwait, bool hasGenAwait) +{ + + auto const& world = tensorrt_llm::mpi::MpiComm::world(); + auto worldRank = world.getRank(); + + // Load dataset + auto const samples = parseWorkloadJson(datasetPath, maxNumSamples, maxPromptLen); + auto const numSamples = samples.size(); + auto recorder = std::make_shared<Recorder>(opCsvFile, benchmarkParams.streaming, beamWidth, + benchmarkParams.enableCollectkvCacheTransferTime, benchmarkParams.enableCollectIterStats); + auto disaggExecutor = std::make_shared<DisaggExecutorServer>(contextEngineDirs, generationEngineDirs, + deviceIdsForInstances, beamWidth, capacitySchedulerPolicy, benchmarkParams, recorder, waitSleep, + logIterationData, hasContextAwait, hasGenAwait); + constexpr size_t numMap = 8; + std::vector<std::unordered_map<tensorrt_llm::executor::IdType, tensorrt_llm::executor::Request>> gidToRequestMaps( + numMap); + std::vector<std::mutex> mtxForMaps(numMap); + + auto fillRequestMap = [&](std::vector<tensorrt_llm::executor::IdType> const& reqIds, + std::vector<tensorrt_llm::executor::Request>&& requests) + { + TLLM_CHECK(reqIds.size() == requests.size()); + for (size_t i = 0; i < reqIds.size(); i++) + { + + size_t mapIdx = reqIds[i] % numMap; + std::scoped_lock<std::mutex> lock(mtxForMaps[mapIdx]); + gidToRequestMaps.at(mapIdx).emplace(reqIds[i], std::move(requests[i])); + } + }; + + auto makeGenRequest = [&](std::vector<ResponseWithId>&& contextResponse) + { + std::vector<tensorrt_llm::executor::IdType> gids; + gids.reserve(contextResponse.size()); + std::vector<tensorrt_llm::executor::Request> genRequest; + genRequest.reserve(contextResponse.size()); + for (auto&& ctxResponse : contextResponse) + { + gids.emplace_back(ctxResponse.gid); + size_t mapIdx = ctxResponse.gid % numMap; + + std::unique_lock<std::mutex> lock(mtxForMaps[mapIdx]); + TLLM_CHECK(gidToRequestMaps.at(mapIdx).find(ctxResponse.gid) != gidToRequestMaps.at(mapIdx).end()); + auto ctxRequest = std::move(gidToRequestMaps.at(mapIdx).at(ctxResponse.gid)); + gidToRequestMaps.at(mapIdx).erase(ctxResponse.gid); + lock.unlock(); + ctxRequest.setRequestType(tensorrt_llm::executor::RequestType::REQUEST_TYPE_GENERATION_ONLY); + ctxRequest.setContextPhaseParams(ctxResponse.response.getResult().contextPhaseParams.value()); + genRequest.emplace_back(std::move(ctxRequest)); + } + return std::make_pair(genRequest, gids); + }; + if (worldRank == 0) + { + { // warmup + TLLM_LOG_INFO("Warmup start"); + + size_t contextNum = contextEngineDirs.size(); + size_t generationNum = generationEngineDirs.size(); + for (auto con = 0; con < contextNum; con++) + { + for (auto gen = 0; gen < generationNum; gen++) + { + std::vector<tensorrt_llm::executor::Request> contextRequests; + contextRequests.reserve(warmUp); + for (int i = 0; i < warmUp; ++i) + { + contextRequests.emplace_back(makeExecutorContextRequest(samples[0], beamWidth, eosId, padId, + benchmarkParams.streaming, returnContextLogits, returnGenerationLogits, std::nullopt, + benchmarkParams.requestLookaheadConfig)); + } + auto reqIds = disaggExecutor->enqueueContext(contextRequests, con, true); + fillRequestMap(reqIds, std::move(contextRequests)); + auto contextResponse = disaggExecutor->waitForContextResponse(warmUp, true); + auto&& [genRequests, gids] = makeGenRequest(std::move(contextResponse)); + disaggExecutor->enqueueGeneration(genRequests, gids, gen, true); + disaggExecutor->waitForGenResponse(warmUp, true); + disaggExecutor->resetNumFinished(); + disaggExecutor->resetNumActive(); + } + } + + auto const warmUpWaitSleep = std::chrono::milliseconds(50); + std::this_thread::sleep_for(warmUpWaitSleep); + TLLM_LOG_INFO("Warmup done"); + } + + { + + auto timeDelays = computeTimeDelays(benchmarkParams, numSamples - 1); + + std::vector<texec::Request> contextRequests; + + for (std::size_t i = 0; i < numSamples; ++i) + { + std::optional<texec::LoraConfig> loraConfig = std::nullopt; + contextRequests.emplace_back(makeExecutorContextRequest(samples[i], beamWidth, eosId, padId, + benchmarkParams.streaming, returnContextLogits, returnGenerationLogits, loraConfig, + benchmarkParams.requestLookaheadConfig)); + } + + bool const hasDelay + = std::any_of(timeDelays.begin(), timeDelays.end(), [](auto const& delay) { return delay > 0.0; }); + disaggExecutor->resetNumFinished(); + disaggExecutor->resetNumActive(); + + recorder->reserve(numSamples); + recorder->initialize(); + if (!staticEmulatedBatchSize) + { + + std::thread waitContextResponseAndEnqueGenThread{[&]() + { + auto numRequest = numSamples; + while (numRequest > 0) + { + auto contextResponseWithIds + = disaggExecutor->getDisaggExecutor()->awaitContextResponses(waitSleep); + if (contextResponseWithIds.empty()) + { + continue; + } + for (auto&& contextResponseWithId : contextResponseWithIds) + { + recorder->recordContextEnd( + contextResponseWithId.gid, contextResponseWithId.response.hasError()); + } + numRequest -= contextResponseWithIds.size(); + auto&& [genReqeust, genGids] = makeGenRequest(std::move(contextResponseWithIds)); + disaggExecutor->enqueueGeneration(genReqeust, genGids); + } + }}; + + std::thread waitGenResponseThread{[&]() { disaggExecutor->waitForGenResponse(numSamples); }}; + int numSentRequests = 0; + while (numSentRequests < numSamples) + { + + if (disaggExecutor->canEnqueue(numSentRequests)) + { + auto gids = disaggExecutor->enqueueContext({contextRequests.at(numSentRequests)}); + fillRequestMap(gids, {contextRequests.at(numSentRequests)}); + + if (hasDelay && numSentRequests < numSamples - 1) + { + std::this_thread::sleep_for( + std::chrono::milliseconds(static_cast<int>(timeDelays.at(numSentRequests) * 1000))); + } + numSentRequests += 1; + } + } + waitContextResponseAndEnqueGenThread.join(); + waitGenResponseThread.join(); + } + else + { + TLLM_CHECK_WITH_INFO( + !hasDelay, "Executor benchmark doesn't support delays with emulated static batch sizes"); + auto numRequests = contextRequests.size(); + int maxBatchSize = staticEmulatedBatchSize.value(); + for (int req = 0; req < numRequests; req += maxBatchSize) + { + auto batchSize = std::min(static_cast<size_t>(maxBatchSize), numRequests - req); + + std::vector<texec::Request> requestsBatch(std::make_move_iterator(contextRequests.begin() + req), + std::make_move_iterator(contextRequests.begin() + req + static_cast<int64_t>(batchSize))); + // Enqueue in batches + + auto reqIds = disaggExecutor->enqueueContext(requestsBatch); + fillRequestMap(reqIds, std::move(requestsBatch)); + auto contextResponse = disaggExecutor->waitForContextResponse(static_cast<SizeType32>(batchSize)); + auto&& [genRequests, genReqIds] = makeGenRequest(std::move(contextResponse)); + disaggExecutor->enqueueGeneration(genRequests, genReqIds); + disaggExecutor->waitForGenResponse(static_cast<SizeType32>(batchSize)); + + // Wait for current batch to be done + } + } + } + recorder->finalize(); + // sleep for collect stats + if (benchmarkParams.enableCollectIterStats || benchmarkParams.enableCollectkvCacheTransferTime) + { + auto const collectWaitSleep = std::chrono::milliseconds(50); + std::this_thread::sleep_for(collectWaitSleep); + } + recorder->calculateMetrics(); + recorder->report(); + recorder->writeOpMetricsToCsv(); + } +} + +int main(int argc, char* argv[]) + +{ + cxxopts::Options options("TensorRT LLM DisaggServer Benchmark"); + options.add_options()("h,help", "Print usage"); + options.add_options()("context_engine_dirs", "Directories that store context engines,separator is a ,", + cxxopts::value<std::vector<std::string>>()); + options.add_options()("generation_engine_dirs", "Directories that store generation engines,separator is a , ", + cxxopts::value<std::vector<std::string>>()); + options.add_options()("device_ids_for_instances", + "device ids for each instances , example: \"[[0,1],[2,3],[4,5,6,7]]\" ", cxxopts::value<std::string>()); + options.add_options()("dataset", "Dataset that is used for benchmarking BatchManager.", + cxxopts::value<std::string>()->default_value("")); + options.add_options()( + "output_csv", "Write output metrics to CSV", cxxopts::value<std::string>()->default_value("")); + options.add_options()("max_num_samples", "maximum number of samples to use from dataset/generate", + cxxopts::value<int>()->default_value("100000")); + options.add_options()( + "beam_width", "Specify beam width you want to benchmark.", cxxopts::value<int>()->default_value("1")); + options.add_options()( + "warm_up", "Specify warm up iterations before benchmark starts.", cxxopts::value<int>()->default_value("2")); + options.add_options()( + "eos_id", "Specify the end-of-sequence token id.", cxxopts::value<TokenIdType>()->default_value("-1")); + options.add_options()("pad_id", "Specify the padding token id.", cxxopts::value<TokenIdType>()); + options.add_options()("max_tokens_in_paged_kvcache", "Max tokens in paged K-V Cache.", cxxopts::value<int>()); + options.add_options()( + "max_attention_window", "Max KV cache length per sequence", cxxopts::value<std::vector<int>>()); + options.add_options()("sink_token_len", "Sink token length in kv cache per sequence.", cxxopts::value<int>()); + options.add_options()( + "random_seed", "integer random seed for exponential time delays.", cxxopts::value<int>()->default_value("420")); + options.add_options()("kv_cache_free_gpu_mem_fractions", "K-V Cache Free Gpu Mem Fraction,each for per instance", + cxxopts::value<std::vector<float>>()); + options.add_options()("request_rate", + "request rate in reqs/sec. Skipping this arg or negative value will trigger offline/0-delay.", + cxxopts::value<float>()); + options.add_options()("concurrency", "Concurrent number of connections with the server.", cxxopts::value<int>()); + options.add_options()("max_batch_sizes", "The max runtime batch size when benchmarking, each for per instance", + cxxopts::value<std::vector<int>>()); + options.add_options()("max_num_tokens_per_instance", + "The max runtime number of tokens per batch when benchmarking, each for per instance", + cxxopts::value<std::vector<int>>()); + options.add_options()( + "enable_batch_size_tuning", "Dynamic tuning of batch size", cxxopts::value<bool>()->default_value("false")); + options.add_options()("enable_exp_delays", "Enables exponential delay distr to mimic real world request arrival", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("streaming", "Operate in streaming mode", cxxopts::value<bool>()->default_value("false")); + options.add_options()( + "enable_kv_cache_reuse", "Enables the KV cache reuse.", cxxopts::value<bool>()->default_value("false")); + options.add_options()("enable_chunked_context_per_instance", "Whether to enable context chunking for per instance", + cxxopts::value<std::vector<bool>>()->default_value("false")); + options.add_options()( + "return_context_logits", "Whether to return context logits.", cxxopts::value<bool>()->default_value("false")); + options.add_options()("return_generation_logits", "Whether to return generation logits.", + cxxopts::value<bool>()->default_value("false")); + + options.add_options()("scheduler_policy", + "Choose scheduler policy between max_utilization/guaranteed_no_evict/static_batch.", + cxxopts::value<std::string>()->default_value("guaranteed_no_evict")); + + options.add_options()("static_emulated_batch_size", + "Emulate static batching performance with the provided batch size.", cxxopts::value<SizeType32>()); + options.add_options()("log_level", "Choose log level between verbose/info/warning/error/internal_error.", + cxxopts::value<std::string>()->default_value("error")); + options.add_options()("log_iteration_data", "On each decoder iteration, print batch state metadata.", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("wait_sleep", "Specify how many milliseconds to sleep each iteration of waitForEmpty loop.", + cxxopts::value<int>()->default_value("25")); + options.add_options()("kv_host_cache_bytes", + "Size of secondary memory pool used for offloading kv cache blocks (in bytes).", + cxxopts::value<size_t>()->default_value("0")); + options.add_options()( + "max_prompt_len", "Truncate all prompts from dataset to the length specified.", cxxopts::value<SizeType32>()); + options.add_options()("gpu_weights_percent", + "Specify the percentage of weights that reside on GPU (from 0.0 to 1.0).", + cxxopts::value<float>()->default_value("1.0")); + options.add_options()( + "medusa_choices", "Medusa choices in the format of [[0], [0, 1], [0, 0, 1]]", cxxopts::value<std::string>()); + options.add_options()("multi_block_mode", + "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel", + cxxopts::value<bool>()->default_value("true")); + options.add_options()("cuda_graph_mode", "When enabled, inference is executed with cuda graph.", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("cuda_graph_cache_size", + "Specify how many cuda graphs are cached in the runtime. Larger cache gives better perf, but consumes more GPU " + "memory.", + cxxopts::value<SizeType32>()->default_value("0")); + options.add_options()("enable_context_fmha_fp32_acc", "Enable FMHA runner FP32 accumulation", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("executor_lookahead_config", + "lookahead config in the format of [max_window_size, max_ngram_size, max_verification_set_size]", + cxxopts::value<std::string>()); + options.add_options()("request_lookahead_config", + "lookahead config in the format of [max_window_size, max_ngram_size, max_verification_set_size], and each <= " + "executor lookahead config", + cxxopts::value<std::string>()); + options.add_options()("context_await", "When enabled, will has a thread to await context response.", + cxxopts::value<bool>()->default_value("true")); + options.add_options()("gen_await", "When enabled,will has a thread to await gen response.", + cxxopts::value<bool>()->default_value("true")); + options.add_options()("enable_collect_kvcache_transfer_time", "When enabled, will collect kvcache transfer time.", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("enable_collect_iter_stats", "When enabled, will collect iteration stats.", + cxxopts::value<bool>()->default_value("false")); + + auto result = options.parse(argc, argv); + + if ((result.count("context_engine_dirs") == 0) || (result.count("generation_engine_dirs") == 0)) + { + std::cout << options.help() << std::endl; + TLLM_LOG_ERROR("Please specify context engine and generation engine directory."); + return 1; + } + // Argument: Log level + auto logger = std::make_shared<TllmLogger>(); + auto const logLevel = result["log_level"].as<std::string>(); + if (logLevel == "verbose") + { + logger->setLevel(trt::ILogger::Severity::kVERBOSE); + } + else if (logLevel == "info") + { + logger->setLevel(trt::ILogger::Severity::kINFO); + } + else if (logLevel == "warning") + { + logger->setLevel(trt::ILogger::Severity::kWARNING); + } + else if (logLevel == "error") + { + logger->setLevel(trt::ILogger::Severity::kERROR); + } + else if (logLevel == "internal_error") + { + logger->setLevel(trt::ILogger::Severity::kINTERNAL_ERROR); + } + else + { + TLLM_LOG_ERROR("Unexpected log level: " + logLevel); + return 1; + } + + initTrtLlmPlugins(logger.get()); + + // Argument: Dataset + auto const datasetPath = result["dataset"].as<std::string>(); + auto const maxNumSamples = result["max_num_samples"].as<int>(); + + // Argument: Output metrics CSV + auto const opCsvFile = result["output_csv"].as<std::string>(); + + // Argument: beam width + auto const beamWidth = result["beam_width"].as<int>(); + TLLM_CHECK_WITH_INFO(beamWidth == 1, "Currently only support beamWidth=1"); + // Argument: wait_sleep + auto const waitSleep = std::chrono::milliseconds(result["wait_sleep"].as<int>()); + auto const hasContextAwait = result["context_await"].as<bool>(); + auto const hasGenAwait = result["gen_await"].as<bool>(); + BenchmarkParams benchmarkParams; + benchmarkParams.enableCollectkvCacheTransferTime = result["enable_collect_kvcache_transfer_time"].as<bool>(); + benchmarkParams.enableCollectIterStats = result["enable_collect_iter_stats"].as<bool>(); + + std::vector<std::string> contextEngineDirs = result["context_engine_dirs"].as<std::vector<std::string>>(); + std::vector<std::string> generationEngineDirs = result["generation_engine_dirs"].as<std::vector<std::string>>(); + if (tensorrt_llm::mpi::MpiComm::world().getRank() == 0) + { + std::string contextEngineStrings; + for (auto&& contextEngineDir : contextEngineDirs) + { + contextEngineStrings += contextEngineDir + ","; + } + std::string generationEnginesStrings; + for (auto&& genEngineDir : generationEngineDirs) + { + generationEnginesStrings += genEngineDir + ","; + } + TLLM_LOG_INFO( + "Will Launch benchmark with %d context engines and %d generation engines. Context Engines:%s ; Generation " + "Engines:%s ;", + contextEngineDirs.size(), generationEngineDirs.size(), contextEngineStrings.c_str(), + generationEnginesStrings.c_str()); + } + std::vector<std::filesystem::path> contextEnigePaths; + std::vector<std::filesystem::path> generationEnginePaths; + + contextEnigePaths.reserve(contextEngineDirs.size()); + + for (auto& contextEngineDir : contextEngineDirs) + { + + contextEnigePaths.emplace_back(contextEngineDir); + } + generationEnginePaths.reserve(generationEngineDirs.size()); + for (auto& genEngineDir : generationEngineDirs) + { + + generationEnginePaths.emplace_back(genEngineDir); + } + + int const instanceNum = contextEngineDirs.size() + generationEngineDirs.size(); + // Argument: Max tokens in paged K-V Cache + if (result.count("max_tokens_in_paged_kvcache")) + { + benchmarkParams.maxTokensInPagedKvCache = result["max_tokens_in_paged_kvcache"].as<int>(); + } + + // Argument: Max KV cache length + if (result.count("max_attention_window")) + { + benchmarkParams.maxAttentionWindowVec = result["max_attention_window"].as<std::vector<int>>(); + } + + // Argument: Sink token length + if (result.count("sink_token_len")) + { + benchmarkParams.sinkTokenLength = result["sink_token_len"].as<int>(); + } + + if (result.count("random_seed")) + { + benchmarkParams.randomSeed = result["random_seed"].as<int>(); + } + + // Argument: K-V Cache Free Gpu Mem Fraction + benchmarkParams.freeGpuMemoryFractions.resize(instanceNum); + if (result.count("kv_cache_free_gpu_mem_fractions")) + { + auto fractions = result["kv_cache_free_gpu_mem_fractions"].as<std::vector<float>>(); + TLLM_CHECK_WITH_INFO(fractions.size() == instanceNum || fractions.size() == 1, + "the number of fraction should be equal to the number of instances or equal to 1"); + for (int i = 0; i < instanceNum; i++) + { + benchmarkParams.freeGpuMemoryFractions.at(i) = fractions.size() == 1 ? fractions[0] : fractions[i]; + } + } + + // Argument: Enable dynamic tuning of batch size + benchmarkParams.enableBatchSizeTuning = result["enable_batch_size_tuning"].as<bool>(); + + // Argument: Enable KV cache reuse + benchmarkParams.enableBlockReuse = result["enable_kv_cache_reuse"].as<bool>(); + + // Argument: streaming + benchmarkParams.streaming = result["streaming"].as<bool>(); + + TLLM_CHECK_WITH_INFO(!(result.count("request_rate") && result.count("concurrency")), + "request_rate and concurrency cannot be specified at the same time."); + + // Argument: request rate + if (result.count("request_rate")) + { + benchmarkParams.requestRate = result["request_rate"].as<float>(); + } + + // Argument: concurrency + if (result.count("concurrency")) + { + benchmarkParams.concurrency = result["concurrency"].as<int>(); + } + + // Argument: max_batch_sizes + benchmarkParams.maxBatchSizes.resize(instanceNum); + if (result.count("max_batch_sizes")) + { + auto batchSizes = result["max_batch_sizes"].as<std::vector<int>>(); + TLLM_CHECK_WITH_INFO(batchSizes.size() == instanceNum || batchSizes.size() == 1, + "the number of batch size should be equal to the number of instances or equal to 1"); + for (int i = 0; i < instanceNum; i++) + { + benchmarkParams.maxBatchSizes.at(i) = batchSizes.size() == 1 ? batchSizes[0] : batchSizes[i]; + } + } + + // Argument: max_num_tokens_per_instance + benchmarkParams.maxNumTokensVec.resize(instanceNum); + if (result.count("max_num_tokens_per_instance")) + { + auto maxNumTokensVec = result["max_num_tokens_per_instance"].as<std::vector<int>>(); + TLLM_CHECK_WITH_INFO(maxNumTokensVec.size() == instanceNum || maxNumTokensVec.size() == 1, + "the number of max_num_tokens should be equal to the number of instances or equal to 1"); + for (int i = 0; i < instanceNum; i++) + { + benchmarkParams.maxNumTokensVec.at(i) + = maxNumTokensVec.size() == 1 ? maxNumTokensVec[0] : maxNumTokensVec[i]; + } + } + + benchmarkParams.enableExpDelays = result["enable_exp_delays"].as<bool>(); + + // Argument: Enable batch stats output + bool logIterationData = result["log_iteration_data"].as<bool>(); + + // Argument: Enable chunked context + benchmarkParams.enableChunekedContextVec.resize(instanceNum); + if (result.count("enable_chunked_context_per_instance")) + { + auto enableChunkedContextVec = result["enable_chunked_context_per_instance"].as<std::vector<bool>>(); + + TLLM_CHECK_WITH_INFO(enableChunkedContextVec.size() == instanceNum || enableChunkedContextVec.size() == 1, + "the number of enable_chunked_context_per_instance should be equal to the number of instances or equal to " + "1"); + for (int i = 0; i < instanceNum; i++) + { + benchmarkParams.enableChunekedContextVec.at(i) + = enableChunkedContextVec.size() == 1 ? enableChunkedContextVec[0] : enableChunkedContextVec[i]; + } + } + // Argument: Enable return context logits + bool returnContextLogits = result["return_context_logits"].as<bool>(); + TLLM_CHECK_WITH_INFO(returnContextLogits == false, "Currently disaggServer don't support returnContextLogits!"); + // Argument: Enable return context logits + bool returnGenerationLogits = result["return_generation_logits"].as<bool>(); + TLLM_CHECK_WITH_INFO( + returnGenerationLogits == false, "Currently disaggServer don't support returnGenerationLogits!"); + + if (result.count("lora_dir")) + { + TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lora!"); + benchmarkParams.loraDir = result["lora_dir"].as<std::string>(); + } + if (result.count("lora_host_cache_bytes")) + { + TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lora!"); + + benchmarkParams.loraHostCacheSize = result["lora_host_cache_bytes"].as<size_t>(); + } + if (result.count("lora_num_device_mod_layers")) + { + TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lora!"); + + benchmarkParams.loraDeviceNumModLayers = result["lora_num_device_mod_layers"].as<SizeType32>(); + } + + // Argument: How many KV cache blocks (as fraction of number of GPU kv cache blocks). + benchmarkParams.kvHostCacheSize = result["kv_host_cache_bytes"].as<size_t>(); + TLLM_CHECK_WITH_INFO( + benchmarkParams.kvHostCacheSize == false, "Currently disaggServer don't support kv_host_cache!"); + + // Argument: Medusa choices for the Medusa speculative decoding. + if (result.count("medusa_choices")) + { + TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support medusa!"); + + benchmarkParams.medusaChoices = parseVectorOfVectors(result["medusa_choices"].as<std::string>()); + } + if (result.count("executor_lookahead_config")) + { + TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lookhead!"); + + benchmarkParams.executorLookaheadConfig + = parseLookaheadConfig(result["executor_lookahead_config"].as<std::string>()); + } + if (result.count("request_lookahead_config")) + { + TLLM_CHECK_WITH_INFO(false, "Currently disaggServer don't support lookhead!"); + + benchmarkParams.requestLookaheadConfig + = parseLookaheadConfig(result["request_lookahead_config"].as<std::string>()); + } + + // Argument: multi_block_mode + benchmarkParams.multiBlockMode = result["multi_block_mode"].as<bool>(); + + // Argument: enable_context_fmha_fp32_acc + benchmarkParams.enableContextFMHAFP32Acc = result["enable_context_fmha_fp32_acc"].as<bool>(); + + // Argument: cuda_graph_mode + benchmarkParams.cudaGraphMode = result["cuda_graph_mode"].as<bool>(); + + // Argument: cuda_graph_cache_size + benchmarkParams.cudaGraphCacheSize = result["cuda_graph_cache_size"].as<SizeType32>(); + + std::optional<TokenIdType> padId; + // Argument: Padding token id + if (result.count("pad_id")) + { + padId = result["pad_id"].as<TokenIdType>(); + } + + // Argument: End-of-sentence token id + std::optional<TokenIdType> eosId = result["eos_id"].as<TokenIdType>(); + + std::optional<std::chrono::milliseconds> batchTimeout; + + std::optional<SizeType32> staticEmulatedBatchSize; + // Argument: Static emulated batch size + if (result.count("static_emulated_batch_size")) + { + staticEmulatedBatchSize = result["static_emulated_batch_size"].as<SizeType32>(); + } + + // Argument: Scheduler policy + texec::CapacitySchedulerPolicy capacitySchedulerPolicy; + auto const capacitySchedulerPolicyArg = result["scheduler_policy"].as<std::string>(); + if (capacitySchedulerPolicyArg == "max_utilization") + { + capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kMAX_UTILIZATION; + } + else if (capacitySchedulerPolicyArg == "guaranteed_no_evict") + { + capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT; + } + else if (capacitySchedulerPolicyArg == "static_batch") + { + capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kSTATIC_BATCH; + } + else + { + TLLM_LOG_ERROR("Unexpected scheduler policy: " + capacitySchedulerPolicyArg); + return 1; + } + + // Argument: max_prompt_len + std::optional<SizeType32> maxPromptLen; + if (result.count("max_prompt_len")) + { + maxPromptLen = result["max_prompt_len"].as<SizeType32>(); + } + + // Argument: GPU weights percentage + auto gpuWeightsPercent = result["gpu_weights_percent"].as<float>(); + if (gpuWeightsPercent < 0 || gpuWeightsPercent > 1) + { + TLLM_LOG_ERROR("--gpu_weights_percent must be between 0.0 and 1.0 but got: %f", gpuWeightsPercent); + return 1; + } + benchmarkParams.gpuWeightsPercent = gpuWeightsPercent; + + std::optional<std::vector<std::vector<int>>> deviceIdsForInstance = std::nullopt; + if (result.count("device_ids_for_instances")) + { + deviceIdsForInstance = parseVectorOfVectors(result["device_ids_for_instances"].as<std::string>()); + } + benchmark(contextEnigePaths, generationEnginePaths, deviceIdsForInstance, datasetPath, opCsvFile, maxNumSamples, + beamWidth, result["warm_up"].as<int>(), eosId, padId, benchmarkParams, capacitySchedulerPolicy, waitSleep, + returnContextLogits, returnContextLogits, staticEmulatedBatchSize, logIterationData, maxPromptLen, + hasContextAwait, hasGenAwait); +} diff --git a/benchmarks/cpp/gptManagerBenchmark.cpp b/benchmarks/cpp/gptManagerBenchmark.cpp new file mode 100644 index 000000000000..287cbba343ce --- /dev/null +++ b/benchmarks/cpp/gptManagerBenchmark.cpp @@ -0,0 +1,1557 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/tensor.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/gptJsonConfig.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tensorrt_llm/runtime/utils/numpyUtils.h" +#include "tensorrt_llm/runtime/worldConfig.h" +#include "utils/utils.h" + +#include <chrono> +#include <cstdint> +#include <cxxopts.hpp> +#include <iostream> +#include <memory> +#include <nlohmann/json.hpp> +#include <numeric> +#include <optional> +#include <string> +#include <thread> +#include <utility> + +using namespace tensorrt_llm::batch_manager; +using namespace tensorrt_llm::runtime; +using namespace tensorrt_llm::benchmark; +namespace texec = tensorrt_llm::executor; +namespace trt = nvinfer1; +namespace fs = std::filesystem; + +namespace +{ + +using TensorPtr = ITensor::SharedPtr; + +class LoraLib +{ +public: + LoraLib(std::string const& loraDir) + : mLoraDir(loraDir) + , mBufferManager(std::make_shared<CudaStream>()) + , mTaskPaths(parseDirPaths(mLoraDir)) + , mLoras(readLoras(mTaskPaths)) + { + } + + TensorPtr getLoraWeights(uint64_t taskId) const + { + return mLoras.at(taskId).first; + } + + TensorPtr getLoraConfig(uint64_t taskId) const + { + return mLoras.at(taskId).second; + } + + void clear() + { + mLoras.clear(); + } + + std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> const& getLoras() + { + return mLoras; + } + +private: + std::string const mLoraDir; + BufferManager mBufferManager; + std::map<uint64_t, fs::path> mTaskPaths; + std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> mLoras; + + std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> readLoras(std::map<uint64_t, fs::path> taskPaths) + { + std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> loras; + for (auto const& [id, p] : taskPaths) + { + TensorPtr loraWeights + = utils::loadNpy(mBufferManager, (p / "model.lora_weights.npy").string(), MemoryType::kCPU); + TensorPtr loraConfig + = utils::loadNpy(mBufferManager, (p / "model.lora_config.npy").string(), MemoryType::kCPU); + loras.insert_or_assign(id, std::make_pair(loraWeights, loraConfig)); + } + return loras; + } + + std::map<uint64_t, fs::path> parseDirPaths(std::string const& loraDir) + { + std::map<uint64_t, fs::path> taskPaths; + if (loraDir == "") + { + return taskPaths; + } + for (auto const& entry : fs::recursive_directory_iterator(loraDir)) + { + if (entry.is_directory()) + { + auto taskId = parseId(entry.path()); + taskPaths.insert_or_assign(taskId, entry.path()); + } + } + return taskPaths; + } + + uint64_t parseId(fs::path p) + { + auto fn = p.filename().string(); + auto dashPos = fn.find_first_of("-"); + std::string idStr = fn; + if (dashPos != std::string::npos) + { + auto idStr = fn.substr(0, dashPos); + } + uint64_t id = static_cast<uint64_t>(std::stoi(idStr)); + return id; + } +}; + +} // namespace + +struct BenchInfo +{ + BenchInfo() = default; + + BenchInfo(int inputLength, std::chrono::time_point<std::chrono::steady_clock> start) + : inputLength(inputLength) + , start(start) + { + } + + int inputLength; + int outputLength{0}; + std::chrono::time_point<std::chrono::steady_clock> start; + std::chrono::time_point<std::chrono::steady_clock> end; + std::chrono::time_point<std::chrono::steady_clock> firstTokenTs; + float latency{}; // millisecond + bool hasError{false}; + float firstTokenLatency{}; + std::optional<float> avgGenT2TLatency{}; + bool firstTokenSeen{false}; + SizeType32 decodingIter{0}; +}; + +class Recorder +{ + using TensorPtr = ITensor::SharedPtr; + +public: + explicit Recorder(std::string opCsvFile, bool streaming = false, int beamWidth = 1, + std::string responsesJsonFile = "", bool excludeInputInOutput = false) + : mOpCsvFile(std::move(opCsvFile)) + , mStreaming(streaming) + , mBeamWidth(beamWidth) + , mRespJsonFile(std::move(responsesJsonFile)) + , mOutputHasInput(!excludeInputInOutput) + { + } + + void initialize() + { + mStart = std::chrono::steady_clock::now(); + mRequestsQueueingLatencies.clear(); + } + + void finalize() + { + mEnd = std::chrono::steady_clock::now(); + } + + void recordQueueLatency(std::vector<float> const& latencies) + { + mRequestsQueueingLatencies.insert(mRequestsQueueingLatencies.end(), latencies.begin(), latencies.end()); + } + + // number of output tokens not calculated from output sequence here, instead set to max_output_len + // - if eos_id == -1 (default behavior), this is correct since output seq will have max permissible length. + // - However, if eos_id != -1, the token size of output sequence may be less than max_output_len, and token + // throughput may be inaccurate + void recordStart( + SizeType32 inputLength, uint64_t requestId, std::chrono::time_point<std::chrono::steady_clock> const& start) + { + TLLM_CHECK_WITH_INFO(mRequestBenchInfos.find(requestId) == mRequestBenchInfos.end(), + "Request %lu already exists in record before start, please report a bug to developers.", requestId); + std::lock_guard<std::mutex> const lock(mRequestBenchInfosMutex); + mRequestBenchInfos[requestId] = BenchInfo(inputLength, start); + } + + void recordToken( + texec::Response const& response, std::chrono::time_point<std::chrono::steady_clock> const& tokenTime) + { + auto const requestId = response.getRequestId(); + auto outputTokenIds = response.getResult().outputTokenIds; + + int32_t outputLength = 1; + for (auto const& beam : outputTokenIds) + { + outputLength = std::max(static_cast<int32_t>(beam.size()), outputLength); + } + + std::lock_guard<std::mutex> const lock(mRequestBenchInfosMutex); + mRequestBenchInfos[requestId].outputLength += outputLength; + + if (!mRequestBenchInfos[requestId].firstTokenSeen) + { + mRequestBenchInfos[requestId].firstTokenTs = tokenTime; + mRequestBenchInfos[requestId].firstTokenSeen = true; + } + + mRequestBenchInfos[requestId].decodingIter += 1; + } + + void recordEnd(texec::Response const& response, std::chrono::time_point<std::chrono::steady_clock> const& end) + { + auto const requestId = response.getRequestId(); + // Get the actual output length + if (!response.hasError()) + { + if (!mStreaming) + { + TLLM_LOG_DEBUG("response.getResult().outputTokenIds"); + auto outputTokenIds = response.getResult().outputTokenIds; + + int32_t outSeqLen = 0; + for (auto const& beam : outputTokenIds) + { + outSeqLen = std::max(static_cast<int32_t>(beam.size()), outSeqLen); + } + if (mOutputHasInput) + { + int inputSeqLen = mRequestBenchInfos[requestId].inputLength; + outSeqLen -= inputSeqLen; + } + std::lock_guard<std::mutex> const lock(mRequestBenchInfosMutex); + mRequestBenchInfos[requestId].outputLength = outSeqLen; + mRequestBenchInfos[requestId].decodingIter = response.getResult().decodingIter; + + // We record the first beam for the response file + mResponseTensors[requestId] = outputTokenIds[0]; + } + else + { + TLLM_CHECK_WITH_INFO(mBeamWidth == 1, "gptManagerBenchmark streaming mode does not support beam > 1"); + this->recordToken(response, end); + } + } + + std::lock_guard<std::mutex> const lock(mRequestBenchInfosMutex); + mRequestBenchInfos[requestId].end = end; + mRequestBenchInfos[requestId].hasError = response.hasError(); + } + + float calcPercentile(std::vector<float> const& latencies, int percentile) + { + int const index = static_cast<int>(std::ceil((percentile / 100.0) * latencies.size())) - 1; + return latencies[index]; + } + + void calculateLatencies() + { + for (auto& reqInfo : mRequestBenchInfos) + { + reqInfo.second.latency + = std::chrono::duration<float, std::milli>(reqInfo.second.end - reqInfo.second.start).count(); + if (mStreaming) + { + reqInfo.second.firstTokenLatency + = std::chrono::duration<float, std::milli>(reqInfo.second.firstTokenTs - reqInfo.second.start) + .count(); + if (reqInfo.second.outputLength > 1) + { + reqInfo.second.avgGenT2TLatency + = std::chrono::duration<float, std::milli>(reqInfo.second.end - reqInfo.second.firstTokenTs) + .count() + / static_cast<float>(reqInfo.second.outputLength - 1); + } + } + } + } + + void calculateMetrics() + { + calculateLatencies(); + + std::vector<float> reqLatencies; + std::vector<float> ftLatencies; + std::vector<float> genT2TLatencies; + std::vector<float> userTokensPerSecond; + + int totalOutputTokens{0}; + int totalDecodingIter{0}; + mNumErrorSamples = 0; + mNumSamples = 0; + for (auto reqInfo : mRequestBenchInfos) + { + if (!reqInfo.second.hasError) + { + reqLatencies.push_back(reqInfo.second.latency); + totalOutputTokens += reqInfo.second.outputLength; + totalDecodingIter += reqInfo.second.decodingIter; + + if (mStreaming) + { + ftLatencies.push_back(reqInfo.second.firstTokenLatency); + + if (reqInfo.second.avgGenT2TLatency) + { + genT2TLatencies.push_back(reqInfo.second.avgGenT2TLatency.value()); + } + if (reqInfo.second.avgGenT2TLatency.value() > 0) + { + userTokensPerSecond.push_back(1000.F / reqInfo.second.avgGenT2TLatency.value()); + } + } + ++mNumSamples; + } + else + { + ++mNumErrorSamples; + } + } + + mTotalLatency = std::chrono::duration<float, std::milli>(mEnd - mStart).count(); + mSeqThroughput = mNumSamples / (mTotalLatency / 1000); + mTokenThroughput = totalOutputTokens / (mTotalLatency / 1000); + mAcceptanceRate = totalDecodingIter + ? (static_cast<float>(totalOutputTokens) / static_cast<float>(totalDecodingIter)) + : 0.0f; + + mAvgSeqLatency = std::accumulate(reqLatencies.begin(), reqLatencies.end(), 0.F) / reqLatencies.size(); + + std::sort(reqLatencies.begin(), reqLatencies.end()); + + mP99SeqLatency = calcPercentile(reqLatencies, 99); + mP90SeqLatency = calcPercentile(reqLatencies, 90); + mP50SeqLatency = calcPercentile(reqLatencies, 50); + mMaxSeqLatency = reqLatencies.back(); + mMinSeqLatency = reqLatencies.front(); + + if (mStreaming) + { + mAvgFtLatency = std::accumulate(ftLatencies.begin(), ftLatencies.end(), 0.F) / ftLatencies.size(); + + std::sort(ftLatencies.begin(), ftLatencies.end()); + + mP99FtLatency = calcPercentile(ftLatencies, 99); + mP90FtLatency = calcPercentile(ftLatencies, 90); + mP50FtLatency = calcPercentile(ftLatencies, 50); + mMaxFtLatency = ftLatencies.back(); + mMinFtLatency = ftLatencies.front(); + + if (!genT2TLatencies.empty()) + { + mAvgGenT2TLatency + = std::accumulate(genT2TLatencies.begin(), genT2TLatencies.end(), 0.F) / genT2TLatencies.size(); + + std::sort(genT2TLatencies.begin(), genT2TLatencies.end()); + + mP99GenT2TLatency = calcPercentile(genT2TLatencies, 99); + mP90GenT2TLatency = calcPercentile(genT2TLatencies, 90); + mP50GenT2TLatency = calcPercentile(genT2TLatencies, 50); + mMaxGenT2TLatency = genT2TLatencies.back(); + mMinGenT2TLatency = genT2TLatencies.front(); + } + + if (!userTokensPerSecond.empty()) + { + mAvgUserTokensPerSecond = std::accumulate(userTokensPerSecond.begin(), userTokensPerSecond.end(), 0.F) + / userTokensPerSecond.size(); + std::sort(userTokensPerSecond.begin(), userTokensPerSecond.end()); + mP99UserTokensPerSecond = calcPercentile(userTokensPerSecond, 99); + mP90UserTokensPerSecond = calcPercentile(userTokensPerSecond, 90); + mP50UserTokensPerSecond = calcPercentile(userTokensPerSecond, 50); + mMaxUserTokensPerSecond = userTokensPerSecond.back(); + mMinUserTokensPerSecond = userTokensPerSecond.front(); + } + + mAvgReqQueueingLatency + = std::accumulate(mRequestsQueueingLatencies.begin(), mRequestsQueueingLatencies.end(), 0.F) + / mRequestsQueueingLatencies.size(); + std::sort(mRequestsQueueingLatencies.begin(), mRequestsQueueingLatencies.end()); + mP99ReqQueueingLatency = calcPercentile(mRequestsQueueingLatencies, 99); + mP90ReqQueueingLatency = calcPercentile(mRequestsQueueingLatencies, 90); + mP50ReqQueueingLatency = calcPercentile(mRequestsQueueingLatencies, 50); + mMaxReqQueueingLatency = mRequestsQueueingLatencies.back(); + mMinReqQueueingLatency = mRequestsQueueingLatencies.front(); + } + } + + void report() + { + + printf("[BENCHMARK] num_samples %d\n", mNumSamples); + printf("[BENCHMARK] num_error_samples %d\n", mNumErrorSamples); + printf("\n[BENCHMARK] num_samples %d\n", mNumSamples); + printf("[BENCHMARK] total_latency(ms) %.2f\n", mTotalLatency); + printf("[BENCHMARK] seq_throughput(seq/sec) %.2f\n", mSeqThroughput); + printf("[BENCHMARK] token_throughput(token/sec) %.2f\n", mTokenThroughput); + printf("[BENCHMARK] avg_acceptance_rate(tokens/decoding steps) %.2f\n\n", mAcceptanceRate); + + printf("[BENCHMARK] avg_sequence_latency(ms) %.2f\n", mAvgSeqLatency); + printf("[BENCHMARK] max_sequence_latency(ms) %.2f\n", mMaxSeqLatency); + printf("[BENCHMARK] min_sequence_latency(ms) %.2f\n", mMinSeqLatency); + printf("[BENCHMARK] p99_sequence_latency(ms) %.2f\n", mP99SeqLatency); + printf("[BENCHMARK] p90_sequence_latency(ms) %.2f\n", mP90SeqLatency); + printf("[BENCHMARK] p50_sequence_latency(ms) %.2f\n\n", mP50SeqLatency); + + if (mStreaming) + { + printf("[BENCHMARK] avg_time_to_first_token(ms) %.2f\n", mAvgFtLatency); + printf("[BENCHMARK] max_time_to_first_token(ms) %.2f\n", mMaxFtLatency); + printf("[BENCHMARK] min_time_to_first_token(ms) %.2f\n", mMinFtLatency); + printf("[BENCHMARK] p99_time_to_first_token(ms) %.2f\n", mP99FtLatency); + printf("[BENCHMARK] p90_time_to_first_token(ms) %.2f\n", mP90FtLatency); + printf("[BENCHMARK] p50_time_to_first_token(ms) %.2f\n\n", mP50FtLatency); + + printf("[BENCHMARK] avg_inter_token_latency(ms) %.2f\n", mAvgGenT2TLatency); + printf("[BENCHMARK] max_inter_token_latency(ms) %.2f\n", mMaxGenT2TLatency); + printf("[BENCHMARK] min_inter_token_latency(ms) %.2f\n", mMinGenT2TLatency); + printf("[BENCHMARK] p99_inter_token_latency(ms) %.2f\n", mP99GenT2TLatency); + printf("[BENCHMARK] p90_inter_token_latency(ms) %.2f\n", mP90GenT2TLatency); + printf("[BENCHMARK] p50_inter_token_latency(ms) %.2f\n\n", mP50GenT2TLatency); + + printf("[BENCHMARK] avg_user_tokens_per_second(tokens/sec/user) %.2f\n", mAvgUserTokensPerSecond); + printf("[BENCHMARK] max_user_tokens_per_second(tokens/sec/user) %.2f\n", mMaxUserTokensPerSecond); + printf("[BENCHMARK] min_user_tokens_per_second(tokens/sec/user) %.2f\n", mMinUserTokensPerSecond); + printf("[BENCHMARK] p99_user_tokens_per_second(tokens/sec/user) %.2f\n", mP99UserTokensPerSecond); + printf("[BENCHMARK] p90_user_tokens_per_second(tokens/sec/user) %.2f\n", mP90UserTokensPerSecond); + printf("[BENCHMARK] p50_user_tokens_per_second(tokens/sec/user) %.2f\n\n", mP50UserTokensPerSecond); + + printf("[BENCHMARK] avg_request_queueing_latency(ms) %.2f\n", mAvgReqQueueingLatency); + printf("[BENCHMARK] max_request_queueing_latency(ms) %.2f\n", mMaxReqQueueingLatency); + printf("[BENCHMARK] min_request_queueing_latency(ms) %.2f\n", mMinReqQueueingLatency); + printf("[BENCHMARK] p99_request_queueing_latency(ms) %.2f\n", mP99ReqQueueingLatency); + printf("[BENCHMARK] p90_request_queueing_latency(ms) %.2f\n", mP90ReqQueueingLatency); + printf("[BENCHMARK] p50_request_queueing_latency(ms) %.2f\n\n", mP50ReqQueueingLatency); + } + } + + void writeOpMetricsToCsv() + { + if (!mOpCsvFile.empty()) + { + std::vector<std::string> headers = {"num_samples", "num_error_samples", "total_latency(ms)", + "seq_throughput(seq/sec)", "token_throughput(token/sec)", "avg_sequence_latency(ms)", + "max_sequence_latency(ms)", "min_sequence_latency(ms)", "p99_sequence_latency(ms)", + "p90_sequence_latency(ms)", "p50_sequence_latency(ms)", "avg_acceptance_rate(tokens/decoding steps)"}; + + if (mStreaming) + { + std::vector<std::string> streamingHeaders = { + "avg_time_to_first_token(ms)", + "max_time_to_first_token(ms)", + "min_time_to_first_token(ms)", + "p99_time_to_first_token(ms)", + "p90_time_to_first_token(ms)", + "p50_time_to_first_token(ms)", + "avg_inter_token_latency(ms)", + "max_inter_token_latency(ms)", + "min_inter_token_latency(ms)", + "p99_inter_token_latency(ms)", + "p90_inter_token_latency(ms)", + "p50_inter_token_latency(ms)", + "avg_user_tokens_per_second(tokens/sec/user)", + "max_user_tokens_per_second(tokens/sec/user)", + "min_user_tokens_per_second(tokens/sec/user)", + "p99_user_tokens_per_second(tokens/sec/user)", + "p90_user_tokens_per_second(tokens/sec/user)", + "p50_user_tokens_per_second(tokens/sec/user)", + }; + + headers.insert(headers.end(), streamingHeaders.begin(), streamingHeaders.end()); + } + + std::ofstream outputFile(mOpCsvFile); + + if (outputFile.is_open()) + { + for (auto const& header : headers) + { + outputFile << header << ","; + } + outputFile << "\n"; + outputFile << mNumSamples << "," << mNumErrorSamples << "," << mTotalLatency << "," << mSeqThroughput + << "," << mTokenThroughput << "," << mAvgSeqLatency << "," << mMaxSeqLatency << "," + << mMinSeqLatency << "," << mP99SeqLatency << "," << mP90SeqLatency << "," << mP50SeqLatency + << "," << mAcceptanceRate; + if (mStreaming) + { + outputFile << "," << mAvgFtLatency << "," << mMaxFtLatency << "," << mMinFtLatency << "," + << mP99FtLatency << "," << mP90FtLatency << "," << mP50FtLatency << "," + << mAvgGenT2TLatency << "," << mMaxGenT2TLatency << "," << mMinGenT2TLatency << "," + << mP99GenT2TLatency << "," << mP90GenT2TLatency << "," << mP50GenT2TLatency << "," + << mAvgUserTokensPerSecond << "," << mMaxUserTokensPerSecond << "," + << mMinUserTokensPerSecond << "," << mP99UserTokensPerSecond << "," + << mP90UserTokensPerSecond << "," << mP50UserTokensPerSecond << ","; + } + + outputFile << "\n"; + } + else + { + std::cerr << "Error opening file '" << mOpCsvFile << "' for writing.\n"; + } + } + } + + void dumpResponseSeqs() + { + if (mRespJsonFile.empty()) + return; + nlohmann::json jsonResponses = nlohmann::json::array(); + for (auto const& [respId, respTokensTensor] : mResponseTensors) + { + auto respTokens = mResponseTensors[respId]; + int respLength = respTokens.size(); + int* respBufferPtr = respTokens.data(); + + if (mOutputHasInput) + { + int inputSeqLen = mRequestBenchInfos[respId].inputLength; + respBufferPtr += inputSeqLen; + respLength -= inputSeqLen; + } + + std::vector<int32_t> outputTokens(respLength); + std::copy(respBufferPtr, respBufferPtr + respLength, outputTokens.begin()); + + nlohmann::json currResp; + currResp["response_id"] = respId; + currResp["response_tokens"] = outputTokens; + jsonResponses.push_back(currResp); + } + std::ofstream outFile(mRespJsonFile); + outFile << jsonResponses; + outFile.close(); + } + +private: + std::unordered_map<uint64_t, BenchInfo> mRequestBenchInfos; + + std::chrono::time_point<std::chrono::steady_clock> mStart; + std::chrono::time_point<std::chrono::steady_clock> mEnd; + int mNumSamples{}; + int mNumErrorSamples{}; + float mTotalLatency{}; + float mSeqThroughput{}; + float mAvgSeqLatency{}; + float mAvgGenT2TLatency{}; + float mAvgUserTokensPerSecond{}; + float mAvgFtLatency{}; + float mTokenThroughput{}; + float mAcceptanceRate{}; + float mP99SeqLatency{}; + float mP90SeqLatency{}; + float mP50SeqLatency{}; + float mMaxSeqLatency{}; + float mMinSeqLatency{}; + float mP99FtLatency{}; + float mP90FtLatency{}; + float mP50FtLatency{}; + float mMaxFtLatency{}; + float mMinFtLatency{}; + float mP99GenT2TLatency{}; + float mP90GenT2TLatency{}; + float mP50GenT2TLatency{}; + float mMaxGenT2TLatency{}; + float mMinGenT2TLatency{}; + float mP99UserTokensPerSecond{}; + float mP90UserTokensPerSecond{}; + float mP50UserTokensPerSecond{}; + float mMaxUserTokensPerSecond{}; + float mMinUserTokensPerSecond{}; + float mAvgReqQueueingLatency{}; + float mP99ReqQueueingLatency{}; + float mP90ReqQueueingLatency{}; + float mP50ReqQueueingLatency{}; + float mMaxReqQueueingLatency{}; + float mMinReqQueueingLatency{}; + std::vector<float> mRequestsQueueingLatencies{}; + + std::string mOpCsvFile; + bool mStreaming; + int mBeamWidth; + std::string mRespJsonFile; + std::unordered_map<uint64_t, texec::VecTokens> mResponseTensors; + bool mOutputHasInput; + std::mutex mRequestBenchInfosMutex; + +}; // class Recorder + +class ExecutorServer +{ +public: + ExecutorServer(std::optional<std::filesystem::path> const& decoderTrtEnginePath, + std::optional<std::filesystem::path> const& encoderTrtEnginePath, texec::BatchingType batchingType, + int32_t maxBeamWidth, texec::CapacitySchedulerPolicy capacitySchedulerPolicy, + BenchmarkParams const& benchmarkParams, std::shared_ptr<Recorder> recorder, std::chrono::milliseconds waitSleep, + bool logIterationData, texec::ModelType executorModelType) + : mRecorder(std::move(recorder)) + , mWaitSleep(waitSleep) + , mConcurrency(benchmarkParams.concurrency) + , mActiveCount(0) + , mNumFinished(0) + , mShutdown(false) + , mLogIterationData(logIterationData) + { + texec::DynamicBatchConfig dynamicBatchConfig( + benchmarkParams.enableBatchSizeTuning, benchmarkParams.enableMaxNumTokensTuning); + texec::SchedulerConfig schedulerConfig(capacitySchedulerPolicy, std::nullopt, dynamicBatchConfig); + + texec::KvCacheConfig kvCacheConfig(benchmarkParams.enableBlockReuse, benchmarkParams.maxTokensInPagedKvCache, + benchmarkParams.maxAttentionWindowVec, benchmarkParams.sinkTokenLength, + benchmarkParams.freeGpuMemoryFraction, benchmarkParams.kvHostCacheSize, + benchmarkParams.crossKvCacheFraction); + texec::PeftCacheConfig peftCacheConfig(0, benchmarkParams.loraDeviceNumModLayers, 8, 64, 4, 4, 4, 24, 8, + std::nullopt, benchmarkParams.loraHostCacheSize); + texec::ExtendedRuntimePerfKnobConfig extendedRuntimePerfKnobConfig(benchmarkParams.multiBlockMode, + benchmarkParams.enableContextFMHAFP32Acc, benchmarkParams.cudaGraphMode, + benchmarkParams.cudaGraphCacheSize); + texec::ExecutorConfig executorConfig( + maxBeamWidth, schedulerConfig, kvCacheConfig, benchmarkParams.enableChunkedContext, true); + executorConfig.setEnableTrtOverlap(benchmarkParams.enableTrtOverlap); + executorConfig.setGpuWeightsPercent(benchmarkParams.gpuWeightsPercent); + executorConfig.setPeftCacheConfig(peftCacheConfig); + executorConfig.setBatchingType(batchingType); + if (benchmarkParams.maxBatchSize) + { + executorConfig.setMaxBatchSize(benchmarkParams.maxBatchSize.value()); + } + if (benchmarkParams.maxNumTokens) + { + executorConfig.setMaxNumTokens(benchmarkParams.maxNumTokens.value()); + } + + auto decodingMode = texec::DecodingMode::Auto(); + if (benchmarkParams.medusaChoices.has_value()) + { + decodingMode = texec::DecodingMode::Medusa(); + } + else if (benchmarkParams.executorLookaheadConfig.has_value()) + { + decodingMode = texec::DecodingMode::Lookahead(); + } + else if (benchmarkParams.eagleConfig.has_value()) + { + decodingMode = texec::DecodingMode::Eagle(); + } + + executorConfig.setDecodingConfig(texec::DecodingConfig(decodingMode, benchmarkParams.executorLookaheadConfig, + benchmarkParams.medusaChoices, benchmarkParams.eagleConfig)); + executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig); + + if (executorModelType == texec::ModelType::kDECODER_ONLY) + { + mExecutor + = std::make_unique<texec::Executor>(decoderTrtEnginePath.value(), executorModelType, executorConfig); + } + else if (executorModelType == texec::ModelType::kENCODER_DECODER) + { + mExecutor = std::make_unique<texec::Executor>( + encoderTrtEnginePath.value(), decoderTrtEnginePath.value(), executorModelType, executorConfig); + } + else if (executorModelType == texec::ModelType::kENCODER_ONLY) + { + mExecutor + = std::make_unique<texec::Executor>(encoderTrtEnginePath.value(), executorModelType, executorConfig); + } + else + { + TLLM_LOG_ERROR("not a supported executor model type in executor server."); + } + + auto const& world = tensorrt_llm::mpi::MpiComm::world(); + auto worldRank = world.getRank(); + if (worldRank == 0) + { + mCollectStatsThread = std::thread(&ExecutorServer::collectStats, this); + } + } + + ~ExecutorServer() + { + mShutdown = true; + if (mCollectStatsThread.joinable()) + { + mCollectStatsThread.join(); + } + } + + void enqueue(std::vector<texec::Request> requests, bool warmup = false) + { + try + { + std::vector<SizeType32> inputLengths; + for (auto const& request : requests) + { + inputLengths.push_back(request.getInputTokenIds().size()); + } + auto const start = std::chrono::steady_clock::now(); + auto reqIds = mExecutor->enqueueRequests(std::move(requests)); + for (int req = 0; req < reqIds.size(); ++req) + { + if (!warmup) + { + mRecorder->recordStart(inputLengths.at(req), reqIds.at(req), start); + } + mActiveCount++; + } + } + catch (std::exception const& e) + { + TLLM_THROW("%s", e.what()); + } + } + + void resetNumFinished() + { + mNumFinished = 0; + } + + bool canEnqueue(int numSentRequests) const + { + return !mConcurrency || (numSentRequests - mNumFinished < mConcurrency); + } + + void waitForResponses(SizeType32 numRequests, bool warmup = false) + { + while (mActiveCount || (mNumFinished < numRequests)) + { + auto responses = mExecutor->awaitResponses(mWaitSleep); + auto const tokenTime = std::chrono::steady_clock::now(); + for (auto const& response : responses) + { + if (response.getResult().isFinal) + { + mActiveCount--; + mNumFinished++; + if (!warmup) + { + mRecorder->recordEnd(response, tokenTime); + } + } + else + { + if (!warmup && !response.hasError()) + { + mRecorder->recordToken(response, tokenTime); + } + } + } + } + } + + void collectStats() const + { + while (!mShutdown) + { + auto iterStats = mExecutor->getLatestIterationStats(); + for (auto const& iterStat : iterStats) + { + SizeType32 numNewActiveRequests = iterStat.numNewActiveRequests; + if (numNewActiveRequests > 0) + { + float avgQueueingTime + = static_cast<float>(iterStat.newActiveRequestsQueueLatencyMS / numNewActiveRequests); + std::vector<float> requestsQueueLatencyMS(numNewActiveRequests, avgQueueingTime); + mRecorder->recordQueueLatency(requestsQueueLatencyMS); + } + if (mLogIterationData) + { + TLLM_LOG_INFO(texec::JsonSerialization::toJsonStr(iterStat)); + } + } + auto const waitSleep = std::chrono::milliseconds(50); + std::this_thread::sleep_for(waitSleep); + } + } + +private: + std::unique_ptr<texec::Executor> mExecutor; + std::thread mCollectStatsThread; + std::shared_ptr<Recorder> mRecorder; + std::chrono::milliseconds mWaitSleep; + std::optional<int> mConcurrency; + std::atomic<uint64_t> mActiveCount; + std::atomic<uint64_t> mNumFinished; + std::atomic<bool> mShutdown; + bool mLogIterationData; +}; // class ExecutorServer + +namespace +{ + +texec::Request makeExecutorRequest(Sample const& sample, SizeType32 const& beamWidth, + std::optional<SizeType32> const& eosId, std::optional<SizeType32> const& padId, bool streaming = false, + bool const& returnContextLogits = false, bool const& returnGenerationLogits = false, + std::optional<texec::LoraConfig> const& loraConfig = std::nullopt, + std::optional<texec::LookaheadDecodingConfig> const& lookaheadConfig = std::nullopt, + std::optional<texec::VecTokens> encoderInputTokenIds = std::nullopt, + std::optional<float> temperature = std::nullopt) +{ + auto samplingConfig = texec::SamplingConfig{beamWidth}; + samplingConfig.setTemperature(temperature); + auto outputConfig = texec::OutputConfig{false, returnContextLogits, returnGenerationLogits, false}; + return texec::Request(sample.inputIds, sample.outputLen, streaming, samplingConfig, outputConfig, eosId, padId, + std::nullopt, // positionIds + std::nullopt, // badWords + std::nullopt, // stopWords + std::nullopt, // embeddingBias + std::nullopt, // speculativeDecoding + std::nullopt, // pTuning + std::nullopt, // multimodalInput + std::nullopt, // multimodalEmbedding + std::nullopt, // mRopeConfig + loraConfig, // loraConfig + lookaheadConfig, // lookaheadConfig + std::nullopt, // kvCacheRetentionConfig + std::nullopt, // logitsPostProcessorName + std::nullopt, // logitsPostProcessor + encoderInputTokenIds.has_value() ? encoderInputTokenIds : std::nullopt, + std::nullopt); // cacheSalt +} + +void benchmarkExecutor(std::optional<std::filesystem::path> const& decoderEngineDir, + std::optional<std::filesystem::path> const& encoderEngineDir, texec::BatchingType batchingType, + std::string const& datasetPath, std::string const& opCsvFile, int maxNumSamples, int beamWidth, int warmUp, + std::optional<int32_t> const& eosId, std::optional<int32_t> const& padId, BenchmarkParams const& benchmarkParams, + texec::CapacitySchedulerPolicy capacitySchedulerPolicy, std::chrono::milliseconds waitSleep, + bool returnContextLogits, bool returnGenerationLogits, std::optional<int> const staticEmulatedBatchSize, + bool logIterationData, std::optional<SizeType32> const maxPromptLen, texec::ModelType executorModelType, + std::string const& responsesJsonFile) +{ + auto const& world = tensorrt_llm::mpi::MpiComm::world(); + auto worldRank = world.getRank(); + + // Load dataset + auto const samples = parseWorkloadJson(datasetPath, maxNumSamples, maxPromptLen); + auto const numSamples = samples.size(); + + auto recorder = std::make_shared<Recorder>(opCsvFile, benchmarkParams.streaming, beamWidth, responsesJsonFile); + int32_t decoderStartTokenId = 0; + std::shared_ptr<ExecutorServer> executorServer; + + if (executorModelType == texec::ModelType::kDECODER_ONLY) + { + TLLM_CHECK_WITH_INFO( + decoderEngineDir.has_value(), "decoder models require a path to decoder engine in executor benchmark."); + executorServer + = std::make_shared<ExecutorServer>(decoderEngineDir.value(), std::nullopt, batchingType, beamWidth, + capacitySchedulerPolicy, benchmarkParams, recorder, waitSleep, logIterationData, executorModelType); + } + else if (executorModelType == texec::ModelType::kENCODER_DECODER) + { + TLLM_CHECK_WITH_INFO(encoderEngineDir.has_value(), + "encoder-decoder models require a path to encoder engine in executor benchmark."); + executorServer = std::make_shared<ExecutorServer>(decoderEngineDir.value(), encoderEngineDir.value(), + batchingType, beamWidth, capacitySchedulerPolicy, benchmarkParams, recorder, waitSleep, logIterationData, + executorModelType); + try + { + std::ifstream decoderJsonConfigPath(decoderEngineDir.value() / "config.json"); + auto const decoderPretrainedConfig + = nlohmann::json::parse(decoderJsonConfigPath, nullptr, true, true).at("pretrained_config"); + decoderStartTokenId = decoderPretrainedConfig.at("decoder_start_token_id").template get<int32_t>(); + } + catch (nlohmann::json::out_of_range& e) + { + TLLM_LOG_ERROR( + "Parameter %s cannot be read from decoder config.json in pretrained_config. Using default id %d.", + std::string("decoder_start_token_id").c_str(), decoderStartTokenId); + } + catch (nlohmann::json::type_error const& e) + { + TLLM_LOG_ERROR( + "Parameter %s has error type in decoder config.json in pretrained_config. Using default id %d.", + std::string("decoder_start_token_id").c_str(), decoderStartTokenId); + } + } + else if (executorModelType == texec::ModelType::kENCODER_ONLY) + { + TLLM_CHECK_WITH_INFO( + encoderEngineDir.has_value(), "encoder models require a path to encoder engine in executor benchmark."); + executorServer + = std::make_shared<ExecutorServer>(std::nullopt, encoderEngineDir.value(), batchingType, beamWidth, + capacitySchedulerPolicy, benchmarkParams, recorder, waitSleep, logIterationData, executorModelType); + } + else + { + TLLM_LOG_ERROR("not a supported executor model type in executor benchmark."); + return; + } + + if (worldRank == 0) + { + if (benchmarkParams.loraDir) + { + auto startLoraLoad = std::chrono::steady_clock::now(); + LoraLib loras(benchmarkParams.loraDir.value()); + std::vector<texec::Request> requests; + for (auto& [taskId, p] : loras.getLoras()) + { + // squeeze lora configs and weights since LoraConfig requires them to be 2D tensors + p.first->squeeze(0); + p.second->squeeze(0); + texec::LoraConfig loraConfig( + taskId, texec::detail::ofITensor(p.first), texec::detail::ofITensor(p.second)); + if (executorModelType == texec::ModelType::kENCODER_DECODER) + { + Sample s{std::vector<int32_t>{decoderStartTokenId}, 1, static_cast<int32_t>(taskId)}; + requests.emplace_back(makeExecutorRequest(s, beamWidth, eosId, padId, false, false, false, + loraConfig, std::nullopt, std::vector<int32_t>{1, 2, 3, 4, 5})); + } + else + { + Sample s{std::vector<int32_t>{1, 2, 3, 4, 5}, 1, static_cast<int32_t>(taskId)}; + requests.emplace_back( + makeExecutorRequest(s, beamWidth, eosId, padId, false, false, false, loraConfig, std::nullopt)); + } + } + executorServer->enqueue(std::move(requests), true); + executorServer->waitForResponses(loras.getLoras().size(), true); + auto endLoraLoad = std::chrono::steady_clock::now(); + printf("[BENCHMARK] time to preload LoRAs(ms) %.2f\n", + std::chrono::duration<float, std::milli>(endLoraLoad - startLoraLoad).count()); + } + // Warm up + { + std::vector<texec::Request> requests; + for (auto i = 0; i < warmUp; ++i) + { + if (executorModelType == texec::ModelType::kENCODER_DECODER) + { + Sample s{std::vector<int32_t>{decoderStartTokenId}, samples[0].outputLen, samples[0].taskId}; + requests.emplace_back(makeExecutorRequest(s, beamWidth, eosId, padId, benchmarkParams.streaming, + returnContextLogits, returnGenerationLogits, std::nullopt, + benchmarkParams.requestLookaheadConfig, samples[0].inputIds)); + } + else + { + requests.emplace_back(makeExecutorRequest(samples[0], beamWidth, eosId, padId, + benchmarkParams.streaming, returnContextLogits, returnGenerationLogits, std::nullopt, + benchmarkParams.requestLookaheadConfig, std::nullopt, benchmarkParams.temperature)); + } + } + executorServer->enqueue(std::move(requests), true); + executorServer->waitForResponses(warmUp, true); + } + + // Benchmark + { + auto timeDelays = computeTimeDelays(benchmarkParams, numSamples - 1); + + // Create requests + recorder->initialize(); + std::vector<texec::Request> requests; + + for (std::size_t i = 0; i < numSamples; ++i) + { + std::optional<texec::LoraConfig> loraConfig; + if (samples[i].taskId >= 0) + { + loraConfig = texec::LoraConfig(samples[i].taskId); + } + if (executorModelType == texec::ModelType::kENCODER_DECODER) + { + Sample s{std::vector<int32_t>{decoderStartTokenId}, samples[i].outputLen, samples[i].taskId}; + requests.emplace_back(makeExecutorRequest(s, beamWidth, eosId, padId, benchmarkParams.streaming, + returnContextLogits, returnGenerationLogits, loraConfig, benchmarkParams.requestLookaheadConfig, + samples[i].inputIds)); + } + else + { + requests.emplace_back(makeExecutorRequest(samples[i], beamWidth, eosId, padId, + benchmarkParams.streaming, returnContextLogits, returnGenerationLogits, loraConfig, + benchmarkParams.requestLookaheadConfig, std::nullopt, benchmarkParams.temperature)); + } + } + + bool const hasDelay + = std::any_of(timeDelays.begin(), timeDelays.end(), [](auto const& delay) { return delay > 0.0; }); + executorServer->resetNumFinished(); + if (!staticEmulatedBatchSize) + { + // Launch a thread that will wait for responses + std::thread waitThread( + [numSamples, executorServer]() { executorServer->waitForResponses(numSamples); }); + + // Enqueue requests one by one + int numSentRequests = 0; + while (numSentRequests < numSamples) + { + if (executorServer->canEnqueue(numSentRequests)) + { + executorServer->enqueue({requests.at(numSentRequests)}); + if (hasDelay && numSentRequests < numSamples - 1) + { + std::this_thread::sleep_for( + std::chrono::milliseconds(static_cast<int>(timeDelays.at(numSentRequests) * 1000))); + } + numSentRequests += 1; + } + } + waitThread.join(); + } + else + { + TLLM_CHECK_WITH_INFO( + !hasDelay, "Executor benchmark doesn't support delays with emulated static batch sizes"); + SizeType32 numRequests = requests.size(); + SizeType32 maxBatchSize = staticEmulatedBatchSize.value(); + for (SizeType32 req = 0; req < numRequests; req += maxBatchSize) + { + auto batchSize = std::min(maxBatchSize, numRequests - req); + + std::vector<texec::Request> requestsBatch(std::make_move_iterator(requests.begin() + req), + std::make_move_iterator(requests.begin() + req + batchSize)); + // Enqueue in batches + executorServer->enqueue(std::move(requestsBatch)); + // Wait for current batch to be done + executorServer->waitForResponses(batchSize); + } + } + } + recorder->finalize(); + recorder->calculateMetrics(); + recorder->report(); + recorder->writeOpMetricsToCsv(); + recorder->dumpResponseSeqs(); + // Send terminateReqId to terminate servers on all ranks + // Sever on rank 0 will broadcast the terminate signal to other servers on multi-GPU cases + } +} + +} // namespace + +int main(int argc, char* argv[]) +{ + cxxopts::Options options( + "TensorRT LLM BatchManager Benchmark", "TensorRT LLM BatchManager Benchmark for GPT and GPT-like models."); + options.add_options()("h,help", "Print usage"); + options.add_options()("engine_dir, decoder_engine_dir", "Directory that store the engines of decoder models.", + cxxopts::value<std::string>()); + options.add_options()( + "encoder_engine_dir", "Directory that store the engines of the encoder models.", cxxopts::value<std::string>()); + options.add_options()( + "api", "API type: gptManager or executor.", cxxopts::value<std::string>()->default_value("executor")); + options.add_options()("type", + "Batching type: choose between inflight/static. (IFB/V1 options are going to be deprecated)", + cxxopts::value<std::string>()->default_value("inflight")); + options.add_options()("dataset", "Dataset that is used for benchmarking BatchManager.", + cxxopts::value<std::string>()->default_value("")); + options.add_options()( + "output_csv", "Write output metrics to CSV", cxxopts::value<std::string>()->default_value("")); + options.add_options()("max_num_samples", "maximum number of samples to use from dataset/generate", + cxxopts::value<int>()->default_value("100000")); + options.add_options()( + "beam_width", "Specify beam width you want to benchmark.", cxxopts::value<int>()->default_value("1")); + options.add_options()( + "warm_up", "Specify warm up iterations before benchmark starts.", cxxopts::value<int>()->default_value("2")); + options.add_options()( + "eos_id", "Specify the end-of-sequence token id.", cxxopts::value<TokenIdType>()->default_value("-1")); + options.add_options()("pad_id", "Specify the padding token id.", cxxopts::value<TokenIdType>()); + options.add_options()("max_tokens_in_paged_kvcache", "Max tokens in paged K-V Cache.", cxxopts::value<int>()); + options.add_options()( + "max_attention_window", "Max KV cache length per sequence", cxxopts::value<std::vector<int>>()); + options.add_options()("sink_token_len", "Sink token length in kv cache per sequence.", cxxopts::value<int>()); + options.add_options()( + "random_seed", "integer random seed for exponential time delays.", cxxopts::value<int>()->default_value("420")); + options.add_options()( + "kv_cache_free_gpu_mem_fraction", "K-V Cache Free Gpu Mem Fraction.", cxxopts::value<float>()); + options.add_options()( + "cross_kv_cache_fraction", "Cross K-V Cache Fraction (from 0.0 to 1.0).", cxxopts::value<float>()); + options.add_options()("request_rate", + "request rate in reqs/sec. Skipping this arg or negative value will trigger offline/0-delay.", + cxxopts::value<float>()); + options.add_options()("concurrency", "Concurrent number of connections with the server.", cxxopts::value<int>()); + options.add_options()("max_batch_size", "The max runtime batch size when benchmarking", cxxopts::value<int>()); + options.add_options()( + "max_num_tokens", "The max runtime number of tokens per batch when benchmarking", cxxopts::value<int>()); + options.add_options()( + "enable_batch_size_tuning", "Dynamic tuning of batch size", cxxopts::value<bool>()->default_value("false")); + options.add_options()("enable_max_num_tokens_tuning", "Dynamic tuning of max num tokens", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("enable_exp_delays", "Enables exponential delay distr to mimic real world request arrival", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("streaming", + "Operate in streaming mode. Note: it reflects time-to-first-token and inter-token-latency", + cxxopts::value<bool>()->default_value("false")); + options.add_options()( + "enable_kv_cache_reuse", "Enables the KV cache reuse.", cxxopts::value<bool>()->default_value("true")); + options.add_options()( + "enable_chunked_context", "Whether to enable context chunking.", cxxopts::value<bool>()->default_value("true")); + options.add_options()( + "return_context_logits", "Whether to return context logits.", cxxopts::value<bool>()->default_value("false")); + options.add_options()("return_generation_logits", "Whether to return generation logits.", + cxxopts::value<bool>()->default_value("false")); + + options.add_options()("scheduler_policy", + "Choose scheduler policy between max_utilization/guaranteed_no_evict/static_batch.", + cxxopts::value<std::string>()->default_value("guaranteed_no_evict")); + + options.add_options()("static_emulated_batch_size", + "Emulate static batching performance with the provided batch size.", cxxopts::value<SizeType32>()); + options.add_options()("log_level", "Choose log level between verbose/info/warning/error/internal_error.", + cxxopts::value<std::string>()->default_value("warning")); + options.add_options()("log_iteration_data", "On each decoder iteration, print batch state metadata.", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("wait_sleep", "Specify how many milliseconds to sleep each iteration of waitForEmpty loop.", + cxxopts::value<int>()->default_value("25")); + options.add_options()("lora_dir", "Directory containing LoRAs", cxxopts::value<std::string>()->default_value("")); + options.add_options()("lora_host_cache_bytes", "LoRA host cache memory in bytes", cxxopts::value<size_t>()); + options.add_options()("lora_num_device_mod_layers", "LoRA number 1d cache rows", cxxopts::value<int>()); + options.add_options()("kv_host_cache_bytes", + "Size of secondary memory pool used for offloading kv cache blocks (in bytes).", + cxxopts::value<size_t>()->default_value("0")); + options.add_options()( + "max_prompt_len", "Truncate all prompts from dataset to the length specified.", cxxopts::value<SizeType32>()); + + options.add_options()("gpu_weights_percent", + "Specify the percentage of weights that reside on GPU (from 0.0 to 1.0).", + cxxopts::value<float>()->default_value("1.0")); + options.add_options()( + "medusa_choices", "Medusa choices in the format of [[0], [0, 1], [0, 0, 1]]", cxxopts::value<std::string>()); + options.add_options()( + "eagle_choices", "Eagle choices in the format of [[0], [0, 1], [0, 0, 1]]", cxxopts::value<std::string>()); + options.add_options()("eagle_posterior_threshold", + "Minimum token probability threshold for typical acceptance. Enables typical acceptance in Eagle", + cxxopts::value<float>()); + options.add_options()("temperature", "Sampling temperature for each request", cxxopts::value<float>()); + options.add_options()( + "eagle_use_dynamic_tree", "Whether to use Eagle-2", cxxopts::value<bool>()->default_value("false")); + options.add_options()("eagle_dynamic_tree_max_top_k", + "The max topK for dynamic tree, also the number of draft tokens that will expand for each node", + cxxopts::value<SizeType32>()); + + options.add_options()("multi_block_mode", + "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel", + cxxopts::value<bool>()->default_value("true")); + options.add_options()("cuda_graph_mode", "When enabled, inference is executed with cuda graph.", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("cuda_graph_cache_size", + "Specify how many cuda graphs are cached in the runtime. Larger cache gives better perf, but consumes more GPU " + "memory.", + cxxopts::value<SizeType32>()->default_value("0")); + options.add_options()("enable_trt_overlap", "Enable TRT Overlap", cxxopts::value<bool>()->default_value("false")); + + options.add_options()("enable_context_fmha_fp32_acc", "Enable FMHA runner FP32 accumulation", + cxxopts::value<bool>()->default_value("false")); + options.add_options()("executor_lookahead_config", + "lookahead config in the format of [max_window_size, max_ngram_size, max_verification_set_size]", + cxxopts::value<std::string>()); + options.add_options()("request_lookahead_config", + "lookahead config in the format of [max_window_size, max_ngram_size, max_verification_set_size], and each <= " + "executor lookahead config", + cxxopts::value<std::string>()); + options.add_options()("responses_json", "Write output response sequences to a json file", + cxxopts::value<std::string>()->default_value("")); + + auto result = options.parse(argc, argv); + + if (result.count("help")) + { + std::cout << options.help() << std::endl; + return 0; + } + + // Argument: Engine directory + if (!result.count("engine_dir") && !result.count("encoder_engine_dir")) + { + std::cout << options.help() << std::endl; + TLLM_LOG_ERROR("Please specify engine directory."); + return 1; + } + + // Argument: Batching Type + auto const type = result["type"].as<std::string>(); + texec::BatchingType batchingType{texec::BatchingType::kINFLIGHT}; + if (type == "V1" || type == "static") + { + if (type == "V1") + { + TLLM_LOG_WARNING("type option \"V1\" is going to be renamed to \"static\"."); + } + bool streaming = result["streaming"].as<bool>(); + if (streaming) + { + TLLM_LOG_ERROR("Streaming is not supported in static batching.\n"); + return 1; + } + batchingType = texec::BatchingType::kSTATIC; + } + else if (type == "IFB" || type == "inflight") + { + if (type == "IFB") + { + TLLM_LOG_WARNING("type option \"IFB\" is going to be renamed to \"inflight\"."); + } + batchingType = texec::BatchingType::kINFLIGHT; + } + else + { + TLLM_LOG_ERROR("Unexpected batching type: %s", type.c_str()); + return 1; + } + + // Argument: Dataset + auto const datasetPath = result["dataset"].as<std::string>(); + auto const maxNumSamples = result["max_num_samples"].as<int>(); + + // Argument: Output metrics CSV + auto const opCsvFile = result["output_csv"].as<std::string>(); + + // Argument: beam width + auto const beamWidth = result["beam_width"].as<int>(); + + // Argument: wait_sleep + auto const waitSleep = std::chrono::milliseconds(result["wait_sleep"].as<int>()); + BenchmarkParams benchmarkParams; + + // Argument: Max tokens in paged K-V Cache + if (result.count("max_tokens_in_paged_kvcache")) + { + benchmarkParams.maxTokensInPagedKvCache = result["max_tokens_in_paged_kvcache"].as<int>(); + } + + // Argument: Max KV cache length + if (result.count("max_attention_window")) + { + benchmarkParams.maxAttentionWindowVec = result["max_attention_window"].as<std::vector<int>>(); + } + + // Argument: Sink token length + if (result.count("sink_token_len")) + { + benchmarkParams.sinkTokenLength = result["sink_token_len"].as<int>(); + } + + if (result.count("random_seed")) + { + benchmarkParams.randomSeed = result["random_seed"].as<int>(); + } + + // Argument: K-V Cache Free Gpu Mem Fraction + if (result.count("kv_cache_free_gpu_mem_fraction")) + { + benchmarkParams.freeGpuMemoryFraction = result["kv_cache_free_gpu_mem_fraction"].as<float>(); + } + // Argument: K-V Cache Cross Attention Fraction. Only applicable to enc-dec models. + if (result.count("encoder_engine_dir") && result.count("decoder_engine_dir")) + { + if (result.count("cross_kv_cache_fraction")) + { + benchmarkParams.crossKvCacheFraction = result["cross_kv_cache_fraction"].as<float>(); + } + else + { + benchmarkParams.crossKvCacheFraction + = 0.5f; // default value if not set. but non enc-dec should not even have this param set + } + } + + // Argument: Enable dynamic tuning of batch size + benchmarkParams.enableBatchSizeTuning = result["enable_batch_size_tuning"].as<bool>(); + + // Argument: Enable dynamic tuning of max num tokens + benchmarkParams.enableMaxNumTokensTuning = result["enable_max_num_tokens_tuning"].as<bool>(); + + // Argument: Enable KV cache reuse + benchmarkParams.enableBlockReuse = result["enable_kv_cache_reuse"].as<bool>(); + + // Argument: streaming + benchmarkParams.streaming = result["streaming"].as<bool>(); + + TLLM_CHECK_WITH_INFO(!(result.count("request_rate") && result.count("concurrency")), + "request_rate and concurrency cannot be specified at the same time."); + + // Argument: request rate + if (result.count("request_rate")) + { + benchmarkParams.requestRate = result["request_rate"].as<float>(); + } + + // Argument: concurrency + if (result.count("concurrency")) + { + benchmarkParams.concurrency = result["concurrency"].as<int>(); + } + + // Argument: request rate + if (result.count("max_batch_size")) + { + benchmarkParams.maxBatchSize = result["max_batch_size"].as<int>(); + } + + // Argument: request rate + if (result.count("max_num_tokens")) + { + benchmarkParams.maxNumTokens = result["max_num_tokens"].as<int>(); + } + + benchmarkParams.enableExpDelays = result["enable_exp_delays"].as<bool>(); + + // Argument: Enable batch stats output + bool logIterationData = result["log_iteration_data"].as<bool>(); + + if (logIterationData) + { + TLLM_LOG_WARNING("Setting log_iteration_data to true adds overheads and may result in lower perf"); + } + + // Argument: Enable chunked context + benchmarkParams.enableChunkedContext = result["enable_chunked_context"].as<bool>(); + + // Argument: Enable return context logits + bool returnContextLogits = result["return_context_logits"].as<bool>(); + + // Argument: Enable return context logits + bool returnGenerationLogits = result["return_generation_logits"].as<bool>(); + + if (result.count("lora_dir")) + { + benchmarkParams.loraDir = result["lora_dir"].as<std::string>(); + } + if (result.count("lora_host_cache_bytes")) + { + benchmarkParams.loraHostCacheSize = result["lora_host_cache_bytes"].as<size_t>(); + } + if (result.count("lora_num_device_mod_layers")) + { + benchmarkParams.loraDeviceNumModLayers = result["lora_num_device_mod_layers"].as<SizeType32>(); + } + + // Argument: How many KV cache blocks (as fraction of number of GPU kv cache blocks). + benchmarkParams.kvHostCacheSize = result["kv_host_cache_bytes"].as<size_t>(); + + // Argument: Medusa choices for the Medusa speculative decoding. + if (result.count("medusa_choices")) + { + benchmarkParams.medusaChoices = parseVectorOfVectors(result["medusa_choices"].as<std::string>()); + } + // Argument: Eagle choices for the Eagle speculative decoding. + if (result.count("eagle_choices") || result.count("eagle_posterior_threshold") + || result.count("eagle_use_dynamic_tree") || result.count("eagle_dynamic_tree_max_top_k")) + { + std::optional<float> posteriorThreshold; + if (result.count("eagle_posterior_threshold")) + { + posteriorThreshold = result["eagle_posterior_threshold"].as<float>(); + } + std::optional<texec::EagleChoices> choices; + if (result.count("eagle_choices")) + { + choices = parseVectorOfVectors(result["eagle_choices"].as<std::string>()); + } + bool eagleUseDynamicTree = false; + if (result.count("eagle_use_dynamic_tree")) + { + eagleUseDynamicTree = result["eagle_use_dynamic_tree"].as<bool>(); + } + std::optional<SizeType32> eagleDynamicTreeMaxTopK; + if (result.count("eagle_dynamic_tree_max_top_k")) + { + eagleDynamicTreeMaxTopK = result["eagle_dynamic_tree_max_top_k"].as<SizeType32>(); + } + benchmarkParams.eagleConfig = texec::EagleConfig( + choices, !posteriorThreshold.has_value(), posteriorThreshold, eagleUseDynamicTree, eagleDynamicTreeMaxTopK); + } + if (result.count("temperature")) + { + benchmarkParams.temperature = result["temperature"].as<float>(); + } + + if (result.count("executor_lookahead_config")) + { + benchmarkParams.executorLookaheadConfig + = parseLookaheadConfig(result["executor_lookahead_config"].as<std::string>()); + } + if (result.count("request_lookahead_config")) + { + benchmarkParams.requestLookaheadConfig + = parseLookaheadConfig(result["request_lookahead_config"].as<std::string>()); + } + + // Argument: multi_block_mode + benchmarkParams.multiBlockMode = result["multi_block_mode"].as<bool>(); + + // Argument: enable_context_fmha_fp32_acc + benchmarkParams.enableContextFMHAFP32Acc = result["enable_context_fmha_fp32_acc"].as<bool>(); + + // Argument: cuda_graph_mode + benchmarkParams.cudaGraphMode = result["cuda_graph_mode"].as<bool>(); + + // Argument: cuda_graph_cache_size + benchmarkParams.cudaGraphCacheSize = result["cuda_graph_cache_size"].as<SizeType32>(); + + // Argument: enable_trt_overlap + benchmarkParams.enableTrtOverlap = result["enable_trt_overlap"].as<bool>(); + + std::optional<TokenIdType> padId; + // Argument: Padding token id + if (result.count("pad_id")) + { + padId = result["pad_id"].as<TokenIdType>(); + } + + // Argument: End-of-sentence token id + std::optional<TokenIdType> eosId = result["eos_id"].as<TokenIdType>(); + + std::optional<SizeType32> staticEmulatedBatchSize; + // Argument: Static emulated batch size + if (result.count("static_emulated_batch_size")) + { + staticEmulatedBatchSize = result["static_emulated_batch_size"].as<SizeType32>(); + } + + // Argument: Scheduler policy + texec::CapacitySchedulerPolicy capacitySchedulerPolicy; + auto const capacitySchedulerPolicyArg = result["scheduler_policy"].as<std::string>(); + if (capacitySchedulerPolicyArg == "max_utilization") + { + capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kMAX_UTILIZATION; + } + else if (capacitySchedulerPolicyArg == "guaranteed_no_evict") + { + capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT; + } + else if (capacitySchedulerPolicyArg == "static_batch") + { + capacitySchedulerPolicy = texec::CapacitySchedulerPolicy::kSTATIC_BATCH; + } + else + { + TLLM_LOG_ERROR("Unexpected scheduler policy: " + capacitySchedulerPolicyArg); + return 1; + } + + // Argument: max_prompt_len + std::optional<SizeType32> maxPromptLen; + if (result.count("max_prompt_len")) + { + maxPromptLen = result["max_prompt_len"].as<SizeType32>(); + } + + // Argument: GPU weights percentage + auto gpuWeightsPercent = result["gpu_weights_percent"].as<float>(); + if (gpuWeightsPercent < 0 || gpuWeightsPercent > 1) + { + TLLM_LOG_ERROR("--gpu_weights_percent must be between 0.0 and 1.0 but got: %f", gpuWeightsPercent); + return 1; + } + benchmarkParams.gpuWeightsPercent = gpuWeightsPercent; + + // Argument: Log level + auto logger = std::make_shared<TllmLogger>(); + auto const logLevel = result["log_level"].as<std::string>(); + if (logLevel == "verbose") + { + logger->setLevel(trt::ILogger::Severity::kVERBOSE); + } + else if (logLevel == "info") + { + logger->setLevel(trt::ILogger::Severity::kINFO); + } + else if (logLevel == "warning") + { + logger->setLevel(trt::ILogger::Severity::kWARNING); + } + else if (logLevel == "error") + { + logger->setLevel(trt::ILogger::Severity::kERROR); + } + else if (logLevel == "internal_error") + { + logger->setLevel(trt::ILogger::Severity::kINTERNAL_ERROR); + } + else + { + TLLM_LOG_ERROR("Unexpected log level: " + logLevel); + return 1; + } + + initTrtLlmPlugins(logger.get()); + + // Argument: output sequences JSON + auto const responsesJsonFile = result["responses_json"].as<std::string>(); + + // Argument: API + auto const api = result["api"].as<std::string>(); + if (api == "executor") + { + texec::ModelType executorModelType; + std::optional<std::string> decoderEngineDir = std::nullopt, encoderEngineDir = std::nullopt; + if (result.count("encoder_engine_dir") && result.count("decoder_engine_dir")) + { + TLLM_CHECK_WITH_INFO(api == "executor", "encoder-decoder only support executor api."); + TLLM_CHECK_WITH_INFO( + batchingType == texec::BatchingType::kINFLIGHT, "encoder-decoder only support inflight batching."); + executorModelType = texec::ModelType::kENCODER_DECODER; + encoderEngineDir = result["encoder_engine_dir"].as<std::string>(); + decoderEngineDir = result["decoder_engine_dir"].as<std::string>(); + } + else if (result.count("engine_dir")) + { + executorModelType = texec::ModelType::kDECODER_ONLY; + decoderEngineDir = result["engine_dir"].as<std::string>(); + } + else + { + executorModelType = texec::ModelType::kENCODER_ONLY; + encoderEngineDir = result["encoder_engine_dir"].as<std::string>(); + } + try + { + benchmarkExecutor(decoderEngineDir, encoderEngineDir, batchingType, datasetPath, opCsvFile, maxNumSamples, + beamWidth, result["warm_up"].as<int>(), eosId, padId, benchmarkParams, capacitySchedulerPolicy, + waitSleep, returnContextLogits, returnGenerationLogits, staticEmulatedBatchSize, logIterationData, + maxPromptLen, executorModelType, responsesJsonFile); + } + catch (std::exception const& e) + { + TLLM_LOG_ERROR(e.what()); + return 1; + } + } + else if (api == "gptManager") + { + TLLM_LOG_ERROR("gptManager is deprecated, please use the executor API."); + return 1; + } + else + { + TLLM_LOG_ERROR("api parameter must be gptManager or executor"); + return 1; + } + + return 0; +} diff --git a/benchmarks/cpp/prepare_dataset.py b/benchmarks/cpp/prepare_dataset.py new file mode 100644 index 000000000000..3b9665fd2902 --- /dev/null +++ b/benchmarks/cpp/prepare_dataset.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from typing import Optional, Tuple + +import click +from pydantic import BaseModel, model_validator +from transformers import AutoTokenizer +from utils.prepare_real_data import dataset +from utils.prepare_synthetic_data import token_norm_dist, token_unif_dist + + +class RootArgs(BaseModel): + tokenizer: str + output: str + random_seed: int + task_id: int + std_out: bool + trust_remote_code: bool = False + rand_task_id: Optional[Tuple[int, int]] + lora_dir: Optional[str] = None + + @model_validator(mode='after') + def validate_tokenizer(self): + try: + tokenizer = AutoTokenizer.from_pretrained( + self.tokenizer, + padding_side='left', + trust_remote_code=self.trust_remote_code) + except EnvironmentError as e: + raise ValueError( + f"Cannot find a tokenizer from the given string because of {e}\nPlease set tokenizer to the directory that contains the tokenizer, or set to a model name in HuggingFace." + ) + tokenizer.pad_token = tokenizer.eos_token + self.tokenizer = tokenizer + + return self + + +@click.group(deprecated=True) +@click.option( + "--tokenizer", + required=True, + type=str, + help= + "Tokenizer dir for the model run by gptManagerBenchmark, or the model name from HuggingFace." +) +@click.option("--output", + type=str, + help="Output json filename.", + default="preprocessed_dataset.json") +@click.option( + "--stdout", + is_flag=True, + help="Print output to stdout with a JSON dataset entry on each line.", + default=False) +@click.option("--random-seed", + required=False, + type=int, + help="random seed for token_ids", + default=420) +@click.option("--task-id", type=int, default=-1, help="LoRA task id") +@click.option("--rand-task-id", + type=int, + default=None, + nargs=2, + help="Random LoRA Tasks") +@click.option("--lora-dir", + type=str, + default=None, + help="Directory containing LoRA adapters") +@click.option("--log-level", + default="info", + type=click.Choice(['info', 'debug']), + help="Logging level.") +@click.option("--trust-remote-code", + is_flag=True, + default=False, + envvar="TRUST_REMOTE_CODE", + help="Trust remote code.") +@click.pass_context +def cli(ctx, **kwargs): + """This script generates dataset input for gptManagerBenchmark.""" + if kwargs['log_level'] == 'info': + logging.basicConfig(level=logging.INFO) + elif kwargs['log_level'] == 'debug': + logging.basicConfig(level=logging.DEBUG) + else: + raise ValueError(f"Unsupported logging level {kwargs['log_level']}") + + ctx.obj = RootArgs(tokenizer=kwargs['tokenizer'], + output=kwargs['output'], + std_out=kwargs['stdout'], + random_seed=kwargs['random_seed'], + task_id=kwargs['task_id'], + rand_task_id=kwargs['rand_task_id'], + lora_dir=kwargs['lora_dir'], + trust_remote_code=kwargs['trust_remote_code']) + + +cli.add_command(dataset) +cli.add_command(token_norm_dist) +cli.add_command(token_unif_dist) + +if __name__ == "__main__": + cli() diff --git a/tensorrt_llm/bench/tuning/__init__.py b/benchmarks/cpp/utils/__init__.py similarity index 100% rename from tensorrt_llm/bench/tuning/__init__.py rename to benchmarks/cpp/utils/__init__.py diff --git a/benchmarks/cpp/utils/convert_nemo_dataset.py b/benchmarks/cpp/utils/convert_nemo_dataset.py new file mode 100644 index 000000000000..6f4884347677 --- /dev/null +++ b/benchmarks/cpp/utils/convert_nemo_dataset.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#!/usr/bin/env python3 + +import argparse +import json + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("input") + parser.add_argument("output") + + args = parser.parse_args() + + output_o = [] + + with open(args.input, 'r') as infile: + for _l in infile: + l = _l.strip() + if len(l) == 0: + continue + o = json.loads(l) + output_o.append({ + "input": o["prompt"], + "instruction": "", + "output": o["completion"] + }) + + with open(args.output, 'w') as outfile: + json.dump(output_o, outfile) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cpp/utils/generate_rand_loras.py b/benchmarks/cpp/utils/generate_rand_loras.py new file mode 100644 index 000000000000..12eb1fdc3648 --- /dev/null +++ b/benchmarks/cpp/utils/generate_rand_loras.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#!/usr/bin/env python3 + +import argparse +import os +from pathlib import Path + +import numpy as np + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("input_lora") + parser.add_argument("output") + parser.add_argument("num_loras", type=int) + + args = parser.parse_args() + + lora_path = Path(args.input_lora) + weights_path = lora_path / "model.lora_weights.npy" + config_path = lora_path / "model.lora_config.npy" + + weights = np.load(weights_path) + config = np.load(config_path) + + for i in range(args.num_loras): + out_path = Path(args.output) / str(i) + os.makedirs(out_path, exist_ok=True) + w = np.random.normal(0, 2, weights.shape).astype(weights.dtype) + np.save(out_path / "model.lora_weights.npy", w) + np.save(out_path / "model.lora_config.npy", config) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cpp/utils/prepare_real_data.py b/benchmarks/cpp/utils/prepare_real_data.py new file mode 100644 index 000000000000..4441c57ee4bc --- /dev/null +++ b/benchmarks/cpp/utils/prepare_real_data.py @@ -0,0 +1,434 @@ +import logging +import random +import re +import tempfile +from pathlib import Path +from typing import Optional + +import click +import datasets +from PIL import Image +from pydantic import BaseModel, model_validator +from utils.utils import (get_norm_dist_lengths, multimodal_dataset_dump, + print_multimodal_dataset, print_text_dataset, + text_dataset_dump) + + +def validate_output_len_dist(ctx, param, value): + """Validate the --output-len-dist option.""" + if value is None: + return value + m = re.match(r"(\d+),(\d+)", value) + if m: + return int(m.group(1)), int(m.group(2)) + else: + raise AssertionError( + "Incorrect specification for --output-len-dist. Correct format: --output-len-dist <output_len_mean>,<output_len_stdev>" + ) + + +class DatasetConfig(BaseModel): + """Dataset configurations.""" + """Name of the dataset on HuggingFace.""" + name: Optional[str] = None + """Config name of the dataset if existing.""" + config_name: Optional[str] = None + """Split of the dataset. Typical values: train, validation, test. Setting to None will include all splits.""" + split: Optional[str] + """The dataset dictionary used for the input sentence.""" + input_key: Optional[str] = None + """The dataset dictionary key used for the prompt of the input sentence. Must not be set when prompt is set.""" + image_key: Optional[str] = None + """The dataset dictionary key used for the images.""" + prompt_key: Optional[str] = None + """The prompt sentence to be added to the input sentence. Must not be set when prompt_key is set.""" + prompt: Optional[str] = None + """The dataset dictionary key used to derive the output sequence length. Set to None if the dataset does not have a key for output.""" + output_key: Optional[str] + """The local path to the dataset to be loaded when using a local cache.""" + local_path: Optional[str] = None + + @model_validator(mode='after') + def check_prompt(self) -> 'DatasetConfig': + if self.prompt_key and self.prompt: + raise AssertionError( + "--prompt-key and --prompt cannot be set at the same time.") + if (not self.prompt_key) and (not self.prompt): + raise AssertionError("Either --prompt-key or --prompt must be set.") + return self + + @model_validator(mode='after') + def check_name_and_local_path(self) -> 'DatasetConfig': + if self.name and self.local_path: + raise AssertionError( + "--dataset-name and --dataset-local-path cannot be set at the same time." + ) + if (not self.name) and (not self.local_path): + raise AssertionError( + "Either --dataset-name or --dataset-local-path must be set.") + return self + + @property + def query(self): + """Generate the query for HuggingFace `datasets.load_dataset()`""" + first_arg = self.local_path if self.local_path else self.name + + if self.config_name: + return [first_arg, self.config_name] + else: + return [first_arg] + + @property + def display_name(self) -> str: + """Returns a human-readable identifier for error messages.""" + # model_validator ensures exactly one of name or local_path is set + if self.name is not None: + return self.name + return self.local_path + + def get_prompt(self, req): + """Get the prompt sentence from the given request.""" + if self.prompt_key: + assert self.prompt_key in req, ( + f"Dataset {self.display_name} does not have key '{self.prompt_key}'. " + "Please set --prompt-key to one of the available keys: " + f"{req.keys()}") + return req[self.prompt_key] + else: + return self.prompt + + def get_input(self, req): + """Get the input sentence from the given request.""" + assert self.input_key in req, ( + f"Dataset {self.display_name} does not have key '{self.input_key}'. " + "Please set --input-key to one of the available keys: " + f"{req.keys()}") + return req[self.input_key] + + def get_images(self, req): + """Get the images from the given request.""" + image_keys = [self.image_key + ] + [f"{self.image_key}_{i}" for i in range(1, 8)] + assert any(key in req for key in image_keys), ( + f"Dataset {self.display_name} does not have key '{self.image_key}'. " + "Please set --dataset-image-key to one of the available keys: " + f"{req.keys()}") + images = [] + for key in image_keys: + if key in req and req[key] is not None: + images.append(req[key]) + return images + + def get_output(self, req): + """Get the output sentence from the given request.""" + if self.output_key is None: + raise RuntimeError( + "--output-key is not set. Please either:\n" + "1. Define output length through --output-len-dist.\n" + f"2. If the dataset {self.display_name} has key for golden output and " + "you wish to set output length to the length of the golden " + "output, set --output-key.") + assert self.output_key in req, ( + f"Dataset {self.display_name} does not have key '{self.output_key}'. " + "Please set --output-key to one of the available keys: " + f"{req.keys()}") + return req[self.output_key] + + +def _create_dataset_load_error(e: ValueError) -> ValueError: + """Create a more informative ValueError from a dataset loading error. + + Args: + e: The original ValueError from datasets.load_dataset(). + Returns: + A new ValueError with additional context. + """ + error_msg = str(e) + if "Config" in error_msg: + error_msg += "\n Please add the config name to the dataset config yaml." + elif "split" in error_msg: + error_msg += "\n Please specify supported split in the dataset config yaml." + return ValueError(error_msg) + + +def load_dataset(dataset_config: DatasetConfig): + """Load dataset from local path or HuggingFace. + Args: + dataset_config: A `DatasetConfig` object that defines the dataset to load. + Returns: + Dataset iterator. + Raises: + ValueError: When dataset loading fails due to incorrect dataset config setting. + """ + if dataset_config.local_path: + return load_dataset_from_local(dataset_config) + else: + return load_dataset_from_hf(dataset_config) + + +def load_dataset_from_hf(dataset_config: DatasetConfig): + """Load dataset from HuggingFace. + + Args: + dataset_config: A `DatasetConfig` object that defines the dataset to load. + Returns: + Dataset iterator. + Raises: + ValueError: When dataset loading fails due to incorrect dataset config setting. + """ + logging.debug( + f"Loading dataset from HF: query={dataset_config.query}, split={dataset_config.split}" + ) + + try: + dataset = iter( + datasets.load_dataset(*dataset_config.query, + split=dataset_config.split, + streaming=True, + trust_remote_code=True)) + except ValueError as e: + raise _create_dataset_load_error(e) + + logging.debug("Finished loading HF dataset") + + return dataset + + +def load_dataset_from_local(dataset_config: DatasetConfig): + """Load dataset from local path. + + Args: + dataset_config: A `DatasetConfig` object that defines the dataset to load. + Returns: + Dataset iterator. + Raises: + FileNotFoundError: When local dataset path does not exist. + ValueError: When dataset loading fails due to incorrect dataset config setting. + """ + + local_path = Path(dataset_config.local_path) + + if not local_path.exists(): + raise FileNotFoundError( + f"Local dataset path {local_path} does not exist.") + + logging.debug( + f"Loading dataset from local path: path={local_path}, query={dataset_config.query}, split={dataset_config.split}" + ) + + # If it's a directory we can use the normal loader, otherwise custom loader + # depends on the file extension + if local_path.is_dir(): + try: + dataset = datasets.load_dataset(*dataset_config.query, + split=dataset_config.split, + trust_remote_code=True) + except ValueError as e: + raise _create_dataset_load_error(e) + else: + format_map = { + ".json": "json", + ".jsonl": "json", + ".csv": "csv", + ".parquet": "parquet", + } + + file_extension = local_path.suffix + dataset_type = format_map.get(file_extension) + + if dataset_type is None: + raise ValueError(f"Unsupported file extension: {file_extension}") + + try: + dataset = datasets.load_dataset(dataset_type, + data_files=str(local_path), + split=dataset_config.split) + except ValueError as e: + raise _create_dataset_load_error(e) + + logging.debug("Finished loading local dataset") + + return iter(dataset) + + +@click.command() +@click.option("--dataset-name", type=str, help="Dataset name in HuggingFace.") +@click.option("--dataset-config-name", + type=str, + default=None, + help="Dataset config name in HuggingFace (if exists).") +@click.option("--dataset-split", + type=str, + required=True, + help="Split of the dataset to use.") +@click.option("--dataset-input-key", + type=str, + help="The dataset dictionary key for input.") +@click.option("--dataset-image-key", + type=str, + default="image", + help="The dataset dictionary key for images.") +@click.option("--dataset-prompt-key", + type=str, + default=None, + help="The dataset dictionary key for prompt (if exists).") +@click.option( + "--dataset-local-path", + type=str, + default=None, + help= + "The local path to the dataset to be loaded when using an offline cache.") +@click.option( + "--dataset-prompt", + type=str, + default=None, + help="The prompt string when there is no prompt key for the dataset.") +@click.option("--dataset-output-key", + type=str, + default=None, + help="The dataset dictionary key for output (if exists).") +@click.option( + "--num-requests", + type=int, + default=None, + help= + "Number of requests to be generated. Will be capped to min(dataset.num_rows, num_requests)." +) +@click.option( + "--max-input-len", + type=int, + default=None, + help= + "Maximum input sequence length for a given request. This will be used to filter out the requests with long input sequence length. Default will include all the requests." +) +@click.option( + "--output-len-dist", + type=str, + default=None, + callback=validate_output_len_dist, + help= + "Output length distribution. Default will be the length of the golden output from the dataset. Format: <output_len_mean>,<output_len_stdev>. E.g. 100,10 will randomize the output length with mean=100 and variance=10." +) +@click.pass_obj +def dataset(root_args, **kwargs): + """Prepare dataset from real dataset.""" + dataset_config = DatasetConfig(**{ + k[8:]: v + for k, v in kwargs.items() if k.startswith('dataset_') + }) + + input_ids = [] + input_lens = [] + output_lens = [] + task_ids = [] + req_cnt = 0 + modality = None + multimodal_texts = [] + multimodal_image_paths = [] + for req in load_dataset(dataset_config): + if any(key in req for key in ['image', 'image_1', 'video']): + # multimodal input + if 'video' in req and req['video'] is not None: + assert "Not supported yet" + assert kwargs['output_len_dist'] is not None, ( + "Output length distribution must be set for multimodal requests." + ) + modality = 'image' + text = dataset_config.get_prompt(req) + images = dataset_config.get_images(req) + image_paths = [] + for image in images: + if image is not None: + if isinstance(image, str): + image_paths.append(image) + elif isinstance(image, Image.Image): + with tempfile.NamedTemporaryFile( + suffix=".jpg", delete=False) as tmp_file: + logging.debug(f"Saving image to {tmp_file.name}") + image = image.convert("RGB") + image.save(tmp_file, "JPEG") + filepath = tmp_file.name + image_paths.append(filepath) + else: + raise ValueError(f"Invalid image path: {image}") + multimodal_texts.append(text) + multimodal_image_paths.append(image_paths) + else: + # text input + prompt = dataset_config.get_prompt( + req) + ' ' + dataset_config.get_input(req) + logging.debug(f"Input sequence: {prompt}") + line = root_args.tokenizer.encode(prompt) + if kwargs['max_input_len'] and len(line) > kwargs['max_input_len']: + continue + input_ids.append(line) + input_lens.append(len(line)) + + # output if fetch from golden + if kwargs['output_len_dist'] is None: + output_lens.append( + len( + root_args.tokenizer.encode( + dataset_config.get_output(req)))) + + # lora task id + task_id = root_args.task_id + if root_args.rand_task_id is not None: + min_id, max_id = root_args.rand_task_id + task_id = random.randint(min_id, max_id) + task_ids.append(task_id) + + req_cnt += 1 + if kwargs['num_requests'] and req_cnt >= kwargs['num_requests']: + break + + if kwargs['num_requests'] and (len(input_ids) if modality is None else len( + multimodal_texts)) < kwargs['num_requests']: + logging.warning( + f"Number of requests={len(input_ids) if modality is None else len(multimodal_texts)} is" + f" smaller than the num-requests user set={kwargs['num_requests']}." + ) + + # output if randomized + if kwargs['output_len_dist'] is not None: + osl_mean, osl_stdev = kwargs['output_len_dist'] + output_lens = get_norm_dist_lengths( + osl_mean, osl_stdev, + len(input_ids) if modality is None else len(multimodal_texts), + root_args.random_seed) + logging.debug(f"Input lengths: {[len(i) for i in input_ids]}") + logging.debug(f"Output lengths: {output_lens}") + if modality is not None: + logging.debug(f"Modality: {modality}") + + if modality is not None: + if not root_args.std_out: + multimodal_dataset_dump( + multimodal_texts, multimodal_image_paths, output_lens, task_ids, + { + "workload_type": "dataset", + "tokenizer": root_args.tokenizer.__class__.__name__, + "num_requests": len(task_ids), + "max_output_len": max(output_lens) + }, root_args.output) + else: + print_multimodal_dataset( + multimodal_texts, + multimodal_image_paths, + output_lens, + ) + else: + if not root_args.std_out: + text_dataset_dump( + input_lens, input_ids, output_lens, task_ids, { + "workload_type": "dataset", + "tokenizer": root_args.tokenizer.__class__.__name__, + "num_requests": len(input_ids), + "max_input_len": max(input_lens), + "max_output_len": max(output_lens) + }, root_args.output) + else: + print_text_dataset( + input_ids, + output_lens, + ) diff --git a/benchmarks/cpp/utils/prepare_synthetic_data.py b/benchmarks/cpp/utils/prepare_synthetic_data.py new file mode 100644 index 000000000000..b072b712d085 --- /dev/null +++ b/benchmarks/cpp/utils/prepare_synthetic_data.py @@ -0,0 +1,164 @@ +import random +import warnings + +import click +from utils.utils import (gen_random_tokens, get_norm_dist_lengths, + get_unif_dist_lengths, print_text_dataset, + text_dataset_dump) + + +def _generate_task_ids_and_lora_config(root_args, num_reqs): + """Generate task IDs and determine LoRA configuration based on root_args.""" + if root_args.rand_task_id is None: + task_ids = [root_args.task_id for _ in range(num_reqs)] + else: + min_id, max_id = root_args.rand_task_id + task_ids = [random.randint(min_id, max_id) for _ in range(num_reqs)] + + use_task_ids = root_args.task_id != -1 or root_args.rand_task_id is not None + + # Determine if LoRA should be used (requires both task IDs and lora_dir) + use_lora = use_task_ids and root_args.lora_dir is not None + + # Warn if task IDs are specified but no LoRA directory is provided + if use_task_ids and not use_lora: + warnings.warn( + "Task IDs require LoRA directory. Use --lora-dir or omit task IDs.", + UserWarning) + + return (task_ids, task_ids if use_task_ids else None, { + "lora_dir": root_args.lora_dir + } if use_lora else None) + + +@click.command() +@click.option("--num-requests", + required=True, + type=int, + help='Number of requests to be generated') +@click.option('--input-mean', + required=True, + type=int, + help='normal dist mean for input tokens') +@click.option('--input-stdev', + required=True, + type=int, + help='normal dist stdev for input tokens') +@click.option('--output-mean', + required=True, + type=int, + help='normal dist mean for output tokens') +@click.option('--output-stdev', + required=True, + type=int, + help='normal dist stdev for output tokens') +@click.pass_obj +def token_norm_dist(root_args, **kwargs): + """Prepare synthetic dataset by generating random tokens with normal dist lengths.""" + input_ids = [] + input_lens = [] + output_lens = [] + + input_lens = get_norm_dist_lengths(kwargs['input_mean'], + kwargs['input_stdev'], + kwargs['num_requests'], + root_args.random_seed) + + num_reqs = len(input_lens) + output_lens = get_norm_dist_lengths(kwargs['output_mean'], + kwargs['output_stdev'], num_reqs, + root_args.random_seed) + + max_input_len = max(input_lens) + max_output_len = max(output_lens) + + input_ids = gen_random_tokens(input_lens, root_args.tokenizer, + root_args.random_seed) + + task_ids, print_task_ids, lora_config = _generate_task_ids_and_lora_config( + root_args, num_reqs) + + if not root_args.std_out: + text_dataset_dump( + input_lens, input_ids, output_lens, task_ids, { + "workload_type": "token-norm-dist", + "input_mean": kwargs['input_mean'], + "input_stdev": kwargs['input_stdev'], + "output_mean": kwargs['output_mean'], + "output_stdev": kwargs['output_stdev'], + "num_requests": kwargs['num_requests'], + "tokenize_vocabsize": root_args.tokenizer.vocab_size, + "max_input_len": max_input_len, + "max_output_len": max_output_len + }, root_args.output) + else: + print_text_dataset(input_ids, + output_lens, + task_ids=print_task_ids, + lora_config=lora_config) + + +@click.command() +@click.option("--num-requests", + required=True, + type=int, + help='Number of requests to be generated') +@click.option('--input-min', + required=True, + type=int, + help='uniform dist (inclusive) min for input tokens') +@click.option('--input-max', + required=True, + type=int, + help='normal dist (inclusive) max for input tokens') +@click.option('--output-min', + required=True, + type=int, + help='normal dist (inclusive) min for output tokens') +@click.option('--output-max', + required=True, + type=int, + help='normal dist (inclusive) max for output tokens') +@click.pass_obj +def token_unif_dist(root_args, **kwargs): + """Prepare synthetic dataset by generating random tokens with normal uniformly lengths.""" + input_ids = [] + input_lens = [] + output_lens = [] + + input_lens = get_unif_dist_lengths(kwargs['input_min'], kwargs['input_max'], + kwargs['num_requests'], + root_args.random_seed) + + num_reqs = len(input_lens) + output_lens = get_unif_dist_lengths(kwargs['output_min'], + kwargs['output_max'], num_reqs, + root_args.random_seed) + + max_input_len = max(input_lens) + max_output_len = max(output_lens) + + input_ids = gen_random_tokens(input_lens, root_args.tokenizer, + root_args.random_seed) + + task_ids, print_task_ids, lora_config = _generate_task_ids_and_lora_config( + root_args, num_reqs) + + if not root_args.std_out: + text_dataset_dump( + input_lens, input_ids, output_lens, task_ids, { + "workload_type": "token-unif-dist", + "input_min": kwargs['input_min'], + "input_max": kwargs['input_max'], + "output_min": kwargs['output_min'], + "output_max": kwargs['output_max'], + "num_requests": kwargs['num_requests'], + "tokenize_vocabsize": root_args.tokenizer.vocab_size, + "max_input_len": max_input_len, + "max_output_len": max_output_len + }, root_args.output) + else: + print_text_dataset(input_ids, + output_lens, + task_ids=print_task_ids, + lora_config=lora_config) diff --git a/benchmarks/cpp/utils/utils.cpp b/benchmarks/cpp/utils/utils.cpp new file mode 100644 index 000000000000..0cbcf1c0468d --- /dev/null +++ b/benchmarks/cpp/utils/utils.cpp @@ -0,0 +1,170 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & + *AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils.h" +#include "tensorrt_llm/common/config.h" +#include "tensorrt_llm/common/logger.h" +#include <random> + +#include <filesystem> +#include <fstream> + +TRTLLM_NAMESPACE_BEGIN + +namespace benchmark +{ + +std::vector<std::vector<SizeType32>> parseVectorOfVectors(std::string const& input) +{ + std::vector<std::vector<SizeType32>> result; + std::regex outer_regex(R"(\[(.*?)\])"); + std::regex inner_regex(R"(\d+)"); + auto outer_begin = std::sregex_iterator(input.begin(), input.end(), outer_regex); + auto outer_end = std::sregex_iterator(); + + for (std::sregex_iterator i = outer_begin; i != outer_end; ++i) + { + std::smatch match = *i; + std::string inner_str = match.str(1); + std::vector<int> inner_vec; + auto inner_begin = std::sregex_iterator(inner_str.begin(), inner_str.end(), inner_regex); + auto inner_end = std::sregex_iterator(); + + for (std::sregex_iterator j = inner_begin; j != inner_end; ++j) + { + std::smatch inner_match = *j; + inner_vec.push_back(std::stoi(inner_match.str())); + } + result.push_back(inner_vec); + } + return result; +} + +texec::LookaheadDecodingConfig parseLookaheadConfig(std::string const& input) +{ + std::regex regex("\\[ *(\\d+) *, *(\\d+) *, *(\\d+) *\\]"); + std::smatch match; + if (std::regex_match(input, match, regex)) + { + TLLM_CHECK(match.size() == 4); + auto w = std::stoi(match[1]); + auto n = std::stoi(match[2]); + auto g = std::stoi(match[3]); + return texec::LookaheadDecodingConfig(w, n, g); + } + else + { + TLLM_LOG_WARNING("cannot parse lookahead config from '%s'", input.c_str()); + return texec::LookaheadDecodingConfig(); + } +} + +Samples parseWorkloadJson( + std::filesystem::path const& datasetPath, int maxNumSamples, std::optional<SizeType32> const maxPromptLen) +{ + auto constexpr allowExceptions = true; + auto constexpr ignoreComments = true; + TLLM_CHECK_WITH_INFO(std::filesystem::exists(datasetPath), "File does not exist: %s", datasetPath.c_str()); + std::ifstream jsonStream(datasetPath); + auto json = nlohmann::json::parse(jsonStream, nullptr, allowExceptions, ignoreComments); + + Samples samples; + + for (auto const& sample : json["samples"]) + { + if (samples.size() >= maxNumSamples) + break; + int32_t taskId = sample.count("task_id") ? sample["task_id"].template get<int32_t>() : -1; + auto input_ids(sample["input_ids"].template get<std::vector<int32_t>>()); + if (maxPromptLen && (input_ids.size() > maxPromptLen.value())) + { + input_ids.resize(maxPromptLen.value()); + } + samples.emplace_back(Sample{std::move(input_ids), sample["output_len"], taskId}); + } + + if (samples.size() < maxNumSamples) + { + TLLM_LOG_WARNING( + "Dataset size %zu is smaller than given max_num_samples " + "%d, max_num_samples will be ignored.\n", + samples.size(), maxNumSamples); + } + return samples; +} + +std::vector<double> generateRandomExponentialValues(int count, float lambda, int seed) +{ + // Set a constant seed for reproducibility + std::mt19937 gen(seed); + + // Create an exponential distribution object + std::exponential_distribution<double> distribution(lambda); + + // Generate random numbers from the exponential distribution + std::vector<double> randomValues; + for (int i = 0; i < count; ++i) + { + double randomValue = distribution(gen); + randomValues.push_back(randomValue); + } + + return randomValues; +} + +std::vector<double> computeTimeDelays(BenchmarkParams const& benchmarkParams, int numDelays) +{ + std::vector<double> timeDelays; + if (benchmarkParams.requestRate.has_value() && benchmarkParams.requestRate.value() > 0.0) + { + if (benchmarkParams.enableExpDelays) + { + timeDelays = generateRandomExponentialValues( + numDelays, benchmarkParams.requestRate.value(), benchmarkParams.randomSeed); + } + else + { + timeDelays.assign(numDelays, 1.0 / benchmarkParams.requestRate.value()); + } + } + else + { + timeDelays.assign(numDelays, 0.0); + } + + return timeDelays; +} + +std::ostream& operator<<(std::ostream& os, RecordTimeMetric const& metric) +{ + os << metric.mAvg << "," << metric.mMax << "," << metric.mMin << "," << metric.mP99 << "," << metric.mP90 << "," + << metric.mP50; + return os; +} + +std::ostream& operator<<(std::ostream& os, RecordBwMetric const& metric) +{ + os << metric.mAvg << "," << metric.mMax << "," << metric.mMin << "," << metric.mP99 << "," << metric.mP90 << "," + << metric.mP50; + return os; +} + +} // namespace benchmark + +TRTLLM_NAMESPACE_END diff --git a/benchmarks/cpp/utils/utils.h b/benchmarks/cpp/utils/utils.h new file mode 100644 index 000000000000..fba30fee69ae --- /dev/null +++ b/benchmarks/cpp/utils/utils.h @@ -0,0 +1,244 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/common/config.h" +#include "tensorrt_llm/executor/executor.h" + +#include <cstdint> +#include <cxxopts.hpp> +#include <iostream> +#include <nlohmann/json.hpp> +#include <numeric> +#include <optional> +#include <string> +#include <utility> + +#pragma once + +TRTLLM_NAMESPACE_BEGIN + +namespace benchmark +{ + +// using namespace tensorrt_llm::batch_manager; +using namespace tensorrt_llm::runtime; + +namespace texec = tensorrt_llm::executor; + +std::vector<std::vector<SizeType32>> parseVectorOfVectors(std::string const& input); + +texec::LookaheadDecodingConfig parseLookaheadConfig(std::string const& input); + +struct BenchmarkParams +{ + std::optional<SizeType32> maxTokensInPagedKvCache{std::nullopt}; + std::optional<float> freeGpuMemoryFraction{std::nullopt}; + std::vector<std::optional<float>> freeGpuMemoryFractions{std::nullopt}; + + std::optional<float> crossKvCacheFraction{std::nullopt}; + bool enableTrtOverlap{false}; + bool enableBatchSizeTuning{false}; + bool enableMaxNumTokensTuning{false}; + bool enableBlockReuse{false}; + bool enableChunkedContext{true}; + bool streaming{false}; + bool enableExpDelays{false}; + std::vector<std::optional<bool>> enableChunekedContextVec{std::nullopt}; + std::optional<float> requestRate{std::nullopt}; + std::optional<int> concurrency{std::nullopt}; + std::optional<SizeType32> maxBatchSize{std::nullopt}; + std::vector<std::optional<SizeType32>> maxBatchSizes{std::nullopt}; + std::optional<SizeType32> maxNumTokens{std::nullopt}; + std::vector<std::optional<SizeType32>> maxNumTokensVec{std::nullopt}; + int randomSeed = 430; + std::optional<std::vector<int>> maxAttentionWindowVec{std::nullopt}; + std::optional<int> sinkTokenLength{std::nullopt}; + bool multiBlockMode{true}; + bool enableContextFMHAFP32Acc{false}; + bool cudaGraphMode{false}; + SizeType32 cudaGraphCacheSize{0}; + + // lora / peft params + std::optional<std::string> loraDir{std::nullopt}; + SizeType32 loraDeviceNumModLayers{0}; + size_t loraHostCacheSize{1024 * 2024 * 1024}; + + // KV cache block offloading + size_t kvHostCacheSize{0}; + + // Weights offloading + float gpuWeightsPercent{1.0}; + + // Decoding params + std::optional<std::vector<std::vector<SizeType32>>> medusaChoices; + + std::optional<texec::EagleConfig> eagleConfig; + std::optional<float> temperature; + + std::optional<texec::LookaheadDecodingConfig> executorLookaheadConfig; + std::optional<texec::LookaheadDecodingConfig> requestLookaheadConfig; + + bool enableCollectkvCacheTransferTime = false; + bool enableCollectIterStats = false; +}; + +struct RecordTimeMetric +{ + + RecordTimeMetric(std::string tag) + : mTag(std::move(tag)) + { + } + + std::string mTag; + + std::vector<float> mDataTimes; + + float mAvg; + float mP99; + float mP95; + float mP90; + float mP50; + float mMax; + float mMin; + + static float calcPercentile(std::vector<float> const& latencies, int percentile) + { + int const index = static_cast<int>(std::ceil((percentile / 100.0) * latencies.size())) - 1; + return latencies[index]; + } + + void calculate() + { + TLLM_CHECK_WITH_INFO(mDataTimes.size() > 0, "No data to calculate for tag:%s", mTag.c_str()); + mAvg = std::accumulate(mDataTimes.begin(), mDataTimes.end(), 0.F) / mDataTimes.size(); + + std::sort(mDataTimes.begin(), mDataTimes.end()); + + mP99 = calcPercentile(mDataTimes, 99); + mP90 = calcPercentile(mDataTimes, 90); + mP50 = calcPercentile(mDataTimes, 50); + mMax = mDataTimes.back(); + mMin = mDataTimes.front(); + } + + void report() const + { + + printf("[BENCHMARK] avg_%s(ms) %.2f\n", mTag.c_str(), mAvg); + printf("[BENCHMARK] max_%s(ms) %.2f\n", mTag.c_str(), mMax); + printf("[BENCHMARK] min_%s(ms) %.2f\n", mTag.c_str(), mMin); + + printf("[BENCHMARK] p99_%s(ms) %.2f\n", mTag.c_str(), mP99); + + printf("[BENCHMARK] p90_%s(ms) %.2f\n", mTag.c_str(), mP90); + + printf("[BENCHMARK] p50_%s(ms) %.2f\n\n", mTag.c_str(), mP50); + } + + std::vector<std::string> genHeaders() const + { + std::string timeTag = mTag + "(ms)"; + return { + "avg_" + timeTag, "max_" + timeTag, "min_" + timeTag, "p99" + timeTag, "p90" + timeTag, "p50" + timeTag}; + } +}; + +struct RecordBwMetric +{ + + RecordBwMetric(std::string tag) + : mTag(std::move(tag)) + { + } + + std::string mTag; + + std::vector<float> mDataTps; + + float mAvg; + float mP99; + float mP95; + float mP90; + float mP50; + float mMax; + float mMin; + + static float calcPercentile(std::vector<float> const& throughputs, int percentile) + { + int const index = static_cast<int>(std::ceil((percentile / 100.0) * throughputs.size())) - 1; + return throughputs[index]; + } + + void calculate() + { + TLLM_CHECK_WITH_INFO(mDataTps.size() > 0, "No data to calculate for tag:%s", mTag.c_str()); + mAvg = std::accumulate(mDataTps.begin(), mDataTps.end(), 0.F) / mDataTps.size(); + + std::sort(mDataTps.begin(), mDataTps.end(), std::greater<float>()); + + mP99 = calcPercentile(mDataTps, 99); + mP90 = calcPercentile(mDataTps, 90); + mP50 = calcPercentile(mDataTps, 50); + mMax = mDataTps.front(); + mMin = mDataTps.back(); + } + + void report() const + { + + printf("[BENCHMARK] avg_%s(Gb/sec) %.8f\n", mTag.c_str(), mAvg); + printf("[BENCHMARK] max_%s(Gb/sec) %.8f\n", mTag.c_str(), mMax); + printf("[BENCHMARK] min_%s(Gb/sec) %.8f\n", mTag.c_str(), mMin); + + printf("[BENCHMARK] p99_%s(Gb/sec) %.8f\n", mTag.c_str(), mP99); + + printf("[BENCHMARK] p90_%s(Gb/sec) %.8f\n", mTag.c_str(), mP90); + + printf("[BENCHMARK] p50_%s(Gb/sec) %.8f\n\n", mTag.c_str(), mP50); + } + + std::vector<std::string> genHeaders() const + { + std::string tpTag = mTag + "(Gb/sec)"; + return {"avg_" + tpTag, "max_" + tpTag, "min_" + tpTag, "p99" + tpTag, "p90" + tpTag, "p50" + tpTag}; + } +}; + +std::ostream& operator<<(std::ostream& os, RecordTimeMetric const& metric); +std::ostream& operator<<(std::ostream& os, RecordBwMetric const& metric); + +struct Sample +{ + std::vector<int32_t> inputIds; + int32_t outputLen; + int32_t taskId; +}; + +using Samples = std::vector<Sample>; + +Samples parseWorkloadJson( + std::filesystem::path const& datasetPath, int maxNumSamples, std::optional<SizeType32> const maxPromptLen); + +std::vector<double> generateRandomExponentialValues(int count, float lambda, int seed); + +std::vector<double> computeTimeDelays(BenchmarkParams const& benchmarkParams, int numDelays); + +} // namespace benchmark + +TRTLLM_NAMESPACE_END diff --git a/benchmarks/cpp/utils/utils.py b/benchmarks/cpp/utils/utils.py new file mode 100644 index 000000000000..c395cf6c9449 --- /dev/null +++ b/benchmarks/cpp/utils/utils.py @@ -0,0 +1,168 @@ +import json +import math +import os +import random +from typing import List, Union + +import numpy as np +from pydantic import BaseModel + + +class TextSample(BaseModel): + input_len: int + input_ids: List[int] + output_len: int + task_id: int + + +class MultimodalSample(BaseModel): + task_id: int + prompt: str + media_paths: List[str] + output_len: int + + +class Workload(BaseModel): + metadata: dict + samples: List[Union[TextSample, MultimodalSample]] = [] + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.setup_workload_name() + + def setup_workload_name(self): + # Keys to ignore + ignore_keys = ['tokenizer'] + # Create a string by concatenating keys and values with "__" + workload_name = '__'.join(f'{key}:{value}' + for key, value in self.metadata.items() + if key not in ignore_keys) + self.metadata.setdefault('workload_name', workload_name) + + +def text_dataset_dump(input_lens, input_ids, output_lens, task_ids, metadata, + output_file): + samples = [] + for i in range(len(input_ids)): + samples.append( + TextSample(input_len=input_lens[i], + input_ids=input_ids[i], + output_len=output_lens[i], + task_id=task_ids[i])) + workload = Workload(metadata=metadata, samples=samples) + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, 'w') as f: + json.dump(workload.model_dump(), f) + + +def multimodal_dataset_dump(multimodal_texts, multimodal_image_paths, + output_lens, task_ids, metadata, output_file): + samples = [] + for i in range(len(multimodal_texts)): + samples.append( + MultimodalSample(task_id=task_ids[i], + prompt=multimodal_texts[i], + media_paths=multimodal_image_paths[i], + output_len=output_lens[i])) + workload = Workload(metadata=metadata, samples=samples) + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, 'w') as f: + json.dump(workload.model_dump(), f) + + +def print_text_dataset(input_ids, output_lens, task_ids=None, lora_config=None): + for i, input_tokens in enumerate(input_ids): + d = { + "task_id": i, + "input_ids": input_tokens, + "output_tokens": output_lens[i] + } + + # Add LoRA request if task_ids indicate LoRA usage + if task_ids is not None and lora_config is not None: + task_id = task_ids[i] + if task_id != -1: # -1 means no LoRA + d["lora_request"] = { + "lora_name": + f"lora_{task_id}", + "lora_int_id": + task_id, + "lora_path": + os.path.join(lora_config.get("lora_dir", "loras"), + str(task_id)) + } + + print(json.dumps(d, separators=(',', ':'), ensure_ascii=False)) + + +def print_multimodal_dataset(multimodal_texts, multimodal_image_paths, + output_lens): + for i, (text, image_paths) in enumerate( + zip(multimodal_texts, multimodal_image_paths)): + d = { + "task_id": i, + "prompt": text, + "media_paths": image_paths, + "output_tokens": output_lens[i] + } + print(json.dumps(d, separators=(',', ':'), ensure_ascii=False)) + + +def get_list_of_delays(delay_dist, mean_time_bet_reqs, num_reqs, random_seed): + if delay_dist == "constant": + delays = [mean_time_bet_reqs] * num_reqs + elif delay_dist == "exponential_dist": + delays = get_exponential_dist_delays(mean_time_bet_reqs, num_reqs, + random_seed) + + return delays + + +def get_exponential_dist_delays(mean_time_bet_reqs, num_reqs, random_seed): + # set seed for determinism + np.random.seed(random_seed) + return np.random.exponential(mean_time_bet_reqs, num_reqs).tolist() + + +def get_norm_dist_lengths(mean, stdev, num_reqs, random_seed): + # set seed for determinism + np.random.seed(random_seed) + numbers_list = np.random.normal(loc=mean, scale=stdev, + size=num_reqs).tolist() + return [max(1, math.ceil(x)) for x in numbers_list] + + +def get_unif_dist_lengths(min_len, max_len, num_reqs, random_seed): + # set seed for determinism + rng = np.random.default_rng(random_seed) + numbers = rng.integers(low=min_len, high=max_len + 1, size=num_reqs) + return numbers.tolist() + + +def gen_random_tokens(ip_lens, tokenizer, random_seed): + + def get_sample_from_population(population_range, sample_size): + # random.sample can not sample a value more than once. hence the check + if sample_size < len(population_range): + sample = random.sample(population_range, sample_size) + else: + sample = random.choices(population_range, k=sample_size) + + return sample + + input_ids = [] + random.seed(random_seed) + for ip_len in ip_lens: + start_ids = get_sample_from_population(range(0, tokenizer.vocab_size), + ip_len) + # Make sure it does not contain EOS token + eos_id = tokenizer.encode(tokenizer.eos_token, add_special_tokens=False) + while set(eos_id).issubset(start_ids): + tmp_id = (eos_id[0] + 1) % tokenizer.vocab_size + start_ids = [ + tmp_id if element == eos_id[0] else element + for element in start_ids + ] + input_ids.append(start_ids) + + return input_ids diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5dec92599ae6..a323b32b82b5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -45,6 +45,7 @@ add_compile_definitions(TRTLLM_ABI_NAMESPACE=${TRTLLM_ABI_NAMESPACE}) # Build options option(BUILD_PYT "Build in PyTorch TorchScript class mode" ON) option(BUILD_TESTS "Build Google tests" ON) +option(BUILD_BENCHMARKS "Build benchmarks" ON) option(BUILD_DEEP_EP "Build the Deep EP module" ON) option(BUILD_DEEP_GEMM "Build the DeepGEMM module" ON) option(BUILD_FLASH_MLA "Build the FlashMLA module" ON) @@ -125,6 +126,12 @@ else() message(STATUS "Not building Google tests") endif() +if(BUILD_BENCHMARKS) + message(STATUS "Building benchmarks") +else() + message(STATUS "Not building benchmarks") +endif() + if(BUILD_MICRO_BENCHMARKS) message(STATUS "Building C++ micro benchmarks") else() @@ -242,6 +249,8 @@ if(ENABLE_MULTI_DEVICE) endif() # TRT dependencies +find_package(TensorRT 10 REQUIRED COMPONENTS OnnxParser) +set(TRT_LIB TensorRT::NvInfer) get_filename_component(TRT_LLM_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR} PATH) @@ -282,6 +291,7 @@ include_directories( ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_INCLUDE_DIRS}/cccl ${CUDNN_ROOT_DIR}/include + $<TARGET_PROPERTY:TensorRT::NvInfer,INTERFACE_INCLUDE_DIRECTORIES> ${maybe_nvtx_includedir} ${CMAKE_BINARY_DIR}/_deps/cutlass-src/include ${CMAKE_BINARY_DIR}/_deps/cutlass-src/tools/util/include @@ -659,6 +669,11 @@ if(BUILD_TESTS) add_subdirectory(tests) endif() +if(BUILD_BENCHMARKS) + add_subdirectory(${TRT_LLM_ROOT_DIR}/benchmarks/cpp + ${CMAKE_BINARY_DIR}/benchmarks) +endif() + if(BUILD_MICRO_BENCHMARKS) add_subdirectory(${TRT_LLM_ROOT_DIR}/cpp/micro_benchmarks ${CMAKE_BINARY_DIR}/micro_benchmarks) @@ -673,6 +688,6 @@ if(MEASURE_BUILD_TIME) endif() set(BUILD_WHEEL_TARGETS - tensorrt_llm + tensorrt_llm;nvinfer_plugin_tensorrt_llm CACHE STRING "Targets used to build wheel") add_custom_target(build_wheel_targets DEPENDS ${BUILD_WHEEL_TARGETS}) diff --git a/cpp/cmake/modules/FindTensorRT.cmake b/cpp/cmake/modules/FindTensorRT.cmake new file mode 100644 index 000000000000..9e7e35b51bae --- /dev/null +++ b/cpp/cmake/modules/FindTensorRT.cmake @@ -0,0 +1,190 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# TensorRT install path in docker image +set(TensorRT_WELL_KNOWN_ROOT /usr/local/tensorrt) + +find_path( + TensorRT_INCLUDE_DIR + NAMES NvInfer.h + PATHS ${TensorRT_WELL_KNOWN_ROOT}/include) + +function(_tensorrt_get_version) + unset(TensorRT_VERSION_STRING PARENT_SCOPE) + set(_hdr_file "${TensorRT_INCLUDE_DIR}/NvInferVersion.h") + + if(NOT EXISTS "${_hdr_file}") + return() + endif() + + file(STRINGS "${_hdr_file}" IS_10_11_NEW_MACRO REGEX "TRT_MAJOR_ENTERPRISE") + if(IS_10_11_NEW_MACRO) + file(STRINGS "${_hdr_file}" VERSION_STRINGS + REGEX "#define TRT_.+_ENTERPRISE.*") + foreach(TYPE MAJOR MINOR PATCH BUILD) + string(REGEX MATCH "TRT_${TYPE}_ENTERPRISE [0-9]+" TRT_TYPE_STRING + ${VERSION_STRINGS}) + string(REGEX MATCH "[0-9]+" TensorRT_VERSION_${TYPE} ${TRT_TYPE_STRING}) + endforeach(TYPE) + else() + file(STRINGS "${_hdr_file}" VERSION_STRINGS REGEX "#define NV_TENSORRT_.*") + foreach(TYPE MAJOR MINOR PATCH BUILD) + string(REGEX MATCH "NV_TENSORRT_${TYPE} [0-9]+" TRT_TYPE_STRING + ${VERSION_STRINGS}) + string(REGEX MATCH "[0-9]+" TensorRT_VERSION_${TYPE} ${TRT_TYPE_STRING}) + endforeach(TYPE) + endif() + + set(TensorRT_VERSION_MAJOR + ${TensorRT_VERSION_MAJOR} + PARENT_SCOPE) + set(TensorRT_VERSION_STRING + "${TensorRT_VERSION_MAJOR}.${TensorRT_VERSION_MINOR}.${TensorRT_VERSION_PATCH}.${TensorRT_VERSION_BUILD}" + PARENT_SCOPE) +endfunction(_tensorrt_get_version) + +_tensorrt_get_version() + +macro(_tensorrt_find_dll VAR) + find_file( + ${VAR} + NAMES ${ARGN} + HINTS ${TensorRT_ROOT} + PATH_SUFFIXES bin) +endmacro(_tensorrt_find_dll) + +find_library( + TensorRT_LIBRARY + NAMES "nvinfer_${TensorRT_VERSION_MAJOR}" nvinfer + PATHS ${TensorRT_WELL_KNOWN_ROOT}/lib) + +if(WIN32) + _tensorrt_find_dll(TensorRT_DLL "nvinfer_${TensorRT_VERSION_MAJOR}.dll" + nvinfer.dll) +endif() + +if(TensorRT_LIBRARY) + set(TensorRT_LIBRARIES ${TensorRT_LIBRARIES} ${TensorRT_LIBRARY}) +endif(TensorRT_LIBRARY) + +if(TensorRT_FIND_COMPONENTS) + list(REMOVE_ITEM TensorRT_FIND_COMPONENTS "nvinfer") + + if("OnnxParser" IN_LIST TensorRT_FIND_COMPONENTS) + find_path( + TensorRT_OnnxParser_INCLUDE_DIR + NAMES NvOnnxParser.h + PATHS ${TensorRT_WELL_KNOWN_ROOT}/include) + + find_library( + TensorRT_OnnxParser_LIBRARY + NAMES "nvonnxparser_${TensorRT_VERSION_MAJOR}" nvonnxparser + PATHS ${TensorRT_WELL_KNOWN_ROOT}/lib) + if(TensorRT_OnnxParser_LIBRARY AND TensorRT_LIBRARIES) + set(TensorRT_LIBRARIES ${TensorRT_LIBRARIES} + ${TensorRT_OnnxParser_LIBRARY}) + set(TensorRT_OnnxParser_FOUND TRUE) + endif() + + if(WIN32) + _tensorrt_find_dll( + TensorRT_OnnxParser_DLL "nvonnxparser_${TensorRT_VERSION_MAJOR}.dll" + nvonnxparser.dll) + endif() + endif() + + if("Plugin" IN_LIST TensorRT_FIND_COMPONENTS) + find_path( + TensorRT_Plugin_INCLUDE_DIR + NAMES NvInferPlugin.h + PATHS ${TensorRT_WELL_KNOWN_ROOT}/include) + + find_library( + TensorRT_Plugin_LIBRARY + NAMES "nvinfer_plugin_${TensorRT_VERSION_MAJOR}" nvinfer_plugin + PATHS ${TensorRT_WELL_KNOWN_ROOT}/lib) + + if(TensorRT_Plugin_LIBRARY AND TensorRT_LIBRARIES) + set(TensorRT_LIBRARIES ${TensorRT_LIBRARIES} ${TensorRT_Plugin_LIBRARY}) + set(TensorRT_Plugin_FOUND TRUE) + endif() + + if(WIN32) + _tensorrt_find_dll( + TensorRT_Plugin_DLL "nvinfer_plugin_${TensorRT_VERSION_MAJOR}.dll" + nvinfer_plugin.dll) + endif() + endif() +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + TensorRT + FOUND_VAR TensorRT_FOUND + REQUIRED_VARS TensorRT_LIBRARY TensorRT_LIBRARIES TensorRT_INCLUDE_DIR + VERSION_VAR TensorRT_VERSION_STRING + HANDLE_COMPONENTS) + +if(NOT TARGET TensorRT::NvInfer) + add_library(TensorRT::NvInfer SHARED IMPORTED) + target_include_directories(TensorRT::NvInfer SYSTEM + INTERFACE "${TensorRT_INCLUDE_DIR}") + if(WIN32) + set_property(TARGET TensorRT::NvInfer PROPERTY IMPORTED_LOCATION + "${TensorRT_DLL}") + set_property(TARGET TensorRT::NvInfer PROPERTY IMPORTED_IMPLIB + "${TensorRT_LIBRARY}") + else() + set_property(TARGET TensorRT::NvInfer PROPERTY IMPORTED_LOCATION + "${TensorRT_LIBRARY}") + endif() +endif() + +if(NOT TARGET TensorRT::OnnxParser AND "OnnxParser" IN_LIST + TensorRT_FIND_COMPONENTS) + add_library(TensorRT::OnnxParser SHARED IMPORTED) + target_include_directories(TensorRT::OnnxParser SYSTEM + INTERFACE "${TensorRT_OnnxParser_INCLUDE_DIR}") + target_link_libraries(TensorRT::OnnxParser INTERFACE TensorRT::NvInfer) + if(WIN32) + set_property(TARGET TensorRT::OnnxParser + PROPERTY IMPORTED_LOCATION "${TensorRT_OnnxParser_DLL}") + set_property(TARGET TensorRT::OnnxParser + PROPERTY IMPORTED_IMPLIB "${TensorRT_OnnxParser_LIBRARY}") + else() + set_property(TARGET TensorRT::OnnxParser + PROPERTY IMPORTED_LOCATION "${TensorRT_OnnxParser_LIBRARY}") + endif() +endif() + +if(NOT TARGET TensorRT::Plugin AND "Plugin" IN_LIST TensorRT_FIND_COMPONENTS) + add_library(TensorRT::Plugin SHARED IMPORTED) + target_include_directories(TensorRT::Plugin SYSTEM + INTERFACE "${TensorRT_Plugin_INCLUDE_DIR}") + target_link_libraries(TensorRT::Plugin INTERFACE TensorRT::NvInfer) + if(WIN32) + set_property(TARGET TensorRT::Plugin PROPERTY IMPORTED_LOCATION + "${TensorRT_Plugin_DLL}") + set_property(TARGET TensorRT::Plugin PROPERTY IMPORTED_IMPLIB + "${TensorRT_Plugin_LIBRARY}") + else() + set_property(TARGET TensorRT::Plugin PROPERTY IMPORTED_LOCATION + "${TensorRT_Plugin_LIBRARY}") + endif() +endif() + +mark_as_advanced(TensorRT_INCLUDE_DIR TensorRT_LIBRARY TensorRT_LIBRARIES) diff --git a/cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h b/cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h index 0c7724c657fc..288af6219fc9 100644 --- a/cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h +++ b/cpp/include/tensorrt_llm/batch_manager/cacheTransceiver.h @@ -22,19 +22,16 @@ #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/batch_manager/rnnCacheTransBuffer.h" #include "tensorrt_llm/batch_manager/rnnStateManager.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cacheCommunicator.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" #include <cstddef> -#include <fstream> #include <future> #include <memory> #include <mutex> #include <optional> #include <pybind11/pybind11.h> -#include <string> #include <torch/csrc/jit/python/pybind_utils.h> #include <torch/custom_class.h> #include <torch/python.h> @@ -250,17 +247,6 @@ class BaseCacheTransceiver [[nodiscard]] virtual bool checkGenTransferComplete() const = 0; virtual bool cancelRequest(std::shared_ptr<LlmRequest> llmRequest) = 0; - - /// Get the serialized DataTransceiverState (CacheState + CommState) for this transceiver. - [[nodiscard]] virtual std::vector<char> getSerializedDataTransceiverState() const - { - return {}; - } - - [[nodiscard]] virtual bool hasPoisonedTransferBuffer() const - { - return false; - } }; class CacheTransceiver : public BaseCacheTransceiver @@ -268,7 +254,7 @@ class CacheTransceiver : public BaseCacheTransceiver public: CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, executor::kv_cache::CacheState::ModelConfig const& cacheStateModelCfg, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, executor::kv_cache::CacheState::AttentionType attentionType = executor::kv_cache::CacheState::AttentionType::kDEFAULT, std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig = std::nullopt, @@ -276,7 +262,7 @@ class CacheTransceiver : public BaseCacheTransceiver CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, std::vector<SizeType32> numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, executor::kv_cache::CacheState::AttentionType attentionType = executor::kv_cache::CacheState::AttentionType::kDEFAULT, std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig = std::nullopt, @@ -306,19 +292,11 @@ class CacheTransceiver : public BaseCacheTransceiver virtual bool cancelRequest(std::shared_ptr<LlmRequest> llmRequest) override; - [[nodiscard]] std::vector<char> getSerializedDataTransceiverState() const override; - - [[nodiscard]] bool hasPoisonedTransferBuffer() const override; - private: void initializeCommState(); void setContextState(LlmRequest* llmRequest); - // Append one row per completed request to the gen-side transfer summary CSV. Opens the file - // lazily on first use; expects timing to already be synced across ranks by the caller. - void writeGenTransferSummary(std::vector<LlmRequest*> const& completedRequests); - std::unique_ptr<CacheSender> mCacheSender; std::unique_ptr<CacheReceiver> mCacheReceiver; // shared_ptr (not raw LlmRequest*) so the futures hold a strong reference for @@ -326,12 +304,9 @@ class CacheTransceiver : public BaseCacheTransceiver // request while a C++ status check still dereferences it. std::vector<std::pair<std::shared_ptr<LlmRequest>, std::future<void>>> mSenderFutures; std::vector<std::pair<std::shared_ptr<LlmRequest>, std::future<void>>> mRequesterFutures; - // Dedup timeout logs separately from accepted cancellation requests so a - // backend that initially declines cancellation is retried on later polls. + // Dedup sets so observe-only timeout WARN logs fire at most once per stuck request. std::unordered_set<LlmRequest::RequestIdType> mTimedOutSenderIds; std::unordered_set<LlmRequest::RequestIdType> mTimedOutRequesterIds; - std::unordered_set<LlmRequest::RequestIdType> mCancelRequestedSenderIds; - std::unordered_set<LlmRequest::RequestIdType> mCancelRequestedRequesterIds; std::unordered_set<LlmRequest::RequestIdType> mCompletedSenderRequestIds; std::unordered_set<LlmRequest::RequestIdType> mFailedSenderRequestIds; std::unordered_map<LlmRequest::RequestIdType, std::shared_ptr<LlmRequest>> mSenderRequestsAwaitingConsensus; @@ -354,13 +329,6 @@ class CacheTransceiver : public BaseCacheTransceiver // TODO(shreyasm): update this to use same container as kv by using base trans buffers instead std::unique_ptr<rnn_state_manager::RnnCacheTransBufferManager> mRnnCacheTransBufferManager{nullptr}; - // Unique instance identifier for CSV file naming (avoids collisions across gen instances) - std::string mInstanceId; - - // Gen-side transfer summary CSV (written after timing sync) - std::ofstream mGenTransferSummaryFile; - std::mutex mGenTransferSummaryMutex; - // library handle to the communicator related features, // this is used to defer dependency resolution until needed. static std::mutex mDllMutex; diff --git a/cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h b/cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h index 600927af9645..bc619a34bc03 100644 --- a/cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h +++ b/cpp/include/tensorrt_llm/batch_manager/createNewDecoderRequests.h @@ -20,7 +20,6 @@ #include "tensorrt_llm/batch_manager/common.h" #include "tensorrt_llm/common/algorithm.h" #include "tensorrt_llm/common/optionalRef.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -67,17 +66,16 @@ class CreateNewDecoderRequests : Algorithm std::vector<executor::LookaheadDecodingConfig>> operator()(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests, - tensorrt_llm::DataType logitsType, DecoderInputBuffers& inputBuffers, - runtime::decoder::DecoderState& decoderState, CudaStream const& runtimeStream, CudaStream const& decoderStream, - SizeType32 maxSequenceLength, SizeType32 beamWidth, OptionalRef<MedusaBuffers const> medusaBuffers) const; + nvinfer1::DataType logitsType, DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, + CudaStream const& runtimeStream, CudaStream const& decoderStream, SizeType32 maxSequenceLength, + SizeType32 beamWidth, OptionalRef<MedusaBuffers const> medusaBuffers) const; [[nodiscard]] std::tuple<std::vector<SharedConstPtr>, std::vector<executor::LookaheadDecodingConfig>> createDecoderRequests(RequestVector const& finishedContextRequests, TensorPtr const& inputIds, executor::DecodingConfig const& decodingConfig, runtime::decoder::DecoderState& decoderState, - tensorrt_llm::DataType logitsType, runtime::ModelConfig const& modelConfig, - runtime::WorldConfig const& worldConfig, runtime::CudaStream const& runtimeStream, - runtime::CudaStream const& decoderStream, SizeType32 maxSequenceLength, - OptionalRef<MedusaBuffers const> medusaBuffers) const; + nvinfer1::DataType logitsType, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + runtime::CudaStream const& runtimeStream, runtime::CudaStream const& decoderStream, + SizeType32 maxSequenceLength, OptionalRef<MedusaBuffers const> medusaBuffers) const; private: bool mSpeculativeDecodingFastLogits; diff --git a/cpp/include/tensorrt_llm/batch_manager/guidedDecoder.h b/cpp/include/tensorrt_llm/batch_manager/guidedDecoder.h new file mode 100644 index 000000000000..9a577b61ad51 --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/guidedDecoder.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/iTensor.h" + +namespace xgrammar +{ +class GrammarMatcher; +class GrammarCompiler; +} // namespace xgrammar + +namespace tensorrt_llm::batch_manager +{ +class DecoderInputBuffers; + +class GuidedDecoder +{ +public: + using TensorPtr = runtime::ITensor::SharedPtr; + using SizeType32 = tensorrt_llm::runtime::SizeType32; + using BitmaskT = uint32_t; + + GuidedDecoder(executor::GuidedDecodingConfig const& guidedDecodingConfig, SizeType32 maxNumSequences, + SizeType32 vocabSizePadded, nvinfer1::DataType logitsDtype, runtime::BufferManager const& runtimeBufferManager); + void build(ScheduledRequests const& scheduledRequests); + void execute(DecoderInputBuffers const& decoderInputBuffers, runtime::BufferManager const& runtimeBufferManager); + +private: + executor::GuidedDecodingConfig::GuidedDecodingBackend mGuidedDecodingBackend; + std::vector<std::shared_ptr<xgrammar::GrammarMatcher>> mXGrammarMatchers; + std::shared_ptr<xgrammar::GrammarCompiler> mXGrammarCompiler; + + SizeType32 mMaxNumSequences; + SizeType32 mVocabSizePadded; + SizeType32 mBitmaskSize; // CeilDiv(vocabSizePadded, 32) + nvinfer1::DataType mLogitsDtype; + + TensorPtr mLogitsBitmask; // [mMaxNumRequests, mBitmaskSize] + TensorPtr mLogitsBitmaskHost; // [mMaxNumRequests, mBitmaskSize] + TensorPtr mLogitsBitmaskPtrVec; // [mMaxNumRequests], pointers to the logitsBitmask in a batch + TensorPtr mLogitsBitmaskPtrVecHost; // [mMaxNumRequests] + TensorPtr mLogitsPtrVec; // [mMaxNumRequests], pointers to the logits in a batch + TensorPtr mLogitsPtrVecHost; // [mMaxNumRequests] + + // BufferManager with a dedicated stream for async copy of buffers for guided decoding. + runtime::BufferManager mCopyBufferManager; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/handleContextLogits.h b/cpp/include/tensorrt_llm/batch_manager/handleContextLogits.h new file mode 100644 index 000000000000..cb77545578c8 --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/handleContextLogits.h @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/common/algorithm.h" +#include "tensorrt_llm/common/optionalRef.h" +#include "tensorrt_llm/runtime/modelConfig.h" + +namespace tensorrt_llm::runtime +{ +class BufferManager; +class CudaStream; +} // namespace tensorrt_llm::runtime + +namespace tensorrt_llm::batch_manager +{ + +class DecoderInputBuffers; +class MedusaBuffers; + +class HandleContextLogits : Algorithm +{ +public: + template <typename T> + using OptionalRef = tensorrt_llm::common::OptionalRef<T>; + + constexpr static auto name{"HandleContextLogits"}; + + HandleContextLogits() = default; + + runtime::SizeType32 operator()(DecoderInputBuffers& inputBuffers, RequestVector const& contextRequests, + runtime::ITensor::SharedPtr const& logits, std::vector<runtime::SizeType32> const& numContextLogitsVec, + runtime::ModelConfig const& modelConfig, runtime::BufferManager const& manager, + OptionalRef<MedusaBuffers> medusaBuffers) const; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/handleGenerationLogits.h b/cpp/include/tensorrt_llm/batch_manager/handleGenerationLogits.h new file mode 100644 index 000000000000..f9fd58800a6f --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/handleGenerationLogits.h @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "common.h" +#include "tensorrt_llm/common/algorithm.h" +#include "tensorrt_llm/common/optionalRef.h" +#include "tensorrt_llm/runtime/modelConfig.h" + +namespace tensorrt_llm::runtime +{ +class BufferManager; +} // namespace tensorrt_llm::runtime + +namespace tensorrt_llm::batch_manager +{ + +class DecoderInputBuffers; +class RuntimeBuffers; +class MedusaBuffers; + +class HandleGenerationLogits : Algorithm +{ +public: + template <typename T> + using OptionalRef = tensorrt_llm::common::OptionalRef<T>; + + constexpr static auto name{"HandleGenerationLogits"}; + + HandleGenerationLogits() = default; + + void operator()(DecoderInputBuffers& inputBuffers, RequestVector const& generationRequests, + runtime::ITensor::SharedPtr const& logits, runtime::SizeType32 logitsIndex, + runtime::ModelConfig const& modelConfig, runtime::BufferManager const& manager, + OptionalRef<RuntimeBuffers> genRuntimeBuffers, OptionalRef<MedusaBuffers> medusaBuffers) const; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h b/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h index 04b6c230e75a..299bf1b570f4 100644 --- a/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h +++ b/cpp/include/tensorrt_llm/batch_manager/kvCacheManager.h @@ -23,7 +23,6 @@ #include "tensorrt_llm/batch_manager/llmRequest.h" // TODO forward declare #include "tensorrt_llm/batch_manager/radixBlockTree.h" #include "tensorrt_llm/common/optionalRef.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/executor/transferAgent.h" #include "tensorrt_llm/kernels/kvCacheIndex.h" @@ -33,6 +32,7 @@ #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/worldConfig.h" +#include <NvInferRuntime.h> #include <algorithm> #include <array> @@ -140,7 +140,7 @@ struct PoolConfiguration { SizeType32 windowSize; SizeType32 sizePerHead; - tensorrt_llm::DataType dtype; + nvinfer1::DataType dtype; }; struct LinearAttentionMetadata @@ -872,7 +872,7 @@ class WindowBlockManager using BlockMap = std::unordered_multimap<size_t, BlockPtr>; using BlockMapIterRange = std::pair<BlockMap::const_iterator, BlockMap::const_iterator>; - explicit WindowBlockManager(tensorrt_llm::DataType dtype, SizeType32 windowSize, + explicit WindowBlockManager(nvinfer1::DataType dtype, SizeType32 windowSize, std::vector<SizeType32> const& managedLayers, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, bool isSWA, SizeType32 blocksInPrimaryPool, SizeType32 blocksInSecondaryPool, SizeType32 maxNumSequences, std::shared_ptr<runtime::CudaStream> stream, @@ -1037,7 +1037,7 @@ class WindowBlockManager //! host pools with mixed precisions when constructed with a per-window //! dtype map. Empty pools or NVFP4-scale pools are routed through the //! per-pool tensor metadata instead. - [[nodiscard]] tensorrt_llm::DataType getDataType() const noexcept + [[nodiscard]] nvinfer1::DataType getDataType() const noexcept { return mDataType; } @@ -1127,7 +1127,7 @@ class WindowBlockManager [[nodiscard]] SizeType32 getNumEltsPerContainer() const { #ifdef ENABLE_FP4 - return mDataType == tensorrt_llm::DataType::kFP4 ? 2 : 1; + return mDataType == nvinfer1::DataType::kFP4 ? 2 : 1; #else return 1; #endif @@ -1192,7 +1192,7 @@ class WindowBlockManager return mLayerToIndexWithinPool.at(layerIdx); } - void setOffsets(kernels::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx, + void setOffsets(kernels::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx, SizeType32 blockIdx, KVCacheBlock::IdType blockId) const; //! \brief Bring offloaded block from secondary to primary memory. @@ -1251,29 +1251,14 @@ class WindowBlockManager return mEnablePartialReuse; } - //! \brief Look up the block chain matching blockKey in the reuse tree. [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey(BlockKey const& blockKey); - //! \brief Same lookup, additionally pinning matched blocks; on a miss all pins are - //! rolled back and pinnedBlockIds is cleared. - [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey( - BlockKey const& blockKey, std::vector<KVCacheBlock::IdType>& pinnedBlockIds); - [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeys( std::vector<BlockKey> const& blockKeys); //! \brief Unpin blocks by block ids directly void unpinBlocksById(std::vector<KVCacheBlock::IdType> const& blockIds); - //! \brief Pin a block: claim it from the eviction policy if free, then take a reference. - //! Safe to call from cache-transceiver threads: block bookkeeping is serialized by the - //! lookup-tree mutex, which every mutating entry point acquires. - void pinBlock(BlockPtr const& block); - - //! \brief Inverse of pinBlock: drop one reference and release the block back to the - //! eviction policy once no references remain. - void unpinBlock(BlockPtr const& block); - void truncateBlocks(LlmRequest::VecTokens const& targetTokens, SizeType32 numTokensToKeep); void resetReuseState() @@ -1289,10 +1274,6 @@ class WindowBlockManager } private: - //! \brief Shared implementation of the findBlocksInReuseTreeByBlockKey overloads. - [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeyImpl( - BlockKey const& blockKey, bool pinBlocks, std::vector<KVCacheBlock::IdType>& pinnedBlockIds); - //! \brief Walk the reuse tree with precomputed per-block keys (no lock; callers must hold mLookupTree->getMutex()). [[nodiscard]] std::shared_ptr<KVCacheBlock> searchReuseTree(std::vector<BlockKey> const& blockKeys); @@ -1372,7 +1353,7 @@ class WindowBlockManager } private: - tensorrt_llm::DataType mDataType; + nvinfer1::DataType mDataType; SizeType32 mWindowSize; // Number of blocks in pools @@ -1500,7 +1481,7 @@ class BlockManager explicit BlockManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkBubbleLength, + std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkBubbleLength, SizeType32 chunkSize, CacheType cacheType = CacheType::kSELF, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt, std::shared_ptr<KVCacheEventManager> eventManager = nullptr, bool enablePartialReuse = true, @@ -1582,7 +1563,7 @@ class BlockManager void releaseLastBlock(GenerationRequest& sequence, SizeType32 windowSize); - void setOffsets(kernels::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx, + void setOffsets(kernels::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx, SizeType32 blockIdx, KVCacheBlock::IdType blockId, SizeType32 windowSize) const; //! \brief Combined prefix reuse analysis — single radix tree walk. @@ -1640,9 +1621,9 @@ class BlockManager //! \brief Convenience: window_size -> dataType, derived from getPoolConfigurations(). //! For one-pool-per-window managers only; multi-pool-per-window will collide. - [[nodiscard]] std::map<SizeType32, tensorrt_llm::DataType> getDataTypePerWindow() const + [[nodiscard]] std::map<SizeType32, nvinfer1::DataType> getDataTypePerWindow() const { - std::map<SizeType32, tensorrt_llm::DataType> result; + std::map<SizeType32, nvinfer1::DataType> result; for (auto const& [windowSize, manager] : mWindowBlockManagers) { result[windowSize] = manager.getDataType(); @@ -1655,7 +1636,7 @@ class BlockManager return mWindowBlockManagers.at(windowSize).getSizePerHead(); } - [[nodiscard]] tensorrt_llm::DataType getDataTypeForWindow(SizeType32 windowSize) const + [[nodiscard]] nvinfer1::DataType getDataTypeForWindow(SizeType32 windowSize) const { return mWindowBlockManagers.at(windowSize).getDataType(); } @@ -1855,12 +1836,6 @@ class BlockManager return mWindowBlockManagers.at(windowSize).findBlocksInReuseTreeByBlockKey(blockKey); } - [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey( - BlockKey const& blockKey, SizeType32 windowSize, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) - { - return mWindowBlockManagers.at(windowSize).findBlocksInReuseTreeByBlockKey(blockKey, pinnedBlockIds); - } - [[nodiscard]] std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeys( std::vector<BlockKey> const& blockKeys, SizeType32 windowSize) { @@ -2214,7 +2189,7 @@ class BaseKVCacheManager /// head_dim=512). Empty vector = uniform @p sizePerHead / @p dtype across all windows. /// @return Map from window size to tuple of (primary blocks, secondary blocks) [[nodiscard]] static BlocksPerWindow calculateMaxNumBlocks(executor::KvCacheConfig const& config, - tensorrt_llm::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, + nvinfer1::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, tensorrt_llm::runtime::WorldConfig const& worldConfig, std::map<SizeType32, std::vector<SizeType32>> const& windowSizeToLayers, uint64_t allottedPrimaryMemBytes, uint64_t allottedSecondaryMemBytes, size_t extraCostMemory, SizeType32 kvFactor, SizeType32 maxBatchSize, @@ -2235,11 +2210,6 @@ class BaseKVCacheManager BlockKey const& blockKey, SizeType32 windowSize) = 0; - //! \brief Pinning lookup: pins matched blocks and records their ids for unpinBlocksById. - [[nodiscard]] virtual std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey( - BlockKey const& blockKey, SizeType32 windowSize, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) - = 0; - [[nodiscard]] virtual std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeys( std::vector<BlockKey> const& blockKeys, SizeType32 windowSize) = 0; @@ -2306,7 +2276,7 @@ class KVCacheManager : public BaseKVCacheManager //! and disagg transfer machinery applies natively. Empty vector = uniform. KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, + std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false, CacheType cacheType = CacheType::kSELF, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt, @@ -2320,7 +2290,7 @@ class KVCacheManager : public BaseKVCacheManager KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, + std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, int64_t stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false, CacheType cacheType = CacheType::kSELF, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt, @@ -2334,7 +2304,7 @@ class KVCacheManager : public BaseKVCacheManager KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, + std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = true, CacheType cacheType = CacheType::kSELF, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority = std::nullopt, @@ -2348,7 +2318,7 @@ class KVCacheManager : public BaseKVCacheManager KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, SizeType32 sinkTokenLength, + std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, int64_t stream, SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse = false, CacheType cacheType = CacheType::kSELF, bool enablePartialReuse = true, bool copyOnpartialReuse = true, bool enableIndexerKCache = false, SizeType32 indexerKCacheQuantBlockSize = 128, @@ -2608,24 +2578,6 @@ class KVCacheManager : public BaseKVCacheManager [[nodiscard]] std::vector<executor::IdType> commitAndGetBlockHashesForRequest( LlmRequest const& llmRequest, SizeType32 windowSize) override; - //! @brief Translate logical block IDs into primary-pool block indices. - //! @details A block ID is stable for the lifetime of a block, but its position inside the - //! memory pool can change after offload/onboard cycles. This function performs - //! that translation. The returned index is the value of - //! `KVCacheBlock::getMemoryPoolBlockIndex()` for each input, with the pool flag - //! stripped (see `kernels::KVCacheIndex::get()`), so it is only meaningful for - //! blocks resident in the primary pool. Every referenced block must therefore be - //! primary; this is asserted. Callers (e.g. the disaggregation cache transceiver - //! on the Python side) cannot check residency themselves, and the invariant holds - //! because allocation onboards offloaded blocks and offload only ever selects free - //! blocks — a violation indicates a block-lifetime bug. - //! @param blockIds IDs to translate. - //! @param windowSize Attention window the IDs belong to (selects the WindowBlockManager). - //! @throws Aborts via TLLM_CHECK_WITH_INFO if any referenced block is not found or is not - //! currently in the primary pool. - [[nodiscard]] std::vector<kernels::KVCacheIndex::UnderlyingType> getMemoryPoolBlockIndicesByBlockIds( - std::vector<KVCacheBlock::IdType> const& blockIds, SizeType32 windowSize) const; - std::optional<KVCacheBlock::IdType> getLastBlockId(LlmRequest::RequestIdType requestId) const override; /// @brief Calculates the number of kv-cache blocks that a sequence will require, for a single beam. @@ -2680,12 +2632,6 @@ class KVCacheManager : public BaseKVCacheManager return mBlockManager.findBlocksInReuseTreeByBlockKey(blockKey, windowSize); } - std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKey( - BlockKey const& blockKey, SizeType32 windowSize, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) override - { - return mBlockManager.findBlocksInReuseTreeByBlockKey(blockKey, windowSize, pinnedBlockIds); - } - std::shared_ptr<KVCacheBlock> findBlocksInReuseTreeByBlockKeys( std::vector<BlockKey> const& blockKeys, SizeType32 windowSize) override { @@ -2718,7 +2664,7 @@ class KVCacheManager : public BaseKVCacheManager SizeType32 mMaxNumSequences; // Maximum beam width SizeType32 mMaxBeamWidth; - tensorrt_llm::DataType mDataType; + nvinfer1::DataType mDataType; // Maximum kv cache length per sequence SizeType32 mMaxAttentionWindow; // Number of tokens per block diff --git a/cpp/include/tensorrt_llm/batch_manager/llmRequest.h b/cpp/include/tensorrt_llm/batch_manager/llmRequest.h index dea7bda60274..bc1ca3e6d012 100644 --- a/cpp/include/tensorrt_llm/batch_manager/llmRequest.h +++ b/cpp/include/tensorrt_llm/batch_manager/llmRequest.h @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iBuffer.h" @@ -84,15 +83,6 @@ enum LlmRequestType class ContextProgress; -// Process-global offset between the local steady clock and the global steady -// clock (rank 0's steady clock). The storage lives in a single translation unit -// (llmRequest.cpp) and is reached through this accessor so that -// libtensorrt_llm.so and the nanobind extension module share one copy across -// .so boundaries. An inline-static member would instead give each shared object -// its own copy, so an offset calibrated on one side would be invisible to the -// other. -std::optional<std::chrono::steady_clock::duration>& globalSteadyClockOffset(); - template <typename TTensor, typename TStream = runtime::BufferManager::CudaStreamPtr> class GenericLlmRequest { @@ -1243,18 +1233,6 @@ class GenericLlmRequest mEstimatedReusableTokens = estimatedReusableTokens; } - //! Get the absolute context positions at which recurrent-state snapshots are expected. - [[nodiscard]] std::vector<SizeType32> const& getExpectedSnapshotPoints() const noexcept - { - return mExpectedSnapshotPoints; - } - - //! Set the absolute context positions at which recurrent-state snapshots are expected. - void setExpectedSnapshotPoints(std::vector<SizeType32> expectedSnapshotPoints) - { - mExpectedSnapshotPoints = std::move(expectedSnapshotPoints); - } - void setDraftTokens(std::shared_ptr<VecTokens> const& draftTokens) { mDraftTokens = draftTokens; @@ -1334,7 +1312,7 @@ class GenericLlmRequest mEncoderOutput = std::move(encoderOutput); } - void allocEncoderOutputHost(SizeType32 encoderHiddenSize, tensorrt_llm::DataType dataType) + void allocEncoderOutputHost(SizeType32 encoderHiddenSize, nvinfer1::DataType dataType) { mEncoderOutputHost = runtime::BufferManager::pinned( runtime::ITensor::makeShape({getEncoderOutputLen(), encoderHiddenSize}), dataType); @@ -1350,13 +1328,13 @@ class GenericLlmRequest return mEncoderHiddenStates; } - void allocEncoderOutput(runtime::BufferManager const& manager, tensorrt_llm::DataType dataType) + void allocEncoderOutput(runtime::BufferManager const& manager, nvinfer1::DataType dataType) { // unique_ptr --> shared_ptr ownership move mEncoderOutput = std::move(manager.emptyTensor(runtime::MemoryType::kGPU, dataType)); } - void allocEncoderHiddenStates(runtime::BufferManager const& manager, tensorrt_llm::DataType dataType) + void allocEncoderHiddenStates(runtime::BufferManager const& manager, nvinfer1::DataType dataType) { // unique_ptr --> shared_ptr ownership move mEncoderHiddenStates = std::move(manager.emptyTensor(runtime::MemoryType::kGPU, dataType)); @@ -1474,7 +1452,7 @@ class GenericLlmRequest mContextLogitsHost = std::move(contextLogitsHost); } - void allocContextLogitsHost(SizeType32 vocabSizePadded, tensorrt_llm::DataType logitsDataType) + void allocContextLogitsHost(SizeType32 vocabSizePadded, nvinfer1::DataType logitsDataType) { mContextLogitsHost = runtime::BufferManager::pinnedPool( runtime::ITensor::makeShape({mPromptLen, vocabSizePadded}), logitsDataType); @@ -1493,7 +1471,7 @@ class GenericLlmRequest mGenerationLogitsHost = std::move(generationLogitsHost); } - void allocGenerationLogitsHost(SizeType32 vocabSizePadded, tensorrt_llm::DataType logitsDataType) + void allocGenerationLogitsHost(SizeType32 vocabSizePadded, nvinfer1::DataType logitsDataType) { if (mIsStreaming) { @@ -1512,7 +1490,7 @@ class GenericLlmRequest } } - void allocTargetModelAcceptedTokenLogitsHost(SizeType32 vocabSizePadded, tensorrt_llm::DataType logitsDataType) + void allocTargetModelAcceptedTokenLogitsHost(SizeType32 vocabSizePadded, nvinfer1::DataType logitsDataType) { mGenerationLogitsHost = runtime::BufferManager::pinnedPool( runtime::ITensor::makeShape({1, getNumDraftTokens() + 1, vocabSizePadded}), logitsDataType); @@ -1866,18 +1844,14 @@ class GenericLlmRequest mDecodingIter = iter; } - // Callers must pass a global-steady-clock time point (getSteadyClockNow(), - // or a value merged from such time points). Normalizing again here would - // apply the global steady clock offset twice, which corrupts cross-node - // min/max merging whenever the offset is non-zero. void setKvCacheTransferStart(TimePoint time) const { - mPerfMetrics.timingMetrics.kvCacheTransferStart = time; + mPerfMetrics.timingMetrics.kvCacheTransferStart = maybeToGlobalSteadyClock(time); } void setKvCacheTransferEnd(TimePoint time) const { - mPerfMetrics.timingMetrics.kvCacheTransferEnd = time; + mPerfMetrics.timingMetrics.kvCacheTransferEnd = maybeToGlobalSteadyClock(time); } TimePoint getKvCacheTransferStart() const @@ -2053,8 +2027,8 @@ class GenericLlmRequest return mUseDraftModel; } - // If the global steady clock offset is set, return a global steady clock time point, otherwise return local steady - // clock time point + // If sGlobalSteadyClockOffset is set, return a global steady clock time point, otherwise return local steady clock + // time point [[nodiscard]] static TimePoint getSteadyClockNow() { return maybeToGlobalSteadyClock(std::chrono::steady_clock::now()); @@ -2084,6 +2058,9 @@ class GenericLlmRequest // current position of the prompt tuning table (only used in chunked prefill mode) SizeType32 mPtableCurrentPosition{0}; + // The offset between local steady clock and global steady clock (at rank 0) + inline static std::optional<Duration> sGlobalSteadyClockOffset{std::nullopt}; + protected: bool mIsStreaming; @@ -2119,9 +2096,6 @@ class GenericLlmRequest // the authoritative mPrepopulatedPromptLen and advances context position. mutable SizeType32 mEstimatedReusableTokens{0}; - // Absolute context positions at which recurrent-state snapshots are expected. - std::vector<SizeType32> mExpectedSnapshotPoints; - SizeType32 mMaxSentTokenLen; std::optional<TensorPtr> mEmbeddingBias{std::nullopt}; @@ -2382,7 +2356,7 @@ class GenericLlmRequest auto const numWords = static_cast<SizeType32>(words.size()); auto const shape = runtime::ITensor::makeShape({2, numWords}); - auto tensor = runtime::BufferManager::pinnedPool(shape, tensorrt_llm::DataType::kINT32); + auto tensor = runtime::BufferManager::pinnedPool(shape, nvinfer1::DataType::kINT32); auto* data = runtime::bufferCast<int32_t>(*tensor); std::memcpy(data, words.data(), numWords * sizeof(int32_t)); std::memcpy(data + numWords, offsets.data(), numWords * sizeof(int32_t)); @@ -2395,10 +2369,9 @@ class GenericLlmRequest static TimePoint maybeToGlobalSteadyClock(TimePoint const& time_point) { - auto const& offset = globalSteadyClockOffset(); - if (offset.has_value()) + if (sGlobalSteadyClockOffset.has_value()) { - return time_point + *offset; + return time_point + *sGlobalSteadyClockOffset; } return time_point; } diff --git a/cpp/include/tensorrt_llm/batch_manager/logitsPostProcessor.h b/cpp/include/tensorrt_llm/batch_manager/logitsPostProcessor.h new file mode 100644 index 000000000000..1916a915e337 --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/logitsPostProcessor.h @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "common.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/common/algorithm.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +namespace tensorrt_llm::runtime +{ +class CudaStream; +} + +namespace tensorrt_llm::batch_manager +{ +class DecoderInputBuffers; + +class LogitsPostProcessor : Algorithm +{ +public: + using CudaStreamPtr = std::shared_ptr<runtime::CudaStream>; + + using LogitsPostProcessorBatched = std::function<void(std::vector<batch_manager::LlmRequest::RequestIdType> const&, + std::vector<batch_manager::LlmRequest::TensorPtr>&, + std::vector<std::reference_wrapper<batch_manager::LlmRequest::BeamTokens const>> const&, CudaStreamPtr const&, + std::vector<std::optional<batch_manager::LlmRequest::RequestIdType>> const&)>; + + constexpr static auto name{"LogitsPostProcessor"}; + + LogitsPostProcessor() = default; + + bool operator()(DecoderInputBuffers& inputBuffers, bool replicateLogitsPostProcessor, + runtime::WorldConfig const& worldConfig, CudaStreamPtr const& stream, + std::optional<LogitsPostProcessorBatched> const& logitsPostProcessorBatched = std::nullopt) const; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h b/cpp/include/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h new file mode 100644 index 000000000000..245f4b4b5286 --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "common.h" +#include "tensorrt_llm/common/algorithm.h" +#include "tensorrt_llm/common/optionalRef.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iGptDecoderBatched.h" +#include "tensorrt_llm/runtime/modelConfig.h" + +namespace tensorrt_llm::runtime::decoder +{ +class DecoderState; +} // namespace tensorrt_llm::runtime::decoder + +namespace tensorrt_llm::batch_manager +{ +class DecoderInputBuffers; +class RuntimeBuffers; + +class MakeDecodingBatchInputOutput : Algorithm +{ +public: + constexpr static auto name{"MakeDecodingBatchInputOutput"}; + + using SizeType32 = tensorrt_llm::runtime::SizeType32; + using TensorPtr = runtime::ITensor::SharedPtr; + template <typename T> + using OptionalRef = tensorrt_llm::common::OptionalRef<T>; + + MakeDecodingBatchInputOutput() = default; + + void operator()(DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, + runtime::ModelConfig const& modelConfig, OptionalRef<RuntimeBuffers> fusedRuntimeBuffers) const; + + static void createDecoderBatchInputs(DecoderInputBuffers& inputBuffers, std::vector<SizeType32> const& activeSlots, + runtime::decoder::DecoderState const& decoderState); +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/medusaBuffers.h b/cpp/include/tensorrt_llm/batch_manager/medusaBuffers.h index 5342591840a8..ba29be6ede81 100644 --- a/cpp/include/tensorrt_llm/batch_manager/medusaBuffers.h +++ b/cpp/include/tensorrt_llm/batch_manager/medusaBuffers.h @@ -22,6 +22,7 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/promptTuningParams.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" #include "tensorrt_llm/runtime/worldConfig.h" namespace tensorrt_llm::batch_manager @@ -35,6 +36,10 @@ class MedusaBuffers using TensorPtr = runtime::ITensor::SharedPtr; using TensorMap = runtime::StringPtrMap<runtime::ITensor>; + MedusaBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::BufferManager const& manager, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + executor::DecodingConfig const& decodingConfig, runtime::TllmRuntime const& runtime); + void reshape(SizeType32 numCtxSequences, SizeType32 numGenSequences, SizeType32 tokensPerStep); void insertInputTensors( diff --git a/cpp/include/tensorrt_llm/batch_manager/peftCacheManager.h b/cpp/include/tensorrt_llm/batch_manager/peftCacheManager.h index ed928e96d811..cf65753783e8 100644 --- a/cpp/include/tensorrt_llm/batch_manager/peftCacheManager.h +++ b/cpp/include/tensorrt_llm/batch_manager/peftCacheManager.h @@ -25,7 +25,7 @@ #include "tensorrt_llm/runtime/workerPool.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <future> #include <memory> @@ -147,7 +147,7 @@ class PeftCacheManager : public BasePeftCacheManager void updateTaskState(uint64_t taskId, uint64_t reqId, bool terminate = false, bool pause = false); static std::pair<uint64_t, uint64_t> getMaxNumSlots(PeftCacheManagerConfig const& config, - tensorrt_llm::DataType dataType, uint64_t pageWidth, uint64_t max1dModSize, + nvinfer1::DataType dataType, uint64_t pageWidth, uint64_t max1dModSize, runtime::BufferManager const& bufferManager); static std::pair<runtime::LoraCachePageManagerConfig, runtime::LoraCachePageManagerConfig> getPageManagerConfig( diff --git a/cpp/include/tensorrt_llm/batch_manager/promptTuningBuffers.h b/cpp/include/tensorrt_llm/batch_manager/promptTuningBuffers.h new file mode 100644 index 000000000000..a1d8849a8811 --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/promptTuningBuffers.h @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/promptTuningParams.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +namespace tensorrt_llm::batch_manager +{ + +class PromptTuningBuffers +{ + +public: + using SizeType32 = tensorrt_llm::runtime::SizeType32; + using ITensor = tensorrt_llm::runtime::ITensor; + using TensorPtr = runtime::ITensor::SharedPtr; + + runtime::PromptTuningParams mPromptTuningParams; + SizeType32 mMaxPromptVocabSize; + + PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); + + PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, bool promptTableOffloading); + + void validate(std::optional<TensorPtr> const& optReqPromptEmbeddingTable, + std::optional<SizeType32> const& optReqPromptVocabSize); + + void fill(RequestVector const& contextRequests, RequestVector const& genRequests, + runtime::BufferManager const& manager, bool packed); + + /* + * The below functions are specific for Chunked Prefill mode + * Chunk Ptable with Ping-Pong Buffer Implementation + * ----------------------------------------------- + * + * Overview: + * The chunk ptable (prompt tuning table) system uses a ping-pong buffer mechanism to efficiently + * manage large embedding tables when operating in context Prefill mode. This allows + * for processing of large embedding tables by loading them in chunks from CPU to GPU memory, + * enabling support for tables that exceed available GPU memory. + * + * Key Components: + * 1. Ping-Pong Buffers (mChunkPtableBuffers): + * - Two alternating GPU buffers that store chunks of the embedding table + * - While the current buffer is being processed by the model, + * the next chunk can be asynchronously loaded into the other buffer + * - Managed through mChunkPtableCurrentIndex (toggles between 0 and 1) + * 2. Start Positions Tracking (mChunkPtableBufferStartPositions): + * - Mainly used for multi-batch processing + * - Maintains the starting position of each batch's data within each buffer + * - Maintained separately for each ping-pong buffer + * + * Memory Optimization: + * - Only two GPU buffers are maintained regardless of total embedding table size + * - Each buffer size is limited to contextChunkSize * hiddenSize + * - Efficient memory usage through chunk-based processing + */ + + bool mPromptTableOffloading; + + bool mChunkPtableInitialized{false}; + std::optional<std::array<TensorPtr, 2>> mChunkPtableBuffers; + std::optional<std::vector<std::vector<SizeType32>>> mChunkPtableBufferStartPositions; + size_t mChunkPtableCurrentIndex{0}; + + void initializeChunkPtableBuffers(runtime::BufferManager const& manager, runtime::ModelConfig const& modelConfig, + SizeType32 contextChunkSize, std::shared_ptr<LlmRequest> const& llmReq); + + void switchChunkPtableBuffer(); + + size_t getChunkPtableCurrentIndex(); + + [[nodiscard]] TensorPtr& getChunkPtableBuffer(size_t index); + + [[nodiscard]] SizeType32 getChunkPtableBufferSliceSize(size_t index, size_t batchIdx); + + [[nodiscard]] SizeType32 getChunkPtableBufferStartPosition(size_t index, size_t batchIdx); + + void updateBufferStartPosition(size_t index, SizeType32 numRows); + + void clearBufferStartPositions(size_t index); +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/rnnStateManager.h b/cpp/include/tensorrt_llm/batch_manager/rnnStateManager.h index c4f97950a6b9..5c0bfe136de2 100644 --- a/cpp/include/tensorrt_llm/batch_manager/rnnStateManager.h +++ b/cpp/include/tensorrt_llm/batch_manager/rnnStateManager.h @@ -17,7 +17,6 @@ #pragma once #include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -43,8 +42,8 @@ class RnnStateManager runtime::WorldConfig const& worldConfig, tensorrt_llm::runtime::BufferManager const& bufferManager); RnnStateManager(SizeType32 dState, SizeType32 dConv, SizeType32 numHeads, SizeType32 nGroups, SizeType32 headDim, - SizeType32 maxBatchSize, runtime::WorldConfig const& worldConfig, int64_t stream, tensorrt_llm::DataType dtype, - tensorrt_llm::DataType ssmCacheDtype, std::vector<SizeType32> const& ppLayers, SizeType32 numLayers); + SizeType32 maxBatchSize, runtime::WorldConfig const& worldConfig, int64_t stream, nvinfer1::DataType dtype, + nvinfer1::DataType ssmCacheDtype, std::vector<SizeType32> const& ppLayers, SizeType32 numLayers); void getPtrBuffers(TensorMap& inputBuffers, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) const; @@ -69,9 +68,9 @@ class RnnStateManager [[nodiscard]] TensorPtr getSsmStates() const; - [[nodiscard]] tensorrt_llm::DataType getConvStateDataType() const noexcept; + [[nodiscard]] nvinfer1::DataType getConvStateDataType() const noexcept; - [[nodiscard]] tensorrt_llm::DataType getSsmStateDataType() const noexcept; + [[nodiscard]] nvinfer1::DataType getSsmStateDataType() const noexcept; [[nodiscard]] executor::kv_cache::CacheState::RnnModelConfig getRnnCacheStateModelConfig() const noexcept; @@ -112,8 +111,8 @@ class RnnStateManager std::vector<SizeType32> mFreeBlocks; std::unordered_map<RequestIdType, SizeType32> mCacheIndex; std::optional<runtime::BufferManager> mBufferManager; - tensorrt_llm::DataType mDtype{tensorrt_llm::DataType::kFLOAT}; - tensorrt_llm::DataType mSsmCacheDtype{tensorrt_llm::DataType::kFLOAT}; + nvinfer1::DataType mDtype{nvinfer1::DataType::kFLOAT}; + nvinfer1::DataType mSsmCacheDtype{nvinfer1::DataType::kFLOAT}; // RNN model config (global values before TP/PP split) SizeType32 mDState{0}; diff --git a/cpp/include/tensorrt_llm/batch_manager/runtimeBuffers.h b/cpp/include/tensorrt_llm/batch_manager/runtimeBuffers.h new file mode 100644 index 000000000000..97a4ae67acdd --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/runtimeBuffers.h @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2023-2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/batch_manager/rnnStateManager.h" +#include "tensorrt_llm/common/optionalRef.h" +#include "tensorrt_llm/runtime/eagleBuffers.h" +#include "tensorrt_llm/runtime/explicitDraftTokensBuffers.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/lookaheadBuffers.h" +#include "tensorrt_llm/runtime/loraManager.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +#include <cstddef> +#include <memory> +#include <optional> +#include <vector> + +namespace tensorrt_llm::runtime +{ +class TllmRuntime; + +namespace decoder +{ +class DecoderState; +} // namespace decoder +} // namespace tensorrt_llm::runtime + +namespace tensorrt_llm::batch_manager +{ + +namespace kv_cache_manager +{ +class BaseKVCacheManager; +} // namespace kv_cache_manager + +class LlmRequest; + +class EncoderBuffers; +class LoraBuffers; +class MedusaBuffers; +class PromptTuningBuffers; +class RnnStateBuffers; +class TransformerBuffers; + +class RuntimeBuffers +{ +public: + static constexpr auto kLogitsTensorName = "logits"; + static constexpr auto kHiddenStatesOutputTensorName = "hidden_states_output"; + static constexpr auto kHiddenStatesInputTensorName = "hidden_states_input"; + static constexpr auto kInputIdsTensorName = "input_ids"; + static constexpr auto kLastTokenIdsTensorName = "last_token_ids"; + static constexpr auto kHostRequestTypesTensorName = "host_request_types"; + static constexpr auto kContextLengthsTensorName = "context_lengths"; + static constexpr auto kHostContextLengthsTensorName = "host_context_lengths"; + static constexpr auto kSequenceLengthsTensorName = "sequence_length"; + static constexpr auto kPromptEmbeddingTableTensorName = "prompt_embedding_table"; + static constexpr auto kTasksTensorName = "tasks"; + static constexpr auto kPromptVocabSizeTensorName = "prompt_vocab_size"; + static constexpr auto kMRopeRotaryCosSinTensorName = "mrope_rotary_cos_sin"; + static constexpr auto kMRopePositionDeltasTensorName = "mrope_position_deltas"; + + using SizeType32 = runtime::SizeType32; + using TensorPtr = runtime::ITensor::SharedPtr; + using TensorMap = runtime::ITensor::TensorMap; + using PeftTable = runtime::LoraManager::PeftTable; + template <typename T> + using OptionalRef = tensorrt_llm::common::OptionalRef<T>; + + [[nodiscard]] SizeType32 constexpr getContextIndex() const noexcept + { + return contextIndex; + }; + + void constexpr setContextIndex(SizeType32 index) noexcept + { + contextIndex = index; + }; + + [[nodiscard]] SizeType32 constexpr getNumContextTokens() const noexcept + { + return numContextTokens; + }; + + [[nodiscard]] BatchState getBatchState() const noexcept + { + return {numContextRequests, numGenRequests, getNumTokens(), maxKvCacheLengthRounded}; + }; + +private: + [[nodiscard]] SizeType32 constexpr getNumRequests() const noexcept + { + return numContextRequests + numGenRequests; + }; + + [[nodiscard]] SizeType32 constexpr getNumSequences() const noexcept + { + return numContextRequests + numGenSequences; + }; + + [[nodiscard]] SizeType32 constexpr getNumTokens() const noexcept + { + return numContextTokens + numGenTokens; + }; + + //! Sizes + SizeType32 numContextRequests{}; + SizeType32 numGenRequests{}; + SizeType32 numGenSequences{}; + SizeType32 numContextTokens{}; + SizeType32 numGenTokens{}; + SizeType32 numLogits{}; + SizeType32 maxKvCacheLengthRounded{}; + + //! General + TensorPtr inputsIds; + + TensorPtr contextLengthsHost; + TensorPtr contextLengthsDevice; + TensorPtr sequenceLengthsHost; + + //! Index of selected runtime context. + SizeType32 contextIndex{}; + SizeType32 maxContextLength{}; + +public: + TensorPtr sequenceLengthsDevice; + bool promptTableOffloading; + + //! Prompt-Tuning + std::unique_ptr<PromptTuningBuffers> promptTuningBuffers; + +private: + //! Runtime + //! Type of host tensor: 0 for context, 1 for generation + TensorPtr requestTypes; + + TensorPtr lastTokenIdsHost; + TensorPtr lastTokenIdsDevice; + TensorPtr logitsIdsHost; + + //! Pipeline-Parallelism + TensorPtr hiddenStates; + + //! Mrope + TensorPtr mropeRotaryCosSin; + TensorPtr mropePositionDeltas; + + //! LoRA + std::unique_ptr<LoraBuffers> loraBuffers; + +public: + //! Additional buffers depending on model type + std::unique_ptr<TransformerBuffers> transformerBuffers; + std::unique_ptr<RnnStateBuffers> rnnStateBuffers; + + //! Encoder-Decoder + std::unique_ptr<EncoderBuffers> encoderBuffers; + + //! Medusa + std::unique_ptr<MedusaBuffers> mMedusaBuffers; + //! Lookahead decoding + std::unique_ptr<runtime::LookaheadRuntimeBuffers> mLookaheadBuffers; + //! Explicit draft tokens decoding + std::unique_ptr<runtime::ExplicitDraftTokensBuffers> mExplicitDraftTokensBuffers; + //! Eagle decoding + std::unique_ptr<runtime::EagleBuffers> mEagleBuffers; + + //! Language adapter routing information if language adapter is presented, [numTokens, numLanguages] + TensorPtr languageAdapterRoutings; + + TensorPtr cacheIndirDecoderIOBatchedCopySrcOffsets; + TensorPtr cacheIndirDecoderIOBatchedCopyDstOffsets; + TensorPtr cacheIndirDecoderIOBatchedCopySizes; + + //! Logits + std::vector<SizeType32> numContextLogits; + TensorPtr logits; + + //! Helper cache for store generation logits + struct GenerationLogitsCache + { + static constexpr auto kCACHE_LENGTH = 8; + + //! Buffer for logits between steps to prevent from being overwritten + //! [kCACHE_LENGTH, maxBatchSize * maxBeamWidth, vocabSizePadded] + TensorPtr logits; + //! Record the usage offset of the cacheGenerationLogits buffer + SizeType32 offset{0}; + + //! Temporarily store the transposed results of multiple fragment logits, [maxBeamWidth, kCACHE_LENGTH] + TensorPtr transposedLogits; + + //! Temporarily store logits buffer address during the transposing, [maxBatchSize, kCACHE_LENGTH] + //! One row per batch slot (same layout as fragmentPointerHost) so concurrent flushes for + //! different requests in the same batch never clobber each other's pointer arrays. + TensorPtr fragmentPointerDevice; + + //! Temporarily store logits buffer address during the transposing, [maxBatchSize, kCACHE_LENGTH] + TensorPtr fragmentPointerHost; + + //! Cycling index for workspace + size_t workIdx{0}; + + void cycleWorkIdx() + { + workIdx = (workIdx + 1) % (fragmentPointerHost->getShape().d[0]); + } + + //! Returns matching host and device pointer rows for the current workIdx, then advances + //! workIdx. Always call this instead of the individual getters to avoid ordering bugs. + [[nodiscard]] std::pair<TensorPtr, TensorPtr> getFragmentPointerSlot() + { + TensorPtr host = runtime::ITensor::slice(fragmentPointerHost, workIdx, 1); + TensorPtr device = runtime::ITensor::slice(fragmentPointerDevice, workIdx, 1); + cycleWorkIdx(); + return {std::move(host), std::move(device)}; + }; + }; + + GenerationLogitsCache generationLogitsCache; + + //! Mapping from batch idx to slot id + TensorPtr seqSlots; + TensorPtr seqSlotsDevice; + + //! Explicitly device-copy src offsets to reduce warp stalls in copy batch kernel invocation + //! [mMaxNumRequests], on gpu + TensorPtr mCacheIndirDecoderIOBatchedCopySrcOffsetsSliceDevice; + //! Explicitly device-copy dst offsets to reduce warp stalls in copy batch kernel invocation + //! [mMaxNumRequests], on gpu + TensorPtr mCacheIndirDecoderIOBatchedCopyDstOffsetsSliceDevice; + //! Explicitly device-copy size to reduce warp stalls in copy batch kernel invocation + //! [mMaxNumRequests], on gpu + TensorPtr mCacheIndirDecoderIOBatchedCopyCopySizesDevice; + +private: + //! Re-capture cuda graph when max kv cache len of the batch has changed on kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE. + static SizeType32 constexpr kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE{256}; + + TensorMap mAdditionalOutputTensors; // Tensors storing additional output tensors. + + //! Engine I/O + TensorMap inputMap; + TensorMap outputMap; + +public: + RuntimeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, + std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, + runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig, executor::DecodingConfig const& decodingConfig, + bool gatherGenerationLogits, std::optional<SizeType32> maxNumTokens = std::nullopt, + std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs = std::nullopt, + bool promptTableOffloading = false); + + RuntimeBuffers(RuntimeBuffers const& other) = delete; + RuntimeBuffers& operator=(RuntimeBuffers const& other) = delete; + RuntimeBuffers(RuntimeBuffers&& other) = delete; + RuntimeBuffers& operator=(RuntimeBuffers&& other) = delete; + + ~RuntimeBuffers(); + + std::tuple<SizeType32, TensorMap const&, TensorMap&> prepareStep(RequestVector const& contextRequests, + RequestVector const& genRequests, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, + runtime::decoder::DecoderState const& decoderState, kv_cache_manager::BaseKVCacheManager* kvCacheManager, + kv_cache_manager::BaseKVCacheManager* crossKvCacheManager, rnn_state_manager::RnnStateManager* rnnStateManager, + PeftTable const& peftTable, runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig, bool gatherGenerationLogits, bool trtOverlap, + OptionalRef<runtime::ITensor const> newOutputTokens = std::nullopt); + + void prepareBuffersForCudaGraph(SizeType32 maxSequenceLength); + + void prepareExplicitDraftTokenBuffers(runtime::ExplicitDraftTokensBuffers::Inputs const& explicitDraftTokensBuffers, + runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig); + + void prepareEagleBuffers(RequestVector const& contextRequests, RequestVector const& genRequests, + runtime::EagleBuffers::Inputs const& eagleBuffers, runtime::TllmRuntime const& runtime, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); + +private: + void create(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, + SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, runtime::TllmRuntime const& runtime, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + executor::DecodingConfig const& decodingConfig, bool gatherGenerationLogits, + std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs = std::nullopt); + + //! @brief set max sizes for pre-allocation + void setMaxBufferSizes(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::ModelConfig const& modelConfig, + std::optional<SizeType32> maxNumRuntimeTokens); + + //! @brief set sizes depending on scheduled requests + void setBufferSizes(RequestVector const& contextRequests, RequestVector const& genRequests); + + void reshape(runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig, bool gatherGenerationLogits); + + void setFromInputs(RequestVector const& contextRequests, RequestVector const& genRequests, SizeType32 maxBeamWidth, + SizeType32 maxAttentionWindow, runtime::decoder::DecoderState const& decoderState, + kv_cache_manager::BaseKVCacheManager* kvCacheManagerPtr, + kv_cache_manager::BaseKVCacheManager* crossKvCacheManagerPtr, + rnn_state_manager::RnnStateManager* rnnStateManagerPtr, PeftTable const& peftTable, + runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig, bool trtOverlap, OptionalRef<runtime::ITensor const> newOutputTokens); + + void fillIOMaps(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/transformerBuffers.h b/cpp/include/tensorrt_llm/batch_manager/transformerBuffers.h new file mode 100644 index 000000000000..b5254c6357b4 --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/transformerBuffers.h @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/batch_manager/kvCacheType.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +namespace tensorrt_llm::runtime +{ +class TllmRuntime; +class MulticastTensor; +} // namespace tensorrt_llm::runtime + +namespace tensorrt_llm::batch_manager +{ + +namespace kv_cache_manager +{ +class BaseKVCacheManager; +} + +class TransformerBuffers +{ +public: + using SizeType32 = runtime::SizeType32; + using TensorPtr = runtime::ITensor::SharedPtr; + using TensorMap = runtime::StringPtrMap<runtime::ITensor>; + + static constexpr auto kCrossAttentionMaskTensorName = "cross_attention_mask"; + static constexpr auto kCrossAttentionPackedMaskTensorName = "cross_attention_packed_mask"; + static constexpr auto kPositionIdsTensorName = "position_ids"; + static constexpr auto kCacheIndirectionsTensorName = "cache_indirection"; + static constexpr auto kHostPastKeyValueLengthsTensorName = "host_past_key_value_lengths"; + static constexpr auto kHostSinkTokenLengthTensorName = "host_sink_token_length"; + static constexpr auto kHostMaxAttentionWindowSizesTensorName = "host_max_attention_window_sizes"; + static constexpr auto kHostContextProgressTensorName = "host_context_progress"; + static constexpr auto kKvCacheBlockOffsetsTensorName = "kv_cache_block_offsets"; + static constexpr auto kHostKvCacheBlockOffsetsTensorName = "host_kv_cache_block_offsets"; + static constexpr auto kCrossKvCacheBlockOffsetsTensorName = "cross_kv_cache_block_offsets"; + static constexpr auto kHostCrossKvCacheBlockOffsetsTensorName = "host_cross_kv_cache_block_offsets"; + static constexpr auto kHostCrossKvCachePoolPointersTensorName = "host_cross_kv_cache_pool_pointers"; + static constexpr auto kHostCrossKvCachePoolMappingTensorName = "host_cross_kv_cache_pool_mapping"; + static constexpr auto kSkipCrossAttentionBlocksTensorName = "skip_cross_attn_blocks"; + + TensorPtr pastKeyValueLengths; // Host tensor + TensorPtr positionIds; + + // max kv cache lengths. + TensorPtr maxAttentionWindows; + // sink token lengths. + TensorPtr sinkTokenLengths; + TensorPtr cacheIndirection; + TensorPtr kvCacheBlockOffsetsHost; // [numPools, maxBatch * maxBeamWidth, 2, maxBlocksPerSeq] + TensorPtr kvCacheBlockOffsetsDevice; // [numPools, maxBatch * maxBeamWidth, 2, maxBlocksPerSeq] + TensorPtr contextProgressHost; + + // Cross attention buffers + TensorPtr crossKvCacheBlockPoolPointers = nullptr; + TensorPtr crossKvCacheBlockPoolMapping = nullptr; + TensorPtr crossKvCacheBlockOffsetsHost = nullptr; + TensorPtr crossKvCacheBlockOffsetsDevice = nullptr; + TensorPtr crossAttentionMaskCopySrcOffsets = nullptr; // [maxNumRequest] pinned memory. + TensorPtr crossAttentionMaskCopyDstOffsets = nullptr; // [maxNumRequest] pinned memory. + TensorPtr crossAttentionMaskCopySizes = nullptr; // [maxNumRequest] pinned memory. + TensorPtr crossAttentionMaskDevice = nullptr; // [maxNumTokens, maxEncoderOutputLen] + // This is created to allow mixed memory types of crossAttentionMask (i.e. CPU and GPU). + TensorPtr crossAttentionMaskPinnedHost = nullptr; // [maxNumTokens, maxEncoderOutputLen] + // See more details in tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmhaPackedMask.cu. + // The attention packed mask for FMHA where each bit represents one mask. + TensorPtr crossAttentionPackedMaskDevice + = nullptr; // [maxBatchSize, maxInputLengthInBatch, roundUp(maxEncoderOutputLen, 32)] + // The number of cumulative Q sequence lengths in the mask input, which is used to get mask offsets for different + // requests. + TensorPtr crossAttentionCuQSeqLensDevice = nullptr; // [maxBatchSize + 1] + // The number of cumulative Q sequence lengths in the packed mask, which is used to get mask offsets for different + // requests. + TensorPtr crossAttentionPackedMaskCuMaskRowsDevice = nullptr; // [maxBatchSize + 1] + + TensorPtr cacheIndirBatchedCopySrcOffsets; + TensorPtr cacheIndirBatchedCopyDstOffsets; + TensorPtr cacheIndirBatchedCopySizes; + + TensorPtr fillValuesAlt; + TensorPtr fillValuesAltDevice; + TensorPtr seqSlotsAlt; + TensorPtr seqSlotsAltDevice; + TensorPtr skipCrossAttnBlocks; + + std::shared_ptr<tensorrt_llm::runtime::MulticastTensor> gemmAllReduceOutput; + + TransformerBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, + std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, + runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig); + + void reshape(SizeType32 numSequences, SizeType32 numInputTokens); + + void reshapeKvTensors(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, SizeType32 maxBlocksPerSeq, + kv_cache_manager::CacheType kvCacheType, SizeType32 numPools, runtime::BufferManager const& manager); + + void getBuffers(TensorMap& inputBuffers, TensorMap& outputBuffers, runtime::ModelConfig const& modelConfig) const; + + void copyPositionIds(runtime::TllmRuntime const& runtime, std::vector<SizeType32> const& positionIdsHost, + bool isChatGlm, TensorPtr const& decoderPositionIds); + + void copyKvBlockOffsets(RequestVector const& contextRequests, RequestVector const& genRequests, + kv_cache_manager::BaseKVCacheManager const* kvCacheManager, + kv_cache_manager::BaseKVCacheManager const* crossKvCacheManager, runtime::BufferManager const& manager); + + // Copy CacheIndirection from `decoderCacheIndirectionOutput` to `this->cacheIndirection` + void copyCacheIndirection(RequestVector const& genRequests, TensorPtr const& decoderCacheIndirectionOutput, + runtime::CudaStream const& stream); + + void copyCrossAttentionMasks(RequestVector const& contextRequests, RequestVector const& genRequests, + TensorPtr const& decoderContextLengthsDevice, TensorPtr const& encoderInputLengths, + SizeType32 maxDecoderContextLength, SizeType32 maxEncoderInputLengthInBatch, + runtime::TllmRuntime const& runtime); + + void copySkipCrossAttnBlocks(bool const& _skipCrossAttnBlocks, runtime::TllmRuntime const& runtime); + +private: + SizeType32 maxInputLen; + SizeType32 maxEncoderOutputLen; + SizeType32 maxNumTokens; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/batch_manager/updateDecoderBuffers.h b/cpp/include/tensorrt_llm/batch_manager/updateDecoderBuffers.h new file mode 100644 index 000000000000..526a756e5546 --- /dev/null +++ b/cpp/include/tensorrt_llm/batch_manager/updateDecoderBuffers.h @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/common/algorithm.h" +#include "tensorrt_llm/runtime/modelConfig.h" + +namespace tensorrt_llm::runtime +{ +class BufferManager; +class CudaEvent; + +namespace decoder +{ +class DecoderState; +} // namespace decoder +} // namespace tensorrt_llm::runtime + +namespace tensorrt_llm::batch_manager +{ + +class DecoderOutputBuffers; + +class UpdateDecoderBuffers : Algorithm +{ +public: + constexpr static auto name{"UpdateDecoderBuffers"}; + + UpdateDecoderBuffers() = default; + + runtime::CudaEvent operator()(runtime::ModelConfig const& modelConfig, DecoderOutputBuffers& decoderOutputBuffers, + runtime::BufferManager const& copyBufferManager, runtime::decoder::DecoderState const& decoderState, + bool returnLogProbs, runtime::CudaEvent const& decoderFinishEvent) const; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/include/tensorrt_llm/common/dataType.h b/cpp/include/tensorrt_llm/common/dataType.h index 9b3bb5fdf0f0..2f19404f9c94 100644 --- a/cpp/include/tensorrt_llm/common/dataType.h +++ b/cpp/include/tensorrt_llm/common/dataType.h @@ -19,7 +19,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/tllmException.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <map> TRTLLM_NAMESPACE_BEGIN @@ -27,61 +27,61 @@ TRTLLM_NAMESPACE_BEGIN namespace common { -constexpr static size_t getDTypeSize(tensorrt_llm::DataType type) +constexpr static size_t getDTypeSize(nvinfer1::DataType type) { switch (type) { - case tensorrt_llm::DataType::kINT64: return 8; - case tensorrt_llm::DataType::kINT32: [[fallthrough]]; - case tensorrt_llm::DataType::kFLOAT: return 4; - case tensorrt_llm::DataType::kBF16: [[fallthrough]]; - case tensorrt_llm::DataType::kHALF: return 2; - case tensorrt_llm::DataType::kBOOL: [[fallthrough]]; - case tensorrt_llm::DataType::kUINT8: [[fallthrough]]; - case tensorrt_llm::DataType::kINT8: [[fallthrough]]; - case tensorrt_llm::DataType::kFP8: return 1; - case tensorrt_llm::DataType::kINT4: TLLM_THROW("Cannot determine size of INT4 data type"); - case tensorrt_llm::DataType::kFP4: TLLM_THROW("Cannot determine size of FP4 data type"); + case nvinfer1::DataType::kINT64: return 8; + case nvinfer1::DataType::kINT32: [[fallthrough]]; + case nvinfer1::DataType::kFLOAT: return 4; + case nvinfer1::DataType::kBF16: [[fallthrough]]; + case nvinfer1::DataType::kHALF: return 2; + case nvinfer1::DataType::kBOOL: [[fallthrough]]; + case nvinfer1::DataType::kUINT8: [[fallthrough]]; + case nvinfer1::DataType::kINT8: [[fallthrough]]; + case nvinfer1::DataType::kFP8: return 1; + case nvinfer1::DataType::kINT4: TLLM_THROW("Cannot determine size of INT4 data type"); + case nvinfer1::DataType::kFP4: TLLM_THROW("Cannot determine size of FP4 data type"); default: TLLM_THROW("Unknown dtype %d", static_cast<int>(type)); } return 0; } -constexpr static size_t getDTypeSizeInBits(tensorrt_llm::DataType type) +constexpr static size_t getDTypeSizeInBits(nvinfer1::DataType type) { switch (type) { - case tensorrt_llm::DataType::kINT64: return 64; - case tensorrt_llm::DataType::kINT32: [[fallthrough]]; - case tensorrt_llm::DataType::kFLOAT: return 32; - case tensorrt_llm::DataType::kBF16: [[fallthrough]]; - case tensorrt_llm::DataType::kHALF: return 16; - case tensorrt_llm::DataType::kBOOL: [[fallthrough]]; - case tensorrt_llm::DataType::kUINT8: [[fallthrough]]; - case tensorrt_llm::DataType::kINT8: [[fallthrough]]; - case tensorrt_llm::DataType::kFP8: return 8; - case tensorrt_llm::DataType::kINT4: [[fallthrough]]; - case tensorrt_llm::DataType::kFP4: return 4; + case nvinfer1::DataType::kINT64: return 64; + case nvinfer1::DataType::kINT32: [[fallthrough]]; + case nvinfer1::DataType::kFLOAT: return 32; + case nvinfer1::DataType::kBF16: [[fallthrough]]; + case nvinfer1::DataType::kHALF: return 16; + case nvinfer1::DataType::kBOOL: [[fallthrough]]; + case nvinfer1::DataType::kUINT8: [[fallthrough]]; + case nvinfer1::DataType::kINT8: [[fallthrough]]; + case nvinfer1::DataType::kFP8: return 8; + case nvinfer1::DataType::kINT4: [[fallthrough]]; + case nvinfer1::DataType::kFP4: return 4; default: TLLM_THROW("Unknown dtype %d", static_cast<int>(type)); } return 0; } -[[maybe_unused]] static std::string getDtypeString(tensorrt_llm::DataType type) +[[maybe_unused]] static std::string getDtypeString(nvinfer1::DataType type) { switch (type) { - case tensorrt_llm::DataType::kFLOAT: return "fp32"; break; - case tensorrt_llm::DataType::kHALF: return "fp16"; break; - case tensorrt_llm::DataType::kINT8: return "int8"; break; - case tensorrt_llm::DataType::kINT32: return "int32"; break; - case tensorrt_llm::DataType::kBOOL: return "bool"; break; - case tensorrt_llm::DataType::kUINT8: return "uint8"; break; - case tensorrt_llm::DataType::kFP8: return "fp8"; break; - case tensorrt_llm::DataType::kBF16: return "bf16"; break; - case tensorrt_llm::DataType::kINT64: return "int64"; break; - case tensorrt_llm::DataType::kINT4: return "int4"; break; - case tensorrt_llm::DataType::kFP4: return "fp4"; break; + case nvinfer1::DataType::kFLOAT: return "fp32"; break; + case nvinfer1::DataType::kHALF: return "fp16"; break; + case nvinfer1::DataType::kINT8: return "int8"; break; + case nvinfer1::DataType::kINT32: return "int32"; break; + case nvinfer1::DataType::kBOOL: return "bool"; break; + case nvinfer1::DataType::kUINT8: return "uint8"; break; + case nvinfer1::DataType::kFP8: return "fp8"; break; + case nvinfer1::DataType::kBF16: return "bf16"; break; + case nvinfer1::DataType::kINT64: return "int64"; break; + case nvinfer1::DataType::kINT4: return "int4"; break; + case nvinfer1::DataType::kFP4: return "fp4"; break; default: throw std::runtime_error("Unsupported data type"); break; } diff --git a/cpp/include/tensorrt_llm/common/logger.h b/cpp/include/tensorrt_llm/common/logger.h index 9073d21f0088..d14b4c02e992 100644 --- a/cpp/include/tensorrt_llm/common/logger.h +++ b/cpp/include/tensorrt_llm/common/logger.h @@ -50,6 +50,8 @@ constexpr std::string_view formatModule(std::string_view module) return "deepgemm"; else if (module == "executor") return "executor"; + else if (module == "executor_worker") + return "exec_wkr"; else if (module == "flash_mla") return "flashmla"; else if (module == "kernels") @@ -58,6 +60,8 @@ constexpr std::string_view formatModule(std::string_view module) return "layers"; else if (module == "nanobind") return "nanobind"; + else if (module == "plugins") + return "plugins"; else if (module == "runtime") return "runtime"; else if (module == "testing") diff --git a/cpp/include/tensorrt_llm/common/tllmDataType.h b/cpp/include/tensorrt_llm/common/tllmDataType.h deleted file mode 100644 index 9e5567ce280e..000000000000 --- a/cpp/include/tensorrt_llm/common/tllmDataType.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/common/config.h" - -#include <cstdint> - -//! \file tllmDataType.h -//! -//! Standalone, TensorRT-free runtime types that replace the \c nvinfer1 -//! types the shared C++ core historically used as common currency: -//! \c tensorrt_llm::common::DataType and \c tensorrt_llm::common::Dims. -//! These are defined here so the retained tree (runtime, batch manager, executor, -//! kernels and the nanobind bridge) compiles and links without the TensorRT -//! library. They are hoisted into the \c tensorrt_llm namespace with -//! using-declarations because they are common currency across the whole tree -//! (\c tensorrt_llm::DataType / \c tensorrt_llm::Dims). -//! -//! The \c DataType enumerator values intentionally mirror the legacy -//! \c nvinfer1::DataType integer values so that previously-serialized executor -//! configs and KV-cache metadata remain byte-compatible. The \c Dims layout -//! mirrors the legacy \c nvinfer1::Dims (\c int32_t \c nbDims followed by -//! \c int64_t \c d[8]) for the same reason. - -TRTLLM_NAMESPACE_BEGIN - -namespace common -{ - -//! \brief Standalone data-type enum. Values mirror the legacy -//! \c nvinfer1::DataType for serialization/format compatibility. -enum class DataType : int32_t -{ - kFLOAT = 0, - kHALF = 1, - kINT8 = 2, - kINT32 = 3, - kBOOL = 4, - kUINT8 = 5, - kFP8 = 6, - kBF16 = 7, - kINT64 = 8, - kINT4 = 9, - kFP4 = 10, - kE8M0 = 11, -}; - -//! \brief Standalone dimensions type. Layout mirrors the legacy -//! \c nvinfer1::Dims (rank plus up to \c MAX_DIMS 64-bit extents) so serialized -//! shapes remain compatible. -class Dims -{ -public: - //! The maximum rank (number of dimensions) supported for a tensor. - static constexpr int32_t MAX_DIMS{8}; - - //! The rank (number of dimensions). - int32_t nbDims; - - //! The extent of each dimension. - int64_t d[MAX_DIMS]; -}; - -} // namespace common - -using common::DataType; -using common::Dims; - -TRTLLM_NAMESPACE_END diff --git a/cpp/include/tensorrt_llm/executor/dataTransceiverState.h b/cpp/include/tensorrt_llm/executor/dataTransceiverState.h index 578e53b81dbf..5067ae61dc83 100644 --- a/cpp/include/tensorrt_llm/executor/dataTransceiverState.h +++ b/cpp/include/tensorrt_llm/executor/dataTransceiverState.h @@ -18,7 +18,6 @@ #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/worldConfig.h" @@ -51,7 +50,7 @@ class CacheState final }; CacheState(ModelConfig modelConfig, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, AttentionType attentionType = AttentionType::kDEFAULT, int kvFactor = 2, bool enableBlockReuse = false, bool enablePartialReuse = false, bool hasIndexerKCache = false, SizeType32 indexerDimPerHead = 0, SizeType32 indexerKCacheQuantBlockSize = 128, bool indexerKCacheUseFp4 = false) @@ -72,7 +71,7 @@ class CacheState final CacheState(std::vector<SizeType32> nbKvHeadPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, SizeType32 tensorParallelism, SizeType32 pipelineParallelism, SizeType32 contextParallelism, - std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, AttentionType attentionType = AttentionType::kDEFAULT, int kvFactor = 2, bool enableAttentionDP = false, int DPrank = 0, int DPsize = 0, bool enableBlockReuse = false, bool enablePartialReuse = false, bool hasIndexerKCache = false, SizeType32 indexerDimPerHead = 0, SizeType32 indexerKCacheQuantBlockSize = 128, @@ -93,7 +92,7 @@ class CacheState final CacheState(SizeType32 nbAttentionLayers, SizeType32 nbKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, SizeType32 tensorParallelism, SizeType32 pipelineParallelism, SizeType32 contextParallelism, - std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, AttentionType attentionType = AttentionType::kDEFAULT, int kvFactor = 2, bool enableAttentionDP = false, int DPrank = 0, int DPsize = 0, bool enableBlockReuse = false, bool enablePartialReuse = false, bool hasIndexerKCache = false, SizeType32 indexerDimPerHead = 0, SizeType32 indexerKCacheQuantBlockSize = 128, @@ -239,8 +238,8 @@ class CacheState final RnnModelConfig mModelConfig; /// Number of RNN layers per pipeline parallelism rank. std::vector<SizeType32> mLayerNumPerPP; - tensorrt_llm::DataType mConvStateDataType; - tensorrt_llm::DataType mSsmStateDataType; + nvinfer1::DataType mConvStateDataType; + nvinfer1::DataType mSsmStateDataType; [[nodiscard]] bool operator==(RnnCacheState const& other) const noexcept { @@ -264,7 +263,7 @@ class CacheState final return mAttentionConfig; } - [[nodiscard]] tensorrt_llm::DataType const& getDataType() const + [[nodiscard]] nvinfer1::DataType const& getDataType() const { return mDataType; } @@ -309,7 +308,7 @@ class CacheState final } void setRnnConfig(RnnModelConfig rnnModelConfig, std::vector<SizeType32> rnnLayerNumPerPP, - tensorrt_llm::DataType convStateDataType, tensorrt_llm::DataType ssmStateDataType) + nvinfer1::DataType convStateDataType, nvinfer1::DataType ssmStateDataType) { mRnnCacheState = RnnCacheState{ std::move(rnnModelConfig), std::move(rnnLayerNumPerPP), convStateDataType, ssmStateDataType}; @@ -326,12 +325,12 @@ class CacheState final return getRnnCacheState().mModelConfig; } - [[nodiscard]] tensorrt_llm::DataType getConvStateDataType() const + [[nodiscard]] nvinfer1::DataType getConvStateDataType() const { return getRnnCacheState().mConvStateDataType; } - [[nodiscard]] tensorrt_llm::DataType getSsmStateDataType() const + [[nodiscard]] nvinfer1::DataType getSsmStateDataType() const { return getRnnCacheState().mSsmStateDataType; } @@ -396,7 +395,7 @@ class CacheState final friend class tensorrt_llm::executor::Serialization; ModelConfig mModelConfig; ParallelConfig mParallelConfig; - tensorrt_llm::DataType mDataType; + nvinfer1::DataType mDataType; AttentionConfig mAttentionConfig; bool mEnableBlockReuse{false}; bool mEnablePartialReuse{false}; @@ -619,22 +618,9 @@ class DataTransceiverState final return mCacheState.has_value() && mCacheState->hasRnnConfig(); } - /// @brief Set only when exported via CacheTransceiver::getSerializedDataTransceiverState: - /// transfers driven by such a state have no LlmRequest on the sender. - [[nodiscard]] bool isArbitraryTransferState() const noexcept - { - return mIsArbitraryTransferState; - } - - void setIsArbitraryTransferState(bool isArbitraryTransferState) noexcept - { - mIsArbitraryTransferState = isArbitraryTransferState; - } - [[nodiscard]] bool operator==(DataTransceiverState const& other) const noexcept { - return mCacheState == other.mCacheState && mCommState == other.mCommState - && mIsArbitraryTransferState == other.mIsArbitraryTransferState; + return mCacheState == other.mCacheState && mCommState == other.mCommState; } [[nodiscard]] std::string toString() const @@ -655,7 +641,6 @@ class DataTransceiverState final friend class Serialization; std::optional<kv_cache::CacheState> mCacheState; std::optional<kv_cache::CommState> mCommState; - bool mIsArbitraryTransferState{false}; }; } // namespace tensorrt_llm::executor diff --git a/cpp/include/tensorrt_llm/executor/disaggServerUtil.h b/cpp/include/tensorrt_llm/executor/disaggServerUtil.h new file mode 100644 index 000000000000..b68dce78738a --- /dev/null +++ b/cpp/include/tensorrt_llm/executor/disaggServerUtil.h @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/executor/executor.h" + +#include <cstdio> +#include <filesystem> +#include <memory> +#include <optional> +#include <vector> + +namespace tensorrt_llm::executor::disagg_executor +{ + +namespace texec = tensorrt_llm::executor; + +struct ResponseWithId +{ + + tensorrt_llm::executor::Response response; + IdType gid; + + ResponseWithId(tensorrt_llm::executor::Response&& response, IdType gid) + : response(std::move(response)) + , gid(gid) + { + } + + ResponseWithId(tensorrt_llm::executor::Response const& response, IdType gid) + : response(response) + , gid(gid) + { + } + + ResponseWithId(ResponseWithId&& other) noexcept + : response(std::move(other.response)) + , gid(other.gid) + { + other.gid = {}; + } + + ResponseWithId(ResponseWithId const& other) = default; + + ResponseWithId& operator=(ResponseWithId&& other) noexcept + { + if (this != &other) + { + response = std::move(other.response); + gid = other.gid; + other.gid = {}; + } + return *this; + } + + ResponseWithId& operator=(ResponseWithId const& other) + { + + if (this != &other) + { + response = other.response; + gid = other.gid; + } + return *this; + } + + ~ResponseWithId() = default; +}; + +class DisaggExecutorOrchestrator +{ +public: + /// @brief Constructs a DisaggExecutorOrchestrator object. + /// + /// @param ctxEnginePaths A vector of file paths to context engine files. + /// @param genEnginePaths A vector of file paths to generation engine files. + /// @param ctxExecutorConfigs A vector of ExecutorConfig for context executors. + /// @param genExecutorConfigs A vector of ExecutorConfig for generation executors. + /// @param hasContextAwaitThreads Whether or not there are threads that receive response for each generation + /// executor. + /// @param hasGenAwaitThreads Whether or not there are threads that receive response for each generation executor. + + DisaggExecutorOrchestrator(std::vector<std::filesystem::path> const& ctxEnginePaths, + std::vector<std::filesystem::path> const& genEnginePaths, + std::vector<executor::ExecutorConfig> const& ctxExecutorConfigs, + std::vector<executor::ExecutorConfig> const& genExecutorConfigs, bool hasContextAwaitThreads, + bool hasGenAwaitThreads); + + /// @brief Enqueue context-only requests to context executors. + /// @param requests A vector of context-only requests. + /// @param selectContextId The index of the context executor to use. If `std::nullopt`, the executor that has the + /// smallest number of inflight requests will be used. + /// @param batch If true,enqueue requests in same context executor.If false, will try to use a different executor + /// for each request. + /// @return A vector of global request ids, corresponding to the order of the requests in `requests`, the id + /// returned may be different from the request id in each executor. + [[nodiscard]] std::vector<IdType> enqueueContext(std::vector<texec::Request> const& requests, + std::optional<int> selectContextId = std::nullopt, bool batch = false); + + /// @brief Enqueue generation-only requests to generation executors. + /// @param requests A vector of generation-only requests. + /// @param globalRequestIds A vector of global request ids, corresponding to the order of the requests,and must be + /// the ids returned by the enqueueContext function. + /// @param selectGenIdx The index of the generation executor to use. If `std::nullopt`, the executor that has the + /// smallest number of inflight requests will be used. + /// @param batch If true,enqueue requests in same generation executor.If false, will try to use a different executor + /// for each request. + + void enqueueGeneration(std::vector<texec::Request> const& requests, std::vector<IdType> const& globalRequestIds, + std::optional<int> selectGenIdx = std::nullopt, bool batch = false); + + /// @brief Await for context responses + /// @param timeout The maximum time to wait for new responses + /// @param contextIdx The index of the context executor to use. If `std::nullopt`, return ready responses in all + /// context executors,if `hasContextAwaitThreads` is true, then this parameter must be std::nullopt. + /// @return A vector of responses with corresponding global request ids + + [[nodiscard]] std::vector<ResponseWithId> awaitContextResponses( + std::optional<std::chrono::milliseconds> const& timeout, std::optional<int> contextIdx = std::nullopt); + + /// @brief Await for generation responses + /// @param timeout The maximum time to wait for new responses. + /// @param genIdx The index of the generation executor to use. If `std::nullopt`, return ready responses in all + /// generation executors,if `hasGenAwaitThreads` is true, then this parameter must be std::nullopt. + /// @return A vector of responses with corresponding global request ids. + [[nodiscard]] std::vector<ResponseWithId> awaitGenerationResponses( + std::optional<std::chrono::milliseconds> const& timeout, std::optional<int> genIdx = std::nullopt); + + /// @brief Indicates if the current process is allowed to enqueueRequests + [[nodiscard]] bool canEnqueue() const; + + /// @brief Get context executors + [[nodiscard]] std::vector<std::unique_ptr<texec::Executor>> const& getContextExecutors() const; + + /// @brief Get generation executors + [[nodiscard]] std::vector<std::unique_ptr<texec::Executor>> const& getGenExecutors() const; + + ~DisaggExecutorOrchestrator(); + +private: + class Impl; + std::unique_ptr<Impl> mImpl; +}; +} // namespace tensorrt_llm::executor::disagg_executor diff --git a/cpp/include/tensorrt_llm/executor/executor.h b/cpp/include/tensorrt_llm/executor/executor.h index acc0efe18966..825b8ad75959 100644 --- a/cpp/include/tensorrt_llm/executor/executor.h +++ b/cpp/include/tensorrt_llm/executor/executor.h @@ -18,7 +18,6 @@ #include "tensorrt_llm/executor/tensor.h" #include "tensorrt_llm/executor/types.h" -#include "tensorrt_llm/executor/version.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/runtimeDefaults.h" @@ -51,11 +50,9 @@ namespace tensorrt_llm::executor using SizeType32 = tensorrt_llm::runtime::SizeType32; /// @brief Version of TRT-LLM -inline char const* version() noexcept -{ - return kTensorRtLlmVersion; -} +char const* version() noexcept; +class Model; class Serialization; class DataTransceiverState; @@ -1236,11 +1233,6 @@ class DebugConfig SizeType32 mDebugTensorsMaxIterations; }; -/// @brief Configuration for the orchestrator communication mode. -/// @deprecated Orchestrator mode is non-functional: the worker binary it spawned -/// (executorWorker) was removed together with the TensorRT backend. This class is -/// retained only for serialization and Python-binding compatibility and is a -/// candidate for removal in a follow-up (needs API-stability review). class OrchestratorConfig { public: @@ -1843,13 +1835,7 @@ using KVCacheEventData = std::variant<KVCacheCreatedData, KVCacheStoredData, KVC struct KVCacheEvent { KVCacheEvent(IdType eventId, KVCacheEventData data, SizeType32 windowSize, - std::optional<SizeType32> attentionDpRank = std::nullopt) - : eventId{eventId} - , data{std::move(data)} - , windowSize{windowSize} - , attentionDpRank{attentionDpRank} - { - } + std::optional<SizeType32> attentionDpRank = std::nullopt); /// @brief The unique id of this event IdType eventId; @@ -1877,6 +1863,119 @@ class KVCacheEventManager std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager> kvCacheManager; }; +/// @brief The executor is responsible for receiving new requests and sending responses, and running the inference +class Executor +{ + +public: + /// @brief + /// @param modelPath Path to the folder that defines the model to run + /// @param modelType The type of model + /// @param executorConfig The configuration for the executor + Executor(std::filesystem::path const& modelPath, ModelType modelType, ExecutorConfig const& executorConfig); + + Executor(std::filesystem::path const& encoderModelPath, std::filesystem::path const& decoderModelPath, + ModelType modelType, ExecutorConfig const& executorConfig); + + Executor(BufferView const& engineBuffer, std::string const& jsonConfigStr, ModelType modelType, + ExecutorConfig const& executorConfig, + std::optional<std::map<std::string, Tensor>> const& managedWeights = std::nullopt); + + Executor(BufferView const& encoderEngineBuffer, std::string const& encoderJsonConfigStr, + BufferView const& decoderEngineBuffer, std::string const& decoderJsonConfigStr, ModelType modelType, + ExecutorConfig const& executorConfig); + + Executor(std::shared_ptr<Model> model, ExecutorConfig const& executorConfig); + + Executor( + std::shared_ptr<Model> encoderModel, std::shared_ptr<Model> decoderModel, ExecutorConfig const& executorConfig); + + ~Executor(); + Executor(Executor const& executor) = delete; + Executor& operator=(Executor const& executor) = delete; + Executor(Executor&&) = default; + Executor& operator=(Executor&&) = default; + + /// @brief Enqueue a new request + /// @param request The LLM request which contains input tokens and request parameters + /// @return A unique id that identifies the request + [[nodiscard]] IdType enqueueRequest(Request const& request); + + /// @brief Enqueue a batch of request + [[nodiscard]] std::vector<IdType> enqueueRequests(std::vector<Request> const& requests); + + /// @brief Await for ready responses + /// + /// This overload awaits for any ready responses. In particular, if several requests + /// have been enqueued, this method will provide any ready responses without order guarantees. + /// @param timeout The maximum time to wait for new responses + /// @return A vector of responses + [[nodiscard]] std::vector<Response> awaitResponses( + std::optional<std::chrono::milliseconds> const& timeout = std::nullopt); + + /// @brief Await for ready responses + /// @param id A request id + /// @param timeout The maximum time to wait for new responses + /// @return A vector of responses + [[nodiscard]] std::vector<Response> awaitResponses( + IdType const& requestId, std::optional<std::chrono::milliseconds> const& timeout = std::nullopt); + + /// @brief Await for multiple ready responses + /// + /// A multiple ID request behaves as if awaitResponses(IdType, timeout) + /// were invoked on all IDs. The returned vector contains + /// a vector of responses per ID in the same order specified by the requestIds. + /// The same behaviour as awaitResponses(IdType, timeout) applies: + /// * Responses may be empty. + /// * If all responses have already been given for one of the requestIds, + /// then this method will hang unless a timeout is specified. + /// @param requestIds Ids requested + /// @param timeout The maximum time to wait for new responses + /// @return A vector of vector of responses + [[nodiscard]] std::vector<std::vector<Response>> awaitResponses( + std::vector<IdType> const& requestIds, std::optional<std::chrono::milliseconds> const& timeout = std::nullopt); + + /// @brief Get the number of ready responses + /// @param requestId An optional request id + /// @return The number of ready responses + [[nodiscard]] SizeType32 getNumResponsesReady(std::optional<IdType> const& requestId = std::nullopt) const; + + /// @brief Cancel the request with provided request id + /// @param id The request id for which to cancel the response + void cancelRequest(IdType requestId); + + /// @brief Signals the server to shutdown. + /// @details This call is blocking. Only returns when all requests have terminated or timeout has been reached + void shutdown(); + + /// @brief Returns the per-iterations statistics computed since last call to getLatestIterationStats. + /// Contains at most iterStatsMaxIterations iterations, or all iterations when set to -1. + /// @return Iteration stats + std::deque<IterationStats> getLatestIterationStats(); + + /// @brief Returns the request stats of each iteration computed since last call to getLatestRequestStats. + /// Contains at most requestStatsMaxIterations iterations, or all iterations when set to -1. + /// @return Request stats grouped by iterations + std::deque<RequestStatsPerIteration> getLatestRequestStats(); + + /// @brief Returns the debug tensors of each iteration computed since last call to getLatestDebugTensors. + /// Contains at most debugTensorsMaxIterations iterations. + /// @return Request debug tensors grouped by iterations + std::deque<DebugTensorsPerIteration> getLatestDebugTensors(); + + /// @brief Indicates if the current process is allowed to enqueueRequests + [[nodiscard]] bool canEnqueueRequests() const; + + /// @brief Indicates if the current process participates in this executor instance + [[nodiscard]] bool isParticipant() const; + + std::optional<std::shared_ptr<KVCacheEventManager>> getKVCacheEventManager() const; + +private: + class Impl; + std::unique_ptr<Impl> mImpl; +}; + /// @brief Class with utility functions to serialize statistics to json string class JsonSerialization { diff --git a/cpp/include/tensorrt_llm/executor/transferAgent.h b/cpp/include/tensorrt_llm/executor/transferAgent.h index 5f175a33723c..532f0ae70c44 100644 --- a/cpp/include/tensorrt_llm/executor/transferAgent.h +++ b/cpp/include/tensorrt_llm/executor/transferAgent.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -216,20 +216,12 @@ struct VmmDescSplitter /// For non-VRAM or addresses not in the map, descs pass through unchanged. [[nodiscard]] static MemoryDescs splitDescsWithRegionMap(MemoryDescs const& descs, VramRegionMap const& regionMap); - /// @brief Split paired src/dst descs at chunk boundaries, then coalesce contiguous pieces. - /// src is split by localRegionMap, dst is split by remoteRegionMap; each piece size is - /// min(srcPiece, dstPiece, remaining). Pairs are sorted by src address, and adjacent pieces - /// whose src AND dst are both contiguous (same deviceId) are merged — but a merged desc never - /// crosses a chunk boundary on either side, and never spans two distinct regions, so every - /// output desc stays within a single registered memory region. Merging requires region - /// metadata: a piece whose address misses the region map on either side is never merged, - /// because two unknown regions are indistinguishable and a merge could cross a chunk or - /// registration boundary. With no region metadata the result is split-only. Non-kVRAM descs - /// pass through unchanged (no region info is available to bound the merge). - /// @param enableCoalesce When false, only split at chunk boundaries without merging pieces. - [[nodiscard]] static std::pair<MemoryDescs, MemoryDescs> splitAndCoalesceTransferDescs(MemoryDescs const& srcDescs, - MemoryDescs const& dstDescs, VramRegionMap const& localRegionMap, VramRegionMap const& remoteRegionMap, - bool enableCoalesce = true); + /// @brief Split paired src/dst descs using local and remote region maps. + /// src is split by localRegionMap, dst is split by remoteRegionMap. + /// The final piece size is min(srcPiece, dstPiece, remaining). + [[nodiscard]] static std::pair<MemoryDescs, MemoryDescs> splitTransferDescsWithRegionMaps( + MemoryDescs const& srcDescs, MemoryDescs const& dstDescs, VramRegionMap const& localRegionMap, + VramRegionMap const& remoteRegionMap); /// @brief Split VRAM descs at VMM chunk boundaries detected via cuMemGetAddressRange. /// For cudaMalloc memory (single allocation), descs pass through unchanged. @@ -364,13 +356,6 @@ class TransferStatus virtual ~TransferStatus() = default; [[nodiscard]] virtual bool isCompleted() const = 0; virtual TransferState wait(int64_t timeout_ms = -1) const = 0; - - /// Release the backend transfer request handle. A true return means the backend accepted the handle release; it - /// does not prove remote memory quiescence. - [[nodiscard]] virtual bool release() - { - return false; - } }; struct BaseAgentConfig diff --git a/cpp/include/tensorrt_llm/plugins/api/tllmPlugin.h b/cpp/include/tensorrt_llm/plugins/api/tllmPlugin.h new file mode 100644 index 000000000000..e3d4613e3d00 --- /dev/null +++ b/cpp/include/tensorrt_llm/plugins/api/tllmPlugin.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <cstdint> +#include <mutex> + +// Forward declarations +namespace nvinfer1 +{ +class ILoggerFinder; +class ILogger; + +namespace v_1_0 +{ +class IPluginCreator; +class IPluginCreatorV3One; +class IPluginCreatorInterface; +} // namespace v_1_0 + +} // namespace nvinfer1 + +namespace tensorrt_llm::plugins::api +{ + +auto constexpr kDefaultNamespace = "tensorrt_llm"; + +class LoggerManager +{ +public: + //! Set the logger finder. + void setLoggerFinder(nvinfer1::ILoggerFinder* finder); + + //! Get the logger. + [[maybe_unused]] nvinfer1::ILogger* logger(); + + static LoggerManager& getInstance() noexcept; + + static nvinfer1::ILogger* defaultLogger() noexcept; + +private: + LoggerManager() = default; + + nvinfer1::ILoggerFinder* mLoggerFinder{nullptr}; + std::mutex mMutex; +}; +} // namespace tensorrt_llm::plugins::api + +extern "C" +{ + // This function is used for explicitly registering the TRT-LLM plugins and the default logger. + bool initTrtLlmPlugins(void* logger = tensorrt_llm::plugins::api::LoggerManager::defaultLogger(), + char const* libNamespace = tensorrt_llm::plugins::api::kDefaultNamespace); + + // The functions below are used by TensorRT to when loading a shared plugin library with automatic registering. + // see https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#generating-plugin-library + [[maybe_unused]] void setLoggerFinder([[maybe_unused]] nvinfer1::ILoggerFinder* finder); + [[maybe_unused]] nvinfer1::v_1_0::IPluginCreator* const* getPluginCreators(std::int32_t& nbCreators); + [[maybe_unused]] nvinfer1::v_1_0::IPluginCreatorInterface* const* getCreators(std::int32_t& nbCreators); +} diff --git a/cpp/include/tensorrt_llm/runtime/bufferManager.h b/cpp/include/tensorrt_llm/runtime/bufferManager.h index 321a96ba321e..8357443dc5ea 100644 --- a/cpp/include/tensorrt_llm/runtime/bufferManager.h +++ b/cpp/include/tensorrt_llm/runtime/bufferManager.h @@ -17,10 +17,10 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" +#include <NvInferRuntime.h> #include <cstring> #include <memory> @@ -62,63 +62,63 @@ class BufferManager } } - static auto constexpr kBYTE_TYPE = tensorrt_llm::DataType::kUINT8; + static auto constexpr kBYTE_TYPE = nvinfer1::DataType::kUINT8; //! \brief Allocates an `IBuffer` of the given size on the GPU, using cudaMallocAsync. - [[nodiscard]] IBufferPtr gpu(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE) const; + [[nodiscard]] IBufferPtr gpu(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE) const; //! \brief Allocates an `ITensor` of the given dimensions on the GPU, using cudaMallocAsync. - [[nodiscard]] ITensorPtr gpu(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE) const; + [[nodiscard]] ITensorPtr gpu(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE) const; //! \brief Allocates an `IBuffer` of the given size on the GPU, using cudaMalloc. - [[nodiscard]] static IBufferPtr gpuSync(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr gpuSync(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates an `ITensor` of the given dimensions on the GPU, using cudaMalloc. - [[nodiscard]] static ITensorPtr gpuSync(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr gpuSync(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates an `IBuffer` of the given size on the CPU. - [[nodiscard]] static IBufferPtr cpu(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr cpu(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates an `ITensor` of the given dimensions on the CPU. - [[nodiscard]] static ITensorPtr cpu(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr cpu(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates a pinned `IBuffer` of the given size on the CPU. - [[nodiscard]] static IBufferPtr pinned(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr pinned(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates a pinned `ITensor` of the given dimensions on the CPU. - [[nodiscard]] static ITensorPtr pinned(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr pinned(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates a pinned `IBuffer` of the given size on the CPU in the default memory pool. - [[nodiscard]] static IBufferPtr pinnedPool(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr pinnedPool(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates a pinned `ITensor` of the given dimensions on the CPU in the default memory pool. - [[nodiscard]] static ITensorPtr pinnedPool(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr pinnedPool(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates an `IBuffer` of the given size in UVM. - [[nodiscard]] static IBufferPtr managed(std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static IBufferPtr managed(std::size_t size, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates an `ITensor` of the given dimensions in UVM. - [[nodiscard]] static ITensorPtr managed(tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE); + [[nodiscard]] static ITensorPtr managed(nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE); //! \brief Allocates an `ITensor` of the given dimensions for NVLS - [[nodiscard]] static ITensorPtr ipcNvls(std::set<int> ranks, tensorrt_llm::Dims dims, tensorrt_llm::DataType type); + [[nodiscard]] static ITensorPtr ipcNvls(std::set<int> ranks, nvinfer1::Dims dims, nvinfer1::DataType type); //! \brief Allocates an `IBuffer` of the given size and memory type. [[nodiscard]] IBufferPtr allocate( - MemoryType memoryType, std::size_t size, tensorrt_llm::DataType type = kBYTE_TYPE) const; + MemoryType memoryType, std::size_t size, nvinfer1::DataType type = kBYTE_TYPE) const; //! \brief Allocates an `ITensor` of the given dimensions and memory type. [[nodiscard]] ITensorPtr allocate( - MemoryType memoryType, tensorrt_llm::Dims dims, tensorrt_llm::DataType type = kBYTE_TYPE) const; + MemoryType memoryType, nvinfer1::Dims dims, nvinfer1::DataType type = kBYTE_TYPE) const; //! \brief Create an empty `IBuffer` of the given memory type. It may be resized later. - [[nodiscard]] IBufferPtr emptyBuffer(MemoryType memoryType, tensorrt_llm::DataType type = kBYTE_TYPE) const + [[nodiscard]] IBufferPtr emptyBuffer(MemoryType memoryType, nvinfer1::DataType type = kBYTE_TYPE) const { return allocate(memoryType, 0, type); } //! \brief Create an empty `ITensor` of the given memory type. It may be reshaped later. - [[nodiscard]] ITensorPtr emptyTensor(MemoryType memoryType, tensorrt_llm::DataType type = kBYTE_TYPE) const + [[nodiscard]] ITensorPtr emptyTensor(MemoryType memoryType, nvinfer1::DataType type = kBYTE_TYPE) const { return allocate(memoryType, ITensor::makeShape({}), type); } @@ -167,7 +167,7 @@ class BufferManager //! \brief Copy `src` into a new `ITensor` with a potentially different memory type. template <typename T> - [[nodiscard]] ITensorPtr copyFrom(T* src, tensorrt_llm::Dims dims, MemoryType memoryType) const + [[nodiscard]] ITensorPtr copyFrom(T* src, nvinfer1::Dims dims, MemoryType memoryType) const { auto buffer = allocate(memoryType, dims, TRTDataType<std::remove_cv_t<T>>::value); copy(src, *buffer); @@ -176,7 +176,7 @@ class BufferManager //! \brief Copy `src` into a new `ITensor` with a potentially different memory type. template <typename T> - [[nodiscard]] ITensorPtr copyFrom(std::vector<T> const& src, tensorrt_llm::Dims dims, MemoryType memoryType) const + [[nodiscard]] ITensorPtr copyFrom(std::vector<T> const& src, nvinfer1::Dims dims, MemoryType memoryType) const { TLLM_CHECK_WITH_INFO(src.size() == ITensor::volumeNonNegative(dims), common::fmtstr("[TensorRT-LLM][ERROR] Incompatible size %lu and dims %s", src.size(), diff --git a/cpp/include/tensorrt_llm/runtime/decoderState.h b/cpp/include/tensorrt_llm/runtime/decoderState.h index ea2c767c0478..95d7ff0ffac9 100644 --- a/cpp/include/tensorrt_llm/runtime/decoderState.h +++ b/cpp/include/tensorrt_llm/runtime/decoderState.h @@ -18,7 +18,6 @@ #include "decodingInput.h" #include "decodingOutput.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/speculativeDecodingMode.h" @@ -53,7 +52,7 @@ class DecoderState //! @brief Setup buffers for the decoder excluding speculative decoding. void setup(SizeType32 maxNumSequences, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, - SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, tensorrt_llm::DataType dtype, + SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager); //! @brief Setup buffers for the cache indirection. @@ -63,7 +62,7 @@ class DecoderState //! @brief Setup buffers for speculative decoding. void setupSpeculativeDecoding(SpeculativeDecodingMode const& speculativeDecodingMode, - SizeType32 maxTokensPerEngineStep, tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, + SizeType32 maxTokensPerEngineStep, nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager); //! @brief Disable lookahead decoding. @@ -200,7 +199,7 @@ class DecoderState [[nodiscard]] DecodingOutput& getJointDecodingOutput() const; private: - void setupBuffers(tensorrt_llm::DataType dtype, BufferManager const& bufferManager); + void setupBuffers(nvinfer1::DataType dtype, BufferManager const& bufferManager); void reshapeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager); @@ -209,8 +208,8 @@ class DecoderState void reshapeCacheIndirectionBuffers( SizeType32 maxBatchSize, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow); - void setupSpeculativeDecodingBuffers(SpeculativeDecodingMode speculativeDecodingMode, tensorrt_llm::DataType dtype, - BufferManager const& bufferManager); + void setupSpeculativeDecodingBuffers( + SpeculativeDecodingMode speculativeDecodingMode, nvinfer1::DataType dtype, BufferManager const& bufferManager); void reshapeSpeculativeDecodingBuffers(SpeculativeDecodingMode const& speculativeDecodingMode, SizeType32 maxTokensPerEngineStep, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager); diff --git a/cpp/include/tensorrt_llm/runtime/gptDecoder.h b/cpp/include/tensorrt_llm/runtime/gptDecoder.h index 5a785e84fe75..7e0cc1bb56d2 100644 --- a/cpp/include/tensorrt_llm/runtime/gptDecoder.h +++ b/cpp/include/tensorrt_llm/runtime/gptDecoder.h @@ -22,7 +22,7 @@ #include "tensorrt_llm/runtime/decodingOutput.h" #include "tensorrt_llm/runtime/samplingConfig.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <curand_kernel.h> #include <memory> @@ -55,7 +55,7 @@ class IGptDecoder /// @param explicitDraftTokensDType is only used by ExplicitDraftTokens model to WAR the lack of bf16 decoder. virtual void setup(SamplingConfig const& samplingConfig, size_t batchSize, TensorConstPtr const& batchSlots, std::optional<DecodingOutput> const& output = std::nullopt, - std::optional<tensorrt_llm::DataType> explicitDraftTokensDType = std::nullopt, + std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt, std::optional<std::vector<TensorConstPtr>> const& lookaheadPrompt = std::nullopt, std::optional<std::vector<executor::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs = std::nullopt) = 0; @@ -70,7 +70,7 @@ class IGptDecoder std::optional<SamplingConfig> const& samplingConfig, SizeType32 batchSize, TensorConstPtr batchSlots) = 0; - static std::unique_ptr<IGptDecoder> create(executor::DecodingMode const& mode, tensorrt_llm::DataType dtype, + static std::unique_ptr<IGptDecoder> create(executor::DecodingMode const& mode, nvinfer1::DataType dtype, size_t maxNumSequences, size_t maxBeamWidth, size_t vocabSize, size_t vocabSizePadded, BufferManager::CudaStreamPtr const& stream, std::shared_ptr<SpeculativeDecodingModule const> const& speculativeDecodingModule = nullptr); @@ -90,7 +90,7 @@ class GptDecoder : public virtual IGptDecoder void setup(SamplingConfig const& samplingConfig, size_t batchSize, TensorConstPtr const& batchSlots, std::optional<DecodingOutput> const& output = std::nullopt, - std::optional<tensorrt_llm::DataType> explicitDraftTokensDType = std::nullopt, + std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt, std::optional<std::vector<TensorConstPtr>> const& lookaheadPrompt = std::nullopt, std::optional<std::vector<executor::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs = std::nullopt) override; @@ -121,17 +121,17 @@ class GptDecoder : public virtual IGptDecoder executor::DecodingMode mDecodingMode; }; -inline std::unique_ptr<IGptDecoder> IGptDecoder::create(executor::DecodingMode const& mode, - tensorrt_llm::DataType dtype, size_t maxNumSequences, size_t maxBeamWidth, size_t vocabSize, size_t vocabSizePadded, +inline std::unique_ptr<IGptDecoder> IGptDecoder::create(executor::DecodingMode const& mode, nvinfer1::DataType dtype, + size_t maxNumSequences, size_t maxBeamWidth, size_t vocabSize, size_t vocabSizePadded, BufferManager::CudaStreamPtr const& stream, std::shared_ptr<SpeculativeDecodingModule const> const& speculativeDecodingModule) { switch (dtype) { - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: return std::make_unique<GptDecoder<float>>( mode, maxNumSequences, maxBeamWidth, vocabSize, vocabSizePadded, stream, speculativeDecodingModule); - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: return std::make_unique<GptDecoder<half>>( mode, maxNumSequences, maxBeamWidth, vocabSize, vocabSizePadded, stream, speculativeDecodingModule); default: diff --git a/cpp/include/tensorrt_llm/runtime/gptDecoderBatched.h b/cpp/include/tensorrt_llm/runtime/gptDecoderBatched.h index d5447f441163..9fcd3262c8ca 100644 --- a/cpp/include/tensorrt_llm/runtime/gptDecoderBatched.h +++ b/cpp/include/tensorrt_llm/runtime/gptDecoderBatched.h @@ -16,7 +16,6 @@ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaEvent.h" #include "tensorrt_llm/runtime/cudaStream.h" @@ -49,7 +48,7 @@ class GptDecoderBatched : public IGptDecoderBatched explicit GptDecoderBatched(CudaStreamPtr stream); void setup(executor::DecodingMode const& mode, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) override; + nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) override; void disableLookahead(RequestVector const& genRequests, TensorPtr const& batchSlots) override; diff --git a/cpp/include/tensorrt_llm/runtime/iBuffer.h b/cpp/include/tensorrt_llm/runtime/iBuffer.h index bf63d3a0da7b..91d5cd739f32 100644 --- a/cpp/include/tensorrt_llm/runtime/iBuffer.h +++ b/cpp/include/tensorrt_llm/runtime/iBuffer.h @@ -22,7 +22,7 @@ #include "tensorrt_llm/kernels/kvCacheIndex.h" #include "tensorrt_llm/runtime/common.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cstdint> #ifdef ENABLE_FP8 @@ -88,13 +88,13 @@ struct MemoryTypeString<MemoryType::kPINNEDPOOL> }; //! \brief For converting a TensorRT data type to a C++ data type. -template <tensorrt_llm::DataType kDataType, bool kIsUnsigned = false, bool kIsPointer = false> +template <nvinfer1::DataType kDataType, bool kIsUnsigned = false, bool kIsPointer = false> struct DataTypeTraits { }; template <> -struct DataTypeTraits<tensorrt_llm::DataType::kFLOAT> +struct DataTypeTraits<nvinfer1::DataType::kFLOAT> { using type = float; static char constexpr name[] = "float"; @@ -102,7 +102,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kFLOAT> }; template <> -struct DataTypeTraits<tensorrt_llm::DataType::kHALF> +struct DataTypeTraits<nvinfer1::DataType::kHALF> { using type = half; static char constexpr name[] = "half"; @@ -110,7 +110,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kHALF> }; template <> -struct DataTypeTraits<tensorrt_llm::DataType::kINT8> +struct DataTypeTraits<nvinfer1::DataType::kINT8> { using type = std::int8_t; static char constexpr name[] = "int8"; @@ -118,7 +118,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kINT8> }; template <> -struct DataTypeTraits<tensorrt_llm::DataType::kINT32> +struct DataTypeTraits<nvinfer1::DataType::kINT32> { using type = std::int32_t; static char constexpr name[] = "int32"; @@ -126,7 +126,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kINT32> }; template <> -struct DataTypeTraits<tensorrt_llm::DataType::kINT64> +struct DataTypeTraits<nvinfer1::DataType::kINT64> { using type = std::int64_t; static char constexpr name[] = "int64"; @@ -134,7 +134,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kINT64> }; template <> -struct DataTypeTraits<tensorrt_llm::DataType::kINT32, true> +struct DataTypeTraits<nvinfer1::DataType::kINT32, true> { using type = std::uint32_t; static char constexpr name[] = "uint32"; @@ -142,7 +142,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kINT32, true> }; template <> -struct DataTypeTraits<tensorrt_llm::DataType::kINT64, true> +struct DataTypeTraits<nvinfer1::DataType::kINT64, true> { using type = std::uint64_t; static char constexpr name[] = "uint64"; @@ -150,7 +150,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kINT64, true> }; template <bool kUnsigned> -struct DataTypeTraits<tensorrt_llm::DataType::kBOOL, kUnsigned> +struct DataTypeTraits<nvinfer1::DataType::kBOOL, kUnsigned> { using type = bool; static char constexpr name[] = "bool"; @@ -158,7 +158,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kBOOL, kUnsigned> }; template <bool kUnsigned> -struct DataTypeTraits<tensorrt_llm::DataType::kUINT8, kUnsigned> +struct DataTypeTraits<nvinfer1::DataType::kUINT8, kUnsigned> { using type = std::uint8_t; static char constexpr name[] = "uint8"; @@ -167,7 +167,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kUINT8, kUnsigned> #ifdef ENABLE_BF16 template <> -struct DataTypeTraits<tensorrt_llm::DataType::kBF16> +struct DataTypeTraits<nvinfer1::DataType::kBF16> { using type = __nv_bfloat16; static char constexpr name[] = "bfloat16"; @@ -177,7 +177,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kBF16> #ifdef ENABLE_FP8 template <> -struct DataTypeTraits<tensorrt_llm::DataType::kFP8> +struct DataTypeTraits<nvinfer1::DataType::kFP8> { using type = __nv_fp8_e4m3; static char constexpr name[] = "fp8"; @@ -185,7 +185,7 @@ struct DataTypeTraits<tensorrt_llm::DataType::kFP8> }; #endif -template <tensorrt_llm::DataType kDataType, bool kUnsigned> +template <nvinfer1::DataType kDataType, bool kUnsigned> struct DataTypeTraits<kDataType, kUnsigned, true> { using type = typename DataTypeTraits<kDataType, kUnsigned, false>::type*; @@ -193,26 +193,26 @@ struct DataTypeTraits<kDataType, kUnsigned, true> static auto constexpr size = sizeof(type); }; -//! \brief A wrapper around `tensorrt_llm::DataType` that provides a support for pointer types. +//! \brief A wrapper around `nvinfer1::DataType` that provides a support for pointer types. class BufferDataType { public: constexpr BufferDataType( // NOLINT(*-explicit-constructor) - tensorrt_llm::DataType dataType, bool _unsigned = false, bool pointer = false) + nvinfer1::DataType dataType, bool _unsigned = false, bool pointer = false) : mDataType{dataType} , mUnsigned{_unsigned} , mPointer{pointer} { } - static auto constexpr kTrtPointerType = tensorrt_llm::DataType::kINT64; + static auto constexpr kTrtPointerType = nvinfer1::DataType::kINT64; - constexpr operator tensorrt_llm::DataType() const noexcept // NOLINT(*-explicit-constructor) + constexpr operator nvinfer1::DataType() const noexcept // NOLINT(*-explicit-constructor) { return mPointer ? kTrtPointerType : mDataType; } - [[nodiscard]] constexpr tensorrt_llm::DataType getDataType() const noexcept + [[nodiscard]] constexpr nvinfer1::DataType getDataType() const noexcept { return mDataType; } @@ -226,24 +226,24 @@ class BufferDataType { switch (mDataType) { - case tensorrt_llm::DataType::kBOOL: [[fallthrough]]; - case tensorrt_llm::DataType::kUINT8: return true; + case nvinfer1::DataType::kBOOL: [[fallthrough]]; + case nvinfer1::DataType::kUINT8: return true; default: return mUnsigned; } } [[nodiscard]] constexpr std::size_t getSize() const noexcept { - return tensorrt_llm::common::getDTypeSize(static_cast<tensorrt_llm::DataType>(*this)); + return tensorrt_llm::common::getDTypeSize(static_cast<nvinfer1::DataType>(*this)); } [[nodiscard]] constexpr std::size_t getSizeInBits() const noexcept { - return tensorrt_llm::common::getDTypeSizeInBits(static_cast<tensorrt_llm::DataType>(*this)); + return tensorrt_llm::common::getDTypeSizeInBits(static_cast<nvinfer1::DataType>(*this)); } private: - tensorrt_llm::DataType mDataType; + nvinfer1::DataType mDataType; bool mUnsigned; bool mPointer; }; @@ -257,62 +257,62 @@ struct TRTDataType template <> struct TRTDataType<float> { - static constexpr auto value = tensorrt_llm::DataType::kFLOAT; + static constexpr auto value = nvinfer1::DataType::kFLOAT; }; template <> struct TRTDataType<half> { - static constexpr auto value = tensorrt_llm::DataType::kHALF; + static constexpr auto value = nvinfer1::DataType::kHALF; }; template <> struct TRTDataType<std::int8_t> { - static constexpr auto value = tensorrt_llm::DataType::kINT8; + static constexpr auto value = nvinfer1::DataType::kINT8; }; template <> struct TRTDataType<std::int32_t> { - static constexpr auto value = tensorrt_llm::DataType::kINT32; + static constexpr auto value = nvinfer1::DataType::kINT32; }; template <> struct TRTDataType<std::uint32_t> { - static constexpr auto value = BufferDataType{tensorrt_llm::DataType::kINT32, true}; + static constexpr auto value = BufferDataType{nvinfer1::DataType::kINT32, true}; }; template <> struct TRTDataType<std::int64_t> { - static constexpr auto value = tensorrt_llm::DataType::kINT64; + static constexpr auto value = nvinfer1::DataType::kINT64; }; template <> struct TRTDataType<std::uint64_t> { - static constexpr auto value = BufferDataType{tensorrt_llm::DataType::kINT64, true}; + static constexpr auto value = BufferDataType{nvinfer1::DataType::kINT64, true}; }; template <> struct TRTDataType<bool> { - static constexpr auto value = tensorrt_llm::DataType::kBOOL; + static constexpr auto value = nvinfer1::DataType::kBOOL; }; template <> struct TRTDataType<std::uint8_t> { - static constexpr auto value = tensorrt_llm::DataType::kUINT8; + static constexpr auto value = nvinfer1::DataType::kUINT8; }; #ifdef ENABLE_BF16 template <> struct TRTDataType<__nv_bfloat16> { - static constexpr auto value = tensorrt_llm::DataType::kBF16; + static constexpr auto value = nvinfer1::DataType::kBF16; }; #endif @@ -320,7 +320,7 @@ struct TRTDataType<__nv_bfloat16> template <> struct TRTDataType<__nv_fp8_e4m3> { - static constexpr auto value = tensorrt_llm::DataType::kFP8; + static constexpr auto value = nvinfer1::DataType::kFP8; }; #endif @@ -380,7 +380,7 @@ class IBuffer using SharedPtr = std::shared_ptr<IBuffer>; using UniqueConstPtr = std::unique_ptr<IBuffer const>; using SharedConstPtr = std::shared_ptr<IBuffer const>; - using DataType = tensorrt_llm::DataType; + using DataType = nvinfer1::DataType; //! //! \brief Returns a pointer to underlying array. diff --git a/cpp/include/tensorrt_llm/runtime/iGptDecoderBatched.h b/cpp/include/tensorrt_llm/runtime/iGptDecoderBatched.h index b664bc007f0e..ab55b754f9be 100644 --- a/cpp/include/tensorrt_llm/runtime/iGptDecoderBatched.h +++ b/cpp/include/tensorrt_llm/runtime/iGptDecoderBatched.h @@ -16,7 +16,6 @@ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/cudaEvent.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -52,7 +51,7 @@ class IGptDecoderBatched //! @brief Setup the decoder before calling `forward()` virtual void setup(executor::DecodingMode const& mode, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) + nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) = 0; //! @brief Disable Lookahead decoding. diff --git a/cpp/include/tensorrt_llm/runtime/iTensor.h b/cpp/include/tensorrt_llm/runtime/iTensor.h index a85291dd8263..eb5c10eeb691 100644 --- a/cpp/include/tensorrt_llm/runtime/iTensor.h +++ b/cpp/include/tensorrt_llm/runtime/iTensor.h @@ -20,7 +20,7 @@ #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <algorithm> #include <cstdint> @@ -33,6 +33,11 @@ #include <string> #include <type_traits> +namespace nvinfer1 +{ +class IExecutionContext; +} + namespace tensorrt_llm::runtime { @@ -45,7 +50,7 @@ class ITensor : virtual public IBuffer using SharedPtr = std::shared_ptr<ITensor>; using UniqueConstPtr = std::unique_ptr<ITensor const>; using SharedConstPtr = std::shared_ptr<ITensor const>; - using Shape = tensorrt_llm::Dims; + using Shape = nvinfer1::Dims; using DimType64 = std::remove_reference_t<decltype(Shape::d[0])>; using TensorMap = runtime::StringPtrMap<runtime::ITensor>; @@ -347,9 +352,9 @@ class ITensor : virtual public IBuffer //! \param shape The shape of the tensor. //! \param capacity The capacity of the buffer. //! \return An `ITensor`. - static UniquePtr wrap(void* data, tensorrt_llm::DataType type, Shape const& shape, std::size_t capacity); + static UniquePtr wrap(void* data, nvinfer1::DataType type, Shape const& shape, std::size_t capacity); - static UniquePtr wrap(void* data, tensorrt_llm::DataType type, Shape const& shape) + static UniquePtr wrap(void* data, nvinfer1::DataType type, Shape const& shape) { return wrap(data, type, shape, volumeNonNegative(shape)); } diff --git a/cpp/include/tensorrt_llm/runtime/lookaheadBuffers.h b/cpp/include/tensorrt_llm/runtime/lookaheadBuffers.h index 26c6e3886be4..ecaa439f2d52 100644 --- a/cpp/include/tensorrt_llm/runtime/lookaheadBuffers.h +++ b/cpp/include/tensorrt_llm/runtime/lookaheadBuffers.h @@ -17,9 +17,9 @@ #pragma once #include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" #include "tensorrt_llm/runtime/worldConfig.h" namespace tensorrt_llm::runtime @@ -37,4 +37,47 @@ class LookaheadDecodingBuffers TensorPtr positionIds; }; +class LookaheadRuntimeBuffers +{ +public: + using TensorPtr = ITensor::SharedPtr; + using TensorMap = StringPtrMap<ITensor>; + + LookaheadRuntimeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, BufferManager const& manager, + ModelConfig const& modelConfig, WorldConfig const& worldConfig, executor::DecodingConfig const& decodingConfig, + TllmRuntime const& runtime); + + void setFromInputs(SizeType32 numCtxSequences, SizeType32 numGenSequences, ITensor const& requestTypes, + ITensor const& seqSlots, LookaheadDecodingBuffers const& decoderLookaheadBuffers, TllmRuntime const& runtime, + ModelConfig const& modelConfig, WorldConfig const& worldConfig) const; + + void reshape(SizeType32 numCtxSequences, SizeType32 numGenSequences, SizeType32 tokensPerStep); + + void insertInputTensors(TensorMap& inputBuffers, TensorMap& outputBuffers, WorldConfig const& worldConfig) const; + + void enableLookaheadDecoding(SizeType32 maxBatchSize, SizeType32 tokensPerStep); + + void disableLookaheadDecoding(); + +public: + TensorPtr cumSumLength; // [1] the cumulative sum of generation length, on pinned + TensorPtr packedMasksDevice; // [forwardBatchSize, tokensPerStep, numPackedMasks], on gpu + TensorPtr generationLengthsDevice; // [forwardBatchSize], on gpu + TensorPtr positionOffsetsDevice; // [forwardBatchSize, tokensPerStep], on gpu + TensorPtr positionIdsDevice; // [forwardBatchSize, tokensPerStep], on gpu + + TensorPtr packedMaskHost; + TensorPtr generationLengthsHost; + TensorPtr positionOffsetsHost; + TensorPtr positionIdsHost; + + TensorPtr packedMaskHostCopy; + TensorPtr generationLengthsHostCopy; + TensorPtr positionOffsetsHostCopy; + TensorPtr positionIdsHostCopy; + TensorPtr useSpecDecoding; + + TensorPtr batchSlotsHostCopy; +}; + } // namespace tensorrt_llm::runtime diff --git a/cpp/include/tensorrt_llm/runtime/loraCache.h b/cpp/include/tensorrt_llm/runtime/loraCache.h index eb4ef57494ea..1d242cdc80c5 100644 --- a/cpp/include/tensorrt_llm/runtime/loraCache.h +++ b/cpp/include/tensorrt_llm/runtime/loraCache.h @@ -25,6 +25,8 @@ #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/worldConfig.h" +#include <NvInferRuntime.h> + #include <deque> #include <list> #include <map> diff --git a/cpp/include/tensorrt_llm/runtime/loraCachePageManagerConfig.h b/cpp/include/tensorrt_llm/runtime/loraCachePageManagerConfig.h index cf1b1a6aac18..a995304e94ec 100644 --- a/cpp/include/tensorrt_llm/runtime/loraCachePageManagerConfig.h +++ b/cpp/include/tensorrt_llm/runtime/loraCachePageManagerConfig.h @@ -20,7 +20,7 @@ #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iBuffer.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <ostream> #include <sstream> @@ -36,7 +36,7 @@ namespace tensorrt_llm::runtime class LoraCachePageManagerConfig { public: - explicit constexpr LoraCachePageManagerConfig(runtime::MemoryType memType, tensorrt_llm::DataType dType, + explicit constexpr LoraCachePageManagerConfig(runtime::MemoryType memType, nvinfer1::DataType dType, SizeType32 totalNumPages, SizeType32 maxPagesPerBlock, SizeType32 slotsPerPage, SizeType32 pageWidth, SizeType32 numCopyStreams) : mMemoryType(memType) @@ -59,12 +59,12 @@ class LoraCachePageManagerConfig mMemoryType = memoryType; } - [[nodiscard]] tensorrt_llm::DataType constexpr getDataType() const noexcept + [[nodiscard]] nvinfer1::DataType constexpr getDataType() const noexcept { return mDataType; } - void constexpr setDataType(tensorrt_llm::DataType const& dtype) noexcept + void constexpr setDataType(nvinfer1::DataType const& dtype) noexcept { mDataType = dtype; } @@ -131,7 +131,7 @@ class LoraCachePageManagerConfig private: runtime::MemoryType mMemoryType; - tensorrt_llm::DataType mDataType; + nvinfer1::DataType mDataType; /* * Number cache pages in the cache. @@ -154,7 +154,7 @@ inline std::ostream& operator<<(std::ostream& os, LoraCachePageManagerConfig con { os << "{" << "memoryType=" << static_cast<typename std::underlying_type<runtime::MemoryType>::type>(c.getMemoryType()) - << " dataType=" << static_cast<typename std::underlying_type<tensorrt_llm::DataType>::type>(c.getDataType()) + << " dataType=" << static_cast<typename std::underlying_type<nvinfer1::DataType>::type>(c.getDataType()) << " totalNumPages=" << c.getTotalNumPages() << " maxPagesPerBlock=" << c.getMaxPagesPerBlock() << " slotsPerPage=" << c.getSlotsPerPage() << " pageWidth=" << c.getPageWidth() << " initToZero=" << c.getInitToZero() << "}"; diff --git a/cpp/include/tensorrt_llm/runtime/modelConfig.h b/cpp/include/tensorrt_llm/runtime/modelConfig.h index b5f18da07f3b..5bfe7bce9d58 100644 --- a/cpp/include/tensorrt_llm/runtime/modelConfig.h +++ b/cpp/include/tensorrt_llm/runtime/modelConfig.h @@ -23,7 +23,7 @@ #include "tensorrt_llm/runtime/speculativeDecodingMode.h" #include "tensorrt_llm/runtime/speculativeDecodingModule.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <array> namespace tensorrt_llm::runtime @@ -101,7 +101,7 @@ class ModelConfig }; explicit ModelConfig(SizeType32 vocabSize, SizeType32 nbLayers, SizeType32 nbAttentionLayers, - SizeType32 nbRnnLayers, SizeType32 nbHeads, SizeType32 hiddenSize, tensorrt_llm::DataType dtype) + SizeType32 nbRnnLayers, SizeType32 nbHeads, SizeType32 hiddenSize, nvinfer1::DataType dtype) : mVocabSize(vocabSize) , mNbLayers(nbLayers) , mNbAttentionLayers(nbAttentionLayers) @@ -137,7 +137,7 @@ class ModelConfig , mUsePositionEmbedding(false) , mUseTokenTypeEmbedding(false) , mSpeculativeDecodingMode(SpeculativeDecodingMode::None()) - , mLogitsDtype(tensorrt_llm::DataType::kFLOAT) + , mLogitsDtype(nvinfer1::DataType::kFLOAT) , mUseShapeInference(true) , mManageWeightsType(ManageWeightsType::kDisabled) , mSkipCrossAttnBlocks(false) @@ -331,7 +331,7 @@ class ModelConfig mSizePerHead = sizePerHead; } - [[nodiscard]] tensorrt_llm::DataType constexpr getDataType() const noexcept + [[nodiscard]] nvinfer1::DataType constexpr getDataType() const noexcept { return mDataType; } @@ -735,20 +735,20 @@ class ModelConfig resetSpeculativeDecodingModule(); } - [[nodiscard]] tensorrt_llm::DataType getKvDataType() const + [[nodiscard]] nvinfer1::DataType getKvDataType() const { if (getQuantMode().hasFp8KvCache()) { - return tensorrt_llm::DataType::kFP8; + return nvinfer1::DataType::kFP8; } if (getQuantMode().hasInt8KvCache()) { - return tensorrt_llm::DataType::kINT8; + return nvinfer1::DataType::kINT8; } else if (getQuantMode().hasFp4KvCache()) { #ifdef ENABLE_FP4 - return tensorrt_llm::DataType::kFP4; + return nvinfer1::DataType::kFP4; #else throw std::runtime_error("Model has FP4 KV cache, but TRT-LLM was not compiled with FP4 enabled."); #endif @@ -800,22 +800,22 @@ class ModelConfig return mSpeculativeDecodingMode; } - void setLogitsDtype(tensorrt_llm::DataType inputDtype) noexcept + void setLogitsDtype(nvinfer1::DataType inputDtype) noexcept { mLogitsDtype = inputDtype; } - [[nodiscard]] tensorrt_llm::DataType constexpr getLogitsDtype() const noexcept + [[nodiscard]] nvinfer1::DataType constexpr getLogitsDtype() const noexcept { return mLogitsDtype; } - void setGemmAllReduceDtype(tensorrt_llm::DataType inputDtype) noexcept + void setGemmAllReduceDtype(nvinfer1::DataType inputDtype) noexcept { mGemmAllReduceDtype = inputDtype; } - [[nodiscard]] tensorrt_llm::DataType constexpr getGemmAllReduceDtype() const noexcept + [[nodiscard]] nvinfer1::DataType constexpr getGemmAllReduceDtype() const noexcept { return mGemmAllReduceDtype; } @@ -945,10 +945,10 @@ class ModelConfig SizeType32 mNbHeads; SizeType32 mHiddenSize; SizeType32 mSizePerHead; - tensorrt_llm::DataType mDataType; + nvinfer1::DataType mDataType; bool mUseGptAttentionPlugin; bool mUseGemmAllReducePlugin; - tensorrt_llm::DataType mGemmAllReduceDtype; + nvinfer1::DataType mGemmAllReduceDtype; bool mUseMambaConv1dPlugin; bool mInputPacked; bool mPagedState; @@ -998,7 +998,7 @@ class ModelConfig SpeculativeDecodingMode mSpeculativeDecodingMode; // Logits datatype - tensorrt_llm::DataType mLogitsDtype; + nvinfer1::DataType mLogitsDtype; bool mUseShapeInference; ManageWeightsType mManageWeightsType; std::string mModelName; diff --git a/cpp/include/tensorrt_llm/runtime/rawEngine.h b/cpp/include/tensorrt_llm/runtime/rawEngine.h new file mode 100644 index 000000000000..b219cbe03382 --- /dev/null +++ b/cpp/include/tensorrt_llm/runtime/rawEngine.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/executor/tensor.h" + +#include <NvInferRuntime.h> +#include <filesystem> +#include <map> +#include <optional> + +namespace tensorrt_llm::runtime +{ + +class RawEngine +{ +public: + enum Type + { + FilePath, + AddressWithSize, + HostMemory + }; + + explicit RawEngine(std::filesystem::path enginePath) noexcept + : mType(FilePath) + , mEnginePath(std::move(enginePath)) + { + } + + explicit RawEngine(void const* engineAddr, std::size_t engineSize) noexcept + : mType(AddressWithSize) + , mEngineAddr(engineAddr) + , mEngineSize(engineSize) + { + } + + explicit RawEngine(nvinfer1::IHostMemory const* engineBuffer) noexcept + : mType(HostMemory) + , mEngineBuffer(engineBuffer) + { + } + + [[nodiscard]] Type getType() const + { + return mType; + } + + [[nodiscard]] std::filesystem::path getPath() const + { + TLLM_CHECK(mEnginePath.has_value()); + return mEnginePath.value(); + } + + [[nodiscard]] std::optional<std::filesystem::path> getPathOpt() const + { + return mEnginePath; + } + + void setPath(std::filesystem::path enginePath) + { + mEnginePath = std::move(enginePath); + } + + [[nodiscard]] std::optional<std::map<std::string, tensorrt_llm::executor::Tensor>> const& + getManagedWeightsMapOpt() const + { + return mManagedWeightsMap; + } + + void setManagedWeightsMap(std::map<std::string, tensorrt_llm::executor::Tensor> managedWeightsMap) + { + mManagedWeightsMap = std::move(managedWeightsMap); + } + + [[nodiscard]] void const* getAddress() const + { + TLLM_CHECK(mType == AddressWithSize); + return mEngineAddr; + } + + [[nodiscard]] std::size_t getSize() const + { + TLLM_CHECK(mType == AddressWithSize); + return mEngineSize; + } + + [[nodiscard]] nvinfer1::IHostMemory const* getHostMemory() const + { + TLLM_CHECK(mType == HostMemory); + return mEngineBuffer; + } + +private: + Type mType; + std::optional<std::filesystem::path> mEnginePath; + + struct + { + void const* mEngineAddr{}; + std::size_t mEngineSize{}; + }; + + nvinfer1::IHostMemory const* mEngineBuffer{}; + std::optional<std::map<std::string, tensorrt_llm::executor::Tensor>> mManagedWeightsMap; +}; + +} // namespace tensorrt_llm::runtime diff --git a/cpp/include/tensorrt_llm/runtime/tllmLogger.h b/cpp/include/tensorrt_llm/runtime/tllmLogger.h new file mode 100644 index 000000000000..dd3806ec5242 --- /dev/null +++ b/cpp/include/tensorrt_llm/runtime/tllmLogger.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <NvInferRuntime.h> + +namespace tensorrt_llm::runtime +{ + +class TllmLogger : public nvinfer1::ILogger +{ +public: + void log(Severity severity, nvinfer1::AsciiChar const* msg) noexcept override; + + Severity getLevel(); + + void setLevel(Severity level); +}; + +} // namespace tensorrt_llm::runtime diff --git a/cpp/include/tensorrt_llm/runtime/utils/debugUtils.h b/cpp/include/tensorrt_llm/runtime/utils/debugUtils.h index 3f6c307a3cde..68064d74c7e2 100644 --- a/cpp/include/tensorrt_llm/runtime/utils/debugUtils.h +++ b/cpp/include/tensorrt_llm/runtime/utils/debugUtils.h @@ -15,7 +15,6 @@ */ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" namespace tensorrt_llm::runtime::utils @@ -25,7 +24,7 @@ template <typename T> bool tensorHasInvalid(ITensor const& tensor, BufferManager const& manager, std::string const& infoStr); bool tensorHasInvalid( - size_t M, size_t K, tensorrt_llm::DataType type, void const* data, cudaStream_t stream, std::string const& infoStr); + size_t M, size_t K, nvinfer1::DataType type, void const* data, cudaStream_t stream, std::string const& infoStr); int stallStream( char const* name, std::optional<cudaStream_t> stream = std::nullopt, std::optional<int> delay = std::nullopt); diff --git a/cpp/include/tensorrt_llm/runtime/worldConfig.h b/cpp/include/tensorrt_llm/runtime/worldConfig.h index 272b0fec5ada..9ff2d0970df7 100644 --- a/cpp/include/tensorrt_llm/runtime/worldConfig.h +++ b/cpp/include/tensorrt_llm/runtime/worldConfig.h @@ -18,6 +18,7 @@ #include "tensorrt_llm/runtime/common.h" +#include <NvInferRuntime.h> #include <optional> #include <vector> diff --git a/cpp/micro_benchmarks/mixtureOfExpertsBackendBenchmarkFixture.h b/cpp/micro_benchmarks/mixtureOfExpertsBackendBenchmarkFixture.h index 354e11184f8d..f466a65e871f 100644 --- a/cpp/micro_benchmarks/mixtureOfExpertsBackendBenchmarkFixture.h +++ b/cpp/micro_benchmarks/mixtureOfExpertsBackendBenchmarkFixture.h @@ -31,7 +31,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/cutlass_kernels/cutlass_preprocessors.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" @@ -381,44 +380,44 @@ class MixtureOfExpertsBenchmark : public ::benchmark::Fixture int64_t mNumExpertsPerNode{}; int64_t mK{}; - constexpr static tensorrt_llm::DataType toDTypeID() + constexpr static nvinfer1::DataType toDTypeID() { if (FP8 || WFP4AFP8) - return tensorrt_llm::DataType::kFP8; + return nvinfer1::DataType::kFP8; if (NVFP4) - return tensorrt_llm::DataType::kFP4; + return nvinfer1::DataType::kFP4; if (INT_QUANT && INT4) - return tensorrt_llm::DataType::kINT4; + return nvinfer1::DataType::kINT4; if (INT_QUANT) - return tensorrt_llm::DataType::kINT8; + return nvinfer1::DataType::kINT8; if (std::is_same_v<DataType, float>) - return tensorrt_llm::DataType::kFLOAT; + return nvinfer1::DataType::kFLOAT; if (std::is_same_v<DataType, half>) - return tensorrt_llm::DataType::kHALF; + return nvinfer1::DataType::kHALF; #ifdef ENABLE_BF16 if (std::is_same_v<DataType, nv_bfloat16>) - return tensorrt_llm::DataType::kBF16; + return nvinfer1::DataType::kBF16; #endif TLLM_THROW("Unrecognised format"); }; - constexpr static tensorrt_llm::DataType toWTypeID() + constexpr static nvinfer1::DataType toWTypeID() { if (FP8) - return tensorrt_llm::DataType::kFP8; + return nvinfer1::DataType::kFP8; if (NVFP4 || WFP4AFP8) - return tensorrt_llm::DataType::kFP4; + return nvinfer1::DataType::kFP4; if (INT_QUANT && INT4) - return tensorrt_llm::DataType::kINT4; + return nvinfer1::DataType::kINT4; if (INT_QUANT) - return tensorrt_llm::DataType::kINT8; + return nvinfer1::DataType::kINT8; if (std::is_same_v<DataType, float>) - return tensorrt_llm::DataType::kFLOAT; + return nvinfer1::DataType::kFLOAT; if (std::is_same_v<DataType, half>) - return tensorrt_llm::DataType::kHALF; + return nvinfer1::DataType::kHALF; #ifdef ENABLE_BF16 if (std::is_same_v<DataType, nv_bfloat16>) - return tensorrt_llm::DataType::kBF16; + return nvinfer1::DataType::kBF16; #endif TLLM_THROW("Unrecognised format"); }; @@ -428,31 +427,31 @@ class MixtureOfExpertsBenchmark : public ::benchmark::Fixture { if constexpr (std::is_same_v<T, SafeFP8>) { - return tensorrt_llm::DataType::kFP8; + return nvinfer1::DataType::kFP8; } else if constexpr (std::is_same_v<T, SafeFP4>) { - return tensorrt_llm::DataType::kFP4; + return nvinfer1::DataType::kFP4; } else if constexpr (std::is_same_v<T, uint8_t>) { - return tensorrt_llm::DataType::kINT8; + return nvinfer1::DataType::kINT8; } else if constexpr (std::is_same_v<T, cutlass::uint4b_t>) { - return tensorrt_llm::DataType::kINT4; + return nvinfer1::DataType::kINT4; } else if constexpr (std::is_same_v<T, nv_bfloat16>) { - return tensorrt_llm::DataType::kBF16; + return nvinfer1::DataType::kBF16; } else if constexpr (std::is_same_v<T, half>) { - return tensorrt_llm::DataType::kHALF; + return nvinfer1::DataType::kHALF; } else if constexpr (std::is_same_v<T, float>) { - return tensorrt_llm::DataType::kFLOAT; + return nvinfer1::DataType::kFLOAT; } else { diff --git a/cpp/tensorrt_llm/CMakeLists.txt b/cpp/tensorrt_llm/CMakeLists.txt index 5f5e37836a05..afd2d3a1f415 100644 --- a/cpp/tensorrt_llm/CMakeLists.txt +++ b/cpp/tensorrt_llm/CMakeLists.txt @@ -144,6 +144,8 @@ add_subdirectory(common) add_subdirectory(kernels) add_subdirectory(layers) add_subdirectory(runtime) +add_subdirectory(testing) +add_subdirectory(executor_worker) set(BATCH_MANAGER_TARGET tensorrt_llm_batch_manager_static) set(BATCH_MANAGER_TARGET_ARCH ${TARGET_ARCH}) @@ -174,6 +176,7 @@ set(TRTLLM_LINK_LIBS ${CUBLASLT_LIB} ${CURAND_LIB} ${CMAKE_DL_LIBS} + ${TRT_LIB} common_src kernels_src flash_mla_src @@ -196,6 +199,7 @@ set(TRTLLM_LINK_LIBS cute_dsl_src layers_src runtime_src + testing_src compressorKernels_src mhcKernels_src userbuffers_src @@ -306,3 +310,5 @@ endif() if(BUILD_FLASH_MLA) add_subdirectory(flash_mla) endif() + +add_subdirectory(plugins) diff --git a/cpp/tensorrt_llm/batch_manager/CMakeLists.txt b/cpp/tensorrt_llm/batch_manager/CMakeLists.txt index c7fea62c023a..1389d0de068f 100644 --- a/cpp/tensorrt_llm/batch_manager/CMakeLists.txt +++ b/cpp/tensorrt_llm/batch_manager/CMakeLists.txt @@ -33,20 +33,34 @@ set(SRCS contextTransferCoordinator.cpp dataTransceiver.cpp decoderBuffers.cpp + encoderBuffers.cpp + guidedDecoder.cpp + handleContextLogits.cpp + handleGenerationLogits.cpp kvCacheManager.cpp kvCacheEventManager.cpp kvCacheTransferManager.cpp kvCacheManagerV2Utils.cpp kvCacheManagerV2Utils.cu llmRequest.cpp + logitsPostProcessor.cpp + loraBuffers.cpp + makeDecodingBatchInputOutput.cpp medusaBuffers.cpp microBatchScheduler.cpp pauseRequests.cpp peftCacheManager.cpp + promptTuningBuffers.cpp + rnnStateBuffers.cpp rnnStateManager.cpp rnnCacheFormatter.cpp rnnCacheTransBuffer.cpp + runtimeBuffers.cpp sequenceSlotManager.cpp + transformerBuffers.cpp + trtEncoderModel.cpp + trtGptModelInflightBatching.cpp + updateDecoderBuffers.cpp utils/debugUtils.cpp utils/inflightBatchingUtils.cpp utils/logitsThread.cpp diff --git a/cpp/tensorrt_llm/batch_manager/baseTransBuffer.cpp b/cpp/tensorrt_llm/batch_manager/baseTransBuffer.cpp index 1fb522afd7be..fe47f6c531ad 100644 --- a/cpp/tensorrt_llm/batch_manager/baseTransBuffer.cpp +++ b/cpp/tensorrt_llm/batch_manager/baseTransBuffer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,62 +20,15 @@ #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/opUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" -#include <chrono> -#include <exception> #include <mutex> namespace tensorrt_llm::batch_manager { -namespace -{ - -char const* bufferKindName(BufferKind kind) -{ - switch (kind) - { - case BufferKind::kKV: return "kv"; - case BufferKind::kKV_INDEXER: return "kv_indexer"; - case BufferKind::kRNN: return "rnn"; - } - return "unknown"; -} - -} // namespace - void BufferIndexHolder::release() noexcept { - if (mMgr == nullptr) - { - return; - } - if (mHeld) - { - try - { - if (mIsRecv) - { - mMgr->freeBufferIndexForRecv(mIndex); - } - else - { - mMgr->freeBufferIndexForSend(mIndex); - } - } - catch (...) - { - // noexcept: swallow so the destructor can never throw. - } - } - mHeld = false; - mMgr = nullptr; -} - -void BufferIndexHolder::poison() noexcept -{ - if (mMgr == nullptr) + if (!mHeld || mMgr == nullptr) { return; } @@ -83,23 +36,22 @@ void BufferIndexHolder::poison() noexcept { if (mIsRecv) { - mMgr->poisonBufferIndexForRecv(mIndex); + mMgr->freeBufferIndexForRecv(mIndex); } else { - mMgr->poisonBufferIndexForSend(mIndex); + mMgr->freeBufferIndexForSend(mIndex); } } catch (...) { - // noexcept: poison is a fail-closed best effort from exception paths. + // noexcept: swallow so the destructor can never throw. } mHeld = false; - mMgr = nullptr; } BaseTransBufferManager::BaseTransBufferManager( - size_t transferBufferSize, tensorrt_llm::DataType dataType, std::optional<size_t> maxNumTokens) + size_t transferBufferSize, nvinfer1::DataType dataType, std::optional<size_t> maxNumTokens) : mDataType{dataType} , mBufferManager{std::make_shared<runtime::CudaStream>()} , mMaxNumTokens{maxNumTokens} @@ -126,11 +78,9 @@ BaseTransBufferManager::BaseTransBufferManager( allocateBuffer(); } -std::optional<int> BaseTransBufferManager::assignBufferIndexForSend( - std::atomic<bool> const* perRequestCancel, int64_t waitSliceMs) +std::optional<int> BaseTransBufferManager::assignBufferIndexForSend() { - return assignBufferIndex( - mConcurrenceSendResource, mSendBufferCount, mOnlyUseDynamicBuffer, perRequestCancel, waitSliceMs); + return assignBufferIndex(mConcurrenceSendResource, mSendBufferCount, mOnlyUseDynamicBuffer); } void BaseTransBufferManager::freeBufferIndexForSend(std::optional<int> bufferId) @@ -138,16 +88,9 @@ void BaseTransBufferManager::freeBufferIndexForSend(std::optional<int> bufferId) freeBufferIndex(mConcurrenceSendResource, bufferId, mSendBufferCount, mOnlyUseDynamicBuffer); } -void BaseTransBufferManager::poisonBufferIndexForSend(std::optional<int> bufferId) noexcept +std::optional<int> BaseTransBufferManager::assignBufferIndexForRecv() { - poisonBufferIndex(mConcurrenceSendResource, bufferId, mSendBufferCount, mOnlyUseDynamicBuffer, "send"); -} - -std::optional<int> BaseTransBufferManager::assignBufferIndexForRecv( - std::atomic<bool> const* perRequestCancel, int64_t waitSliceMs) -{ - return assignBufferIndex( - mConcurrenceRecvResource, mRecvBufferCount, mOnlyUseDynamicBuffer, perRequestCancel, waitSliceMs); + return assignBufferIndex(mConcurrenceRecvResource, mRecvBufferCount, mOnlyUseDynamicBuffer); } void BaseTransBufferManager::freeBufferIndexForRecv(std::optional<int> bufferId) @@ -155,11 +98,6 @@ void BaseTransBufferManager::freeBufferIndexForRecv(std::optional<int> bufferId) freeBufferIndex(mConcurrenceRecvResource, bufferId, mRecvBufferCount, mOnlyUseDynamicBuffer); } -void BaseTransBufferManager::poisonBufferIndexForRecv(std::optional<int> bufferId) noexcept -{ - poisonBufferIndex(mConcurrenceRecvResource, bufferId, mRecvBufferCount, mOnlyUseDynamicBuffer, "recv"); -} - std::tuple<std::vector<runtime::ITensor::SharedPtr>, size_t, bool> BaseTransBufferManager::getOrAllocateSendBuffers( std::optional<int> bufferId, int targetNum, std::vector<size_t> const& requestedNumberOfElements, runtime::BufferManager const& bufferManagerToUse) @@ -311,51 +249,16 @@ void BaseTransBufferManager::allocateBuffer() } } -std::optional<int> BaseTransBufferManager::assignBufferIndex(ConcurrenceResource& resource, size_t bufferCount, - bool onlyUseDynamicBuffer, std::atomic<bool> const* perRequestCancel, int64_t waitSliceMs) +std::optional<int> BaseTransBufferManager::assignBufferIndex( + ConcurrenceResource& resource, size_t bufferCount, bool onlyUseDynamicBuffer) { - auto const isCancelled = [perRequestCancel]() - { return perRequestCancel != nullptr && perRequestCancel->load(std::memory_order_relaxed); }; - if (isCancelled()) - { - TLLM_THROW("Cache transfer buffer acquisition cancelled"); - } if (onlyUseDynamicBuffer) { - TLLM_CHECK_WITH_INFO(!resource.mPoisoned.load(std::memory_order_relaxed), - "Cannot assign dynamic cache transfer buffer kind=%s because the transfer buffer pool is poisoned", - bufferKindName(getBufferKind())); return std::nullopt; } std::unique_lock lk(resource.mBuffersMutex); - auto const predicate = [&resource, bufferCount]() - { - return resource.mPoisoned.load(std::memory_order_relaxed) - || static_cast<size_t>(resource.mConcurrence) < bufferCount; - }; - if (perRequestCancel == nullptr) - { - resource.mBuffersCV.wait(lk, predicate); - } - else - { - auto const slice = std::chrono::milliseconds{waitSliceMs}; - while (!predicate()) - { - resource.mBuffersCV.wait_for(lk, slice); - if (isCancelled()) - { - TLLM_THROW("Cache transfer buffer acquisition cancelled"); - } - } - } - if (isCancelled()) - { - TLLM_THROW("Cache transfer buffer acquisition cancelled"); - } - TLLM_CHECK_WITH_INFO(!resource.mPoisoned.load(std::memory_order_relaxed), - "Cannot assign cache transfer buffer kind=%s because the transfer buffer pool is poisoned", - bufferKindName(getBufferKind())); + resource.mBuffersCV.wait( + lk, [&resource, bufferCount]() { return static_cast<size_t>(resource.mConcurrence) < bufferCount; }); int bufferId = -1; for (size_t i = 0; i < bufferCount; i++) { @@ -385,12 +288,6 @@ void BaseTransBufferManager::freeBufferIndex( TLLM_CHECK(static_cast<size_t>(bufferId.value()) < bufferCount); { std::scoped_lock lk(resource.mBuffersMutex); - if (resource.mBufferIndexFlag[bufferId.value()] == 2) - { - TLLM_LOG_ERROR("Refusing to free poisoned cache transfer buffer kind=%s index=%d", - bufferKindName(getBufferKind()), bufferId.value()); - return; - } resource.mBufferIndexFlag[bufferId.value()] = 0; } resource.mConcurrence--; @@ -398,48 +295,6 @@ void BaseTransBufferManager::freeBufferIndex( } } -void BaseTransBufferManager::poisonBufferIndex(ConcurrenceResource& resource, std::optional<int> bufferId, - size_t bufferCount, bool onlyUseDynamicBuffer, char const* direction) noexcept -{ - resource.mPoisoned.store(true, std::memory_order_relaxed); - - if (onlyUseDynamicBuffer) - { - TLLM_LOG_ERROR("Poisoned dynamic %s cache transfer buffer kind=%s; process restart is required", direction, - bufferKindName(getBufferKind())); - resource.mBuffersCV.notify_all(); - return; - } - - try - { - if (bufferId.has_value()) - { - TLLM_CHECK(static_cast<size_t>(bufferId.value()) < bufferCount); - { - std::scoped_lock lk(resource.mBuffersMutex); - if (resource.mBufferIndexFlag[bufferId.value()] == 1) - { - resource.mBufferIndexFlag[bufferId.value()] = 2; - } - } - } - TLLM_LOG_ERROR("Poisoned %s cache transfer buffer kind=%s index=%d; process restart is required", direction, - bufferKindName(getBufferKind()), bufferId.value_or(-1)); - } - catch (std::exception const& e) - { - TLLM_LOG_ERROR("Exception while poisoning %s cache transfer buffer kind=%s index=%d: %s", direction, - bufferKindName(getBufferKind()), bufferId.value_or(-1), e.what()); - } - catch (...) - { - TLLM_LOG_ERROR("Unknown exception while poisoning %s cache transfer buffer kind=%s index=%d", direction, - bufferKindName(getBufferKind()), bufferId.value_or(-1)); - } - resource.mBuffersCV.notify_all(); -} - size_t BaseTransBufferManager::getRecvBufferCount() { return mRecvBufferCount; diff --git a/cpp/tensorrt_llm/batch_manager/baseTransBuffer.h b/cpp/tensorrt_llm/batch_manager/baseTransBuffer.h index 88585818d4e5..2cbf9f514bd7 100644 --- a/cpp/tensorrt_llm/batch_manager/baseTransBuffer.h +++ b/cpp/tensorrt_llm/batch_manager/baseTransBuffer.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,6 @@ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -50,9 +49,8 @@ enum class BufferKind : uint8_t class BaseTransBufferManager; /// @brief RAII holder for an index from BaseTransBufferManager::assignBufferIndexFor{Send,Recv}. -/// Releases on destruction (incl. exception unwind). A dynamic-buffer holder has no -/// concrete index, but retains its manager binding so poison() can fail closed. -/// Move-only; call release() on the happy path or detach() when ownership is handed off downstream. +/// Releases on destruction (incl. exception unwind). Move-only; call release() on +/// the happy path or detach() when ownership is handed off downstream. class BufferIndexHolder { public: @@ -84,7 +82,6 @@ class BufferIndexHolder , mIsRecv(other.mIsRecv) { other.mHeld = false; - other.mMgr = nullptr; } BufferIndexHolder& operator=(BufferIndexHolder&& other) noexcept @@ -97,7 +94,6 @@ class BufferIndexHolder mHeld = other.mHeld; mIsRecv = other.mIsRecv; other.mHeld = false; - other.mMgr = nullptr; } return *this; } @@ -112,27 +108,18 @@ class BufferIndexHolder return mHeld; } - [[nodiscard]] bool isBoundTo(BaseTransBufferManager const& manager) const noexcept - { - return mMgr == &manager; - } - /// @brief Relinquish ownership without releasing. Use when a downstream /// owner (e.g. the formatter inside receiveSync) takes over the /// release responsibility on the happy path. std::optional<int> detach() noexcept { mHeld = false; - mMgr = nullptr; return mIndex; } /// @brief Release the slot now and disarm the destructor. Safe to call multiple times. void release() noexcept; - /// @brief Fail-closed release for an exit path where transfer-buffer quiescence is unknown. - void poison() noexcept; - private: BaseTransBufferManager* mMgr{nullptr}; std::optional<int> mIndex{}; @@ -140,8 +127,6 @@ class BufferIndexHolder bool mIsRecv{true}; }; -inline constexpr int64_t kBufferAcquireSliceMs = 100; - /// @brief Base class for cache transfer buffer management. /// Handles buffer pool allocation, index assignment, and slicing. /// Derived classes provide cache-specific size calculations. @@ -154,28 +139,20 @@ class BaseTransBufferManager /// @brief Assign a buffer index for sending. /// @return Assigned buffer index, or nullopt if using dynamic buffers. - std::optional<int> assignBufferIndexForSend( - std::atomic<bool> const* perRequestCancel = nullptr, int64_t waitSliceMs = kBufferAcquireSliceMs); + std::optional<int> assignBufferIndexForSend(); /// @brief Free a buffer index used for sending. /// @param bufferId The buffer index to free. void freeBufferIndexForSend(std::optional<int> bufferId); - /// @brief Poison a send buffer index after an unquiesced transfer exit. - void poisonBufferIndexForSend(std::optional<int> bufferId) noexcept; - /// @brief Assign a buffer index for receiving. /// @return Assigned buffer index, or nullopt if using dynamic buffers. - std::optional<int> assignBufferIndexForRecv( - std::atomic<bool> const* perRequestCancel = nullptr, int64_t waitSliceMs = kBufferAcquireSliceMs); + std::optional<int> assignBufferIndexForRecv(); /// @brief Free a buffer index used for receiving. /// @param bufferId The buffer index to free. void freeBufferIndexForRecv(std::optional<int> bufferId); - /// @brief Poison a receive buffer index after an unquiesced transfer exit. - void poisonBufferIndexForRecv(std::optional<int> bufferId) noexcept; - /// @brief Get or allocate send buffers for cache transfer. /// @param bufferId The assigned buffer ID. /// @param targetNum Number of target sequences. @@ -214,19 +191,13 @@ class BaseTransBufferManager return mMaxNumTokens; } - [[nodiscard]] bool hasPoisonedBuffer() const noexcept - { - return mConcurrenceSendResource.mPoisoned.load(std::memory_order_relaxed) - || mConcurrenceRecvResource.mPoisoned.load(std::memory_order_relaxed); - } - protected: /// @brief Constructor - derived classes call this after computing buffer sizes. /// @param transferBufferSize Size of each transfer buffer in bytes. /// @param dataType Data type for the buffers. /// @param maxNumTokens Optional max tokens for sizing. BaseTransBufferManager( - size_t transferBufferSize, tensorrt_llm::DataType dataType, std::optional<size_t> maxNumTokens = std::nullopt); + size_t transferBufferSize, nvinfer1::DataType dataType, std::optional<size_t> maxNumTokens = std::nullopt); struct ConcurrenceResource { @@ -235,7 +206,6 @@ class BaseTransBufferManager std::mutex mBuffersMutex; std::condition_variable mBuffersCV; std::atomic<int> mConcurrence{0}; - std::atomic<bool> mPoisoned{false}; }; std::tuple<std::vector<runtime::ITensor::SharedPtr>, size_t, bool> getOrAllocateBuffers(std::optional<int> bufferId, @@ -243,12 +213,9 @@ class BaseTransBufferManager runtime::BufferManager const& bufferManagerToUse, ConcurrenceResource& concurrenceResource); void allocateBuffer(); - std::optional<int> assignBufferIndex(ConcurrenceResource& resource, size_t bufferCount, bool onlyUseDynamicBuffer, - std::atomic<bool> const* perRequestCancel = nullptr, int64_t waitSliceMs = kBufferAcquireSliceMs); + std::optional<int> assignBufferIndex(ConcurrenceResource& resource, size_t bufferCount, bool onlyUseDynamicBuffer); void freeBufferIndex( ConcurrenceResource& resource, std::optional<int> bufferId, size_t bufferCount, bool onlyUseDynamicBuffer); - void poisonBufferIndex(ConcurrenceResource& resource, std::optional<int> bufferId, size_t bufferCount, - bool onlyUseDynamicBuffer, char const* direction) noexcept; size_t mPreAllocBufferSize; size_t mRecvBufferCount; @@ -257,7 +224,7 @@ class BaseTransBufferManager bool mOnlyUseDynamicBuffer; bool mUseFabricMemory; size_t mNumberOfElements; - tensorrt_llm::DataType mDataType; + nvinfer1::DataType mDataType; ConcurrenceResource mConcurrenceSendResource; ConcurrenceResource mConcurrenceRecvResource; runtime::BufferManager mBufferManager; diff --git a/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp b/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp index 6165ec7386b4..e9daca6f5447 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheFormatter.cpp @@ -183,7 +183,7 @@ void sendAllBuffers(TransferSession& session, int deviceId, namespace tensorrt_llm::batch_manager::kv_cache_manager { -BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, std::optional<LlmRequest const*> llmRequest, +BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, LlmRequest const& llmRequest, BlockKey const& lastBlockKey, int32_t indexFromEnd, bool recvSideHasCP, SizeType32 ppSize) { auto poolNum = cacheManager->getBlockManager().getNumPools( @@ -197,10 +197,9 @@ BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, std::option || lastBlockKey.uniqueTokens.size() == 0 || recvSideHasCP || ppSize > 1) { // disable reuse path, and vwsa don't support reuse. - TLLM_CHECK_WITH_INFO(llmRequest.has_value(), "LlmRequest required for non-reuse-tree transfer path"); bool needSendAllForWindow = common::getEnvKVCacheTransferAllBlocksForWindow(); - auto blockRange = BlockRange::fromAllBlockIds(*cacheManager, (*llmRequest)->mRequestId); + auto blockRange = BlockRange::fromAllBlockIds(*cacheManager, llmRequest.mRequestId); auto const& windowsMetadata = cacheManager->getBlockManager().getWindowSizesMetadata(); @@ -236,20 +235,16 @@ BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, std::option TLLM_CHECK_WITH_INFO(lastBlockKey.uniqueTokens.size() > 0, "lastBlockKey must be non-empty when reuse is enabled"); - // No request on the reuse-tree path: fall through to the plain lastBlockKey lookup. - if (llmRequest.has_value()) + auto multimodalHashes = llmRequest.getMultimodalHashes(); + bool isMultimodal = multimodalHashes.has_value() && *multimodalHashes && !(*multimodalHashes)->empty(); + if (isMultimodal) { - auto multimodalHashes = (*llmRequest)->getMultimodalHashes(); - bool isMultimodal = multimodalHashes.has_value() && *multimodalHashes && !(*multimodalHashes)->empty(); - if (isMultimodal) - { - auto tokensPerBlock = cacheManager->getBlockManager().getTokensPerBlock(); - auto const usableSize = static_cast<SizeType32>(lastBlockKey.uniqueTokens.size()); - auto blockedUniqueTokens = chopVectorIntoBlocks<UniqueToken>( - lastBlockKey.uniqueTokens, usableSize, tokensPerBlock, /*allowPartial=*/true); - auto blockKeys = buildBlockKeys(blockedUniqueTokens, **llmRequest); - return BlockRange::fromReuseTree(*cacheManager, blockKeys, indexFromEnd); - } + auto tokensPerBlock = cacheManager->getBlockManager().getTokensPerBlock(); + auto const usableSize = static_cast<SizeType32>(lastBlockKey.uniqueTokens.size()); + auto blockedUniqueTokens = chopVectorIntoBlocks<UniqueToken>( + lastBlockKey.uniqueTokens, usableSize, tokensPerBlock, /*allowPartial=*/true); + auto blockKeys = buildBlockKeys(blockedUniqueTokens, llmRequest); + return BlockRange::fromReuseTree(*cacheManager, blockKeys, indexFromEnd); } return BlockRange::fromReuseTree(*cacheManager, lastBlockKey, indexFromEnd); @@ -368,15 +363,11 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio { NVTX3_SCOPED_RANGE(CacheFormatter_format); session.setTime(TransferSession::kTimeFormatter); - auto llmRequest = session.getLlmRequest(); - if (llmRequest.has_value()) - { - TLLM_LOG_DEBUG( - mpi::MpiComm::world().getRank(), "Start sending KV cache for request ID: %ld.", (*llmRequest)->mRequestId); - TLLM_CHECK_WITH_INFO( - (*llmRequest)->mSamplingConfig.beamWidth == 1, "Currently, only beam width 1 is supported."); - } + auto const& llmRequest = session.getLlmRequest(); + TLLM_LOG_DEBUG( + mpi::MpiComm::world().getRank(), "Start sending KV cache for request ID: %ld.", llmRequest.mRequestId); + TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently, only beam width 1 is supported."); auto const& connections = session.getConnections(); auto const& selfConfig = session.getSelfState().getCacheState().value(); auto const& destConfig = session.getOtherState().getCacheState().value(); @@ -394,6 +385,7 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio size_t targetNum = pickUpConnections.size(); if (targetNum == 0) { + TLLM_LOG_DEBUG("No targets to send KV cache to for request ID: %ld", llmRequest.mRequestId); return; } @@ -421,15 +413,13 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio SizeType32 const numKvPools = static_cast<SizeType32>(kvWindowSizes.size()); - TLLM_LOG_DEBUG("CacheFormatter::format: allWindowSizes=%zu, kvWindowSizes=%d, numPools=%d, requestId=%s", - allWindowSizes.size(), numKvPools, numPools, - llmRequest.has_value() ? std::to_string((*llmRequest)->mRequestId).c_str() : "<request-free>"); + TLLM_LOG_DEBUG("CacheFormatter::format: allWindowSizes=%zu, kvWindowSizes=%d, numPools=%d, requestId=%lu", + allWindowSizes.size(), numKvPools, numPools, llmRequest.mRequestId); bool layerWise = common::getEnvDisaggLayerwise() && numKvPools == 1; if (layerWise) { - TLLM_CHECK_WITH_INFO(llmRequest.has_value(), "LlmRequest required for layer-wise transfer"); - auto& progress = (*llmRequest)->getContextProgress(); + auto& progress = llmRequest.getContextProgress(); SizeType32 const numLayers = blockManager.getNumLayers(); runtime::ITensor::Shape offset = runtime::ITensor::makeShape({0, 0}); for (SizeType32 layerIdx = 0; layerIdx < numLayers; layerIdx++) @@ -525,11 +515,8 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio } } } - if (llmRequest.has_value()) - { - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", - (*llmRequest)->mRequestId); - } + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", + llmRequest.mRequestId); return; } @@ -542,9 +529,7 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio // cache blocks to the corresponding buffer. // 5. send the buffer to the corresponding target. Ideally, we send only once (one buffer) for each target. - auto const* sendCancelFlag - = common::getEnvDisaggEnableInflightCancel() ? &session.getDataContext().getTransferTerminate() : nullptr; - auto cacheBufferId = mCacheTransBufferManager->assignBufferIndexForSend(sendCancelFlag); + auto cacheBufferId = mCacheTransBufferManager->assignBufferIndexForSend(); BufferIndexHolder sendHolder(*mCacheTransBufferManager, cacheBufferId, /*isRecv=*/false); int peerDuplicateHeadFactor = targetInfo.mPeerDupHeadFactor; auto bufferTargetNum = targetNum / peerDuplicateHeadFactor; @@ -624,44 +609,23 @@ void CacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& sessio == inputKvCacheBlocksPerWindow.begin()->second.front()->getDataType()); } - if (sendCancelFlag != nullptr && sendCancelFlag->load(std::memory_order_relaxed)) - { - TLLM_THROW("KV cache transfer cancelled before NIXL submission"); - } - - try - { - sendAllBuffers(session, deviceId, outputSplitCaches, bufferCoverTargetNum, preAllocSendBuffer, - bufferManager, targetInfo, pickUpConnections); - } - catch (...) - { - if (agentConnection != nullptr && common::getEnvDisaggEnableInflightCancel()) - { - sendHolder.poison(); - } - throw; - } + sendAllBuffers(session, deviceId, outputSplitCaches, bufferCoverTargetNum, preAllocSendBuffer, bufferManager, + targetInfo, pickUpConnections); session.setTime(TransferSession::kTimeTransmissions); sendHolder.release(); session.setTime(TransferSession::kTimePostprocess); } - if (llmRequest.has_value()) - { - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID:%ld ", - (*llmRequest)->mRequestId); - } + TLLM_LOG_DEBUG( + mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID:%ld ", llmRequest.mRequestId); } void CacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& session) { NVTX3_SCOPED_RANGE(CacheFormatter_unformat); session.setTime(TransferSession::kTimeFormatter); - auto llmRequestOpt = session.getLlmRequest(); - TLLM_CHECK_WITH_INFO(llmRequestOpt.has_value(), "LlmRequest required for receiving KV cache"); - auto const& llmRequest = **llmRequestOpt; + auto const& llmRequest = session.getLlmRequest(); auto const ctxReqId = llmRequest.getContextPhaseParams().value().getReqId(); TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "Start receiving KV cache for request ID: %ld, context request ID: %ld.", llmRequest.mRequestId, ctxReqId); @@ -911,16 +875,12 @@ void CacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& sess if (preAssignedKvId.has_value()) { cacheBufferId = static_cast<int>(*preAssignedKvId); - if (!session.hasReservedRecvBuffer(*mCacheTransBufferManager)) - { - recvHolder = BufferIndexHolder(*mCacheTransBufferManager, cacheBufferId, /*isRecv=*/true); - } } else { cacheBufferId = mCacheTransBufferManager->assignBufferIndexForRecv(); - recvHolder = BufferIndexHolder(*mCacheTransBufferManager, cacheBufferId, /*isRecv=*/true); } + recvHolder = BufferIndexHolder(*mCacheTransBufferManager, cacheBufferId, /*isRecv=*/true); auto [recvSplitCachestmp, bufferCoverTargetNumtmp, onlyUseDynamicBuffer] = mCacheTransBufferManager->getOrAllocateRecvBuffers( cacheBufferId, static_cast<int>(targetNum), bufferEleSizes, bufferManager); @@ -1054,7 +1014,6 @@ void CacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& sess recvSplitCaches, outputBuffersPerWindow, destConfig, selfConfig, selfIdx, bufferManager); bufferManager.getStream().synchronize(); - (void) session.releaseReservedRecvBuffer(*mCacheTransBufferManager); recvHolder.release(); } session.setTime(TransferSession::kTimePostprocess); diff --git a/cpp/tensorrt_llm/batch_manager/cacheFormatter.h b/cpp/tensorrt_llm/batch_manager/cacheFormatter.h index 21356f3dd6c0..458cac8d4382 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheFormatter.h +++ b/cpp/tensorrt_llm/batch_manager/cacheFormatter.h @@ -28,6 +28,7 @@ #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include <NvInferRuntimeBase.h> #include <cstddef> #include <cstdint> #include <fstream> @@ -176,7 +177,7 @@ inline std::pair<std::vector<size_t>, std::vector<size_t>> pickRecvConnections(s namespace tensorrt_llm::batch_manager::kv_cache_manager { -BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, std::optional<LlmRequest const*> llmRequest, +BlockRange getBlockRangeForSending(BaseKVCacheManager* cacheManager, LlmRequest const& llmRequest, BlockKey const& lastBlockKey, SizeType32 indexFromEnd, bool recvSideHasCP = false, SizeType32 ppSize = 1); using DataContext = tensorrt_llm::executor::kv_cache::DataContext; diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp index e06198f9ea60..772c9555f0f3 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransBuffer.cpp @@ -21,7 +21,7 @@ #include "tensorrt_llm/common/opUtils.h" #include "tensorrt_llm/executor/executor.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntimeBase.h> #include <mutex> namespace tensorrt_llm::batch_manager::kv_cache_manager @@ -194,7 +194,7 @@ bool FabricMemory::supportFabricMemory() size_t CacheTransBufferManager::computeTransferBufferSize( KVCacheManager::BaseKVCacheManager* cacheManager, std::optional<size_t> maxNumTokens, bool transferIndexerKCache) { - tensorrt_llm::DataType dataType; + nvinfer1::DataType dataType; if (transferIndexerKCache) { dataType = cacheManager->getIndexerKCachePool()->getDataType(); diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp index c4acce47593f..c25e81632f71 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransceiver.cpp @@ -48,22 +48,15 @@ #include "tensorrt_llm/batch_manager/rnnStateManager.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/mpi_utils/connection.h" #include "tensorrt_llm/executor/dataTransceiverState.h" -#include "tensorrt_llm/executor/serialization.h" #include "tensorrt_llm/executor/serializeUtils.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" #include <algorithm> #include <chrono> #include <cstddef> -#include <filesystem> -#include <fstream> -#include <iomanip> #include <numeric> -#include <random> -#include <sstream> #include <thread> #include <unordered_map> #include <unordered_set> @@ -71,28 +64,6 @@ namespace tensorrt_llm::batch_manager { -namespace -{ - -/// Generate a UUID-like hex string (e.g. "a1b2c3d4-e5f6-7890-abcd-ef1234567890") -/// to uniquely identify a CacheTransceiver instance across gen instances. -std::string generateInstanceId() -{ - // The RNG state is comparatively expensive to construct/seed, so keep one - // per thread instead of building it on every call. - static thread_local std::mt19937_64 gen{std::random_device{}()}; - std::uniform_int_distribution<uint64_t> dis; - uint64_t a = dis(gen); - uint64_t b = dis(gen); - std::ostringstream oss; - oss << std::hex << std::setfill('0') << std::setw(8) << (a >> 32) << "-" << std::setw(4) << ((a >> 16) & 0xFFFF) - << "-" << std::setw(4) << (a & 0xFFFF) << "-" << std::setw(4) << (b >> 48) << "-" << std::setw(12) - << (b & 0xFFFFFFFFFFFF); - return oss.str(); -} - -} // anonymous namespace - std::mutex CacheTransceiver::mDllMutex; namespace @@ -121,57 +92,20 @@ enum class TransferConsensusState : std::uint64_t { kCompleted = 1, kFailed = 2, - kTimedOut = 3, }; struct TransferStateCounts { int completedCount{0}; int failedCount{0}; - int timedOutCount{0}; }; struct TransferConsensusOutcome { std::unordered_set<RequestIdType> completedRequestIds; std::unordered_set<RequestIdType> failedRequestIds; - std::unordered_set<RequestIdType> timedOutRequestIds; }; -template <typename CancelFn> -bool requestCancellationNoThrow(RequestIdType requestId, char const* transferKind, CancelFn&& cancelFn) noexcept -{ - try - { - return cancelFn(); - } - catch (std::exception const& error) - { - TLLM_LOG_ERROR( - "%s cancellation for request %ld failed and will be retried: %s", transferKind, requestId, error.what()); - } - catch (...) - { - TLLM_LOG_ERROR("%s cancellation for request %ld failed with an unknown error and will be retried", transferKind, - requestId); - } - return false; -} - -long getTransferElapsedMs(std::shared_ptr<LlmRequest> const& request, LlmRequest::TimePoint end) -{ - auto const elapsed - = std::chrono::duration_cast<std::chrono::milliseconds>(end - request->getKvCacheTransferStart()); - return static_cast<long>(elapsed.count()); -} - -std::vector<RequestIdType> sortedRequestIds(std::unordered_set<RequestIdType> const& requestIds) -{ - std::vector<RequestIdType> result(requestIds.begin(), requestIds.end()); - std::sort(result.begin(), result.end()); - return result; -} - void appendPackedTransferState( std::vector<std::uint64_t>& packedStates, RequestIdType requestId, TransferConsensusState state) { @@ -211,11 +145,10 @@ std::vector<std::uint64_t> gatherPackedTransferStates( TransferConsensusOutcome reduceTransferStates(std::shared_ptr<CacheTransceiverComm> const& comm, std::unordered_set<RequestIdType> const& completedRequestIds, - std::unordered_set<RequestIdType> const& failedRequestIds, - std::unordered_set<RequestIdType> const& timedOutRequestIds) + std::unordered_set<RequestIdType> const& failedRequestIds) { std::vector<std::uint64_t> localStates; - localStates.reserve((completedRequestIds.size() + failedRequestIds.size() + timedOutRequestIds.size()) * 2); + localStates.reserve((completedRequestIds.size() + failedRequestIds.size()) * 2); for (auto const requestId : completedRequestIds) { if (failedRequestIds.find(requestId) == failedRequestIds.end()) @@ -227,10 +160,6 @@ TransferConsensusOutcome reduceTransferStates(std::shared_ptr<CacheTransceiverCo { appendPackedTransferState(localStates, requestId, TransferConsensusState::kFailed); } - for (auto const requestId : timedOutRequestIds) - { - appendPackedTransferState(localStates, requestId, TransferConsensusState::kTimedOut); - } int const syncSize = (comm != nullptr) ? comm->getSize() : 1; auto const gatheredStates @@ -250,7 +179,6 @@ TransferConsensusOutcome reduceTransferStates(std::shared_ptr<CacheTransceiverCo { case TransferConsensusState::kCompleted: counts.completedCount++; break; case TransferConsensusState::kFailed: counts.failedCount++; break; - case TransferConsensusState::kTimedOut: counts.timedOutCount++; break; } } @@ -258,11 +186,7 @@ TransferConsensusOutcome reduceTransferStates(std::shared_ptr<CacheTransceiverCo for (auto const& [requestId, counts] : stateCounts) { auto const terminalCount = counts.completedCount + counts.failedCount; - if (counts.timedOutCount > 0) - { - outcome.timedOutRequestIds.insert(requestId); - } - if (terminalCount == syncSize && (counts.failedCount > 0 || counts.timedOutCount > 0)) + if (terminalCount == syncSize && counts.failedCount > 0) { outcome.failedRequestIds.insert(requestId); } @@ -277,13 +201,10 @@ TransferConsensusOutcome reduceTransferStates(std::shared_ptr<CacheTransceiverCo TransferConsensusOutcome reduceTransferStates(std::shared_ptr<CacheTransceiverComm> const& firstComm, std::shared_ptr<CacheTransceiverComm> const& secondComm, std::unordered_set<RequestIdType> const& completedRequestIds, - std::unordered_set<RequestIdType> const& failedRequestIds, - std::unordered_set<RequestIdType> const& timedOutRequestIds) + std::unordered_set<RequestIdType> const& failedRequestIds) { - auto const firstOutcome - = reduceTransferStates(firstComm, completedRequestIds, failedRequestIds, timedOutRequestIds); - return reduceTransferStates( - secondComm, firstOutcome.completedRequestIds, firstOutcome.failedRequestIds, firstOutcome.timedOutRequestIds); + auto const firstOutcome = reduceTransferStates(firstComm, completedRequestIds, failedRequestIds); + return reduceTransferStates(secondComm, firstOutcome.completedRequestIds, firstOutcome.failedRequestIds); } void recordLocalTransferOutcome(RequestIdType requestId, std::shared_ptr<LlmRequest> request, bool failed, @@ -323,9 +244,6 @@ std::unique_ptr<BaseCacheTransceiver> CacheTransceiverFactory::createCacheTransc TLLM_LOG_INFO("CacheTransceiver is disabled."); return nullptr; } - TLLM_CHECK_WITH_INFO(!common::getEnvDisaggEnableInflightCancel(), - "TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL=1 is supported only by the PyExecutor C++ NIXL transceiver path; " - "the legacy C++ executor does not provide the required deferred cleanup and poison escalation."); auto backendType = cacheTransceiverConfig.value().getBackendType(); if (backendType.value() == executor::CacheTransceiverConfig::BackendType::DEFAULT) { @@ -374,37 +292,13 @@ std::unique_ptr<BaseCacheTransceiver> CacheTransceiverFactory::createCacheTransc CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheManager, executor::kv_cache::CacheState::ModelConfig const& cacheStateModelCfg, runtime::WorldConfig const& worldConfig, - std::vector<SizeType32> const& attentionLayerNumPerPP, tensorrt_llm::DataType dataType, + std::vector<SizeType32> const& attentionLayerNumPerPP, nvinfer1::DataType dataType, executor::kv_cache::CacheState::AttentionType attentionType, std::optional<executor::CacheTransceiverConfig> cacheTransceiverConfig, std::vector<SizeType32> const& rnnLayerNumPerPP) : mCacheTransceiverConfig{cacheTransceiverConfig} { using tensorrt_llm::batch_manager::kv_cache_manager::CacheFormatter; - TLLM_CHECK_WITH_INFO(mCacheTransceiverConfig.has_value(), "CacheTransceiverConfig is not set."); - auto const backendType = mCacheTransceiverConfig.value().getBackendType(); - TLLM_CHECK_WITH_INFO( - backendType.has_value() && (backendType.value() != executor::CacheTransceiverConfig::BackendType::DEFAULT), - " CacheTransceiverConfig::BackendType is not set."); - if (common::getEnvDisaggEnableInflightCancel()) - { - auto const nixlBackend = common::getEnvNixlBackend(); - TLLM_CHECK_WITH_INFO( - backendType.value() == executor::CacheTransceiverConfig::BackendType::NIXL && nixlBackend == "UCX", - "TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL=1 is experimental and currently supported only with the " - "NIXL cache transceiver and the UCX NIXL backend."); - TLLM_CHECK_WITH_INFO(mCacheTransceiverConfig->getKvTransferTimeoutMs().has_value(), - "TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL=1 requires kv_transfer_timeout_ms to enforce a finite deadline."); - TLLM_CHECK_WITH_INFO(!common::getEnvDisableKVCacheTransferOverlap(), - "TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL=1 requires asynchronous KV cache transfer; " - "TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=1 is not supported."); - TLLM_CHECK_WITH_INFO(!common::getEnvDisaggLayerwise(), - "TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL=1 does not support layer-wise KV cache transfer."); - TLLM_CHECK_WITH_INFO(!common::getEnvTryZCopyForKVCacheTransfer(), - "TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL=1 does not support zero-copy KV cache transfer because request " - "blocks cannot be quarantined after an unquiesced cancellation."); - } - if (useMPI()) { mGroupComm = std::make_shared<CacheTransceiverComm>(std::addressof(tensorrt_llm::mpi::MpiComm::session())); @@ -414,89 +308,6 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa mGroupComm = std::make_shared<CacheTransceiverComm>(tensorrt_llm::pg_utils::get_world_pg()); } - // Generate instance ID on rank 0 and broadcast to all ranks in the session - // so every rank in the same gen/ctx instance shares the same ID. - { - if (mGroupComm->getRank() == 0) - { - mInstanceId = generateInstanceId(); - } - if (useMPI()) - { - int len = static_cast<int>(mInstanceId.size()); - tensorrt_llm::mpi::MpiComm::session().bcast(&len, 1, mpi::MpiType::kINT32, 0); - mInstanceId.resize(len); - tensorrt_llm::mpi::MpiComm::session().bcast(mInstanceId.data(), len, mpi::MpiType::kCHAR, 0); - } - else - { - // PG path: rank 0 sends via allgather, others receive. - constexpr int kUuidLen = 36; - std::vector<char> sendBuf(kUuidLen, '\0'); - if (mGroupComm->getRank() == 0) - { - std::copy_n(mInstanceId.begin(), std::min<size_t>(mInstanceId.size(), kUuidLen), sendBuf.begin()); - } - std::vector<char> recvBuf(kUuidLen * mGroupComm->getSize(), '\0'); - mGroupComm->allgather(std::ref(sendBuf), std::ref(recvBuf), {}); - // Take rank 0's segment. - mInstanceId = std::string(recvBuf.begin(), recvBuf.begin() + kUuidLen); - } - } - - // Calibrate steady_clock across ranks so that cross-node allgather - // in batchUpdateKVCacheTransferBW can compare time points. - // globalSteadyClockOffset() reads a single process-global copy shared with - // the nanobind module, so if the Python runtime already calibrated the offset - // (PyExecutor::_set_global_steady_clock_offset) it is visible here and we skip; - // the pure-C++ path performs the calibration below. - // The check-and-set is guarded by a mutex so that CacheTransceiver instances - // constructed concurrently in the same process (e.g. multi-engine serving) do - // not race on the shared offset or issue mismatched collectives. - { - static std::mutex sSteadyClockCalibrationMutex; - std::lock_guard<std::mutex> lock(sSteadyClockCalibrationMutex); - if (!globalSteadyClockOffset().has_value()) - { - using Duration = LlmRequest::Duration; - // Synchronize all ranks immediately before sampling the local clock so - // every rank measures from a consistent point. - if (useMPI()) - { - tensorrt_llm::mpi::MpiComm::session().barrier(); - } - else - { - // CacheTransceiverComm exposes no barrier primitive, so use a cheap - // allgather as a pseudo-barrier for the process-group path. - int64_t const dummy = 0; - std::vector<int64_t> dummyRecv(mGroupComm->getSize(), 0); - mGroupComm->allgather(dummy, std::ref(dummyRecv), {}); - } - auto localNow = std::chrono::steady_clock::now(); - auto localNs = std::chrono::duration_cast<std::chrono::nanoseconds>(localNow.time_since_epoch()).count(); - - // Allgather timestamps from all ranks - std::vector<int64_t> allNs(mGroupComm->getSize(), 0); - if (useMPI()) - { - tensorrt_llm::mpi::MpiComm::session().allgather(&localNs, allNs.data(), 1, mpi::MpiType::kINT64); - } - else - { - mGroupComm->allgather(localNs, std::ref(allNs), {}); - } - - // Offset = rank0's timestamp - my timestamp (same formula as Python) - auto offsetNs = allNs[0] - localNs; - globalSteadyClockOffset() = Duration(offsetNs); - - TLLM_LOG_INFO(mGroupComm->getRank(), - "CacheTransceiver: set global steady clock offset = %.6f sec for rank %d", - static_cast<double>(offsetNs) / 1e9, mGroupComm->getRank()); - } - } - if (worldConfig.isTensorParallel() || worldConfig.isContextParallel()) { mGroupTensorParaComm = std::make_shared<CacheTransceiverComm>( @@ -538,6 +349,12 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa } } bool isMLA = attentionType == executor::kv_cache::CacheState::AttentionType::kMLA; + TLLM_CHECK_WITH_INFO(mCacheTransceiverConfig.has_value(), "CacheTransceiverConfig is not set."); + auto backendType = mCacheTransceiverConfig.value().getBackendType(); + TLLM_CHECK_WITH_INFO( + backendType.has_value() && (backendType.value() != executor::CacheTransceiverConfig::BackendType::DEFAULT), + " CacheTransceiverConfig::BackendType is not set."); + std::optional<size_t> maxNumTokens = mCacheTransceiverConfig.value().getMaxTokensInBuffer(); mCacheTransBufferManagers.push_back( @@ -570,20 +387,20 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa // Pool dtype is UINT8 (raw byte storage), so we cannot use pool->getDataType(). // Only the byte size matters for split/concat kernel stride calculations — the actual // dtype enum is not interpreted numerically, just used for getDTypeSize() dispatch. - auto dtypeFromSize = [](SizeType32 size) -> tensorrt_llm::DataType + auto dtypeFromSize = [](SizeType32 size) -> nvinfer1::DataType { switch (size) { - case 4: return tensorrt_llm::DataType::kFLOAT; - case 2: return tensorrt_llm::DataType::kBF16; - case 1: return tensorrt_llm::DataType::kFP8; + case 4: return nvinfer1::DataType::kFLOAT; + case 2: return nvinfer1::DataType::kBF16; + case 1: return nvinfer1::DataType::kFP8; default: TLLM_THROW("Unsupported RNN state dtype size: %d", size); } }; TLLM_CHECK_WITH_INFO(linearMeta->rnnSsmDtypeSize > 0, "rnnSsmDtypeSize not set in LinearAttentionMetadata"); TLLM_CHECK_WITH_INFO(linearMeta->rnnConvDtypeSize > 0, "rnnConvDtypeSize not set in LinearAttentionMetadata"); - tensorrt_llm::DataType ssmDtype = dtypeFromSize(linearMeta->rnnSsmDtypeSize); - tensorrt_llm::DataType convDtype = dtypeFromSize(linearMeta->rnnConvDtypeSize); + nvinfer1::DataType ssmDtype = dtypeFromSize(linearMeta->rnnSsmDtypeSize); + nvinfer1::DataType convDtype = dtypeFromSize(linearMeta->rnnConvDtypeSize); mCacheState->setRnnConfig(rnnModelCfg, rnnLayerNumPerPP, convDtype, ssmDtype); // Create RnnCacheTransBufferManager for unified pool path. @@ -678,10 +495,8 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa auto makeCacheTransferLayer = [&]() { return CacheTransferLayer(*mCacheState, makeFormatter(), makeRnnFormatter()); }; - mCacheSender - = std::make_unique<CacheSender>(mManager.get(), worldConfig.getRank(), makeCacheTransferLayer(), mInstanceId); - mCacheReceiver - = std::make_unique<CacheReceiver>(mManager.get(), worldConfig.getRank(), makeCacheTransferLayer(), mInstanceId); + mCacheSender = std::make_unique<CacheSender>(mManager.get(), worldConfig.getRank(), makeCacheTransferLayer()); + mCacheReceiver = std::make_unique<CacheReceiver>(mManager.get(), worldConfig.getRank(), makeCacheTransferLayer()); // Keep automatic enablement within the currently qualified C++ NIXL/UCX TP1/CP1 pipeline topology. bool const coordinatorTopologyEligible = worldConfig.getPipelineParallelism() > 1 && useMPI() @@ -693,7 +508,7 @@ CacheTransceiver::CacheTransceiver(kv_cache_manager::BaseKVCacheManager* cacheMa TLLM_CHECK(mGroupPipeParaComm != nullptr); constexpr std::uint64_t kCoordinatorProtocolVersion = 1; std::uint64_t const localVersion = coordinatorTopologyEligible ? kCoordinatorProtocolVersion : 0; - bool const cancellationEnabled = common::getEnvDisaggEnableInflightCancel(); + constexpr bool cancellationEnabled = false; std::uint64_t const localProtocolMode = (localVersion << 1) | static_cast<std::uint64_t>(cancellationEnabled); std::vector<std::uint64_t> protocolModes(static_cast<std::size_t>(mGroupPipeParaComm->getSize())); mGroupPipeParaComm->allgather(&localProtocolMode, protocolModes.data(), 1, mpi::MpiType::kUINT64); @@ -734,17 +549,6 @@ void CacheTransceiver::initializeCommState() mCommState = std::addressof(mCacheSender->getCommState()); } -std::vector<char> CacheTransceiver::getSerializedDataTransceiverState() const -{ - TLLM_CHECK(mCommState != nullptr && mCacheState != nullptr); - executor::DataTransceiverState state; - state.setCommState(*mCommState); - state.setCacheState(*mCacheState); - // Only this API marks the state; context responses leave it unset. - state.setIsArbitraryTransferState(true); - return executor::Serialization::serialize(state); -} - void CacheTransceiver::setContextState(LlmRequest* llmRequest) { TLLM_CHECK(llmRequest && llmRequest->isContextOnlyRequest()); @@ -819,14 +623,6 @@ void CacheTransceiver::requestAndReceiveSync(std::shared_ptr<LlmRequest> llmRequ contextRequestId, err.what()); return; } - catch (...) - { - llmRequest->setState(LlmRequestState::kDISAGG_TRANS_ERROR); - llmRequest->setKvCacheTransferEnd(LlmRequest::getSteadyClockNow()); - TLLM_LOG_ERROR("Synchronous KV cache receive request %zu, context request %zu failed with an unknown error", - requestId, contextRequestId); - return; - } if (llmRequest->getState() == LlmRequestState::kDISAGG_TRANS_ERROR) { TLLM_LOG_ERROR("Synchronous KV cache receive request %zu, context request %zu completed with an error state.", @@ -886,109 +682,67 @@ std::vector<LlmRequest::RequestIdType> gatherRequestIds( return retData; } -void batchUpdateKVCacheTransferBW( - std::shared_ptr<CacheTransceiverComm> const& comm, std::vector<LlmRequest*> const& requests) +void updateKVCacheTransferBW(std::shared_ptr<CacheTransceiverComm> const& mComm, LlmRequest* request) { - // Key-based merge: each rank serializes (requestId, start, end, size) - // tuples and we use allgatherv so ranks may have different request counts. - // The merge matches by requestId, not by position — this tolerates - // ordering differences and count mismatches across ranks. - namespace su = executor::serialize_utils; - int const worldSize = comm->getSize(); - - // --- Serialize local entries keyed by requestId --- - std::size_t const numReqs = requests.size(); + int worldSize = mComm->getSize(); std::ostringstream oStream; - su::serialize(numReqs, oStream); - for (auto* req : requests) - { - su::serialize(req->getContextPhaseParams().value().getReqId(), oStream); - su::serialize(req->getKvCacheTransferStart(), oStream); - su::serialize(req->getKvCacheTransferEnd(), oStream); - su::serialize(req->getKvCacheSize(), oStream); - } + su::serialize(request->getKvCacheTransferStart(), oStream); + su::serialize(request->getKvCacheTransferEnd(), oStream); auto str = oStream.str(); std::vector<char> sendBuffer(str.begin(), str.end()); - int const sendSize = static_cast<int>(sendBuffer.size()); + auto sendBufferSize = sendBuffer.size(); + auto recvBufferSize = sendBufferSize * worldSize; + std::vector<char> recvBuffer(recvBufferSize); - // --- Step 1: allgather per-rank buffer sizes --- - std::vector<int> recvCounts(worldSize, 0); if (useMPI()) { - comm->allgather(&sendSize, recvCounts.data(), 1, mpi::MpiType::kINT32); + mComm->allgather(sendBuffer.data(), recvBuffer.data(), sendBufferSize, mpi::MpiType::kCHAR); } else { - comm->allgather(sendSize, std::ref(recvCounts), {}); + mComm->allgather(std::ref(sendBuffer), std::ref(recvBuffer), {}); } - // --- Step 2: allgatherv the serialized data --- - std::vector<int> displs(worldSize, 0); - int totalRecvSize = 0; - for (int r = 0; r < worldSize; ++r) + su::VectorWrapBuf<char> strbuf(recvBuffer); + std::istream is(&strbuf); + + auto minStartTime = executor::RequestPerfMetrics::TimePoint::max(); + auto maxEndTime = executor::RequestPerfMetrics::TimePoint::min(); + + for (int rank = 0; rank < worldSize; rank++) { - displs[r] = totalRecvSize; - totalRecvSize += recvCounts[r]; + minStartTime = std::min(su::deserialize<executor::RequestPerfMetrics::TimePoint>(is), minStartTime); + maxEndTime = std::max(su::deserialize<executor::RequestPerfMetrics::TimePoint>(is), maxEndTime); } - std::vector<char> recvBuffer(totalRecvSize, 0); + + // Handle KV cache size separately - gather all sizes to the leader rank + std::size_t localKVCacheSize = request->getKvCacheSize(); + std::vector<std::size_t> allKVCacheSizes(worldSize, 0); if (useMPI()) { - comm->allgatherv(sendBuffer.data(), sendSize, mpi::MpiType::kCHAR, recvBuffer.data(), recvCounts, displs, - mpi::MpiType::kCHAR); + mComm->allgather(&localKVCacheSize, allKVCacheSizes.data(), 1, mpi::MpiType::kUINT64); } else { - comm->allgatherv(std::ref(sendBuffer), std::ref(recvBuffer), recvCounts, {}); + mComm->allgather(&localKVCacheSize, std::ref(allKVCacheSizes), {}); } - // --- Step 3: Deserialize and merge by requestId --- - using TimePoint = executor::RequestPerfMetrics::TimePoint; - using ReqIdType = LlmRequest::RequestIdType; - - struct MergedEntry - { - TimePoint minStart = TimePoint::max(); - TimePoint maxEnd = TimePoint::min(); - std::size_t totalSize = 0; - }; - - std::unordered_map<ReqIdType, MergedEntry> merged; - - su::VectorWrapBuf<char> strbuf(recvBuffer); - std::istream is(&strbuf); - - for (int rank = 0; rank < worldSize; ++rank) + std::size_t totalKVCacheSize = 0; + for (int rank = 0; rank < worldSize; rank++) { - auto rankNumReqs = su::deserialize<std::size_t>(is); - for (std::size_t i = 0; i < rankNumReqs; ++i) - { - auto rid = su::deserialize<ReqIdType>(is); - auto start = su::deserialize<TimePoint>(is); - auto end = su::deserialize<TimePoint>(is); - auto size = su::deserialize<std::size_t>(is); - - auto& entry = merged[rid]; - entry.minStart = std::min(entry.minStart, start); - entry.maxEnd = std::max(entry.maxEnd, end); - entry.totalSize += size; - } + totalKVCacheSize += allKVCacheSizes[rank]; } - // --- Step 4: Update local requests --- - for (auto* req : requests) + // Update the latest KV cache transfer time for leader rank + if (mComm->getRank() == 0) { - auto reqId = req->getContextPhaseParams().value().getReqId(); - auto it = merged.find(reqId); - if (it != merged.end()) - { - req->setKvCacheTransferStart(it->second.minStart); - req->setKvCacheTransferEnd(it->second.maxEnd); - req->setKvCacheSize(it->second.totalSize); - } + request->setKvCacheTransferStart(minStartTime); + request->setKvCacheTransferEnd(maxEndTime); + request->setKvCacheSize(totalKVCacheSize); } } @@ -996,9 +750,6 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( std::optional<int> const& atLeastRequestNum, bool markComplete) { bool const blockAll = !atLeastRequestNum.has_value(); - bool const inflightCancelEnabled = common::getEnvDisaggEnableInflightCancel(); - TLLM_CHECK_WITH_INFO(!inflightCancelEnabled || !blockAll, - "In-flight cancellation requires a finite context-transfer status poll; pass 0 for a nonblocking poll."); std::optional<int> senderFutureTimeoutMs = std::nullopt; if (mCacheTransceiverConfig.has_value()) { @@ -1006,9 +757,8 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( } bool const needsProgress = atLeastRequestNum.value_or(0) > 0; auto const futureWaitInterval = getTransferFutureWaitInterval(senderFutureTimeoutMs, needsProgress); - // Without the opt-in flag, deadline checks remain observe-only. With the - // flag, timeout IDs participate in the same topology consensus as terminal - // outcomes and request cancellation is requested on every nonterminal rank. + // Observe-only: WARN per-request when the wall-clock transfer time exceeds + // kvTransferTimeoutMs. No cancellation, eviction, or state transition. std::optional<int> kvTransferTimeoutMs = std::nullopt; if (mCacheTransceiverConfig.has_value()) { @@ -1064,15 +814,6 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( toCompleteIdSet.insert(request->mRequestId); } - auto recordTimeout = [&](RequestIdType const requestId) - { - bool const inserted = mTimedOutSenderIds.insert(requestId).second; - if (inserted && inflightCancelEnabled && mContextTransferCoordinator) - { - mContextTransferCoordinator->publishTimeout(requestId); - } - return inserted; - }; auto recordOutcome = [&](RequestIdType const requestId, std::shared_ptr<LlmRequest> const& request, bool const failed) { @@ -1091,20 +832,17 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( { auto& [request, future] = *it; auto const requestId = request->mRequestId; - if (kvTransferTimeoutMs.has_value() - && future.wait_for(std::chrono::milliseconds(0)) != std::future_status::ready) + if (kvTransferTimeoutMs.has_value()) { - auto const elapsedMs = getTransferElapsedMs(request, LlmRequest::getSteadyClockNow()); - if (elapsedMs > kvTransferTimeoutMs.value()) + auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>( + LlmRequest::getSteadyClockNow() - request->getKvCacheTransferStart()); + auto elapsedMs = static_cast<long>(elapsed.count()); + if (elapsedMs > kvTransferTimeoutMs.value() && mTimedOutSenderIds.insert(requestId).second) { - if (recordTimeout(requestId)) - { - TLLM_LOG_WARNING( - "Context KV cache transfer for request %ld exceeded configured timeout: " - "elapsed %ld ms > limit %d ms (%s).", - requestId, elapsedMs, kvTransferTimeoutMs.value(), - inflightCancelEnabled ? "requesting cancellation" : "observe-only"); - } + TLLM_LOG_WARNING( + "Context KV cache transfer for request %ld exceeded configured timeout: " + "elapsed %ld ms > limit %d ms (observe-only).", + requestId, elapsedMs, kvTransferTimeoutMs.value()); } } if (blockAll || (toCompleteIdSet.find(requestId) != toCompleteIdSet.end())) @@ -1117,18 +855,6 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( if (status == std::future_status::ready) { future.get(); - if (kvTransferTimeoutMs.has_value()) - { - auto const elapsedMs = getTransferElapsedMs(request, request->getKvCacheTransferEnd()); - if (elapsedMs > kvTransferTimeoutMs.value() && recordTimeout(requestId)) - { - TLLM_LOG_WARNING( - "Context KV cache transfer for request %ld completed after its deadline: " - "elapsed %ld ms > limit %d ms (%s).", - requestId, elapsedMs, kvTransferTimeoutMs.value(), - inflightCancelEnabled ? "failing request" : "observe-only"); - } - } failed = request->getState() == LlmRequestState::kDISAGG_TRANS_ERROR; terminal = true; } @@ -1185,8 +911,6 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( coordinatorOutcome.completedRequestIds.begin(), coordinatorOutcome.completedRequestIds.end()); consensusOutcome.failedRequestIds.insert( coordinatorOutcome.failedRequestIds.begin(), coordinatorOutcome.failedRequestIds.end()); - consensusOutcome.timedOutRequestIds.insert( - coordinatorOutcome.timedOutRequestIds.begin(), coordinatorOutcome.timedOutRequestIds.end()); }; do { @@ -1200,46 +924,13 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( } while (blockAll && consensusOutcome.completedRequestIds.size() + consensusOutcome.failedRequestIds.size() < mSenderRequestsAwaitingConsensus.size()); - - if (inflightCancelEnabled) - { - // A timeout update is transmitted once, but cancellation may be declined transiently. Keep the globally - // observed timeout active so rank-local cancellation is retried on every poll until terminal commit. - consensusOutcome.timedOutRequestIds.insert(mTimedOutSenderIds.begin(), mTimedOutSenderIds.end()); - } } else { - consensusOutcome = reduceTransferStates(syncComm, mGroupPipeParaComm, mCompletedSenderRequestIds, - mFailedSenderRequestIds, inflightCancelEnabled ? mTimedOutSenderIds : std::unordered_set<RequestIdType>{}); - } - if (inflightCancelEnabled) - { - for (auto const requestId : consensusOutcome.timedOutRequestIds) - { - // Persist the global timeout even if this rank has not registered its local future yet. The one-shot - // coordinator update must remain actionable when that future appears on a later scheduler poll. - mTimedOutSenderIds.insert(requestId); - auto const futureIt = std::find_if(mSenderFutures.begin(), mSenderFutures.end(), - [requestId](auto const& entry) { return entry.first->mRequestId == requestId; }); - if (futureIt == mSenderFutures.end() - || futureIt->second.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready - || mCancelRequestedSenderIds.find(requestId) != mCancelRequestedSenderIds.end()) - { - continue; - } - if (requestCancellationNoThrow( - requestId, "Context", [&]() { return mCacheSender->cancelRequest(*futureIt->first); })) - { - mCancelRequestedSenderIds.insert(requestId); - } - else - { - TLLM_LOG_DEBUG("Context cancellation for request %ld was not accepted; will retry", requestId); - } - } + consensusOutcome + = reduceTransferStates(syncComm, mGroupPipeParaComm, mCompletedSenderRequestIds, mFailedSenderRequestIds); } - for (auto const requestId : sortedRequestIds(consensusOutcome.failedRequestIds)) + for (auto const requestId : consensusOutcome.failedRequestIds) { auto const requestIt = mSenderRequestsAwaitingConsensus.find(requestId); if (requestIt == mSenderRequestsAwaitingConsensus.end()) @@ -1249,11 +940,10 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( requestIt->second->setState(LlmRequestState::kDISAGG_TRANS_ERROR); requestsStatus.errorRequestIds.insert(requestId); mTimedOutSenderIds.erase(requestId); - mCancelRequestedSenderIds.erase(requestId); eraseLocalTransferOutcome( requestId, mCompletedSenderRequestIds, mFailedSenderRequestIds, mSenderRequestsAwaitingConsensus); } - for (auto const requestId : sortedRequestIds(consensusOutcome.completedRequestIds)) + for (auto const requestId : consensusOutcome.completedRequestIds) { auto const requestIt = mSenderRequestsAwaitingConsensus.find(requestId); if (requestIt == mSenderRequestsAwaitingConsensus.end()) @@ -1266,7 +956,6 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( requestIt->second->setState(LlmRequestState::kDISAGG_CONTEXT_COMPLETE); } mTimedOutSenderIds.erase(requestId); - mCancelRequestedSenderIds.erase(requestId); eraseLocalTransferOutcome( requestId, mCompletedSenderRequestIds, mFailedSenderRequestIds, mSenderRequestsAwaitingConsensus); } @@ -1277,9 +966,6 @@ RequestStatuses CacheTransceiver::checkContextTransferStatus( void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastRequestNum) { bool const blockAll = !atLeastRequestNum.has_value(); - bool const inflightCancelEnabled = common::getEnvDisaggEnableInflightCancel(); - TLLM_CHECK_WITH_INFO(!inflightCancelEnabled || !blockAll, - "In-flight cancellation requires a finite generation-transfer status poll; pass 0 for a nonblocking poll."); bool const needsProgress = atLeastRequestNum.value_or(0) > 0; std::optional<int> genTransferPollIntervalMs = std::nullopt; if (mCacheTransceiverConfig.has_value()) @@ -1369,7 +1055,7 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR atLeastRequestNum.value_or(0)); } - // Gen-side mirror of the context deadline/consensus path. + // Observe-only: gen-side mirror of the context-side timeout WARN. std::optional<int> kvTransferTimeoutMs = std::nullopt; if (mCacheTransceiverConfig.has_value()) { @@ -1379,20 +1065,17 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR { auto& request = it->first; auto const requestId = request->mRequestId; - if (kvTransferTimeoutMs.has_value() - && it->second.wait_for(std::chrono::milliseconds(0)) != std::future_status::ready) + if (kvTransferTimeoutMs.has_value()) { - auto const elapsedMs = getTransferElapsedMs(request, LlmRequest::getSteadyClockNow()); - if (elapsedMs > kvTransferTimeoutMs.value()) + auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>( + LlmRequest::getSteadyClockNow() - request->getKvCacheTransferStart()); + auto elapsedMs = static_cast<long>(elapsed.count()); + if (elapsedMs > kvTransferTimeoutMs.value() && mTimedOutRequesterIds.insert(requestId).second) { - if (mTimedOutRequesterIds.insert(requestId).second) - { - TLLM_LOG_WARNING( - "Generation KV cache transfer for request %ld exceeded configured timeout: " - "elapsed %ld ms > limit %d ms (%s).", - requestId, elapsedMs, kvTransferTimeoutMs.value(), - inflightCancelEnabled ? "requesting cancellation" : "observe-only"); - } + TLLM_LOG_WARNING( + "Generation KV cache transfer for request %ld exceeded configured timeout: " + "elapsed %ld ms > limit %d ms (observe-only).", + requestId, elapsedMs, kvTransferTimeoutMs.value()); } } if (blockAll || toCompleteIdSet.find(requestId) != toCompleteIdSet.end()) @@ -1403,19 +1086,14 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR if (status == std::future_status::ready) { it->second.get(); - if (kvTransferTimeoutMs.has_value()) + bool const failed = request->getState() == LlmRequestState::kDISAGG_TRANS_ERROR; + if (failed) { - auto const elapsedMs = getTransferElapsedMs(request, request->getKvCacheTransferEnd()); - if (elapsedMs > kvTransferTimeoutMs.value() && mTimedOutRequesterIds.insert(requestId).second) - { - TLLM_LOG_WARNING( - "Generation KV cache transfer for request %ld completed after its deadline: " - "elapsed %ld ms > limit %d ms (%s).", - requestId, elapsedMs, kvTransferTimeoutMs.value(), - inflightCancelEnabled ? "failing request" : "observe-only"); - } + // The receiver uses the error state as a local transfer-failed signal. + // Keep that signal local until the consensus outcome commits it globally. + request->setState(LlmRequestState::kDISAGG_GENERATION_TRANS_IN_PROGRESS); } - recordLocalTransferOutcome(requestId, request, /*failed=*/false, mCompletedRequesterRequestIds, + recordLocalTransferOutcome(requestId, request, failed, mCompletedRequesterRequestIds, mFailedRequesterRequestIds, mRequesterRequestsAwaitingConsensus); } else if (status == std::future_status::timeout) @@ -1440,12 +1118,6 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR recordLocalTransferOutcome(requestId, request, /*failed=*/true, mCompletedRequesterRequestIds, mFailedRequesterRequestIds, mRequesterRequestsAwaitingConsensus); } - catch (...) - { - TLLM_LOG_ERROR("Unknown error occurred during generation transfer for request %ld", requestId); - recordLocalTransferOutcome(requestId, request, /*failed=*/true, mCompletedRequesterRequestIds, - mFailedRequesterRequestIds, mRequesterRequestsAwaitingConsensus); - } if (useMPI()) { TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), @@ -1467,33 +1139,8 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR } auto const consensusOutcome - = reduceTransferStates(syncComm, mCompletedRequesterRequestIds, mFailedRequesterRequestIds, - inflightCancelEnabled ? mTimedOutRequesterIds : std::unordered_set<RequestIdType>{}); - if (inflightCancelEnabled) - { - for (auto const requestId : consensusOutcome.timedOutRequestIds) - { - auto const futureIt = std::find_if(mRequesterFutures.begin(), mRequesterFutures.end(), - [requestId](auto const& entry) { return entry.first->mRequestId == requestId; }); - if (futureIt == mRequesterFutures.end() - || futureIt->second.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready - || mCancelRequestedRequesterIds.find(requestId) != mCancelRequestedRequesterIds.end()) - { - continue; - } - mTimedOutRequesterIds.insert(requestId); - if (requestCancellationNoThrow( - requestId, "Generation", [&]() { return mCacheReceiver->cancelRequest(*futureIt->first); })) - { - mCancelRequestedRequesterIds.insert(requestId); - } - else - { - TLLM_LOG_DEBUG("Generation cancellation for request %ld was not accepted; will retry", requestId); - } - } - } - for (auto const requestId : sortedRequestIds(consensusOutcome.failedRequestIds)) + = reduceTransferStates(syncComm, mCompletedRequesterRequestIds, mFailedRequesterRequestIds); + for (auto const requestId : consensusOutcome.failedRequestIds) { auto const requestIt = mRequesterRequestsAwaitingConsensus.find(requestId); if (requestIt == mRequesterRequestsAwaitingConsensus.end()) @@ -1502,15 +1149,10 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR } requestIt->second->setState(LlmRequestState::kDISAGG_TRANS_ERROR); mTimedOutRequesterIds.erase(requestId); - mCancelRequestedRequesterIds.erase(requestId); eraseLocalTransferOutcome( requestId, mCompletedRequesterRequestIds, mFailedRequesterRequestIds, mRequesterRequestsAwaitingConsensus); } - - // Collect consensus-completed requests so timing can be synced across ranks in a single - // batched allgather (instead of one collective per request). - std::vector<LlmRequest*> completedRequests; - for (auto const requestId : sortedRequestIds(consensusOutcome.completedRequestIds)) + for (auto const requestId : consensusOutcome.completedRequestIds) { auto const requestIt = mRequesterRequestsAwaitingConsensus.find(requestId); if (requestIt == mRequesterRequestsAwaitingConsensus.end()) @@ -1518,44 +1160,16 @@ void CacheTransceiver::checkGenTransferStatus(std::optional<int> const& atLeastR continue; } requestIt->second->setState(LlmRequestState::kDISAGG_GENERATION_TRANS_COMPLETE); - completedRequests.push_back(requestIt->second.get()); + + // Gather the kv cache transfer time from all workers and update to leader rank. + if (!common::getEnvKVCacheTimeOutputPath().empty()) + { + updateKVCacheTransferBW(syncComm, requestIt->second.get()); + } mTimedOutRequesterIds.erase(requestId); - mCancelRequestedRequesterIds.erase(requestId); eraseLocalTransferOutcome( requestId, mCompletedRequesterRequestIds, mFailedRequesterRequestIds, mRequesterRequestsAwaitingConsensus); } - - // Batch-sync timing across ranks in one allgather (instead of per-request), then write - // the gen-side transfer summary CSV. - if (!completedRequests.empty() && !common::getEnvKVCacheTimeOutputPath().empty()) - { - batchUpdateKVCacheTransferBW(syncComm, completedRequests); - writeGenTransferSummary(completedRequests); - } -} - -void CacheTransceiver::writeGenTransferSummary(std::vector<LlmRequest*> const& completedRequests) -{ - std::lock_guard<std::mutex> lock(mGenTransferSummaryMutex); - if (!mGenTransferSummaryFile.is_open()) - { - namespace fs = std::filesystem; - auto outputPath = fs::path(common::getEnvKVCacheTimeOutputPath()); - fs::create_directories(outputPath); - int rank = useMPI() ? mpi::MpiComm::world().getRank() : tensorrt_llm::pg_utils::get_world_pg()->getRank(); - auto filePath = outputPath / (mInstanceId + "_" + std::to_string(rank) + "_gen_transfer_summary.csv"); - mGenTransferSummaryFile.open(filePath); - TLLM_CHECK_WITH_INFO(mGenTransferSummaryFile.is_open(), "Failed to open gen transfer summary file: %s", - filePath.string().c_str()); - mGenTransferSummaryFile << "RequestID,gen_side_transfer_time(ms),kv_cache_size" << '\n'; - } - for (auto* req : completedRequests) - { - auto reqId = req->getContextPhaseParams().value().getReqId(); - mGenTransferSummaryFile << reqId << "," << req->getKvCacheTransferTimeMS() << "," << req->getKvCacheSize() - << '\n'; - } - mGenTransferSummaryFile << std::flush; } bool CacheTransceiver::checkGenTransferComplete() const @@ -1563,19 +1177,8 @@ bool CacheTransceiver::checkGenTransferComplete() const return mRequesterFutures.empty() && mCompletedRequesterRequestIds.empty() && mFailedRequesterRequestIds.empty(); } -bool CacheTransceiver::hasPoisonedTransferBuffer() const -{ - return std::any_of(mCacheTransBufferManagerPtrs.begin(), mCacheTransBufferManagerPtrs.end(), - [](BaseTransBufferManager const* manager) { return manager != nullptr && manager->hasPoisonedBuffer(); }); -} - bool CacheTransceiver::cancelRequest(std::shared_ptr<LlmRequest> llmRequest) { - if (llmRequest == nullptr) - { - TLLM_LOG_WARNING("Cannot cancel a null KV cache transfer request"); - return false; - } if (llmRequest->isContextOnlyRequest()) { return mCacheSender->cancelRequest(*llmRequest); diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.cpp b/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.cpp index b2e0a21537d1..d0a54dbb7d3c 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.cpp +++ b/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.cpp @@ -21,7 +21,6 @@ #include "tensorrt_llm/batch_manager/rnnCacheFormatter.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include "tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h" @@ -122,8 +121,7 @@ void CacheTransferLayer::unformat(TransferSession& session) const } void CacheTransferLayer::setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, - tensorrt_llm::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, nvinfer1::DataType ssmStateDataType) { mCacheState.setRnnConfig( std::move(rnnModelConfig), std::move(rnnLayerNumPerPP), convStateDataType, ssmStateDataType); diff --git a/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.h b/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.h index 48a171a65c9c..0506e98197e6 100644 --- a/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.h +++ b/cpp/tensorrt_llm/batch_manager/cacheTransferLayer.h @@ -18,7 +18,6 @@ #pragma once #include "tensorrt_llm/batch_manager/rnnCacheFormatter.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/runtime/common.h" @@ -80,8 +79,8 @@ class CacheTransferLayer /// @brief Update the RNN config on the internal CacheState. /// Used by CppMambaHybridCacheManager path where RNN config is set after construction. void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, - tensorrt_llm::DataType ssmStateDataType); + std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, + nvinfer1::DataType ssmStateDataType); [[nodiscard]] kv_cache_manager::BaseKVCacheManager* getCacheManager() const noexcept; diff --git a/cpp/tensorrt_llm/batch_manager/createNewDecoderRequests.cpp b/cpp/tensorrt_llm/batch_manager/createNewDecoderRequests.cpp index 04c3760be5c7..5c5d3e11a01c 100644 --- a/cpp/tensorrt_llm/batch_manager/createNewDecoderRequests.cpp +++ b/cpp/tensorrt_llm/batch_manager/createNewDecoderRequests.cpp @@ -33,7 +33,7 @@ #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/speculativeChoicesUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntimeBase.h> using namespace tensorrt_llm::runtime; @@ -93,7 +93,7 @@ void copySequenceLengths(RequestVector const& contextRequests, DecoderInputBuffe /// @brief Retrieve the embedding bias from the request. This potentially makes a copy of the tensor /// to the appropriate type if the input tensor does not match it. -[[nodiscard]] TensorPtr getEmbeddingBias(tensorrt_llm::DataType logitsType, TensorPtr const& tensor) +[[nodiscard]] TensorPtr getEmbeddingBias(nvinfer1::DataType logitsType, TensorPtr const& tensor) { // Check that embedding bias type is same as logits type. If so, we can return the tensor right away if (tensor->getDataType() == logitsType) @@ -102,7 +102,7 @@ void copySequenceLengths(RequestVector const& contextRequests, DecoderInputBuffe } // Support FP32 input for FP16 embedding bias (in the case of FP8 models) - if (tensor->getDataType() == tensorrt_llm::DataType::kFLOAT && logitsType == tensorrt_llm::DataType::kHALF) + if (tensor->getDataType() == nvinfer1::DataType::kFLOAT && logitsType == nvinfer1::DataType::kHALF) { // Do a deep copy of the tensor to the expected type TLLM_LOG_WARNING( @@ -133,10 +133,10 @@ void copySequenceLengths(RequestVector const& contextRequests, DecoderInputBuffe std::tuple<TensorPtr, std::vector<runtime::SamplingConfig>, std::vector<runtime::ITensor::SharedConstPtr>, std::vector<executor::LookaheadDecodingConfig>> CreateNewDecoderRequests::operator()(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, - executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests, - tensorrt_llm::DataType logitsType, DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, - CudaStream const& runtimeStream, CudaStream const& decoderStream, SizeType32 maxSequenceLength, - SizeType32 beamWidth, OptionalRef<MedusaBuffers const> medusaBuffers) const + executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests, nvinfer1::DataType logitsType, + DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, CudaStream const& runtimeStream, + CudaStream const& decoderStream, SizeType32 maxSequenceLength, SizeType32 beamWidth, + OptionalRef<MedusaBuffers const> medusaBuffers) const { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); NVTX3_SCOPED_RANGE(CreateNewDecoderRequests); @@ -235,7 +235,7 @@ void initializeBeamSearch(DecodingInput& dJointInput, DecodingOutput& dJointOutp } void initializeEmbeddingBias(DecodingInput& dJointInput, SizeType32 batchSlot, - std::optional<TensorPtr> const& embeddingBias, tensorrt_llm::DataType logitsType, + std::optional<TensorPtr> const& embeddingBias, nvinfer1::DataType logitsType, runtime::ModelConfig const& modelConfig, BufferManager const& manager) { TensorPtr const embeddingBiasSlice = ITensor::slice(constPointerCast(dJointInput.embeddingBias), batchSlot, 1); @@ -631,7 +631,7 @@ void newRequestSpeculativeDecoding(DecodingInput& jointDecodingInput, DecodingOu std::tuple<std::vector<runtime::ITensor::SharedConstPtr>, std::vector<executor::LookaheadDecodingConfig>> CreateNewDecoderRequests::createDecoderRequests(RequestVector const& finishedContextRequests, TensorPtr const& inputIds, executor::DecodingConfig const& decodingConfig, runtime::decoder::DecoderState& decoderState, - tensorrt_llm::DataType logitsType, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + nvinfer1::DataType logitsType, runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, runtime::CudaStream const& runtimeStream, runtime::CudaStream const& decoderStream, SizeType32 maxSequenceLength, OptionalRef<MedusaBuffers const> medusaBuffers) const { diff --git a/cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp b/cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp index 0f8ded65613f..2ec86df47e99 100644 --- a/cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp +++ b/cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp @@ -20,16 +20,15 @@ #include "tensorrt_llm/batch_manager/cacheFormatter.h" #include "tensorrt_llm/batch_manager/common.h" #include "tensorrt_llm/batch_manager/kvCacheUtils.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/tllmException.h" #include "tensorrt_llm/common/utils.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include "tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include <algorithm> #include <chrono> #include <future> #include <map> @@ -37,7 +36,6 @@ #include <optional> #include <stdexcept> #include <unordered_map> -#include <variant> namespace tensorrt_llm::batch_manager { @@ -82,11 +80,6 @@ void TransferSession::send(size_t idx, void const* data, size_t size) } catch (std::exception const& e) { - // Request-free (llmRequest-agnostic) transfer: there is no valid ID to attach. - if (mRequest == nullptr) - { - TLLM_THROW("%s", e.what()); - } throw common::RequestSpecificException( __FILE__, __LINE__, e.what(), mRequest->mRequestId, common::RequestErrorCode::kNETWORK_ERROR); } @@ -100,23 +93,15 @@ void TransferSession::recv(size_t idx, void* data, size_t size) } catch (std::exception const& e) { - // Request-free (llmRequest-agnostic) transfer: there is no valid ID to attach. - if (mRequest == nullptr) - { - TLLM_THROW("%s", e.what()); - } throw common::RequestSpecificException( __FILE__, __LINE__, e.what(), mRequest->mRequestId, common::RequestErrorCode::kNETWORK_ERROR); } } -std::optional<LlmRequest const*> TransferSession::getLlmRequest() const +LlmRequest const& TransferSession::getLlmRequest() const { - if (mRequest == nullptr) - { - return std::nullopt; - } - return mRequest; + TLLM_CHECK(mRequest != nullptr); + return *mRequest; } void TransferSession::setLlmRequest(LlmRequest const& llmRequest) @@ -140,53 +125,9 @@ void TransferSession::appendMeasure(LlmRequest::TimePoint start, LlmRequest::Tim } } -void TransferSession::setReservedRecvBuffers(std::vector<BufferIndexHolder> holders) -{ - TLLM_CHECK(mReservedRecvBuffers.empty()); - mReservedRecvBuffers = std::move(holders); -} - -bool TransferSession::hasReservedRecvBuffer(BaseTransBufferManager const& manager) const noexcept -{ - return std::any_of(mReservedRecvBuffers.begin(), mReservedRecvBuffers.end(), - [&manager](BufferIndexHolder const& holder) { return holder.isBoundTo(manager); }); -} - -bool TransferSession::releaseReservedRecvBuffer(BaseTransBufferManager const& manager) noexcept -{ - auto const holderIt = std::find_if(mReservedRecvBuffers.begin(), mReservedRecvBuffers.end(), - [&manager](BufferIndexHolder const& holder) { return holder.isBoundTo(manager); }); - if (holderIt == mReservedRecvBuffers.end()) - { - return false; - } - holderIt->release(); - mReservedRecvBuffers.erase(holderIt); - return true; -} - -void TransferSession::releaseReservedRecvBuffers() noexcept -{ - for (auto& holder : mReservedRecvBuffers) - { - holder.release(); - } - mReservedRecvBuffers.clear(); -} - -void TransferSession::poisonReservedRecvBuffers() noexcept -{ - for (auto& holder : mReservedRecvBuffers) - { - holder.poison(); - } - mReservedRecvBuffers.clear(); -} - void TransferSession::exportMeasure(std::ofstream& outFile, bool isContext) const { - // Request-free transfers are excluded: the exported row is keyed by the LlmRequest. - if (!mTimes || mTimes->measures.empty() || mRequest == nullptr) + if (!mTimes || mTimes->measures.empty()) { return; } @@ -244,7 +185,7 @@ int32_t tagFromRequestId(LlmRequest::RequestIdType requestId) return ((requestId & 0xFFF) << 8) | (kDATA_TAG & 0xFF); } -std::filesystem::path getTransferOutputPath(char const* tag, std::string const& instanceId = "") +std::filesystem::path getTransferOutputPath(char const* tag) { namespace fs = std::filesystem; auto outputPath = common::getEnvKVCacheTimeOutputPath(); @@ -253,9 +194,7 @@ std::filesystem::path getTransferOutputPath(char const* tag, std::string const& auto rank = mpi::MpiComm::world().getRank(); auto path = fs::path(outputPath); fs::create_directories(path); - std::string prefix - = instanceId.empty() ? "rank_" + std::to_string(rank) : instanceId + "_" + std::to_string(rank); - return path / (prefix + "_" + tag + ".csv"); + return path / ("rank_" + std::to_string(rank) + "_" + tag + ".csv"); } return {}; } @@ -292,7 +231,7 @@ RequestInfo::RequestInfo(LlmRequest::RequestIdType requestId, executor::DataTran bool RequestInfo::operator==(RequestInfo const& rhs) const { return mRequestId == rhs.mRequestId && mIndexFromEnd == rhs.mIndexFromEnd && mLastBlockKey == rhs.mLastBlockKey - && mIsArbitraryTransfer == rhs.mIsArbitraryTransfer && mTransState == rhs.mTransState; + && mTransState == rhs.mTransState; } LlmRequest::RequestIdType RequestInfo::getRequestId() const noexcept @@ -311,7 +250,6 @@ void RequestInfo::serialize(RequestInfo const& requestInfo, std::ostream& os) su::serialize(requestInfo.mRequestId, os); su::serialize(requestInfo.mIndexFromEnd, os); su::serialize(requestInfo.mLastBlockKey, os); - su::serialize(requestInfo.mIsArbitraryTransfer, os); su::serialize(requestInfo.mTransState, os); } @@ -321,11 +259,8 @@ RequestInfo RequestInfo::deserialize(std::istream& is) auto requestId = su::deserialize<decltype(mRequestId)>(is); auto indexFromEnd = su::deserialize<decltype(mIndexFromEnd)>(is); auto lastBlockKey = su::deserialize<decltype(mLastBlockKey)>(is); - auto isArbitraryTransfer = su::deserialize<decltype(mIsArbitraryTransfer)>(is); auto transState = su::deserialize<decltype(mTransState)>(is); - auto requestInfo = RequestInfo{requestId, std::move(transState), indexFromEnd, lastBlockKey}; - requestInfo.setIsArbitraryTransfer(isArbitraryTransfer); - return requestInfo; + return RequestInfo{requestId, std::move(transState), indexFromEnd, lastBlockKey}; } std::size_t RequestInfo::serializedSize(RequestInfo const& requestInfo) @@ -335,7 +270,6 @@ std::size_t RequestInfo::serializedSize(RequestInfo const& requestInfo) totalSize += su::serializedSize(requestInfo.mRequestId); totalSize += su::serializedSize(requestInfo.mIndexFromEnd); totalSize += su::serializedSize(requestInfo.mLastBlockKey); - totalSize += su::serializedSize(requestInfo.mIsArbitraryTransfer); totalSize += su::serializedSize(requestInfo.mTransState); return totalSize; } @@ -345,13 +279,11 @@ class CacheSender::Impl public: using RequestIdType = LlmRequest::RequestIdType; - Impl(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer, - std::string instanceId = "") + Impl(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer) : mManager{manager} , mSelfState{cacheLayer.getCacheState(), executor::kv_cache::CommState{manager->getCommState()}} , mCacheTransferLayer{std::move(cacheLayer)} , mBufferManager{std::make_shared<runtime::CudaStream>()} - , mInstanceId{std::move(instanceId)} { TLLM_CHECK(mManager); TLLM_CHECK(mManager->getCommState().getSelfIdx() == selfIndex); @@ -371,10 +303,6 @@ class CacheSender::Impl std::promise<void> promise; auto future = promise.get_future(); llmRequest->setKvCacheTransferStart(LlmRequest::getSteadyClockNow()); - if (common::getEnvDisaggEnableInflightCancel()) - { - (void) getOrCreateInFlightCancelFlag(llmRequest->mRequestId); - } { std::scoped_lock lock(mSenderMutex); TLLM_CHECK_WITH_INFO( @@ -388,19 +316,6 @@ class CacheSender::Impl return future; } - std::shared_ptr<std::atomic<bool>> getOrCreateInFlightCancelFlag(RequestIdType requestId) - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - auto it = mInFlightCancelFlags.find(requestId); - if (it != mInFlightCancelFlags.end()) - { - return it->second; - } - auto flag = std::make_shared<std::atomic<bool>>(false); - mInFlightCancelFlags.emplace(requestId, flag); - return flag; - } - [[nodiscard]] executor::kv_cache::CommState const& getCommState() const { return mSelfState.getCommState().value(); @@ -421,31 +336,24 @@ class CacheSender::Impl void release(LlmRequest::RequestIdType requestId) { + std::unique_lock<std::mutex> lk(mMtxForMap); + auto it = mRequestToSession.find(requestId); + TLLM_CHECK(it != mRequestToSession.end()); + if (!common::getEnvKVCacheTimeOutputPath().empty()) { - std::unique_lock<std::mutex> lk(mMtxForMap); - auto it = mRequestToSession.find(requestId); - TLLM_CHECK(it != mRequestToSession.end()); - if (!common::getEnvKVCacheTimeOutputPath().empty()) + if (!mMeasuresFile.is_open()) { - if (!mMeasuresFile.is_open()) - { - auto outputPath = getTransferOutputPath("send", mInstanceId); - mMeasuresFile.open(outputPath); - TLLM_CHECK_WITH_INFO(mMeasuresFile.is_open(), "Failed to open transfer output file: %s", - outputPath.string().c_str()); - } - it->second.exportMeasure(mMeasuresFile, true); + auto outputPath = getTransferOutputPath("send"); + mMeasuresFile.open(outputPath); + TLLM_CHECK_WITH_INFO( + mMeasuresFile.is_open(), "Failed to open transfer output file: %s", outputPath.string().c_str()); } - mRequestToSession.erase(it); - } - if (common::getEnvDisaggEnableInflightCancel()) - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - mInFlightCancelFlags.erase(requestId); + it->second.exportMeasure(mMeasuresFile, true); } + mRequestToSession.erase(it); } - void discardTransferState(LlmRequest::RequestIdType requestId) noexcept + void discardSession(LlmRequest::RequestIdType requestId) noexcept { try { @@ -454,28 +362,7 @@ class CacheSender::Impl } catch (std::exception const& e) { - TLLM_LOG_WARNING("Failed to discard sender session for request %ld: %s", requestId, e.what()); - } - catch (...) - { - TLLM_LOG_WARNING("Failed to discard sender session for request %ld: unknown exception", requestId); - } - if (!common::getEnvDisaggEnableInflightCancel()) - { - return; - } - try - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - mInFlightCancelFlags.erase(requestId); - } - catch (std::exception const& e) - { - TLLM_LOG_WARNING("Failed to discard in-flight cancel flag for request %ld: %s", requestId, e.what()); - } - catch (...) - { - TLLM_LOG_WARNING("Failed to discard in-flight cancel flag for request %ld: unknown exception", requestId); + TLLM_LOG_ERROR("Failed to discard KV cache transfer session for request %zu: %s", requestId, e.what()); } } @@ -491,6 +378,7 @@ class CacheSender::Impl : mManager->recvConnect(DataContext{TransceiverTag::kID_TAG, mTerminate}, &id, sizeof(id)); if (connection == nullptr) { + TLLM_LOG_WARNING("recvRequestInfo connection is nullptr, maybe the server is terminating"); return std::nullopt; } @@ -518,25 +406,15 @@ class CacheSender::Impl TLLM_CHECK_WITH_INFO(peerIdx < static_cast<int>(allCounterparts.size()), "Peer rank %d not found in expected counterparts", peerSelfIdx); - std::shared_ptr<std::atomic<bool>> cancelFlag; - if (common::getEnvDisaggEnableInflightCancel()) - { - cancelFlag = getOrCreateInFlightCancelFlag(requestId); - } { std::unique_lock<std::mutex> lk(mMtxForMap); auto it = mRequestToSession.find(requestId); if (it == mRequestToSession.end()) { - auto session = cancelFlag != nullptr - ? TransferSession(std::vector<Connection const*>(allCounterparts.size(), nullptr), - DataContext{tagFromRequestId(requestId), *cancelFlag}, allCounterparts, mSelfState, - info.getTransState(), mBufferManager, info.getIndexFromEnd(), info.getLastBlockKey(), nullptr, - !common::getEnvKVCacheTimeOutputPath().empty()) - : TransferSession(std::vector<Connection const*>(allCounterparts.size(), nullptr), - DataContext{tagFromRequestId(requestId), mTerminate}, allCounterparts, mSelfState, - info.getTransState(), mBufferManager, info.getIndexFromEnd(), info.getLastBlockKey(), nullptr, - !common::getEnvKVCacheTimeOutputPath().empty()); + auto session = TransferSession(std::vector<Connection const*>(allCounterparts.size(), nullptr), + DataContext{tagFromRequestId(requestId), mTerminate}, allCounterparts, mSelfState, + info.getTransState(), mBufferManager, info.getIndexFromEnd(), info.getLastBlockKey(), nullptr, + !common::getEnvKVCacheTimeOutputPath().empty()); session.setTime(TransferSession::kTimeRequestInfo); it = mRequestToSession.emplace(requestId, std::move(session)).first; } @@ -563,52 +441,19 @@ class CacheSender::Impl bool cancelRequest(LlmRequest const& llmRequest) { - bool const inflightCancelEnabled = common::getEnvDisaggEnableInflightCancel(); bool isCancelled = false; - bool isCurrentRequest = false; + std::scoped_lock lock(mSenderMutex); + auto it = mReadyResponses.find(llmRequest.mRequestId); + // If the request is not the current request and already in the ready queue, we can cancel it. + if (it != mReadyResponses.end() + && (!mCurrentRequest.has_value() || mCurrentRequest.value() != llmRequest.mRequestId)) { - std::scoped_lock lock(mSenderMutex); - auto it = mReadyResponses.find(llmRequest.mRequestId); - if (it != mReadyResponses.end()) - { - isCurrentRequest = mCurrentRequest.has_value() && mCurrentRequest.value() == llmRequest.mRequestId; - // The legacy path cannot interrupt a ready/active transfer, so - // preserve its false return until the opt-in is enabled. - if (!isCurrentRequest || inflightCancelEnabled) - { - mCancelledRequests.insert(llmRequest.mRequestId); - isCancelled = true; - if (inflightCancelEnabled && !isCurrentRequest) - { - // Keep only the request ID as a tombstone so a late peer - // receives ready=false without retaining the request. - failResponse(it->second, - std::make_exception_ptr( - TLLM_REQUEST_EXCEPTION(llmRequest.mRequestId, common::RequestErrorCode::kNETWORK_ERROR, - "Context KV cache request cancelled before a peer was ready for request %zu", - llmRequest.mRequestId))); - mReadyResponses.erase(it); - } - } - } - } - if (inflightCancelEnabled && (!isCancelled || isCurrentRequest)) - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - auto flagIt = mInFlightCancelFlags.find(llmRequest.mRequestId); - if (flagIt != mInFlightCancelFlags.end()) - { - flagIt->second->store(true, std::memory_order_relaxed); - isCancelled = true; - } - } - if (!isCancelled) - { - TLLM_LOG_WARNING("Cannot cancel request %zu", llmRequest.mRequestId); + mCancelledRequests.insert(llmRequest.mRequestId); + isCancelled = true; } else { - mSenderCv.notify_all(); + TLLM_LOG_WARNING("Cannot cancel request %zu", llmRequest.mRequestId); } return isCancelled; } @@ -648,23 +493,10 @@ class CacheSender::Impl private: struct Response { - // An LlmRequest (co-owned until the promise resolves) for normal transfers, or - // just the request id for llmRequest-agnostic reuse-tree transfers. - std::variant<std::shared_ptr<LlmRequest>, RequestIdType> mRequestOrId; + // shared_ptr so this struct co-owns the request until the promise resolves; + // protects worker-side dereferences and the promise itself from premature destruction. + std::shared_ptr<LlmRequest> mRequest; std::promise<void> mPromise; - std::vector<kv_cache_manager::KVCacheBlock::IdType> mPinnedBlockIds; - - [[nodiscard]] LlmRequest* getRequest() const - { - auto const* request = std::get_if<std::shared_ptr<LlmRequest>>(&mRequestOrId); - return request != nullptr ? request->get() : nullptr; - } - - [[nodiscard]] RequestIdType getRequestId() const - { - auto const* request = getRequest(); - return request != nullptr ? request->mRequestId : std::get<RequestIdType>(mRequestOrId); - } }; struct AsyncSendResource @@ -697,69 +529,38 @@ class CacheSender::Impl resp = std::move(resource.mSendQueue.front()); resource.mSendQueue.pop_front(); } - // Read before std::move(resp): argument evaluations are indeterminately sequenced. - auto const requestId = resp.getRequestId(); - sendAndRemoveResponse(requestId, std::move(resp)); + // Sequence the read before the move: argument initializations + // are indeterminately sequenced, so inlining resp.mRequest->... + // alongside std::move(resp) is UB once mRequest is a shared_ptr. + TLLM_CHECK(resp.mRequest != nullptr); + auto const reqId = resp.mRequest->mRequestId; + sendAndRemoveResponse(reqId, std::move(resp)); } } - //! Must not throw: called from noexcept send/failure paths. - void releasePinnedBlocks(Response& response) noexcept - { - if (response.mPinnedBlockIds.empty()) - { - return; - } - try - { - mCacheTransferLayer.getCacheManager()->unpinBlocksById(response.mPinnedBlockIds); - } - catch (std::exception const& err) - { - TLLM_LOG_ERROR("Failed to unpin reuse-tree blocks: %s", err.what()); - } - response.mPinnedBlockIds.clear(); - } - void sendAndRemoveResponse(RequestIdType id, Response resp) noexcept { try { TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); - if (auto* llmRequest = resp.getRequest(); llmRequest != nullptr) - { - sendSync(*llmRequest); - } - else - { - // Reuse tree path — no LlmRequest - sendSyncFromReuseTree(id); - } + sendSync(*resp.mRequest); release(id); resp.mPromise.set_value(); } catch (tensorrt_llm::common::RequestSpecificException const& e) { + discardSession(id); TLLM_LOG_ERROR("Exception in sendAndRemoveResponse: %s ", e.what()); - discardTransferState(id); auto new_exception = TLLM_REQUEST_EXCEPTION(id, e.getErrorCode(), "%s", e.what()); failResponse(resp, std::make_exception_ptr(new_exception)); } catch (std::exception const& e) { auto const exception = std::current_exception(); + discardSession(id); TLLM_LOG_ERROR("Exception in sendAndRemoveResponse: %s request id: %ld", e.what(), id); - discardTransferState(id); failResponse(resp, exception); } - catch (...) - { - auto const exception = std::current_exception(); - TLLM_LOG_ERROR("Unknown exception in sendAndRemoveResponse for request id: %ld", id); - discardTransferState(id); - failResponse(resp, exception); - } - releasePinnedBlocks(resp); } void asyncSendAndRemoveResponse(RequestIdType id, Response resp) noexcept @@ -773,13 +574,6 @@ class CacheSender::Impl catch (std::exception const& err) { TLLM_LOG_ERROR("Failed to queue asynchronous KV cache send for request %zu: %s", id, err.what()); - discardTransferState(id); - failResponse(resp, std::current_exception()); - } - catch (...) - { - TLLM_LOG_ERROR("Unknown error while queueing asynchronous KV cache send for request %zu", id); - discardTransferState(id); failResponse(resp, std::current_exception()); } } @@ -787,44 +581,21 @@ class CacheSender::Impl void sendResponse(RequestIdType reqId) { bool isReady = true; - bool allCounterpartsReady = false; - std::optional<Response> cancelledResponse; { std::scoped_lock lock(mSenderMutex); TLLM_CHECK(mCurrentRequest.has_value() && mCurrentRequest.value() == reqId); - auto responseIt = mReadyResponses.find(reqId); - bool const isCancelled = mCancelledRequests.find(reqId) != mCancelledRequests.end(); - TLLM_CHECK(responseIt != mReadyResponses.end() || isCancelled); + TLLM_CHECK(mReadyResponses.find(reqId) != mReadyResponses.end()); auto countIt = mRemainSendCount.find(reqId); TLLM_CHECK(countIt != mRemainSendCount.end()); auto const count = --countIt->second; TLLM_CHECK(count >= 0); - if (isCancelled && responseIt != mReadyResponses.end()) - { - cancelledResponse.emplace(std::move(responseIt->second)); - mReadyResponses.erase(responseIt); - } if (count > 0) { mCurrentRequest = std::nullopt; + return; } - else - { - mRemainSendCount.erase(countIt); - isReady = !isCancelled; - allCounterpartsReady = true; - } - } - - if (cancelledResponse.has_value()) - { - failResponse(*cancelledResponse, - std::make_exception_ptr(TLLM_REQUEST_EXCEPTION(reqId, common::RequestErrorCode::kNETWORK_ERROR, - "KV cache transfer for request %zu was cancelled", reqId))); - } - if (!allCounterpartsReady) - { - return; + mRemainSendCount.erase(countIt); + isReady = mCancelledRequests.find(reqId) == mCancelledRequests.end(); } // Keep mCurrentRequest set while notifying the peer so cancellation cannot change the decision after it has @@ -835,12 +606,9 @@ class CacheSender::Impl { std::scoped_lock lock(mSenderMutex); auto it = mReadyResponses.find(reqId); - if (isReady) - { - TLLM_CHECK(it != mReadyResponses.end()); - response = std::move(it->second); - mReadyResponses.erase(it); - } + TLLM_CHECK(it != mReadyResponses.end()); + response = std::move(it->second); + mReadyResponses.erase(it); mCancelledRequests.erase(reqId); mCurrentRequest = std::nullopt; } @@ -862,39 +630,11 @@ class CacheSender::Impl } else { - discardTransferState(reqId); - } - } - - void sendSyncFromReuseTree(RequestIdType requestId) - { - TransferSession* session = nullptr; - { - std::unique_lock<std::mutex> lk(mMtxForMap); - auto it = mRequestToSession.find(requestId); - TLLM_CHECK(it != mRequestToSession.end()); - session = std::addressof(it->second); - } - // READY was already sent by response(); the receiver consumes exactly one per transfer. - mCacheTransferLayer.format(*session); - } - - // Pin the requested chain in the reuse tree; an empty result means no full match. - // The caller must unpin once the transfer settles. - std::vector<kv_cache_manager::KVCacheBlock::IdType> pinReuseTreeBlocks(RequestIdType requestId) - { - std::unique_lock<std::mutex> lk(mMtxForMap); - auto it = mRequestToSession.find(requestId); - auto const& lastBlockKey = it->second.getLastBlockKey(); - auto* cacheManager = mCacheTransferLayer.getCacheManager(); - auto windowSize = cacheManager->getBlockManager().getWindowSizesMetadata().begin()->first; - std::vector<kv_cache_manager::KVCacheBlock::IdType> pinnedIds; - auto lastBlock = cacheManager->findBlocksInReuseTreeByBlockKey(lastBlockKey, windowSize, pinnedIds); - if (lastBlock == nullptr) - { - return {}; + discardSession(reqId); + failResponse(response, + std::make_exception_ptr(TLLM_REQUEST_EXCEPTION(reqId, common::RequestErrorCode::kNETWORK_ERROR, + "KV cache transfer for request %zu was cancelled", reqId))); } - return pinnedIds; } void response() noexcept @@ -906,13 +646,15 @@ class CacheSender::Impl TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); while (true) { - if (mTerminate) { - break; + std::unique_lock lock(mSenderMutex); + mSenderCv.wait(lock, [this]() { return mTerminate || !mReadyResponses.empty(); }); + if (mTerminate) + { + break; + } } - // Arbitrary transfers arrive without a pre-registered response; do not gate on - // mReadyResponses. auto requestInfo = recvRequestInfo(); if (!requestInfo.has_value() || mTerminate || !mManager->isRunning()) { @@ -925,70 +667,18 @@ class CacheSender::Impl mRemainSendCount[reqId] = getCounterpartsCount(reqId); } - if (requestInfo->isArbitraryTransfer()) { - // No LlmRequest will ever be registered; serve from the reuse tree off-thread. - { - std::scoped_lock lock(mSenderMutex); - mCurrentRequest = reqId; - } - auto countIt = mRemainSendCount.find(reqId); - auto const count = --countIt->second; - TLLM_CHECK(count >= 0); - if (count == 0) + std::unique_lock lock(mSenderMutex); + mCurrentRequest = reqId; + mSenderCv.wait(lock, + [this, reqId]() { return mTerminate || mReadyResponses.find(reqId) != mReadyResponses.end(); }); + if (mTerminate) { - mRemainSendCount.erase(countIt); - auto pinnedIds = pinReuseTreeBlocks(reqId); - if (pinnedIds.empty()) - { - TLLM_LOG_ERROR( - "Requested blocks do not exist in the source's reuse tree (request id: %lu). Notifying " - "receiver.", - reqId); - sendReadySignal(reqId, false); - discardTransferState(reqId); - } - else - { - sendReadySignal(reqId, true); - std::promise<void> promise; - // Id-only response: the reuse-tree path has no LlmRequest. - Response resp{reqId, std::move(promise), std::move(pinnedIds)}; - if (dynamic_cast<executor::kv_cache::AgentConnectionManager*>(mManager) != nullptr) - { - sendAndRemoveResponse(reqId, std::move(resp)); - } - else - { - asyncSendAndRemoveResponse(reqId, std::move(resp)); - } - } - } - { - std::scoped_lock lock(mSenderMutex); mCurrentRequest = std::nullopt; + break; } } - else - { - // The RequestInfo may race ahead of sendAsync; wait for the specific response. - { - std::unique_lock lock(mSenderMutex); - mCurrentRequest = reqId; - mSenderCv.wait(lock, - [this, reqId]() - { - return mTerminate || mReadyResponses.find(reqId) != mReadyResponses.end() - || mCancelledRequests.find(reqId) != mCancelledRequests.end(); - }); - if (mTerminate) - { - mCurrentRequest = std::nullopt; - break; - } - } - sendResponse(reqId); - } + sendResponse(reqId); } } catch (std::exception const& err) @@ -996,11 +686,6 @@ class CacheSender::Impl TLLM_LOG_ERROR("Exception in CacheSender response: %s", err.what()); responseException = std::current_exception(); } - catch (...) - { - TLLM_LOG_ERROR("Unknown exception in CacheSender response"); - responseException = std::current_exception(); - } if (!responseException) { @@ -1021,16 +706,6 @@ class CacheSender::Impl std::scoped_lock lock(mSenderMutex); mTerminate = true; } - if (common::getEnvDisaggEnableInflightCancel()) - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - for (auto& [id, flag] : mInFlightCancelFlags) - { - flag->store(true, std::memory_order_relaxed); - } - } - // Wake the sender loop and make in-flight agent transfers observe termination through - // their per-request cancellation flags. mSenderCv.notify_all(); if (mResponseFuture.valid()) { @@ -1066,7 +741,6 @@ class CacheSender::Impl { TLLM_LOG_ERROR("Failed to set CacheSender response exception: %s", err.what()); } - releasePinnedBlocks(response); } void failPendingResponses(std::exception_ptr const& exception) noexcept @@ -1087,8 +761,8 @@ class CacheSender::Impl public: void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, - tensorrt_llm::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, + nvinfer1::DataType ssmStateDataType) { mCacheTransferLayer.setRnnConfig(rnnModelConfig, rnnLayerNumPerPP, convStateDataType, ssmStateDataType); mSelfState.setCacheState(mCacheTransferLayer.getCacheState()); @@ -1114,21 +788,16 @@ class CacheSender::Impl std::mutex mMtxForMap; runtime::BufferManager mBufferManager; std::ofstream mMeasuresFile; - std::mutex mInFlightCancelMutex; - std::unordered_map<LlmRequest::RequestIdType, std::shared_ptr<std::atomic<bool>>> mInFlightCancelFlags; - std::string mInstanceId; }; class CacheReceiver::Impl { public: - Impl(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer, - std::string instanceId = "") + Impl(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer) : mManager{manager} , mSelfState{cacheLayer.getCacheState(), executor::kv_cache::CommState{manager->getCommState()}} , mCacheTransferLayer{std::move(cacheLayer)} , mBufferManager{std::make_shared<runtime::CudaStream>()} - , mInstanceId{std::move(instanceId)} { TLLM_CHECK(mManager); TLLM_CHECK(mManager->getCommState().getSelfIdx() == selfIndex); @@ -1166,16 +835,9 @@ class CacheReceiver::Impl mRequestFutures.emplace_back(std::move(requestFuture)); } auto& asyncResource = mInstanceToAsyncResource.at(processInfo); - std::shared_ptr<std::atomic<bool>> cancelFlag; - if (common::getEnvDisaggEnableInflightCancel()) - { - cancelFlag = std::make_shared<std::atomic<bool>>(false); - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - mInFlightCancelFlags[llmRequest->mRequestId] = cancelFlag; - } { std::unique_lock<std::mutex> lck(asyncResource->mMtxForQueue); - asyncResource->mRequestsQueue.emplace_back(llmRequest, std::move(promise), cancelFlag); + asyncResource->mRequestsQueue.emplace_back(llmRequest, std::move(promise)); } asyncResource->mCVforQueue.notify_all(); return future; @@ -1188,34 +850,22 @@ class CacheReceiver::Impl void receiveSync(TransferSession& session) { - try + mCacheTransferLayer.unformat(session); + if (!common::getEnvKVCacheTimeOutputPath().empty()) { - mCacheTransferLayer.unformat(session); - if (!common::getEnvKVCacheTimeOutputPath().empty()) + std::unique_lock<std::mutex> lock(mMeasuresFileMutex); + if (!mMeasuresFile.is_open()) { - std::unique_lock<std::mutex> lock(mMeasuresFileMutex); - if (!mMeasuresFile.is_open()) - { - auto outputPath = getTransferOutputPath("recv", mInstanceId); - mMeasuresFile.open(outputPath); - TLLM_CHECK_WITH_INFO(mMeasuresFile.is_open(), "Failed to open transfer output file: %s", - outputPath.string().c_str()); - } - session.exportMeasure(mMeasuresFile, false); + auto outputPath = getTransferOutputPath("recv"); + mMeasuresFile.open(outputPath); + TLLM_CHECK_WITH_INFO( + mMeasuresFile.is_open(), "Failed to open transfer output file: %s", outputPath.string().c_str()); } - session.releaseReservedRecvBuffers(); - } - catch (...) - { - if (common::getEnvDisaggEnableInflightCancel()) - { - session.poisonReservedRecvBuffers(); - } - throw; + session.exportMeasure(mMeasuresFile, false); } } - TransferSession sendRequestInfo(LlmRequest const& llmRequest, std::atomic<bool> const* perRequestCancel = nullptr) + TransferSession sendRequestInfo(LlmRequest const& llmRequest) { uint64_t requestId = llmRequest.getContextPhaseParams().value().getReqId(); auto const& contextState = llmRequest.getDataTransceiverState(); @@ -1255,31 +905,14 @@ class CacheReceiver::Impl requestInfo = RequestInfo(requestId, mSelfState, indexFromEnd, lastBlockKey); } } - // The state's provenance marks llmRequest-agnostic transfers: only - // getSerializedDataTransceiverState sets it; context responses leave it unset. - requestInfo.setIsArbitraryTransfer(contextState.isArbitraryTransferState()); auto* agentConnectionManager = dynamic_cast<executor::kv_cache::AgentConnectionManager*>(mManager); - std::vector<BufferIndexHolder> recvHolders; std::vector<std::optional<size_t>> cacheBufferIds; if (agentConnectionManager) { - auto const* bufferCancel = common::getEnvDisaggEnableInflightCancel() ? perRequestCancel : nullptr; - auto const& managers = agentConnectionManager->getCacheTransBufferManagers(); - recvHolders.reserve(managers.size()); - cacheBufferIds.reserve(managers.size()); - for (auto& cacheTransBufferManager : managers) + for (auto& cacheTransBufferManager : agentConnectionManager->getCacheTransBufferManagers()) { - auto rawIdx = cacheTransBufferManager->assignBufferIndexForRecv(bufferCancel); - recvHolders.emplace_back(*cacheTransBufferManager, rawIdx, /*isRecv=*/true); - if (rawIdx.has_value()) - { - cacheBufferIds.push_back(static_cast<size_t>(rawIdx.value())); - } - else - { - cacheBufferIds.push_back(std::nullopt); - } + cacheBufferIds.push_back(cacheTransBufferManager->assignBufferIndexForRecv()); } TLLM_CHECK(!cacheBufferIds.empty()); } @@ -1308,102 +941,70 @@ class CacheReceiver::Impl allConnections.emplace_back(connection); } - if (common::getEnvDisaggEnableInflightCancel() && perRequestCancel != nullptr - && perRequestCancel->load(std::memory_order_relaxed)) + for (size_t ci = 0; ci < allCounterparts.size(); ci++) { - TLLM_THROW("KV cache receive request cancelled before publishing receive buffers"); - } + auto rank = allCounterparts[ci]; + auto const* connection = connections.at(rank); - try - { - for (size_t ci = 0; ci < allCounterparts.size(); ci++) - { - auto rank = allCounterparts[ci]; - auto const* connection = connections.at(rank); + bool isKvCounterpart + = std::find(kvCounterParts.begin(), kvCounterParts.end(), rank) != kvCounterParts.end(); + bool isRnnCounterpart + = hasRnn && std::find(rnnCounterParts.begin(), rnnCounterParts.end(), rank) != rnnCounterParts.end(); - bool isKvCounterpart - = std::find(kvCounterParts.begin(), kvCounterParts.end(), rank) != kvCounterParts.end(); - bool isRnnCounterpart = hasRnn - && std::find(rnnCounterParts.begin(), rnnCounterParts.end(), rank) != rnnCounterParts.end(); - - if (agentConnectionManager) + if (agentConnectionManager) + { + auto idsForRank = cacheBufferIds; + auto const& managers = agentConnectionManager->getCacheTransBufferManagers(); + for (size_t i = 0; i < idsForRank.size(); i++) { - auto idsForRank = cacheBufferIds; - auto const& managers = agentConnectionManager->getCacheTransBufferManagers(); - for (size_t i = 0; i < idsForRank.size(); i++) - { - auto kind = managers[i]->getBufferKind(); - bool include = (kind != BufferKind::kRNN) ? isKvCounterpart : isRnnCounterpart; - if (!include) - { - idsForRank[i] = std::nullopt; - } - } - - int validConnectionIdx = 0; - if (isKvCounterpart) + auto kind = managers[i]->getBufferKind(); + bool include = (kind != BufferKind::kRNN) ? isKvCounterpart : isRnnCounterpart; + if (!include) { - auto kvCpIdx - = std::find(kvCounterParts.begin(), kvCounterParts.end(), rank) - kvCounterParts.begin(); - auto [pickUpIdx, localRankIdx] = mCacheTransferLayer.getKvFormatter()->pickRecvConnections( - allCounterparts.size(), mSelfState.getCacheState().value(), - mSelfState.getCommState().value().getSelfIdx(), destCacheState, allCounterparts); - validConnectionIdx - = std::find(localRankIdx.begin(), localRankIdx.end(), kvCpIdx) - localRankIdx.begin(); + idsForRank[i] = std::nullopt; } - else if (isRnnCounterpart) - { - auto rnnTargetInfo = executor::kv_cache::targetIRanksForRnn(destCacheState, - mCacheTransferLayer.getCacheState(), mSelfState.getCommState().value().getSelfIdx()); - auto rnnCpIdx - = std::find(rnnCounterParts.begin(), rnnCounterParts.end(), rank) - rnnCounterParts.begin(); - auto [pickUpIdx, localRankIdx] - = cache_formatter_utils::pickRecvConnections(rnnCounterParts.size(), - mCacheTransferLayer.getCacheState(), mSelfState.getCommState().value().getSelfIdx(), - destCacheState, rnnCounterParts, rnnTargetInfo); - validConnectionIdx - = std::find(localRankIdx.begin(), localRankIdx.end(), rnnCpIdx) - localRankIdx.begin(); - } - - auto* agentConnection = dynamic_cast<executor::kv_cache::AgentConnection const*>(connection); - TLLM_CHECK(agentConnection != nullptr); + } - const_cast<executor::kv_cache::AgentConnection*>(agentConnection) - ->sendRequestAndBufferInfo(requestInfo, idsForRank, validConnectionIdx, perRequestCancel); + int validConnectionIdx = 0; + if (isKvCounterpart) + { + auto kvCpIdx + = std::find(kvCounterParts.begin(), kvCounterParts.end(), rank) - kvCounterParts.begin(); + auto [pickUpIdx, localRankIdx] = mCacheTransferLayer.getKvFormatter()->pickRecvConnections( + allCounterparts.size(), mSelfState.getCacheState().value(), + mSelfState.getCommState().value().getSelfIdx(), destCacheState, allCounterparts); + validConnectionIdx + = std::find(localRankIdx.begin(), localRankIdx.end(), kvCpIdx) - localRankIdx.begin(); } - else + else if (isRnnCounterpart) { - sendRequestInfo(connection, requestInfo); + auto rnnTargetInfo = executor::kv_cache::targetIRanksForRnn(destCacheState, + mCacheTransferLayer.getCacheState(), mSelfState.getCommState().value().getSelfIdx()); + auto rnnCpIdx + = std::find(rnnCounterParts.begin(), rnnCounterParts.end(), rank) - rnnCounterParts.begin(); + auto [pickUpIdx, localRankIdx] = cache_formatter_utils::pickRecvConnections(rnnCounterParts.size(), + mCacheTransferLayer.getCacheState(), mSelfState.getCommState().value().getSelfIdx(), + destCacheState, rnnCounterParts, rnnTargetInfo); + validConnectionIdx + = std::find(localRankIdx.begin(), localRankIdx.end(), rnnCpIdx) - localRankIdx.begin(); } - } - auto const& resource = getReceiveCacheResource(llmRequest); - TransferSession session = perRequestCancel != nullptr - ? TransferSession(std::move(allConnections), - DataContext{tagFromRequestId(requestId), *perRequestCancel}, std::move(allCounterparts), mSelfState, - contextState, resource->mBufferManager, requestInfo.getIndexFromEnd(), - requestInfo.getLastBlockKey(), &llmRequest, !common::getEnvKVCacheTimeOutputPath().empty()) - : TransferSession(std::move(allConnections), DataContext{tagFromRequestId(requestId), mTerminate}, - std::move(allCounterparts), mSelfState, contextState, resource->mBufferManager, - requestInfo.getIndexFromEnd(), requestInfo.getLastBlockKey(), &llmRequest, - !common::getEnvKVCacheTimeOutputPath().empty()); - if (!recvHolders.empty()) - { - session.setReservedRecvBuffers(std::move(recvHolders)); + auto* agentConnection = dynamic_cast<executor::kv_cache::AgentConnection const*>(connection); + TLLM_CHECK(agentConnection != nullptr); + + const_cast<executor::kv_cache::AgentConnection*>(agentConnection) + ->sendRequestAndBufferInfo(requestInfo, idsForRank, validConnectionIdx); } - return session; - } - catch (...) - { - if (common::getEnvDisaggEnableInflightCancel()) + else { - for (auto& holder : recvHolders) - { - holder.poison(); - } + sendRequestInfo(connection, requestInfo); } - throw; } + auto const& resource = getReceiveCacheResource(llmRequest); + return TransferSession(std::move(allConnections), DataContext{tagFromRequestId(requestId), mTerminate}, + std::move(allCounterparts), mSelfState, contextState, resource->mBufferManager, + requestInfo.getIndexFromEnd(), requestInfo.getLastBlockKey(), &llmRequest, + !common::getEnvKVCacheTimeOutputPath().empty()); } std::unique_ptr<ReceiveCacheResource> const& getReceiveCacheResource(LlmRequest const& llmRequest) @@ -1442,27 +1043,11 @@ class CacheReceiver::Impl std::string processInfo = kDefaultProcessInfo; if (common::getEnvRequestKVCacheConcurrent()) { - auto const& commState = llmRequest.getDataTransceiverState().getCommState(); - if (!commState.has_value()) - { - TLLM_LOG_WARNING("Cannot cancel request %zu: the request has no data-transceiver communication state", - llmRequest.mRequestId); - return false; - } - processInfo = commState->toString(); - } - - auto const resourceIt = mInstanceToAsyncResource.find(processInfo); - if (resourceIt == mInstanceToAsyncResource.end()) - { - TLLM_LOG_WARNING("Cannot cancel request %zu: receive worker %s is not registered", llmRequest.mRequestId, - processInfo.c_str()); - return false; + processInfo = llmRequest.getDataTransceiverState().getCommState()->toString(); } bool isCancelled = false; - auto& asyncResource = resourceIt->second; - std::optional<LlmRequest::RequestIdType> queuedCancelledReqId; + auto& asyncResource = mInstanceToAsyncResource.at(processInfo); { std::unique_lock<std::mutex> lck(asyncResource->mMtxForQueue); auto it = std::find_if(asyncResource->mRequestsQueue.begin(), asyncResource->mRequestsQueue.end(), @@ -1489,116 +1074,43 @@ class CacheReceiver::Impl } asyncResource->mRequestsQueue.erase(it); isCancelled = true; - queuedCancelledReqId = llmRequest.mRequestId; } - } - if (common::getEnvDisaggEnableInflightCancel() && queuedCancelledReqId.has_value()) - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - mInFlightCancelFlags.erase(*queuedCancelledReqId); - } - if (!isCancelled && common::getEnvDisaggEnableInflightCancel()) - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - auto flagIt = mInFlightCancelFlags.find(llmRequest.mRequestId); - if (flagIt != mInFlightCancelFlags.end()) + else { - flagIt->second->store(true, std::memory_order_relaxed); - isCancelled = true; + TLLM_LOG_WARNING("Cannot cancel request %zu", llmRequest.mRequestId); } } - if (!isCancelled) - { - TLLM_LOG_WARNING("Cannot cancel request %zu", llmRequest.mRequestId); - } return isCancelled; } - enum class ReadySignalResult - { - kReady, - kNotReady, - kMixed, - kCancelled, - }; - - ReadySignalResult receiveReadySignalDetailed(TransferSession& session, std::atomic<bool> const& perRequestCancel) + bool receiveReadySignal(TransferSession& session) { + bool isReadyFinal = true; bool isReady = false; - bool anyReady = false; - bool anyNotReady = false; auto const& connections = session.getConnections(); for (size_t i = 0; i < connections.size(); i++) { - if (perRequestCancel.load(std::memory_order_relaxed)) - { - return ReadySignalResult::kCancelled; - } auto* agentConnectionManager = dynamic_cast<executor::kv_cache::AgentConnectionManager*>(mManager); if (agentConnectionManager) { auto* agentConnection = dynamic_cast<executor::kv_cache::AgentConnection const*>(connections.at(i)); TLLM_CHECK(agentConnection); - auto ready = agentConnection->recvReadySignalWithStatus( - executor::kv_cache::DataContext{session.getDataContext().getTag(), perRequestCancel}); - if (!ready.has_value()) - { - return ReadySignalResult::kCancelled; - } - isReady = ready.value(); + isReady = agentConnection->recvReadySignal(session.getDataContext()); } else { connections.at(i)->recv( executor::kv_cache::DataContext{TransceiverTag::kREADY_SIGNAL_TAG}, &isReady, sizeof(isReady)); - if (perRequestCancel.load(std::memory_order_relaxed)) - { - return ReadySignalResult::kCancelled; - } } - anyReady |= isReady; - anyNotReady |= !isReady; + isReadyFinal &= isReady; } - if (anyReady && anyNotReady) - { - return ReadySignalResult::kMixed; - } - return anyReady ? ReadySignalResult::kReady : ReadySignalResult::kNotReady; - } - - bool receiveReadySignal(TransferSession& session) - { - auto const result = receiveReadySignalDetailed(session, mTerminate); - if (result == ReadySignalResult::kNotReady) - { - session.releaseReservedRecvBuffers(); - } - else if (result == ReadySignalResult::kMixed) - { - if (common::getEnvDisaggEnableInflightCancel()) - { - session.poisonReservedRecvBuffers(); - } - else - { - session.releaseReservedRecvBuffers(); - } - } - return result == ReadySignalResult::kReady; + return isReadyFinal; } ~Impl() { mTerminate.store(true); - if (common::getEnvDisaggEnableInflightCancel()) - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - for (auto& [id, flag] : mInFlightCancelFlags) - { - flag->store(true, std::memory_order_relaxed); - } - } for (auto&& [processInfo, asyncResource] : mInstanceToAsyncResource) { asyncResource->mTerminate = true; @@ -1611,127 +1123,71 @@ class CacheReceiver::Impl } private: - void requestSync(LlmRequest& llmRequest, std::atomic<bool> const& perRequestCancel) + void requestSync(LlmRequest& llmRequest) { auto const requestId = llmRequest.mRequestId; auto const contextRequestId = llmRequest.getContextPhaseParams().value().getReqId(); char const* phase = "request-info"; TLLM_LOG_DEBUG("KV cache receive request %zu, context request %zu started.", requestId, contextRequestId); - if (llmRequest.getKvCacheTransferStart() == LlmRequest::TimePoint{}) - { - llmRequest.setKvCacheTransferStart(LlmRequest::getSteadyClockNow()); - } - - std::optional<TransferSession> session; + llmRequest.setKvCacheTransferStart(LlmRequest::getSteadyClockNow()); try { - if (perRequestCancel.load(std::memory_order_relaxed) || mTerminate.load(std::memory_order_relaxed)) - { - TLLM_THROW("KV cache receive request %zu cancelled before request-info", requestId); - } TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); TLLM_LOG_DEBUG("KV cache receive request %zu, context request %zu phase=%s begin.", requestId, contextRequestId, phase); - auto const* cancelFlag = common::getEnvDisaggEnableInflightCancel() ? &perRequestCancel : nullptr; - session.emplace(sendRequestInfo(llmRequest, cancelFlag)); - session->setTime(TransferSession::kTimeRequestInfo); + auto session = sendRequestInfo(llmRequest); + session.setTime(TransferSession::kTimeRequestInfo); TLLM_LOG_DEBUG( "KV cache receive request %zu, context request %zu phase=%s end.", requestId, contextRequestId, phase); phase = "ready-signal"; TLLM_LOG_DEBUG("KV cache receive request %zu, context request %zu phase=%s begin.", requestId, contextRequestId, phase); - auto readyResult = receiveReadySignalDetailed(*session, perRequestCancel); - TLLM_LOG_DEBUG("KV cache receive request %zu, context request %zu phase=%s end: result=%d.", requestId, - contextRequestId, phase, static_cast<int>(readyResult)); - if (readyResult == ReadySignalResult::kCancelled) - { - if (common::getEnvDisaggEnableInflightCancel()) - { - session->poisonReservedRecvBuffers(); - } - TLLM_THROW("KV cache receive request %zu cancelled while waiting for the ready signal", requestId); - } - if (readyResult == ReadySignalResult::kNotReady) + bool const isReady = receiveReadySignal(session); + TLLM_LOG_DEBUG("KV cache receive request %zu, context request %zu phase=%s end: ready=%d.", requestId, + contextRequestId, phase, static_cast<int>(isReady)); + if (!isReady) { - session->releaseReservedRecvBuffers(); - TLLM_THROW("KV cache receive request %zu was rejected by the context peer", requestId); - } - if (readyResult == ReadySignalResult::kMixed) - { - if (common::getEnvDisaggEnableInflightCancel()) - { - session->poisonReservedRecvBuffers(); - } - else - { - session->releaseReservedRecvBuffers(); - } - TLLM_THROW("KV cache receive request %zu received inconsistent ready signals from its context peers", - requestId); + // Reuse the error state for the cancelled request. + llmRequest.setState(LlmRequestState::kDISAGG_TRANS_ERROR); + llmRequest.setKvCacheTransferEnd(LlmRequest::getSteadyClockNow()); + return; } phase = "transfer-completion-notification"; TLLM_LOG_DEBUG("KV cache receive request %zu, context request %zu phase=%s begin.", requestId, contextRequestId, phase); - receiveSync(*session); + receiveSync(session); TLLM_LOG_DEBUG( "KV cache receive request %zu, context request %zu phase=%s end.", requestId, contextRequestId, phase); llmRequest.setKvCacheTransferEnd(LlmRequest::getSteadyClockNow()); } catch (std::exception const& err) { - if (common::getEnvDisaggEnableInflightCancel() && session.has_value()) - { - session->poisonReservedRecvBuffers(); - } - llmRequest.setKvCacheTransferEnd(LlmRequest::getSteadyClockNow()); TLLM_LOG_ERROR("KV cache receive request %zu, context request %zu failed in phase=%s: %s", requestId, contextRequestId, phase, err.what()); throw; } - catch (...) - { - if (common::getEnvDisaggEnableInflightCancel() && session.has_value()) - { - session->poisonReservedRecvBuffers(); - } - llmRequest.setKvCacheTransferEnd(LlmRequest::getSteadyClockNow()); - TLLM_LOG_ERROR( - "KV cache receive request %zu, context request %zu failed in phase=%s with an unknown " - "exception", - requestId, contextRequestId, phase); - throw; - } TLLM_LOG_DEBUG("KV cache receive request %zu, context request %zu completed.", requestId, contextRequestId); } - void requestSync(LlmRequest& llmRequest) - { - requestSync(llmRequest, mTerminate); - } - struct RequestAndPromise { // shared_ptr so this struct co-owns the request until the promise resolves; // protects worker-side dereferences and the promise itself from premature destruction. std::shared_ptr<LlmRequest> mRequest; std::unique_ptr<std::promise<void>> mPromise; - std::shared_ptr<std::atomic<bool>> mCancelFlag; RequestAndPromise() : mRequest(nullptr) , mPromise(nullptr) - , mCancelFlag(nullptr) { } - RequestAndPromise(std::shared_ptr<LlmRequest> request, std::unique_ptr<std::promise<void>>&& promise, - std::shared_ptr<std::atomic<bool>> cancelFlag) + RequestAndPromise(std::shared_ptr<LlmRequest> request, std::unique_ptr<std::promise<void>>&& promise) : mRequest(std::move(request)) , mPromise(std::move(promise)) - , mCancelFlag(std::move(cancelFlag)) { } @@ -1740,7 +1196,6 @@ class CacheReceiver::Impl RequestAndPromise(RequestAndPromise&& other) noexcept : mRequest(std::move(other.mRequest)) , mPromise(std::move(other.mPromise)) - , mCancelFlag(std::move(other.mCancelFlag)) { } @@ -1756,7 +1211,6 @@ class CacheReceiver::Impl mRequest = std::move(other.mRequest); mPromise = std::move(other.mPromise); - mCancelFlag = std::move(other.mCancelFlag); } return *this; } @@ -1800,9 +1254,7 @@ class CacheReceiver::Impl try { TLLM_CHECK_WITH_INFO(requestAndPromise.mRequest != nullptr, "requestAndPromise.mRequest is null"); - auto const& cancelFlag - = requestAndPromise.mCancelFlag != nullptr ? *requestAndPromise.mCancelFlag : mTerminate; - requestSync(*requestAndPromise.mRequest, cancelFlag); + requestSync(*requestAndPromise.mRequest); requestAndPromise.mPromise->set_value(); } catch (tensorrt_llm::common::RequestSpecificException const& err) @@ -1821,27 +1273,14 @@ class CacheReceiver::Impl requestAndPromise.mRequest->getContextPhaseParams().value().getReqId(), err.what()); requestAndPromise.mPromise->set_exception(std::current_exception()); } - catch (...) - { - TLLM_LOG_ERROR("Unknown exception in CacheReceiver request() loop"); - if (requestAndPromise.mPromise) - { - requestAndPromise.mPromise->set_exception(std::current_exception()); - } - } - if (common::getEnvDisaggEnableInflightCancel() && requestAndPromise.mRequest != nullptr) - { - std::lock_guard<std::mutex> lg(mInFlightCancelMutex); - mInFlightCancelFlags.erase(requestAndPromise.mRequest->mRequestId); - } } } } public: void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, - tensorrt_llm::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, + nvinfer1::DataType ssmStateDataType) { mCacheTransferLayer.setRnnConfig(rnnModelConfig, rnnLayerNumPerPP, convStateDataType, ssmStateDataType); mSelfState.setCacheState(mCacheTransferLayer.getCacheState()); @@ -1861,9 +1300,6 @@ class CacheReceiver::Impl std::ofstream mMeasuresFile; std::mutex mMeasuresFileMutex; std::atomic<bool> mTerminate{false}; - std::mutex mInFlightCancelMutex; - std::unordered_map<LlmRequest::RequestIdType, std::shared_ptr<std::atomic<bool>>> mInFlightCancelFlags; - std::string mInstanceId; }; void CacheSender::ImplDeleter::operator()(Impl* ptr) @@ -1876,10 +1312,9 @@ void CacheReceiver::ImplDeleter::operator()(Impl* ptr) delete ptr; } -CacheSender::CacheSender(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, - CacheTransferLayer cacheLayer, std::string instanceId) - : mImpl{ - std::unique_ptr<Impl, ImplDeleter>(new Impl(manager, selfIndex, std::move(cacheLayer), std::move(instanceId)))} +CacheSender::CacheSender( + executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer) + : mImpl{std::unique_ptr<Impl, ImplDeleter>(new Impl(manager, selfIndex, std::move(cacheLayer)))} { } @@ -1923,16 +1358,14 @@ void CacheSender::sendReadySignal(LlmRequest::RequestIdType requestId, bool isRe } void CacheSender::setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, - tensorrt_llm::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, nvinfer1::DataType ssmStateDataType) { mImpl->setRnnConfig(std::move(rnnModelConfig), std::move(rnnLayerNumPerPP), convStateDataType, ssmStateDataType); } -CacheReceiver::CacheReceiver(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, - CacheTransferLayer cacheLayer, std::string instanceId) - : mImpl{ - std::unique_ptr<Impl, ImplDeleter>(new Impl(manager, selfIndex, std::move(cacheLayer), std::move(instanceId)))} +CacheReceiver::CacheReceiver( + executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer) + : mImpl{std::unique_ptr<Impl, ImplDeleter>(new Impl(manager, selfIndex, std::move(cacheLayer)))} { } @@ -1964,8 +1397,7 @@ bool CacheReceiver::receiveReadySignal(TransferSession& session) } void CacheReceiver::setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, - tensorrt_llm::DataType ssmStateDataType) + std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, nvinfer1::DataType ssmStateDataType) { mImpl->setRnnConfig(std::move(rnnModelConfig), std::move(rnnLayerNumPerPP), convStateDataType, ssmStateDataType); } diff --git a/cpp/tensorrt_llm/batch_manager/dataTransceiver.h b/cpp/tensorrt_llm/batch_manager/dataTransceiver.h index 778e5e80c7f4..3362574da902 100644 --- a/cpp/tensorrt_llm/batch_manager/dataTransceiver.h +++ b/cpp/tensorrt_llm/batch_manager/dataTransceiver.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,18 +19,15 @@ #include <fstream> #include <future> #include <map> -#include <optional> #include <string> #include <vector> -#include "tensorrt_llm/batch_manager/baseTransBuffer.h" #include "tensorrt_llm/batch_manager/cacheTransceiver.h" #include "tensorrt_llm/batch_manager/cacheTransferLayer.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cacheCommunicator.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/serializeUtils.h" @@ -122,7 +119,7 @@ class TransferSession void recv(size_t idx, void* data, size_t size); - [[nodiscard]] std::optional<LlmRequest const*> getLlmRequest() const; + [[nodiscard]] LlmRequest const& getLlmRequest() const; // in CacheSender, the LlmRequest is not available until the sendSync is called void setLlmRequest(LlmRequest const& llmRequest); @@ -131,20 +128,6 @@ class TransferSession void appendMeasure(LlmRequest::TimePoint start, LlmRequest::TimePoint end, size_t size); - /// @brief Transfer ownership of pre-assigned receive buffers to this session. - void setReservedRecvBuffers(std::vector<BufferIndexHolder> holders); - - [[nodiscard]] bool hasReservedRecvBuffer(BaseTransBufferManager const& manager) const noexcept; - - /// @brief Release one formatter's pre-assigned buffer after its receive and postprocessing complete. - bool releaseReservedRecvBuffer(BaseTransBufferManager const& manager) noexcept; - - /// @brief Release all pre-assigned receive buffers after the full receive pipeline completes. - void releaseReservedRecvBuffers() noexcept; - - /// @brief Fail closed when receive-buffer quiescence cannot be established. - void poisonReservedRecvBuffers() noexcept; - // TODO: 1. use global id instead of context request id; 2. export to llm metrics instead of file void exportMeasure(std::ofstream& outFile, bool isContext) const; @@ -177,7 +160,6 @@ class TransferSession runtime::BufferManager const* mBufferManager; LlmRequest const* mRequest; std::unique_ptr<KVCacheTimes> mTimes; - std::vector<BufferIndexHolder> mReservedRecvBuffers; int32_t mIndexFromEnd{0}; BlockKey mLastBlockKey{}; }; @@ -235,17 +217,6 @@ class RequestInfo return mLastBlockKey; } - /// @brief Arbitrary (llmRequest-agnostic) transfer served from the sender's reuse tree. - [[nodiscard]] bool isArbitraryTransfer() const noexcept - { - return mIsArbitraryTransfer; - } - - void setIsArbitraryTransfer(bool isArbitraryTransfer) noexcept - { - mIsArbitraryTransfer = isArbitraryTransfer; - } - /// @brief Serialization. /// @param requestInfo Request information to be serialized. /// @param os The output stream to which the serialization result points. @@ -269,9 +240,6 @@ class RequestInfo // Last block key, used to derive other block keys on receiver BlockKey mLastBlockKey{}; - // True for arbitrary (llmRequest-agnostic) transfers served from the sender's reuse tree. - bool mIsArbitraryTransfer{false}; - // The state of the data transceiver. executor::DataTransceiverState mTransState; }; @@ -283,8 +251,7 @@ class CacheSender /// @param manager The connection manager. /// @param selfIndex The sequential index of the current executor process. /// @param cacheLayer The cache layer bundling all cache states and formatters. - CacheSender(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer, - std::string instanceId = ""); + CacheSender(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer); CacheSender() = default; @@ -323,8 +290,8 @@ class CacheSender /// @brief Update the RNN config on the internal CacheState copies. /// Used by CppMambaHybridCacheManager path where RNN config is set after construction. void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, - tensorrt_llm::DataType ssmStateDataType); + std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, + nvinfer1::DataType ssmStateDataType); /// @brief Destructor. virtual ~CacheSender(); @@ -347,8 +314,7 @@ class CacheReceiver /// @param manager The connection manager. /// @param selfIndex The sequential index of the current executor process. /// @param cacheLayer The cache layer bundling all cache states and formatters. - CacheReceiver(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer, - std::string instanceId = ""); + CacheReceiver(executor::kv_cache::ConnectionManager* manager, SizeType32 selfIndex, CacheTransferLayer cacheLayer); CacheReceiver() = default; @@ -375,8 +341,8 @@ class CacheReceiver /// @brief Update the RNN config on the internal CacheState copies. /// Used by CppMambaHybridCacheManager path where RNN config is set after construction. void setRnnConfig(executor::kv_cache::CacheState::RnnModelConfig rnnModelConfig, - std::vector<SizeType32> rnnLayerNumPerPP, tensorrt_llm::DataType convStateDataType, - tensorrt_llm::DataType ssmStateDataType); + std::vector<SizeType32> rnnLayerNumPerPP, nvinfer1::DataType convStateDataType, + nvinfer1::DataType ssmStateDataType); /// @brief Destructor. virtual ~CacheReceiver(); diff --git a/cpp/tensorrt_llm/batch_manager/decoderBuffers.cpp b/cpp/tensorrt_llm/batch_manager/decoderBuffers.cpp index fecc0851d361..fd67bb55e89d 100644 --- a/cpp/tensorrt_llm/batch_manager/decoderBuffers.cpp +++ b/cpp/tensorrt_llm/batch_manager/decoderBuffers.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/batch_manager/decoderBuffers.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/decoderState.h" @@ -71,21 +70,21 @@ DecoderOutputBuffers::DecoderOutputBuffers(SizeType32 maxNumSequences, SizeType3 auto constexpr TRTTokenIdType = runtime::TRTDataType<runtime::TokenIdType>::value; sequenceLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), nvinfer1::DataType::kINT32); - finishedSumHost = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + finishedSumHost = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); newOutputTokensHost = BufferManager::pinned(ITensor::makeShape({maxTokensPerStep, maxNumSequences, maxBeamWidth}), TRTTokenIdType); cumLogProbsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), nvinfer1::DataType::kFLOAT); logProbsHost = BufferManager::pinned( - ITensor::makeShape({maxNumSequences, maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({maxNumSequences, maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kFLOAT); finishReasonsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), tensorrt_llm::DataType::kUINT8); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences, maxBeamWidth}), nvinfer1::DataType::kUINT8); } void DecoderOutputBuffers::enableLookaheadDecoding(SizeType32 maxNumSequences, SizeType32 maxTokensPerStep) @@ -116,9 +115,9 @@ void DecoderOutputBuffers::setupSpeculativeDecoding( if (speculativeDecodingMode.variableDraftLength()) { nextDraftTokensLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); prevDraftTokensLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); } } } @@ -308,18 +307,17 @@ DecoderSlotAsyncSend::~DecoderSlotAsyncSend() SlotDecoderBuffers::SlotDecoderBuffers(SizeType32 maxBeamWidth, SizeType32 maxSeqLen, BufferManager const& manager) { - outputIds = manager.gpu(ITensor::makeShape({maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kINT32); - outputIdsHost - = BufferManager::pinned(ITensor::makeShape({maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kINT32); + outputIds = manager.gpu(ITensor::makeShape({maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kINT32); + outputIdsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kINT32); - sequenceLengths = manager.gpu(ITensor::makeShape({maxBeamWidth}), tensorrt_llm::DataType::kINT32); - sequenceLengthsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth}), tensorrt_llm::DataType::kINT32); + sequenceLengths = manager.gpu(ITensor::makeShape({maxBeamWidth}), nvinfer1::DataType::kINT32); + sequenceLengthsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth}), nvinfer1::DataType::kINT32); - cumLogProbs = manager.gpu(ITensor::makeShape({maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); - cumLogProbsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); + cumLogProbs = manager.gpu(ITensor::makeShape({maxBeamWidth}), nvinfer1::DataType::kFLOAT); + cumLogProbsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth}), nvinfer1::DataType::kFLOAT); - logProbs = manager.gpu(ITensor::makeShape({maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kFLOAT); - logProbsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth, maxSeqLen}), tensorrt_llm::DataType::kFLOAT); + logProbs = manager.gpu(ITensor::makeShape({maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kFLOAT); + logProbsHost = BufferManager::pinned(ITensor::makeShape({maxBeamWidth, maxSeqLen}), nvinfer1::DataType::kFLOAT); } } // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/encoderBuffers.cpp b/cpp/tensorrt_llm/batch_manager/encoderBuffers.cpp new file mode 100644 index 000000000000..56fd393c68d7 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/encoderBuffers.cpp @@ -0,0 +1,560 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "encoderBuffers.h" + +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" + +#include <valarray> + +using namespace tensorrt_llm::runtime; + +namespace tensorrt_llm::batch_manager +{ + +EncoderBuffers::EncoderBuffers( + SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, TllmRuntime const& runtime) +{ + // init empty buffers on cpu/gpu/pinned + init(maxBatchSize, modelConfig, worldConfig, runtime); + + // pre-allocate based on max buffer sizes + // Note: pre-allocation can be done directly instead of empty-->reshape, but it is ok extract the common reshape() + // utility because the buffer shapes can be dynamically set during runtime as well + initBufferSizes(maxBatchSize, modelConfig, worldConfig, runtime); +} + +void EncoderBuffers::init( + SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const& manager = runtime.getBufferManager(); + + auto hiddenStatesType = modelConfig.getDataType(); + + inputFeatures = manager.emptyTensor(MemoryType::kGPU, hiddenStatesType); + inputIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + + // in PP, only rank 0 needs the following input fields + if (modelConfig.usePositionEmbedding() && worldConfig.isFirstPipelineParallelRank()) + { + positionIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + positionIdsReserved.resize(maxBatchSize * modelConfig.getMaxInputLen()); + std::iota(positionIdsReserved.begin(), positionIdsReserved.end(), 0); + } + if (modelConfig.useTokenTypeEmbedding() && worldConfig.isFirstPipelineParallelRank()) + { + tokenTypeIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + tokenTypeIdsReserved.resize(maxBatchSize * modelConfig.getMaxInputLen()); + std::fill(tokenTypeIdsReserved.begin(), tokenTypeIdsReserved.end(), 0); + } + + inputLengths = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + maxInputLength = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + + if (worldConfig.isPipelineParallel()) + { + hiddenStates = manager.emptyTensor(MemoryType::kGPU, hiddenStatesType); + } + if (worldConfig.isLastPipelineParallelRank()) + { + encoderOutput = manager.emptyTensor(MemoryType::kGPU, hiddenStatesType); + } + + if (modelConfig.useLanguageAdapter()) + { + languageAdapterRoutings = manager.emptyTensor(MemoryType::kGPU, TRTDataType<SizeType32>::value); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::initBufferSizes( + SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // get buffer shape based on max values + numRequests = maxBatchSize; + encoderInputLen = maxBatchSize * modelConfig.getMaxInputLen(); + encoderOutputLen = maxBatchSize * modelConfig.getMaxInputLen(); // assume output length <= input length + maxInputLengthInBatch = modelConfig.getMaxInputLen(); + + // update buffer shapes + reshape(runtime, modelConfig, worldConfig); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::updateBufferSizes(RequestVector const& requests, ModelConfig const& modelConfig, + WorldConfig const& worldConfig, TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + numRequests = requests.size(); + encoderInputLen = 0; + encoderOutputLen = 0; + maxInputLengthInBatch = 0; + + // get buffer shape based on actual batched requests + for (auto const& req : requests) + { + encoderInputLen += req->getEncoderInputLen(); + encoderOutputLen += req->getEncoderOutputLen(); + maxInputLengthInBatch + = std::max(maxInputLengthInBatch, req->getEncoderInputLen()); // Decoder input is encoder output + } + + // update buffer shapes + reshape(runtime, modelConfig, worldConfig); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::reshape(TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + if (modelConfig.isMultiModal()) + { + return; // multimodal models do not need to set position id, etc. or any output tensors + } + + inputIds->reshape(ITensor::makeShape({encoderInputLen})); + if (positionIds) + { + if (modelConfig.isWhisper()) + { + positionIds->reshape(ITensor::makeShape({encoderOutputLen})); + } + else + { + positionIds->reshape(ITensor::makeShape({encoderInputLen})); + } + } + if (tokenTypeIds) + { + tokenTypeIds->reshape(ITensor::makeShape({encoderInputLen})); + } + + inputLengths->reshape(ITensor::makeShape({numRequests})); + maxInputLength->reshape(ITensor::makeShape({maxInputLengthInBatch})); + + if (worldConfig.isPipelineParallel()) + { + hiddenStates->reshape( + ITensor::makeShape({encoderOutputLen, modelConfig.getHiddenSize() * worldConfig.getTensorParallelism()})); + } + if (worldConfig.isLastPipelineParallelRank()) + { + encoderOutput->reshape( + ITensor::makeShape({encoderOutputLen, modelConfig.getHiddenSize() * worldConfig.getTensorParallelism()})); + } + if (modelConfig.useLanguageAdapter()) + { + languageAdapterRoutings->reshape(ITensor::makeShape({encoderInputLen, 1})); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::setFromInputs(RequestVector const& requests, ModelConfig const& modelConfig, + WorldConfig const& worldConfig, TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(encoderBuffersSetFromInputs); + + if (!worldConfig.isFirstPipelineParallelRank()) + { + return; + } + + auto const& manager = runtime.getBufferManager(); + + std::vector<TokenIdType> inputIdsAll; + std::vector<SizeType32> positionIdsAll; + std::vector<SizeType32> tokenTypeIdsAll; + std::vector<SizeType32> inputLengthsAll; + std::vector<SizeType32> languageAdapterRoutingAll; + // use shape to indicates max input length, content is not important + // TODO: change to a scalar value for this from engine side + std::vector<SizeType32> maxInputLengthAll(maxInputLengthInBatch, 0); + + if (requests.front()->getEncoderInputFeatures()) + { + if (modelConfig.isMultiModal()) + { + auto batchedInputShape = requests.front()->getEncoderInputFeatures()->getShape(); // [1, 3, H, W] + batchedInputShape.d[0] = encoderInputLen; // [batch_size, 3, H, W] + inputFeatures->reshape(batchedInputShape); + } + else + { + SizeType32 const featureDim = requests.front()->getEncoderInputFeatures()->getShape().d[1]; + TLLM_LOG_DEBUG("EncoderBuffers::setFromInputs - featureDim = %d", featureDim); + inputFeatures->reshape(ITensor::makeShape({encoderInputLen, featureDim})); + } + } + + SizeType32 offset = 0; + + for (auto const& llmReq : requests) + { + SizeType32 const inputLength = llmReq->getEncoderInputLen(); + SizeType32 const outputLength = llmReq->getEncoderOutputLen(); + if (llmReq->getEncoderInputFeatures()) + { + auto const& reqFeatures + = llmReq + ->getEncoderInputFeatures(); // whisper: [length, featureDim]; Vision: [batch_size, channel, W, H] + TLLM_LOG_DEBUG("EncoderBuffers::setFromInputs - request id = %d, input features length = %d", + llmReq->mRequestId, inputLength); + manager.copy(*reqFeatures, *ITensor::slice(inputFeatures, offset, inputLength)); + offset += inputLength; + } + else + { + auto const& reqTokens = *llmReq->getEncoderTokens().value(); + inputIdsAll.insert(inputIdsAll.end(), reqTokens.begin(), reqTokens.end()); + if (tokenTypeIds) + { + tokenTypeIdsAll.insert( + tokenTypeIdsAll.end(), tokenTypeIdsReserved.begin(), tokenTypeIdsReserved.begin() + inputLength); + } + } + if (positionIds) + { + SizeType32 const length = modelConfig.isWhisper() ? outputLength : inputLength; + positionIdsAll.insert( + positionIdsAll.end(), positionIdsReserved.begin(), positionIdsReserved.begin() + length); + } + if (modelConfig.useLanguageAdapter()) + { + auto const languageAdapterRouting + = llmReq->getLanguageAdapterRouting(modelConfig.getNumLanguages().value(), inputLength); + languageAdapterRoutingAll.insert( + languageAdapterRoutingAll.end(), std::begin(languageAdapterRouting), std::end(languageAdapterRouting)); + } + inputLengthsAll.push_back(inputLength); + } + + // copy inputs from host to device + { + NVTX3_SCOPED_RANGE(bufferCopies); + if (requests.front()->getEncoderTokens()) + { + manager.copy(inputIdsAll.data(), *inputIds); + if (tokenTypeIds) + { + manager.copy(tokenTypeIdsAll.data(), *tokenTypeIds); + } + manager.copy(maxInputLengthAll.data(), *maxInputLength); + } + if (positionIds) + { + manager.copy(positionIdsAll.data(), *positionIds); + } + manager.copy(inputLengthsAll.data(), *inputLengths); + if (modelConfig.useLanguageAdapter()) + { + manager.copy(languageAdapterRoutingAll.data(), *languageAdapterRoutings); + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::fillIOMaps(ModelConfig const& modelConfig, WorldConfig const& worldConfig) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(runtimeBuffersFillIOMaps); + + inputMap.clear(); + outputMap.clear(); + + // inputs + if (modelConfig.isMultiModal()) + { + inputMap.insert_or_assign("input", inputFeatures); + } + else if (modelConfig.isWhisper()) + { + inputMap.insert_or_assign("input_features", inputFeatures); + inputMap.insert_or_assign("input_lengths", inputLengths); + inputMap.insert_or_assign("position_ids", positionIds); + } + else + { + if (worldConfig.isFirstPipelineParallelRank()) + { + inputMap.insert_or_assign("input_ids", inputIds); + if (positionIds) + { + inputMap.insert_or_assign("position_ids", positionIds); + } + if (tokenTypeIds) + { + inputMap.insert_or_assign("token_type_ids", tokenTypeIds); + } + } + else + { + inputMap.insert_or_assign("hidden_states_input", hiddenStates); + } + inputMap.insert_or_assign("input_lengths", inputLengths); + inputMap.insert_or_assign("max_input_length", maxInputLength); + if (modelConfig.useLanguageAdapter()) + { + inputMap.insert_or_assign("language_adapter_routings", languageAdapterRoutings); + } + } + + // outputs + if (worldConfig.isLastPipelineParallelRank()) + { + outputMap.insert_or_assign("encoder_output", encoderOutput); + } + else + { + outputMap.insert_or_assign("hidden_states_output", hiddenStates); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +std::pair<EncoderBuffers::TensorMap const&, EncoderBuffers::TensorMap&> EncoderBuffers::prepareIO( + RequestVector const& requests, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + updateBufferSizes(requests, modelConfig, worldConfig, runtime); + + setFromInputs(requests, modelConfig, worldConfig, runtime); + + fillIOMaps(modelConfig, worldConfig); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + + return {inputMap, outputMap}; +} + +void EncoderBuffers::rearrangeOutputs(RequestVector const& requests, ModelConfig const& modelConfig, + WorldConfig const& worldConfig, TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(encoderBuffersRearrangeOutput); + + auto const& manager = runtime.getBufferManager(); + + SizeType32 offset = 0, size = 0; + + updateReqOutputShape(requests, runtime, worldConfig, modelConfig); + + for (auto const& req : requests) + { + // copy from internal buffer to request-owned external buffers + size = req->getEncoderOutputLen(); + TLLM_LOG_DEBUG("EncoderBuffers::rearrangeOutputs - req: %d, encoderOutput shape = (%d, %d)", req->mClientId, + req->getEncoderOutput()->getShape().d[0], req->getEncoderOutput()->getShape().d[1]); + TLLM_LOG_DEBUG("EncoderBuffers::rearrangeOutputs - req: %d, enc output size = %d", req->mClientId, size); + + if (worldConfig.isPipelineParallel()) + { + manager.copy(*ITensor::slice(hiddenStates, offset, size), *req->getEncoderHiddenStates()); + } + if (worldConfig.isLastPipelineParallelRank()) + { + if (modelConfig.isMultiModal()) + { + manager.copy( + *ITensor::slice(encoderOutput, offset, size), *(req->getPromptEmbeddingTableMutable().value())); + } + else + { + manager.copy(*ITensor::slice(encoderOutput, offset, size), *req->getEncoderOutput()); + } + } + offset += size; + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::updateReqOutputShape(RequestVector const& requests, TllmRuntime const& runtime, + WorldConfig const& worldConfig, ModelConfig const& modelConfig) +{ + auto const& manager = runtime.getBufferManager(); + + for (auto const& req : requests) + { + if (modelConfig.isMultiModal()) + { + auto shape = encoderOutput->getShape(); // [batch_size, prompt_vocab_size, feature_dim] + shape.d[0] = req->getEncoderOutputLen(); + req->getPromptEmbeddingTableMutable() = manager.emptyTensor(MemoryType::kGPU, encoderOutput->getDataType()); + req->getPromptEmbeddingTableMutable().value()->reshape(shape); + req->setPromptVocabSize(shape.d[1]); + // TODO: extra ids for kv cache reuse + } + else + { + auto encOutLen = req->getEncoderOutputLen(); + // update request-owned external buffer for each request + if (worldConfig.isPipelineParallel()) + { + req->getEncoderHiddenStates()->reshape( + ITensor::makeShape({encOutLen, modelConfig.getHiddenSize() * worldConfig.getTensorParallelism()})); + } + if (worldConfig.isLastPipelineParallelRank()) + { + req->getEncoderOutput()->reshape( + ITensor::makeShape({encOutLen, modelConfig.getHiddenSize() * worldConfig.getTensorParallelism()})); + } + } + } +} + +void EncoderBuffers::create(SizeType32 maxBatchSize, ModelConfig const& modelConfig, TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const& manager = runtime.getBufferManager(); + + inputLengths = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + maxInputLength = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + + hiddenSize = modelConfig.getEncoderHiddenSize(); // full hidden size + // assume encoder & decoder use the same data type + encoderOutput = manager.emptyTensor(MemoryType::kGPU, modelConfig.getDataType()); + encoderOutputReserved = manager.gpu(ITensor::makeShape({1, hiddenSize}), modelConfig.getDataType()); + + crossKvCacheGen = manager.gpu(ITensor::makeShape({1}), nvinfer1::DataType::kBOOL); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::setMaxBufferSizes(SizeType32 maxBatchSize, runtime::ModelConfig const& modelConfig) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + numRequests = maxBatchSize; + encoderInputLen = maxBatchSize * modelConfig.getMaxEncoderLen(); + encoderOutputLen = maxBatchSize * modelConfig.getMaxEncoderLen(); + maxInputLengthInBatch = modelConfig.getMaxEncoderLen(); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::setBufferSizes(RequestVector const& contextRequests, RequestVector const& genRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + numRequests = 0; /// total number of requests that need encoder information (context requests + + /// generation requests * beam width) + encoderInputLen = 0; + encoderOutputLen = 0; + maxInputLengthInBatch = 1; /// maximum encoder length in a batch + + for (auto const& llmReq : contextRequests) + { + numRequests += 1; + encoderInputLen += llmReq->getEncoderInputLen(); + encoderOutputLen += llmReq->getEncoderOutputLen(); + maxInputLengthInBatch = std::max(maxInputLengthInBatch, llmReq->getEncoderInputLen()); + } + + for (auto const& llmReq : genRequests) + { + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + numRequests += reqBeamWidth; // tile by beam width + maxInputLengthInBatch = std::max(maxInputLengthInBatch, llmReq->getEncoderInputLen()); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::reshape() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + inputLengths->reshape(ITensor::makeShape({numRequests})); + maxInputLength->reshape(ITensor::makeShape({maxInputLengthInBatch})); + encoderOutput->reshape(ITensor::makeShape({encoderOutputLen, hiddenSize})); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::fill( + RequestVector const& ctxRequests, RequestVector const& genRequests, runtime::BufferManager const& manager) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(encoderBufferCopies); + + std::vector<SizeType32> inputLengthsAll; + std::vector<SizeType32> maxInputLengthAll(maxInputLength->getShape().d[0], 0); + + SizeType32 offset = 0, size = 0; + for (auto const& requests : {ctxRequests, genRequests}) + { + for (auto const& llmReq : requests) + { + // 1. only ctx requests should gather the encoder output + // 2. only gen requests should tile encoder input lengths info by beam width + bool isCtx = llmReq->isContextInitState(); + if (isCtx) + { + size = llmReq->getEncoderOutputLen(); + auto const encoderOutputSlice = runtime::ITensor::slice(encoderOutput, offset, size); + manager.copy(*llmReq->getEncoderOutput(), *encoderOutputSlice); + offset += size; + + inputLengthsAll.emplace_back(size); + } + else + { + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + std::fill_n(std::back_inserter(inputLengthsAll), reqBeamWidth, + llmReq->getEncoderOutputLen()); // although encoder output is not needed, gen phase still needs the + // encoder length info for cross kv cache. Also tile by beam width + } + } + } + manager.copy(inputLengthsAll.data(), *inputLengths); + manager.copy(maxInputLengthAll.data(), *maxInputLength); + // crossKvCacheGen unused in engine for now, use default tensor + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EncoderBuffers::insertInputTensors(TensorMap& inputMap) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + inputMap.insert_or_assign("encoder_output", encoderOutput); + inputMap.insert_or_assign("encoder_input_lengths", inputLengths); + inputMap.insert_or_assign("encoder_max_input_length", maxInputLength); + inputMap.insert_or_assign("cross_kv_cache_gen", crossKvCacheGen); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/encoderBuffers.h b/cpp/tensorrt_llm/batch_manager/encoderBuffers.h new file mode 100644 index 000000000000..64d416280f21 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/encoderBuffers.h @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +namespace tensorrt_llm::batch_manager +{ + +class EncoderBuffers +{ +public: + using SizeType32 = tensorrt_llm::runtime::SizeType32; + using ITensor = tensorrt_llm::runtime::ITensor; + using TensorPtr = runtime::ITensor::SharedPtr; + using TensorMap = runtime::StringPtrMap<runtime::ITensor>; + using ModelConfig = runtime::ModelConfig; + using WorldConfig = runtime::WorldConfig; + using TllmRuntime = runtime::TllmRuntime; + + TensorPtr inputIds; + TensorPtr positionIds = nullptr; + TensorPtr tokenTypeIds = nullptr; + + TensorPtr inputLengths; // [numEncoderRequests] + TensorPtr maxInputLength; // [maxInputLengthInBatch] + + // intermediate states in pipeline parallelism + TensorPtr hiddenStates; // [numTokens, hiddenSize] + + // features for multimodal encoders (audio, image, etc.) + TensorPtr + inputFeatures; // [totalNumOfFeatures, featureDim] if remove_padding else [batchSize, featureDim, featureLength] + + // language adapter routing information for encoders if language adapter is presented. + TensorPtr languageAdapterRoutings; // [numTokens, numLanguages] + + // encoder output + TensorPtr encoderOutput; // [numEncoderTokens, hiddenSize] + + // output buffer owned by llmRequest, such that it's per-request output buffer + // encoderBuffers class can init and reshape each buffer, without maintaining a list/set of inflight buffers + // TODO in progress: to support BS>1 encoder, need (1) internal scratch space tensors to save the contiguous + // batched output (2) copy from CONTIGUOUS scratch tensor to individual request's DISCRETE output tensor after + // execution To standardize the implementation, for both BS=1 and BS>1, we use internal buffer to store BS=1/BS>1 + // results, and copy to request's external buffers. For BS=1, this introduces a redundancy copy, but ok for now. + + EncoderBuffers() = default; + EncoderBuffers(SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + TllmRuntime const& runtime); + + std::pair<EncoderBuffers::TensorMap const&, EncoderBuffers::TensorMap&> prepareIO(RequestVector const& requests, + ModelConfig const& modelConfig, WorldConfig const& worldConfig, TllmRuntime const& runtime); + + void rearrangeOutputs(RequestVector const& requests, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + TllmRuntime const& runtime); + + //! @brief set shape of individual request's encoder output (Ptuning embedding table if multimodal) + void updateReqOutputShape(RequestVector const& requests, TllmRuntime const& runtime, WorldConfig const& worldConfig, + ModelConfig const& modelConfig); + +private: + SizeType32 numRequests{}; + SizeType32 encoderInputLen{}; + SizeType32 encoderOutputLen{}; + SizeType32 maxInputLengthInBatch{}; // max input length in a batch + + // prefilled with deterministic values to avoid runtime creation + std::vector<SizeType32> positionIdsReserved; + std::vector<SizeType32> tokenTypeIdsReserved; + + // engine I/O + TensorMap inputMap; + TensorMap outputMap; + + void init(SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + TllmRuntime const& runtime); + + //! @brief pre-allocate max buffer sizes during init + void initBufferSizes(SizeType32 maxBatchSize, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + TllmRuntime const& runtime); + + //! @brief update actual buffer usage of requests during runtime + void updateBufferSizes(RequestVector const& requests, ModelConfig const& modelConfig, + WorldConfig const& worldConfig, TllmRuntime const& runtime); + + void reshape(TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig); + + void setFromInputs(RequestVector const& requests, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + TllmRuntime const& runtime); + + void fillIOMaps(ModelConfig const& modelConfig, WorldConfig const& worldConfig); + + // additional members that are Encoder-Decoder specific +private: + TensorPtr encoderOutputReserved; // [1, hiddenSize], dummy tensor for gen phase + TensorPtr crossKvCacheGen; // [1] + SizeType32 hiddenSize; // full hidden size (after multiplying tensor parallelism) + +public: + void create(SizeType32 maxBatchSize, ModelConfig const& modelConfig, TllmRuntime const& runtime); + + SizeType32 getMaxInputLengthInBatch() const + { + return maxInputLengthInBatch; + }; + + void setMaxBufferSizes(SizeType32 maxBatchSize, runtime::ModelConfig const& modelConfig); + + void setBufferSizes(RequestVector const& contextRequests, RequestVector const& genRequests); + + void reshape(); + + void fill( + RequestVector const& ctxRequests, RequestVector const& genRequests, runtime::BufferManager const& manager); + + void insertInputTensors(TensorMap& inputMap); +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/guidedDecoder.cpp b/cpp/tensorrt_llm/batch_manager/guidedDecoder.cpp new file mode 100644 index 000000000000..cb2264ec8003 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/guidedDecoder.cpp @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/guidedDecoder.h" +#include "tensorrt_llm/batch_manager/decoderBuffers.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/kernels/logitsBitmask.h" + +#include <nlohmann/json.hpp> +#include <xgrammar/xgrammar.h> + +using namespace tensorrt_llm::runtime; + +namespace tensorrt_llm::batch_manager +{ + +GuidedDecoder::GuidedDecoder(executor::GuidedDecodingConfig const& guidedDecodingConfig, SizeType32 maxNumSequences, + SizeType32 vocabSizePadded, nvinfer1::DataType logitsDtype, BufferManager const& runtimeBufferManager) + : mGuidedDecodingBackend{guidedDecodingConfig.getBackend()} + , mMaxNumSequences{maxNumSequences} + , mVocabSizePadded{vocabSizePadded} + , mBitmaskSize{common::ceilDiv(mVocabSizePadded, 32)} + , mLogitsDtype{logitsDtype} + , mCopyBufferManager{std::make_shared<CudaStream>()} +{ + TLLM_CHECK_WITH_INFO(mGuidedDecodingBackend != executor::GuidedDecodingConfig::GuidedDecodingBackend::kLLGUIDANCE, + "LLGuidance is not supported for guided decoding in C++ runtime."); + if (mGuidedDecodingBackend == executor::GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR) + { + mXGrammarMatchers.resize(mMaxNumSequences); + xgrammar::VocabType vocabType = xgrammar::VocabType::RAW; + bool addPrefixSpace = false; + auto const& tokenizerStr = guidedDecodingConfig.getTokenizerStr(); + if (tokenizerStr) + { + auto const& metadata = xgrammar::TokenizerInfo::DetectMetadataFromHF(tokenizerStr.value()); + auto const& metadataJson = nlohmann::json::parse(metadata); + vocabType = metadataJson.at("vocab_type").template get<xgrammar::VocabType>(); + addPrefixSpace = metadataJson.at("add_prefix_space").template get<bool>(); + } + auto const& tokenizerInfo = xgrammar::TokenizerInfo(guidedDecodingConfig.getEncodedVocab().value(), vocabType, + mVocabSizePadded, guidedDecodingConfig.getStopTokenIds(), addPrefixSpace); + + auto const cacheLimitGb = common::getFloatEnv("XGRAMMAR_CACHE_LIMIT_GB"); + mXGrammarCompiler = std::make_shared<xgrammar::GrammarCompiler>(tokenizerInfo, /*max_threads=*/8, + /*cache_enabled=*/true, + /*cache_limit_bytes=*/static_cast<long long>(cacheLimitGb.value_or(1.0f) * 1024 * 1024 * 1024)); + + auto const logitsPtrDtype = BufferDataType{mLogitsDtype, false, true}; + auto constexpr bitmaskDtype = TRTDataType<BitmaskT>::value; + auto constexpr bitmaskPtrDtype = TRTDataType<BitmaskT*>::value; + + mLogitsBitmask = runtimeBufferManager.gpu(ITensor::makeShape({mMaxNumSequences, mBitmaskSize}), bitmaskDtype); + mLogitsBitmaskHost = BufferManager::pinned(ITensor::makeShape({mMaxNumSequences, mBitmaskSize}), bitmaskDtype); + mLogitsBitmaskPtrVec = runtimeBufferManager.gpu(ITensor::makeShape({mMaxNumSequences}), bitmaskPtrDtype); + mLogitsBitmaskPtrVecHost = BufferManager::pinned(ITensor::makeShape({mMaxNumSequences}), bitmaskPtrDtype); + mLogitsPtrVec = runtimeBufferManager.gpu(ITensor::makeShape({mMaxNumSequences}), logitsPtrDtype); + mLogitsPtrVecHost = BufferManager::pinned(ITensor::makeShape({mMaxNumSequences}), logitsPtrDtype); + } +} + +void GuidedDecoder::build(ScheduledRequests const& scheduledRequests) +{ + if (mGuidedDecodingBackend == executor::GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR) + { + for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + auto const& guidedDecodingParams = llmReq->getGuidedDecodingParams(); + if (!guidedDecodingParams.has_value()) + { + continue; + } + auto const seqSlot = llmReq->mSeqSlot.value(); + if (llmReq->isContextInitState() && llmReq->isFirstContextChunk()) + { + // The request is in the first context forward step (considering kv cache reuse). + auto const& guideType = guidedDecodingParams->getGuideType(); + auto const& guide = guidedDecodingParams->getGuide(); + switch (guideType) + { + case executor::GuidedDecodingParams::GuideType::kJSON: + { + mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( + mXGrammarCompiler->CompileBuiltinJSONGrammar()); + break; + } + case executor::GuidedDecodingParams::GuideType::kJSON_SCHEMA: + { + mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( + mXGrammarCompiler->CompileJSONSchema(guide.value())); + break; + } + case executor::GuidedDecodingParams::GuideType::kREGEX: + { + mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( + mXGrammarCompiler->CompileRegex(guide.value())); + break; + } + case executor::GuidedDecodingParams::GuideType::kEBNF_GRAMMAR: + { + mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( + mXGrammarCompiler->CompileGrammar(guide.value())); + break; + } + case executor::GuidedDecodingParams::GuideType::kSTRUCTURAL_TAG: + { + mXGrammarMatchers.at(seqSlot) = std::make_shared<xgrammar::GrammarMatcher>( + mXGrammarCompiler->CompileStructuralTag(guide.value())); + break; + } + default: + { + TLLM_THROW("Unsupported guide type."); + } + } + } + else if (llmReq->isGenerationInProgressState()) + { + // The request is in a generation forward step. + // Currently, guided decoding does not support with beam search. + mXGrammarMatchers.at(seqSlot)->AcceptToken(llmReq->getLastTokens(0)); + } + else + { + continue; + } + + // Fill the bitmask on host and asynchorously copy to device using mCopyBufferManager. + auto const logitsBitmask = ITensor::at(mLogitsBitmask, {seqSlot}); + auto const logitsBitmaskHost = ITensor::at(mLogitsBitmaskHost, {seqSlot}); + + std::array<int64_t, 1> bitmaskShape{mBitmaskSize}; + DLTensor logitsBitmaskDlt{logitsBitmaskHost->data(), DLDevice{kDLCPU, 0}, 1, DLDataType{kDLInt, 32, 1}, + bitmaskShape.data(), nullptr, 0}; + mXGrammarMatchers.at(seqSlot)->FillNextTokenBitmask(&logitsBitmaskDlt); + mCopyBufferManager.copy(*logitsBitmaskHost, *logitsBitmask); + } + } + } +} + +void GuidedDecoder::execute(DecoderInputBuffers const& decoderInputBuffers, BufferManager const& runtimeBufferManager) +{ + auto const& stream = runtimeBufferManager.getStream(); + + // Wait for mCopyBufferManager finishing the H2D copy of logitsBitmask + // TODO(enweiz): Move the H2D copy of logitsBitmaskPtrVec to buildGuidedDecoding. + // This may not bring too much perf gain because of the small size of logitsBitmaskPtrVec. + // TODO(enweiz): For chunked context, we currently build mask cache at the first context chunk, and apply + // the mask at the last context chunk. So, ideally we should sync the stream at the last context chunk. + CudaEvent event{}; + mCopyBufferManager.getStream().record(event); + stream.wait(event); + + if (mGuidedDecodingBackend == executor::GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR + && !decoderInputBuffers.decoderRequests.empty()) + { + SizeType32 batchIdx{0}; + for (size_t requestIdx = 0; requestIdx < decoderInputBuffers.decoderRequests.size(); ++requestIdx) + { + auto const& llmReq = decoderInputBuffers.decoderRequests.at(requestIdx); + + auto const& guidedDecodingParams = llmReq->getGuidedDecodingParams(); + if (guidedDecodingParams.has_value()) + { + auto const seqSlot = llmReq->mSeqSlot.value(); + + auto const& logits = decoderInputBuffers.decoderLogits.at(requestIdx); + auto const logitsBitmask = ITensor::at(mLogitsBitmask, {seqSlot}); + + // Use void* to unify the code for different mLogitsDtype + *reinterpret_cast<void**>(ITensor::at(mLogitsPtrVecHost, {batchIdx})->data()) = logits->data(); + *reinterpret_cast<void**>(ITensor::at(mLogitsBitmaskPtrVecHost, {batchIdx})->data()) + = logitsBitmask->data(); + + ++batchIdx; + } + } + if (batchIdx > 0) + { + runtimeBufferManager.copy( + *ITensor::slice(mLogitsPtrVecHost, 0, batchIdx), *ITensor::slice(mLogitsPtrVec, 0, batchIdx)); + runtimeBufferManager.copy(*ITensor::slice(mLogitsBitmaskPtrVecHost, 0, batchIdx), + *ITensor::slice(mLogitsBitmaskPtrVec, 0, batchIdx)); + + auto logitsBitmaskPtrVec = bufferCast<BitmaskT const*>(*mLogitsBitmaskPtrVec); + if (mLogitsDtype == nvinfer1::DataType::kFLOAT) + { + auto logitsPtrVec = bufferCast<float*>(*mLogitsPtrVec); + tensorrt_llm::kernels::invokeLogitsBitmask<float>( + logitsPtrVec, logitsBitmaskPtrVec, batchIdx, mVocabSizePadded, stream.get()); + } + else if (mLogitsDtype == nvinfer1::DataType::kHALF) + { + auto logitsPtrVec = bufferCast<half*>(*mLogitsPtrVec); + tensorrt_llm::kernels::invokeLogitsBitmask<half>( + logitsPtrVec, logitsBitmaskPtrVec, batchIdx, mVocabSizePadded, stream.get()); + } + else + { + TLLM_THROW("Unsupported logits data type."); + } + } + } +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/handleContextLogits.cpp b/cpp/tensorrt_llm/batch_manager/handleContextLogits.cpp new file mode 100644 index 000000000000..6f4a541ffcbb --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/handleContextLogits.cpp @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/handleContextLogits.h" + +#include "tensorrt_llm/batch_manager/decoderBuffers.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/medusaBuffers.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/runtimeKernels.h" +#include "tensorrt_llm/runtime/utils/debugUtils.h" + +namespace tr = tensorrt_llm::runtime; +namespace tru = tensorrt_llm::runtime::utils; + +namespace tensorrt_llm::batch_manager +{ + +using BufferManager = tensorrt_llm::runtime::BufferManager; +using TensorPtr = runtime::ITensor::SharedPtr; +using ITensor = runtime::ITensor; +using SizeType32 = tensorrt_llm::runtime::SizeType32; + +namespace +{ + +//! @brief Copy logits from context phase to beginning of generation logits. +//! @details Usually, this concerns logits of 1 token. In speculative decoding this concerns draftLen + 1 tokens. +void copyLastContextLogits(TensorPtr const& contextLogits, LlmRequest& llmReq, BufferManager const& bufferManager) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + auto const numLogits = contextLogits->getShape().d[0]; + for (int beam = 0; beam < llmReq.getBeamWidthByIter(); beam++) + { + // [beamWidth, mMaxNewTokens, vocabSizePadded] -> [numLogits, vocabSizePadded] + auto beamHostTensorPtr = ITensor::slice(llmReq.getGenerationLogitsHost(), {beam, 0}, numLogits); + bufferManager.copy(*contextLogits, *beamHostTensorPtr); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void setupMedusaLogits(std::vector<TensorPtr>& medusaLogitsHeads, TensorPtr const& medusaLogitsDevice, + SizeType32 medusaHeads, SizeType32 logitsIndex, SizeType32 numLogits) +{ + for (SizeType32 hi = 0; hi < medusaHeads; ++hi) + { + TensorPtr logitsHead = ITensor::slice(medusaLogitsDevice, hi, 1); + logitsHead->squeeze(0); + medusaLogitsHeads[hi] = ITensor::slice(logitsHead, logitsIndex, numLogits); + } +} + +} // namespace + +SizeType32 HandleContextLogits::operator()(DecoderInputBuffers& inputBuffers, RequestVector const& contextRequests, + tr::ITensor::SharedPtr const& logits, std::vector<tr::SizeType32> const& numContextLogitsVec, + tr::ModelConfig const& modelConfig, tr::BufferManager const& manager, + OptionalRef<MedusaBuffers> medusaBuffers) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(HandleContextLogits); + + auto& decoderRequests = inputBuffers.decoderRequests; + decoderRequests.clear(); + decoderRequests.reserve(contextRequests.size()); + auto& allDecoderLogits = inputBuffers.decoderLogits; + allDecoderLogits.clear(); + allDecoderLogits.reserve(contextRequests.size()); + + SizeType32 batchIndex{0}; + SizeType32 logitsIndex{0}; + // Copy logits into decoderBuffers.logits + for (auto const& llmReq : contextRequests) + { + auto const numContextLogits = numContextLogitsVec.at(batchIndex); + auto const draftLength = llmReq->isLastContextChunk() ? llmReq->getNumDraftTokens() : 0; + + TLLM_LOG_DEBUG("logitsIndex: %d", logitsIndex); + TLLM_LOG_DEBUG("numContextLogits %d", numContextLogits); + TLLM_LOG_DEBUG("draftLength: %d", draftLength); + + if (modelConfig.computeContextLogits()) + { + // Since the computational graph has been modified, only the last token is needed. + TLLM_CHECK_WITH_INFO(!modelConfig.getSpeculativeDecodingMode().isMedusa() + && !modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding(), + "Return context logits is not supported with Medusa and Lookahead decoding"); + + if (llmReq->getReturnContextLogits()) + { + if (llmReq->getPrepopulatedPromptLen() > 0) + { + TLLM_LOG_WARNING( + "Because of KV cache reuse, not all context logits could be produced for request %lu.", + llmReq->mRequestId); + } + TensorPtr contextLogitsDeviceView = ITensor::slice(logits, logitsIndex, numContextLogits); + TensorPtr contextLogitsHostView = ITensor::slice( + llmReq->getContextLogitsHost(), llmReq->getContextCurrentPosition(), numContextLogits); + // Copy to host directly + manager.copy(*contextLogitsDeviceView, *contextLogitsHostView); + } + } + logitsIndex += numContextLogits + draftLength; + + // Get the logits from the last context token and draft tokens + auto const numDecoderLogits = 1 + draftLength; + auto const seqSlot = llmReq->mSeqSlot.value(); + TensorPtr logitsView = ITensor::slice(logits, logitsIndex - numDecoderLogits, numDecoderLogits); + + if (modelConfig.getSpeculativeDecodingMode().hasDraftLogits()) + { + auto& medusaLogitsHeads = inputBuffers.predictedDraftLogits.at(seqSlot); + TLLM_CHECK(medusaBuffers); + setupMedusaLogits(medusaLogitsHeads, medusaBuffers->medusaLogitsDevice, + modelConfig.getSpeculativeDecodingModule().getMaxDraftPathLen(), logitsIndex - numDecoderLogits, + numDecoderLogits); + } + + // Save the last token logits of context into generation logits or + // save the accepted token logits from target model + if (llmReq->getReturnGenerationLogits()) + { + copyLastContextLogits(logitsView, *llmReq, manager); + } + + TLLM_CHECK_DEBUG_WITH_INFO(tru::tensorHasInvalid<float>(*logitsView, manager, "logits") == false, + "Found invalid number (NaN or Inf) in logits"); + + if (llmReq->isLastContextChunk()) + { + TensorPtr decoderLogits; + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + if (reqBeamWidth > 1) + { + // Tile logits of context requests + auto const& logitsShape = logitsView->getShape(); + auto const logitsType = logitsView->getDataType(); + decoderLogits = manager.gpu(ITensor::makeShape({reqBeamWidth, logitsShape.d[1]}), logitsType); + tensorrt_llm::runtime::kernels::tileTensor( + *decoderLogits, *logitsView, reqBeamWidth, manager.getStream()); + decoderLogits->unsqueeze(0); + } + else + { + decoderLogits = logitsView; + decoderLogits->unsqueeze(1); + } + decoderRequests.push_back(llmReq); + allDecoderLogits.emplace_back(std::move(decoderLogits)); + } + + ++batchIndex; + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return logitsIndex; +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/handleGenerationLogits.cpp b/cpp/tensorrt_llm/batch_manager/handleGenerationLogits.cpp new file mode 100644 index 000000000000..e2a7486b050a --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/handleGenerationLogits.cpp @@ -0,0 +1,161 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/handleGenerationLogits.h" + +#include "tensorrt_llm/batch_manager/decoderBuffers.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/medusaBuffers.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" +#include "tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/utils/debugUtils.h" + +namespace tr = tensorrt_llm::runtime; +namespace tru = tensorrt_llm::runtime::utils; + +namespace tensorrt_llm::batch_manager +{ + +using BufferManager = tensorrt_llm::runtime::BufferManager; +using TensorPtr = runtime::ITensor::SharedPtr; +using ITensor = runtime::ITensor; +using SizeType32 = tensorrt_llm::runtime::SizeType32; + +namespace +{ + +//! @brief Copy logits from generation phase under streaming mode. +void copyStreamingGenerationLogits(BufferManager const& bufferManager, LlmRequest& llmReq) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + // If llmRequest is streaming, directly copy to host. + // Only one token's logits needs to be copied each time. + TLLM_CHECK(llmReq.getGenerationLogitsFragmentsSize() == 1); + + SizeType32 numGenerationToken = llmReq.getMaxBeamNumTokens() - llmReq.mPromptLen; + TensorPtr const& generationLogitsHost + = llmReq.getGenerationLogitsHost(); // [mMaxNewTokens (or 1), beamWidth, vocabSizePadded] + + TensorPtr hostTensorPtr + = ITensor::slice(generationLogitsHost, numGenerationToken, 1); // [1, beamWidth, vocabSizePadded] + TensorPtr deviceTensorPtr = *(llmReq.getGenerationLogitsFragments().begin()); + + bufferManager.copy(*deviceTensorPtr, *hostTensorPtr); + llmReq.clearGenerationLogitsFragments(); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void setupMedusaLogits(std::vector<TensorPtr>& medusaLogitsHeads, TensorPtr const& medusaLogitsDevice, + SizeType32 medusaHeads, SizeType32 logitsIndex, SizeType32 numLogits) +{ + for (SizeType32 hi = 0; hi < medusaHeads; ++hi) + { + TensorPtr logitsHead = ITensor::slice(medusaLogitsDevice, hi, 1); + logitsHead->squeeze(0); + medusaLogitsHeads[hi] = ITensor::slice(logitsHead, logitsIndex, numLogits); + } +} + +} // namespace + +void HandleGenerationLogits::operator()(DecoderInputBuffers& inputBuffers, RequestVector const& generationRequests, + tr::ITensor::SharedPtr const& logits, tr::SizeType32 logitsIndex, tr::ModelConfig const& modelConfig, + tr::BufferManager const& manager, OptionalRef<RuntimeBuffers> genRuntimeBuffers, + OptionalRef<MedusaBuffers> medusaBuffers) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(HandleGenerationLogits); + + auto& decoderRequests = inputBuffers.decoderRequests; + decoderRequests.reserve(decoderRequests.size() + generationRequests.size()); + auto& allDecoderLogits = inputBuffers.decoderLogits; + allDecoderLogits.reserve(allDecoderLogits.size() + generationRequests.size()); + + for (auto const& llmReq : generationRequests) + { + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + auto const seqSlot = llmReq->mSeqSlot.value(); + + auto const draftLength = llmReq->getNumDraftTokens(); + auto const numLogits = draftLength + reqBeamWidth; + + TLLM_CHECK(draftLength == 0 || reqBeamWidth == 1); + + TLLM_LOG_DEBUG("logitsIndex: %d", logitsIndex); + TLLM_LOG_DEBUG("draftLength: %d", draftLength); + TLLM_LOG_DEBUG("reqBeamWidth: %d", reqBeamWidth); + + // genRuntimeBuffers.logits shape: [numGen*reqBeamWidth, vocabSize] + // logitsView shape: [numLogits, vocabSize] + TensorPtr logitsView = ITensor::slice(logits, logitsIndex, numLogits); + TLLM_CHECK_DEBUG_WITH_INFO(tru::tensorHasInvalid<float>(*logitsView, manager, "logits") == false, + "Found invalid number (NaN or Inf) in logits"); + + TLLM_CHECK(llmReq->isGenerationInProgressState()); + TensorPtr decoderLogits; + if (reqBeamWidth > 1) + { + decoderLogits = logitsView; + decoderLogits->unsqueeze(0); + } + else + { + decoderLogits = logitsView; + decoderLogits->unsqueeze(1); + } + decoderRequests.push_back(llmReq); + allDecoderLogits.emplace_back(std::move(decoderLogits)); + + if (llmReq->getReturnGenerationLogits()) + { + TLLM_CHECK_WITH_INFO(modelConfig.getSpeculativeDecodingMode().isNone() + || modelConfig.getSpeculativeDecodingMode().isDraftTokensExternal(), + "Only speculative decoding with external draft tokens supports returning generation logits"); + + // Push into fragments vector + llmReq->addGenerationLogitsFragment(logitsView); + TLLM_CHECK( + llmReq->getGenerationLogitsFragmentsSize() <= RuntimeBuffers::GenerationLogitsCache::kCACHE_LENGTH); + if (llmReq->isStreaming()) + { + copyStreamingGenerationLogits(manager, *llmReq); + } + // Copy back to host for every kCACHE_LENGTH steps to mitigate GPU memory pressure + else if (llmReq->getGenerationLogitsFragmentsSize() == RuntimeBuffers::GenerationLogitsCache::kCACHE_LENGTH) + { + TLLM_CHECK(genRuntimeBuffers); + auto constexpr beforeDecoder = true; + utils::copyGenerationLogits(genRuntimeBuffers->generationLogitsCache, manager, *llmReq, beforeDecoder); + } + } + if (modelConfig.getSpeculativeDecodingMode().hasDraftLogits()) + { + auto& medusaLogitsHeads = inputBuffers.predictedDraftLogits.at(seqSlot); + TLLM_CHECK(medusaBuffers); + setupMedusaLogits(medusaLogitsHeads, medusaBuffers->medusaLogitsDevice, + modelConfig.getSpeculativeDecodingModule().getMaxDraftPathLen(), logitsIndex, draftLength); + } + logitsIndex += numLogits; + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp b/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp index 8c1ffb70e372..c67a4711cc03 100644 --- a/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/kvCacheManager.cpp @@ -27,7 +27,6 @@ #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/kernels/kvCacheIndex.h" #include "tensorrt_llm/runtime/common.h" @@ -587,7 +586,7 @@ std::map<SizeType32, float> BlockManager::calculateWindowSizeToShare( BlockManager::BlockManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, CudaStreamPtr stream, SizeType32 maxSequenceLength, SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, - tensorrt_llm::DataType dtype, SizeType32 sinkBubbleLength, SizeType32 chunkSize, CacheType cacheType, + nvinfer1::DataType dtype, SizeType32 sinkBubbleLength, SizeType32 chunkSize, CacheType cacheType, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority, std::shared_ptr<KVCacheEventManager> eventManager, bool enablePartialReuse, bool copyOnPartialReuse, std::shared_ptr<kv_connector::KvCacheConnectorManager> kvCacheConnectorManager, @@ -741,7 +740,7 @@ BlockManager::BlockManager(std::vector<SizeType32> const& numKvHeadsPerLayer, Si "Maybe you tried changing either of them to an std::unordered_map?"); } -WindowBlockManager::WindowBlockManager(tensorrt_llm::DataType dtype, SizeType32 windowSize, +WindowBlockManager::WindowBlockManager(nvinfer1::DataType dtype, SizeType32 windowSize, std::vector<SizeType32> const& managedLayers, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, bool isSWA, SizeType32 blocksInPrimaryPool, SizeType32 blocksInSecondaryPool, SizeType32 maxNumSequences, std::shared_ptr<runtime::CudaStream> stream, @@ -841,7 +840,7 @@ WindowBlockManager::WindowBlockManager(tensorrt_llm::DataType dtype, SizeType32 // to specify FP4 related parameters (scale dtypes, etc)? This can also be passed // in the constructor. constexpr SizeType32 kQuantBlockSizeNVFP4 = 16; - if (dtype == tensorrt_llm::DataType::kFP4) + if (dtype == nvinfer1::DataType::kFP4) { createBlockScalePools(kQuantBlockSizeNVFP4); } @@ -937,7 +936,6 @@ bool BlockManager::verifyQueueIntegrity(SizeType32 windowSize) const bool WindowBlockManager::verifyQueueIntegrity() const { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); return mEvictionPolicy->verifyQueueIntegrity(); } @@ -1094,7 +1092,7 @@ void BlockManager::allocatePools(bool useUvm) void WindowBlockManager::allocatePools(bool useUvm) { - constexpr tensorrt_llm::DataType kScaleDtypeNVFP4 = tensorrt_llm::DataType::kFP8; + constexpr nvinfer1::DataType kScaleDtypeNVFP4 = nvinfer1::DataType::kFP8; bool const requestFabricMemory = tc::getEnvKVCachePoolUseFabricMemory(); bool const fabricMemorySupported = FabricMemory::supportFabricMemory(); @@ -1120,21 +1118,21 @@ void WindowBlockManager::allocatePools(bool useUvm) auto blockSize = pool.blockSize; auto poolDtype = pool.containsBlockScales ? kScaleDtypeNVFP4 : mDataType; #ifdef ENABLE_FP4 - auto const poolIsFP4 = poolDtype == tensorrt_llm::DataType::kFP4; + auto const poolIsFP4 = poolDtype == nvinfer1::DataType::kFP4; #else auto const poolIsFP4 = false; #endif if (poolIsFP4) { - poolDtype = tensorrt_llm::DataType::kINT8; + poolDtype = nvinfer1::DataType::kINT8; } if (pool.containsIndexerKCache) { - poolDtype = tensorrt_llm::DataType::kUINT8; + poolDtype = nvinfer1::DataType::kUINT8; } - tensorrt_llm::Dims cacheShape = isRecurrentState() + nvinfer1::Dims cacheShape = isRecurrentState() ? ITensor::makeShape({pool.numLayers, mNumPrimaryBlocks, mKVFactor, blockSize}) : ITensor::makeShape({mNumPrimaryBlocks, pool.numLayers, mKVFactor, blockSize}); pool.layerFirstLayout = isRecurrentState(); @@ -1168,7 +1166,7 @@ void WindowBlockManager::allocatePools(bool useUvm) if (mNumSecondaryBlocks > 0) { - tensorrt_llm::Dims cacheShapeOffload = isRecurrentState() + nvinfer1::Dims cacheShapeOffload = isRecurrentState() ? ITensor::makeShape({pool.numLayers, mNumSecondaryBlocks, mKVFactor, blockSize}) : ITensor::makeShape({mNumSecondaryBlocks, pool.numLayers, mKVFactor, blockSize}); TLLM_LOG_DEBUG("[%s] Allocating secondary pool with %d blocks for %d layers with %d kv heads", @@ -1220,7 +1218,6 @@ void BlockManager::startScheduling() void WindowBlockManager::startScheduling() { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); mSchedulingNumFreeBlocks = mEvictionPolicy->getNumFreeBlocks(kPrimaryLevel); for (auto& [requestId, slotAllocatedBlocks] : mAllocatedBlocksPerSeq) { @@ -1240,7 +1237,6 @@ void WindowBlockManager::freeLeafBlock(BlockPtr const& block) void WindowBlockManager::releaseSubtree(BlockPtr const& block) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); // Iterative pre-order DFS over `block` and all its descendants. Collect // first, then detach in reverse order: cascade-prune in freeLeafBlock() // removes empty parent nodes from the trie, so leaves must be detached @@ -1278,7 +1274,6 @@ BlockPtr WindowBlockManager::getFreeBlock(GenerationRequest& sequence, executor: std::optional<std::chrono::milliseconds> durationMs, executor::KvCacheTransferMode mode, std::string const& directory, bool wantPlaceholder) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); // eviction policy get free primary block auto [block, canOffload] = mEvictionPolicy->getFreeBlock(kPrimaryLevel, wantPlaceholder); if (block->getUniqueTokens().empty()) @@ -1349,7 +1344,7 @@ BlockPtr WindowBlockManager::getFreeBlock(GenerationRequest& sequence, executor: return block; } -void WindowBlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, +void WindowBlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx, SizeType32 blockIdx, KVCacheBlock::IdType blockId) const { auto constexpr kIdx = 0; @@ -1387,7 +1382,7 @@ void WindowBlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, tensorrt_llm:: } } -void BlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, tensorrt_llm::Dims const& offsetsShape, SizeType32 beamIdx, +void BlockManager::setOffsets(tk::KVCacheIndex* offsetsPtr, nvinfer1::Dims const& offsetsShape, SizeType32 beamIdx, SizeType32 blockIdx, KVCacheBlock::IdType blockId, SizeType32 windowSize) const { mWindowBlockManagers.at(windowSize).setOffsets(offsetsPtr, offsetsShape, beamIdx, blockIdx, blockId); @@ -1402,7 +1397,6 @@ void BlockManager::onboardBlock(GenerationRequest& sequence, BlockPtr const& off void WindowBlockManager::onboardBlock(GenerationRequest& sequence, BlockPtr const& offloadBlock, executor::KvCacheTransferMode mode, std::string const& directory) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); if (!offloadBlock->isPlaceholder() && !offloadBlock->isPrimary()) { auto block = getFreeBlock( @@ -1431,7 +1425,6 @@ void BlockManager::offloadBlock( void WindowBlockManager::offloadBlock( BlockPtr const& block, executor::KvCacheTransferMode mode, std::string const& directory) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); // The current default behavior is to offload the out-of-window block // to secondary block pool to allow more free primary blocks for reuse. // However, such behavior does not take account whether the offloaded @@ -2137,19 +2130,6 @@ bool WindowBlockManager::blockInRadixTree(BlockPtr const& block) } std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKey(BlockKey const& blockKey) -{ - std::vector<KVCacheBlock::IdType> unusedPinnedBlockIds; - return findBlocksInReuseTreeByBlockKeyImpl(blockKey, /*pinBlocks=*/false, unusedPinnedBlockIds); -} - -std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKey( - BlockKey const& blockKey, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) -{ - return findBlocksInReuseTreeByBlockKeyImpl(blockKey, /*pinBlocks=*/true, pinnedBlockIds); -} - -std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKeyImpl( - BlockKey const& blockKey, bool pinBlocks, std::vector<KVCacheBlock::IdType>& pinnedBlockIds) { std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto blockedUniqueTokens @@ -2162,42 +2142,7 @@ std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKe blockKeys.emplace_back(blockKey.usesExtraIds, blockKey.loraTaskId, blockedUniqueTokensList, blockKey.extraKeys, blockKey.cacheSalt); } - auto searchRoot = mCachedBlocksRoot; - std::vector<BlockPtr> pinnedInScope; - for (auto const& blockKey : blockKeys) - { - auto [partialMatch, numMatched, matchingBlock] = searchRoot != nullptr - ? searchRoot->findMatchingBlock(blockKey, true, true) - : std::make_tuple(false, 0, nullptr); - - // A prefix-only match lacks KV for the requested tail, and pinning (transfer) - // lookups read primary-pool buffers directly, so offloaded and placeholder blocks - // are misses too (isPlaceholder first: isPrimary asserts on placeholders). - bool const fullyMatched - = matchingBlock != nullptr && numMatched == static_cast<SizeType32>(blockKey.uniqueTokens.size()); - bool const transferable - = fullyMatched && (!pinBlocks || (!matchingBlock->isPlaceholder() && matchingBlock->isPrimary())); - if (!transferable) - { - // Roll back pins taken during the partial walk. - for (auto const& block : pinnedInScope) - { - unpinBlock(block); - } - pinnedBlockIds.clear(); - return nullptr; - } - - if (pinBlocks) - { - pinBlock(matchingBlock); - pinnedInScope.push_back(matchingBlock); - pinnedBlockIds.push_back(matchingBlock->getBlockId()); - } - - searchRoot = std::move(matchingBlock); - } - return searchRoot; + return searchReuseTree(blockKeys); } std::shared_ptr<KVCacheBlock> WindowBlockManager::findBlocksInReuseTreeByBlockKeys( @@ -2246,7 +2191,6 @@ void BlockManager::refreshBlocks() void WindowBlockManager::refreshBlocks() { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); mEvictionPolicy->refresh(); mTransferManager->syncTransfers(); } @@ -2377,7 +2321,6 @@ bool BlockManager::copyLinearAttentionBlock(GenerationRequest& sequence, LlmRequ bool WindowBlockManager::tryAllocatePlaceholderForLinearAttention(GenerationRequest& sequence, bool shareAmongBeams) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const beamWidth = sequence.getBeamWidth(); auto const newBlockIdx = sequence.getCacheBlockIds(mWindowSize).at(0).size(); // The first block is not a placeholder. @@ -2467,7 +2410,6 @@ bool WindowBlockManager::tryAllocatePlaceholderForLinearAttention(GenerationRequ void WindowBlockManager::allocateBlock(GenerationRequest& sequence, bool shareAmongBeams) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const beamWidth = sequence.getBeamWidth(); auto const requiredBlocks = shareAmongBeams ? 1 : beamWidth; @@ -2749,7 +2691,16 @@ std::pair<SizeType32, std::vector<KVCacheBlock::IdType>> WindowBlockManager::sto if (pinBlocks) { - pinBlock(prevBlock); + // If the block has no refs it sits in the eviction policy's free + // queue. Claim it first so that the later unpinBlocksById / + // releaseBlock cycle does not create a duplicate queue entry. + // Pass the block's existing priority and duration so that + // claimBlock does not clear its retention/expiry metadata. + if (!prevBlock->hasRefs()) + { + mEvictionPolicy->claimBlock(prevBlock, prevBlock->getPriority(), prevBlock->getDurationMs()); + } + prevBlock->incRefCount(); pinnedBlockIds.push_back(prevBlock->getBlockId()); } } @@ -2784,7 +2735,6 @@ void BlockManager::replaceSharedBlock(GenerationRequest& sequence, SizeType32 wi void WindowBlockManager::replaceSharedBlock(GenerationRequest& sequence, SizeType32 blockIdx) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const requestId = sequence.getRequestId(); auto const beamWidth = sequence.getBeamWidth(); auto& allocatedBlocks = mAllocatedBlocksPerSeq.at(requestId); @@ -2836,7 +2786,6 @@ void BlockManager::releaseLastBlock(GenerationRequest& sequence, SizeType32 wind void WindowBlockManager::releaseLastBlock(GenerationRequest& sequence) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); if (isRecurrentState()) { // In recurrent state, the last block always contains the current state and should not be released. @@ -2885,7 +2834,6 @@ void WindowBlockManager::releaseLastBlock(GenerationRequest& sequence) [[nodiscard]] SizeType32 WindowBlockManager::getNumFreeBlocks() const { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const numFree = mEvictionPolicy->getNumFreeBlocks(kPrimaryLevel); TLLM_CHECK_WITH_INFO(numFree <= getMaxNumBlocks(), "%s::getNumFreeBlocks - primary free block count (%d) exceeds total block count (%d). " @@ -2897,7 +2845,6 @@ void WindowBlockManager::releaseLastBlock(GenerationRequest& sequence) [[nodiscard]] SizeType32 WindowBlockManager::getNumFreeSecondaryBlocks() const noexcept { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); return mEvictionPolicy->getNumFreeBlocks(kSecondaryLevel); } @@ -3028,41 +2975,18 @@ void BlockManager::unpinBlocksById(std::vector<KVCacheBlock::IdType> const& bloc firstManager.unpinBlocksById(blockIds); } -void WindowBlockManager::pinBlock(BlockPtr const& block) -{ - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); - // Claim free blocks out of the eviction queue first, keeping their retention metadata. - if (!block->hasRefs()) - { - mEvictionPolicy->claimBlock(block, block->getPriority(), block->getDurationMs()); - } - block->incRefCount(); -} - -void WindowBlockManager::unpinBlock(BlockPtr const& block) -{ - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); - block->decRefCount(); - if (!block->hasRefs()) - { - mEvictionPolicy->releaseBlock(block); - } -} - void WindowBlockManager::pinBlocks(GenerationRequest& sequence) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const requestId = sequence.getRequestId(); auto& allocatedBlocks = mAllocatedBlocksPerSeq.at(requestId); for (auto& block : allocatedBlocks) { - pinBlock(block); + block->incRefCount(); } } void WindowBlockManager::unpinBlocksById(std::vector<KVCacheBlock::IdType> const& blockIds) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); if (blockIds.empty()) { return; @@ -3075,7 +2999,11 @@ void WindowBlockManager::unpinBlocksById(std::vector<KVCacheBlock::IdType> const auto block = mAllBlocksById[blockId]; if (block && block->getBlockId() != KVCacheBlock::kCachedBlocksRootId) { - unpinBlock(block); + block->decRefCount(); + if (!block->hasRefs()) + { + mEvictionPolicy->releaseBlock(block); + } } } } @@ -3091,7 +3019,6 @@ void BlockManager::storeNewBlock(GenerationRequest& sequence, OptionalRef<LlmReq void WindowBlockManager::storeNewBlock(GenerationRequest& sequence, OptionalRef<LlmRequest const> llmRequest) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto constexpr beamIdx = 0; auto const& uniqueTokens = llmRequest->getUniqueTokens(beamIdx); @@ -3198,7 +3125,6 @@ std::vector<KVCacheBlock::IdType> WindowBlockManager::storeBlocksForReuse( std::optional<KVCacheBlock::IdType> WindowBlockManager::releaseBlocks( GenerationRequest& sequence, OptionalRef<LlmRequest const> llmRequest) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); auto const requestId = sequence.getRequestId(); TLLM_LOG_DEBUG("%s::releaseBlocks - requestId=%lu, llmRequest.id=%s", mLogPrefix.c_str(), requestId, llmRequest.has_value() ? std::to_string(llmRequest->mRequestId).c_str() : "null"); @@ -3303,7 +3229,7 @@ void WindowBlockManager::schedulingReleaseBlocks(RequestIdType requestId) KVCacheManager::KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, - SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, + SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, int64_t stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, bool enablePartialReuse, bool copyOnPartialReuse, bool enableIndexerKCache, SizeType32 indexerKCacheQuantBlockSize, SizeType32 indexerKCacheIndexHeadDim, @@ -3320,7 +3246,7 @@ KVCacheManager::KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, Size KVCacheManager::KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, - SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, + SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, int64_t stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority, std::shared_ptr<KVCacheEventManager> eventManager, bool enablePartialReuse, bool copyOnPartialReuse, @@ -3339,7 +3265,7 @@ KVCacheManager::KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer KVCacheManager::KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, - SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, + SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, CudaStreamPtr stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority, std::shared_ptr<KVCacheEventManager> eventManager, bool enablePartialReuse, bool copyOnPartialReuse, @@ -3381,7 +3307,7 @@ KVCacheManager::KVCacheManager(std::vector<SizeType32> const& numKvHeadsPerLayer KVCacheManager::KVCacheManager(SizeType32 numLayers, SizeType32 numKvHeads, SizeType32 sizePerHead, SizeType32 tokensPerBlock, BlocksPerWindow const& blocksPerWindow, SizeType32 maxNumSequences, - SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, tensorrt_llm::DataType dtype, + SizeType32 maxBeamWidth, std::vector<SizeType32> const& maxAttentionWindowVec, nvinfer1::DataType dtype, SizeType32 sinkTokenLength, CudaStreamPtr stream, runtime::SizeType32 maxSequenceLength, SizeType32 chunkSize, bool enableBlockReuse, CacheType cacheType, std::optional<executor::RetentionPriority> secondaryOffloadMinPriority, std::shared_ptr<KVCacheEventManager> eventManager, bool enablePartialReuse, bool copyOnPartialReuse, @@ -3414,7 +3340,7 @@ void KVCacheManager::allocatePools(bool useUvm) // a future per-window override map can mix precisions inside a single manager. auto const poolDataType = primaryPool->getDataType(); #ifdef ENABLE_FP4 - auto const isFp4 = poolDataType == tensorrt_llm::DataType::kFP4; + auto const isFp4 = poolDataType == nvinfer1::DataType::kFP4; #else auto const isFp4 = false; #endif @@ -3785,7 +3711,6 @@ bool KVCacheManager::copyLinearAttentionBlockBatch(std::vector<std::shared_ptr<L void WindowBlockManager::detachFrontBlock(GenerationRequest& sequence) { - std::lock_guard<std::recursive_mutex> lock(mLookupTree->getMutex()); // streamLLM is not supported at the moment. The out of window block will // always be the 0th block. TLLM_CHECK_WITH_INFO( @@ -4068,28 +3993,6 @@ tle::RetentionPriority KVCacheManager::getPriorityByBlockId(KVCacheBlock::IdType } } -std::vector<kernels::KVCacheIndex::UnderlyingType> KVCacheManager::getMemoryPoolBlockIndicesByBlockIds( - std::vector<KVCacheBlock::IdType> const& blockIds, SizeType32 windowSize) const -{ - std::vector<kernels::KVCacheIndex::UnderlyingType> indices; - indices.reserve(blockIds.size()); - for (auto const blockId : blockIds) - { - BlockPtr const& block = mBlockManager.getBlockById(blockId, windowSize); - TLLM_CHECK_WITH_INFO(block != nullptr, "Block not found (blockId=%d, windowSize=%d)", blockId, windowSize); - // Invariant, not a recoverable condition: blocks referenced here are held by a live - // request, allocation onboards offloaded blocks, and offload only selects free blocks. - // A secondary block therefore indicates a block-lifetime bug, and the returned index - // (pool flag stripped) would silently alias a primary slot. - TLLM_CHECK_WITH_INFO(block->isPrimary(), - "Block is not in the primary pool (blockId=%d, windowSize=%d); the returned index is only " - "valid for primary-pool blocks", - blockId, windowSize); - indices.push_back(block->getMemoryPoolBlockIndex()); - } - return indices; -} - SizeType32 KVCacheManager::copyBlockOffsets(ITensor& output, SizeType32 outputSlotOffset, RequestIdType requestId) const { auto const& sequence = getSequence(requestId); @@ -4358,7 +4261,7 @@ std::map<SizeType32, float> computeWindowSizeShares( } // namespace BlocksPerWindow BaseKVCacheManager::calculateMaxNumBlocks(executor::KvCacheConfig const& config, - tensorrt_llm::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, + nvinfer1::DataType dtype, std::vector<SizeType32> const& numKvHeadsPerLayer, SizeType32 sizePerHead, SizeType32 tokensPerBlock, WorldConfig const& worldConfig, std::map<SizeType32, std::vector<SizeType32>> const& windowSizeToLayers, uint64_t allottedPrimaryMemBytes, uint64_t allottedSecondaryMemBytes, size_t extraCostMemory, SizeType32 kvFactor, SizeType32 maxBatchSize, diff --git a/cpp/tensorrt_llm/batch_manager/kvCacheTransferManager.cpp b/cpp/tensorrt_llm/batch_manager/kvCacheTransferManager.cpp index 33ad5b4a1675..c28d1e476137 100644 --- a/cpp/tensorrt_llm/batch_manager/kvCacheTransferManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/kvCacheTransferManager.cpp @@ -22,7 +22,6 @@ #include "tensorrt_llm/batch_manager/kvCacheEventManager.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/kernels/kvCachePartialCopy.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -164,8 +163,8 @@ void KVCacheTransferManager::copyBlock(BlockPtr const& src, BlockPtr const& dst, // If no partial tokens or if the dataType is not supported for partial copy, copy entire block. // Note that nvfp4 kv cache SFs use an interleaved layout, so we need to copy the entire block. - if (numTokensToCopy <= 0 || srcPtr->getDataType() == tensorrt_llm::DataType::kINT4 - || srcPtr->getDataType() == tensorrt_llm::DataType::kFP4 || containsBlockScales) + if (numTokensToCopy <= 0 || srcPtr->getDataType() == nvinfer1::DataType::kINT4 + || srcPtr->getDataType() == nvinfer1::DataType::kFP4 || containsBlockScales) { // For partial copy not implemented with these data types, // just do a full copy. @@ -462,8 +461,8 @@ std::size_t KVCacheTransferManager::computeBlockTransferBytes( // Mirror the logic in copyBlock: a partial copy only happens when numTokensToCopy > 0, // the data type supports it (not kINT4/kFP4), not block scales, and numTokensToCopy < tokensPerBlock. - bool const isPartialCopy = numTokensToCopy > 0 && dataType != tensorrt_llm::DataType::kINT4 - && dataType != tensorrt_llm::DataType::kFP4 && !pool.containsBlockScales + bool const isPartialCopy = numTokensToCopy > 0 && dataType != nvinfer1::DataType::kINT4 + && dataType != nvinfer1::DataType::kFP4 && !pool.containsBlockScales && numTokensToCopy < pool.tokensPerBlock; if (isPartialCopy) diff --git a/cpp/tensorrt_llm/batch_manager/llmRequest.cpp b/cpp/tensorrt_llm/batch_manager/llmRequest.cpp index d5466b4a7539..e51fad8ba149 100644 --- a/cpp/tensorrt_llm/batch_manager/llmRequest.cpp +++ b/cpp/tensorrt_llm/batch_manager/llmRequest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,16 +22,6 @@ namespace tensorrt_llm::batch_manager { -// Single, process-global storage for the steady-clock offset. Keeping it in this -// translation unit (rather than as an inline-static member reachable from every -// shared object) guarantees that libtensorrt_llm.so and the nanobind extension -// module observe the same value once either side calibrates it. -std::optional<std::chrono::steady_clock::duration>& globalSteadyClockOffset() -{ - static std::optional<std::chrono::steady_clock::duration> offset{std::nullopt}; - return offset; -} - template <typename TTensor, typename TStream> runtime::SizeType32 GenericLlmRequest<TTensor, TStream>::getBeamWidthByIter(bool const forNextIteration) { diff --git a/cpp/tensorrt_llm/batch_manager/logitsPostProcessor.cpp b/cpp/tensorrt_llm/batch_manager/logitsPostProcessor.cpp new file mode 100644 index 000000000000..95b324f0f2ec --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/logitsPostProcessor.cpp @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/logitsPostProcessor.h" + +#include "tensorrt_llm/batch_manager/decoderBuffers.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/runtime/iTensor.h" + +namespace tr = tensorrt_llm::runtime; + +namespace tensorrt_llm::batch_manager +{ + +using TensorPtr = runtime::ITensor::SharedPtr; +using ITensor = runtime::ITensor; +using SizeType32 = tensorrt_llm::runtime::SizeType32; + +bool LogitsPostProcessor::operator()(DecoderInputBuffers& inputBuffers, bool replicateLogitsPostProcessor, + tr::WorldConfig const& worldConfig, CudaStreamPtr const& stream, + std::optional<LogitsPostProcessorBatched> const& logitsPostProcessorBatched) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(LogitsPostProcessor); + + // Arguments for batched processor + std::vector<LlmRequest::RequestIdType> reqIdsVec; + std::vector<LlmRequest::TensorPtr> logitsVec; + std::vector<std::reference_wrapper<LlmRequest::BeamTokens const>> beamTokensVec; + std::vector<std::optional<LlmRequest::RequestIdType>> clientIdsVec; + + bool logitsPostProcessorIsApplied = false; + for (size_t batchIdx = 0; batchIdx < inputBuffers.decoderRequests.size(); ++batchIdx) + { + auto const& llmReq = inputBuffers.decoderRequests.at(batchIdx); + auto& logits = inputBuffers.decoderLogits.at(batchIdx); + + // Invoke non-batched processor or collect arguments for batched processor + if (llmReq->mLogitsPostProcessor) + { + logitsPostProcessorIsApplied = true; + if (replicateLogitsPostProcessor || worldConfig.isFirstTensorParallelRank()) + { + (*llmReq->mLogitsPostProcessor)( + llmReq->mRequestId, logits, llmReq->getTokens(), stream, llmReq->mClientId); + } + } + else if (llmReq->mApplyLogitsPostProcessorBatched) + { + reqIdsVec.push_back(llmReq->mRequestId); + logitsVec.push_back(logits); + beamTokensVec.emplace_back(llmReq->getTokens()); + clientIdsVec.push_back(llmReq->mClientId); + } + } + + // Invoke batched processor + if (!reqIdsVec.empty()) + { + logitsPostProcessorIsApplied = true; + if (replicateLogitsPostProcessor || worldConfig.isFirstTensorParallelRank()) + { + (*logitsPostProcessorBatched)(reqIdsVec, logitsVec, beamTokensVec, stream, clientIdsVec); + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + + return logitsPostProcessorIsApplied; +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/loraBuffers.cpp b/cpp/tensorrt_llm/batch_manager/loraBuffers.cpp new file mode 100644 index 000000000000..b67b72f6c49a --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/loraBuffers.cpp @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "loraBuffers.h" + +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/runtime/loraUtils.h" + +namespace tensorrt_llm::batch_manager +{ + +LoraBuffers::LoraBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::TllmRuntime const& tllmRuntime, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) +{ + auto const localNbLayers + = modelConfig.getNbAttentionLayers(worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); + auto const firstLayerId = worldConfig.getPipelineParallelRank() * localNbLayers; + + auto nbModelConfigs = static_cast<SizeType32>(modelConfig.getLoraModules().size()); + + // there are 3 pointers: LoRA A, LoRA B, and a DoRA magnitude (null if not DoRA) + auto loraWeightsPtrsShape + = runtime::ITensor::makeShape({nbModelConfigs, localNbLayers, maxBatchSize * maxBeamWidth, 3}); + auto loraAdapterSizesShape + = runtime::ITensor::makeShape({nbModelConfigs, localNbLayers, maxBatchSize * maxBeamWidth}); + + auto firstModuleName = std::string(modelConfig.getLoraModules().front().name()); + auto ptrsFieldName = firstModuleName + "_lora_weights_pointers_" + std::to_string(firstLayerId); + auto rankFieldName = firstModuleName + "_lora_ranks_" + std::to_string(firstLayerId); + auto weightsPtrDtype = tllmRuntime.getEngine().getTensorDataType(ptrsFieldName.c_str()); + auto ranksDtype = tllmRuntime.getEngine().getTensorDataType(rankFieldName.c_str()); + + mLoraManager.create(modelConfig); + + mLoraWeightsPointersHost = runtime::BufferManager::pinned(loraWeightsPtrsShape, weightsPtrDtype); + mLoraAdapterSizesHost = runtime::BufferManager::pinned(loraAdapterSizesShape, ranksDtype); +} + +void LoraBuffers::fill(RequestVector const& contextRequests, RequestVector const& genRequests, + PeftTable const& peftTable, runtime::BufferManager const& manager, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig) +{ + manager.setZero(*mLoraWeightsPointersHost); + manager.setZero(*mLoraAdapterSizesHost); + + SizeType32 batchIdx{0}; + for (auto const& requests : {contextRequests, genRequests}) + { + for (auto const& llmReq : requests) + { + auto const optReqLoraWeights = llmReq->getLoraWeights(); + auto const optReqLoraConfig = llmReq->getLoraConfig(); + + auto const isContextRequest = llmReq->isContextInitState(); + auto const beamWidth = isContextRequest ? 1 : llmReq->mSamplingConfig.beamWidth; + auto const peftIt = peftTable.find(llmReq->mRequestId); + if (peftIt != peftTable.end()) + { + auto const& peftValues = peftIt->second; + if (!peftValues.empty()) + { + mLoraManager.fillInputTensors(mLoraWeightsPointersHost, mLoraAdapterSizesHost, peftIt->second, + batchIdx, beamWidth, modelConfig, worldConfig); + } + } + ++batchIdx; + } + } +} + +void LoraBuffers::validate(std::optional<std::uint64_t> const& optTaskId, + std::optional<TensorPtr> const& optReqLoraWeights, std::optional<TensorPtr> const& optReqLoraConfig, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) +{ + runtime::lora::loraValidateRequestTensors(optTaskId, optReqLoraWeights, optReqLoraConfig, modelConfig, worldConfig); +} + +void LoraBuffers::insertInputTensors(TensorMap& inputTensors, TensorPtr weightsPtrs, TensorPtr adapterSizes, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) const +{ + mLoraManager.insertInputTensors(inputTensors, weightsPtrs, adapterSizes, modelConfig, worldConfig); +} + +void LoraBuffers::reshape(SizeType32 numSequences) +{ + auto weightsPtrsShape = mLoraWeightsPointersHost->getShape(); + weightsPtrsShape.d[2] = numSequences; + mLoraWeightsPointersHost->reshape(weightsPtrsShape); + + auto adapterSizesShape = mLoraAdapterSizesHost->getShape(); + adapterSizesShape.d[2] = numSequences; + mLoraAdapterSizesHost->reshape(adapterSizesShape); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/loraBuffers.h b/cpp/tensorrt_llm/batch_manager/loraBuffers.h new file mode 100644 index 000000000000..3ba68995518f --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/loraBuffers.h @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/loraManager.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +namespace tensorrt_llm::batch_manager +{ + +class LoraBuffers +{ +public: + using SizeType32 = tensorrt_llm::runtime::SizeType32; + using PeftTable = runtime::LoraManager::PeftTable; + using TensorPtr = runtime::ITensor::SharedPtr; + using TensorMap = runtime::StringPtrMap<runtime::ITensor>; + + TensorPtr mLoraWeightsPointersHost; + TensorPtr mLoraAdapterSizesHost; + + runtime::LoraManager mLoraManager; + + LoraBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::TllmRuntime const& tllmRuntime, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); + + static void validate(std::optional<std::uint64_t> const& optTaskId, + std::optional<TensorPtr> const& optReqLoraWeights, std::optional<TensorPtr> const& optReqLoraConfig, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); + + void fill(RequestVector const& contextRequests, RequestVector const& genRequests, PeftTable const& peftTable, + runtime::BufferManager const& manager, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig); + + void insertInputTensors(TensorMap& inputTensors, TensorPtr weightsPtrs, TensorPtr adapterSizes, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) const; + + void reshape(SizeType32 numSequences); +}; +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.cpp b/cpp/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.cpp new file mode 100644 index 000000000000..3e494a6383ec --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.cpp @@ -0,0 +1,198 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h" +#include "tensorrt_llm/batch_manager/decoderBuffers.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/runtime/decoderState.h" +#include "tensorrt_llm/runtime/iGptDecoderBatched.h" + +namespace tr = tensorrt_llm::runtime; + +namespace tensorrt_llm::batch_manager +{ +using SizeType32 = MakeDecodingBatchInputOutput::SizeType32; +using TensorPtr = MakeDecodingBatchInputOutput::TensorPtr; + +void MakeDecodingBatchInputOutput::createDecoderBatchInputs(DecoderInputBuffers& inputBuffers, + std::vector<SizeType32> const& activeSlots, runtime::decoder::DecoderState const& decoderState) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const& numDecodingEngineTokens = decoderState.getNumDecodingEngineTokens(); + auto const& maxDecodingEngineTokens = decoderState.getMaxDecodingEngineTokens(); + auto const& maxDecodingDecoderTokens = decoderState.getMaxDecodingDecoderTokens(); + auto const maxDecoderSteps = common::ceilDiv(maxDecodingEngineTokens, maxDecodingDecoderTokens); + + auto& batchSlots = inputBuffers.forwardBatchSlots; + auto& decoderLogits = inputBuffers.decoderLogits; + + for (SizeType32 step = 0; step < maxDecoderSteps; ++step) + { + batchSlots.at(step)->resize(activeSlots.size()); + } + + auto constexpr singleRequest = 1; + + std::vector<SizeType32> batchSizes(maxDecoderSteps); + std::vector<std::vector<tr::ITensor::SharedConstPtr>> batchLogits(maxDecoderSteps); + auto maxActiveDecoderSteps = 1; + for (size_t batchIdx = 0; batchIdx < activeSlots.size(); ++batchIdx) + { + auto const slot = activeSlots.at(batchIdx); + auto const& logits = decoderLogits.at(batchIdx); + + auto const numDecoderSteps = common::ceilDiv(numDecodingEngineTokens.at(slot), maxDecodingDecoderTokens); + maxActiveDecoderSteps = std::max(maxActiveDecoderSteps, numDecoderSteps); + for (SizeType32 step = 0; step < numDecoderSteps; ++step) + { + auto batchSlotsRange = tr::BufferRange<SizeType32>(*batchSlots.at(step)); + batchSlotsRange[batchSizes[step]] = slot; + batchSizes[step]++; + auto logitsSlice = tr::ITensor::slice(logits, step, singleRequest); + batchLogits[step].emplace_back(std::move(logitsSlice)); + } + } + + for (SizeType32 step = 0; step < maxDecoderSteps; ++step) + { + batchSlots.at(step)->resize(batchSizes[step]); + } + batchLogits.resize(maxActiveDecoderSteps); + + inputBuffers.maxDecoderSteps = maxActiveDecoderSteps; + inputBuffers.batchLogits = batchLogits; + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +namespace +{ + +std::pair<std::vector<SizeType32>, std::vector<SizeType32>> getActiveSlots(RequestVector const& decoderRequests) +{ + std::vector<SizeType32> activeSlots; + std::vector<SizeType32> generationSteps; + for (auto const& llmReq : decoderRequests) + { + activeSlots.push_back(llmReq->mSeqSlot.value()); + generationSteps.push_back(llmReq->getDecodingIter()); + } + + return {activeSlots, generationSteps}; +} + +//! @brief Sets inputs for explicit draft tokens. +void setExplicitDraftTokensInputs(tr::DecodingInput& dInput, RuntimeBuffers const& fusedRuntimeBuffers) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK(fusedRuntimeBuffers.mExplicitDraftTokensBuffers); + auto const& explicitDraftTokensInputs = fusedRuntimeBuffers.mExplicitDraftTokensBuffers->engineOutputs; + auto const& explicitDraftTokensLastInputs = fusedRuntimeBuffers.mExplicitDraftTokensBuffers->engineInputs; + + dInput.explicitDraftTokensInputs = tr::DecodingInput::ExplicitDraftTokensInputs(); + dInput.explicitDraftTokensInputs->nextDraftTokens = explicitDraftTokensInputs.nextDraftTokens; + dInput.explicitDraftTokensInputs->nextFlatTokens = explicitDraftTokensInputs.nextFlatTokens; + dInput.explicitDraftTokensInputs->nextDraftIndices = explicitDraftTokensInputs.nextDraftIndices; + dInput.explicitDraftTokensInputs->nextDraftProbs = explicitDraftTokensInputs.nextDraftProbs; + dInput.explicitDraftTokensInputs->lastDraftTokens = explicitDraftTokensLastInputs.draftTokens; + dInput.explicitDraftTokensInputs->lastDraftIndices = explicitDraftTokensLastInputs.draftIndices; + dInput.explicitDraftTokensInputs->lastPositionIdsBase = explicitDraftTokensLastInputs.positionIdsBase; + dInput.explicitDraftTokensInputs->masks = explicitDraftTokensInputs.masks; + dInput.explicitDraftTokensInputs->packedPositionIds = explicitDraftTokensInputs.packedPositionIds; + dInput.explicitDraftTokensInputs->bestPathLengths = explicitDraftTokensInputs.bestPathLengths; + dInput.explicitDraftTokensInputs->bestPathIndices = explicitDraftTokensInputs.bestPathIndices; + dInput.explicitDraftTokensInputs->nextGenerationLengths = explicitDraftTokensInputs.nextGenerationLengths; + dInput.explicitDraftTokensInputs->lastGenerationLengths = explicitDraftTokensLastInputs.generationLengths; + dInput.explicitDraftTokensInputs->maxGenLengthDevice = explicitDraftTokensInputs.maxGenToken; + // Slots in request order + dInput.explicitDraftTokensInputs->seqSlots = fusedRuntimeBuffers.seqSlots; + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +//! @brief Sets inputs for eagle decoding. +void setEagleInputs(tr::DecodingInput& dInput, RuntimeBuffers const& fusedRuntimeBuffers) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK(fusedRuntimeBuffers.mEagleBuffers); + auto const& eagleInputs = fusedRuntimeBuffers.mEagleBuffers->engineOutputs; + auto const& eagleLastInputs = fusedRuntimeBuffers.mEagleBuffers->engineInputs; + + dInput.eagleInputs = tr::DecodingInput::EagleInputs(); + dInput.eagleInputs->nextDraftTokens = eagleInputs.nextDraftTokens; + dInput.eagleInputs->nextDraftLens = eagleInputs.nextDraftLens; + dInput.eagleInputs->nextDraftPaths = eagleInputs.nextDraftPaths; + dInput.eagleInputs->lastDraftTokens = eagleLastInputs.draftTokens; + dInput.eagleInputs->lastDraftLens = eagleLastInputs.draftLens; + dInput.eagleInputs->lastDraftPaths = eagleLastInputs.draftPaths; + dInput.eagleInputs->acceptedTokens = eagleInputs.acceptedTokens; + dInput.eagleInputs->acceptedLens = eagleInputs.acceptedLens; + dInput.eagleInputs->acceptedPathIds = eagleInputs.acceptedPaths; + dInput.eagleInputs->chunkedContextNextTokens = eagleInputs.chunkedContextNextTokens; + // Slots in request order + dInput.eagleInputs->seqSlots = fusedRuntimeBuffers.seqSlots; + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +} // namespace + +void MakeDecodingBatchInputOutput::operator()(DecoderInputBuffers& inputBuffers, + runtime::decoder::DecoderState& decoderState, runtime::ModelConfig const& modelConfig, + OptionalRef<RuntimeBuffers> fusedRuntimeBuffers) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto [activeSlots, generationSteps] = getActiveSlots(inputBuffers.decoderRequests); + + createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + + auto const maxBeamWidth = decoderState.getMaxBeamWidth(); + if (maxBeamWidth > 1) + { + // For Variable-Beam-Width-Search + decoderState.getJointDecodingInput().generationSteps = generationSteps; + } + + if (modelConfig.getSpeculativeDecodingMode().hasDraftLogits()) + { + decoderState.getJointDecodingInput().medusaInputs->medusaLogits = inputBuffers.predictedDraftLogits; + } + + if (modelConfig.getSpeculativeDecodingMode().isExplicitDraftTokens()) + { + TLLM_CHECK(fusedRuntimeBuffers); + // requires mCtxGenFusion == true + setExplicitDraftTokensInputs(decoderState.getJointDecodingInput(), *fusedRuntimeBuffers); + } + else if (modelConfig.getSpeculativeDecodingMode().isEagle()) + { + TLLM_CHECK(fusedRuntimeBuffers); + // requires mCtxGenFusion == true + setEagleInputs(decoderState.getJointDecodingInput(), *fusedRuntimeBuffers); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/medusaBuffers.cpp b/cpp/tensorrt_llm/batch_manager/medusaBuffers.cpp index 32935e683b83..eb40208739cf 100644 --- a/cpp/tensorrt_llm/batch_manager/medusaBuffers.cpp +++ b/cpp/tensorrt_llm/batch_manager/medusaBuffers.cpp @@ -17,10 +17,99 @@ #include "tensorrt_llm/batch_manager/medusaBuffers.h" #include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/medusaModule.h" +#include "tensorrt_llm/runtime/utils/speculativeChoicesUtils.h" namespace tensorrt_llm::batch_manager { +MedusaBuffers::MedusaBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, runtime::BufferManager const& manager, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + executor::DecodingConfig const& decodingConfig, runtime::TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + TLLM_CHECK_WITH_INFO(maxBeamWidth == 1, "Medusa does not support beam search"); + + auto const& engine = runtime.getEngine(); + + auto const maxNumSequences = maxBatchSize; + + auto const medusaModule = std::dynamic_pointer_cast<tensorrt_llm::runtime::MedusaModule const>( + modelConfig.getSpeculativeDecodingModulePtr()); + + auto const medusaHeads = medusaModule->getMaxDraftPathLen(); + auto const maxPathLen = medusaModule->getMaxPathLen(); // medusaHeads + 1 + auto const maxMedusaTokens = medusaModule->getMaxDecodingDraftTokens(); + auto const maxDecodingTokens = medusaModule->getMaxDecodingTokens(); // maxMedusaTokens + 1 + auto const numPackedMasks = medusaModule->getNumPackedMasks(); + + auto const vocabSizePadded = modelConfig.getVocabSizePadded(worldConfig.getSize()); + + if (worldConfig.isLastPipelineParallelRank()) + { + auto logitsType = engine.getTensorDataType("medusa_logits"); + medusaLogitsDevice = manager.gpu( + ITensor::makeShape({medusaHeads, maxBatchSize, maxDecodingTokens, vocabSizePadded}), logitsType); + } + + // Note: reserved for variable sequence length support. + medusaGenerationLengthsHost + = runtime::BufferManager::pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + // TODO: pack batch and tokensPerStep into one dim to support variable sequence length without padddings. + attentionPackedMaskHost = runtime::BufferManager::pinned( + ITensor::makeShape({maxNumSequences, maxDecodingTokens, numPackedMasks}), nvinfer1::DataType::kINT32); + medusaPositionOffsetsHost = runtime::BufferManager::pinned( + ITensor::makeShape({maxNumSequences, maxDecodingTokens}), nvinfer1::DataType::kINT32); + medusaTreeIdsHost = runtime::BufferManager::pinned( + ITensor::makeShape({maxNumSequences, maxMedusaTokens}), nvinfer1::DataType::kINT32); + medusaPathsHost = runtime::BufferManager::pinned( + ITensor::makeShape({maxNumSequences, maxDecodingTokens, maxPathLen}), nvinfer1::DataType::kINT32); + + TensorPtr medusaPositionOffsetsHostSlice = ITensor::slice(medusaPositionOffsetsHost, 0, 1); + medusaPositionOffsetsHostSlice->squeeze(0); + TensorPtr medusaTreeIdsHostSlice = ITensor::slice(medusaTreeIdsHost, 0, 1); + medusaTreeIdsHostSlice->squeeze(0); + TensorPtr medusaPathsHostSlice = ITensor::slice(medusaPathsHost, 0, 1); + medusaPathsHostSlice->squeeze(0); + TensorPtr attentionPackedMaskHostSlice = ITensor::slice(attentionPackedMaskHost, 0, 1); + attentionPackedMaskHostSlice->squeeze(0); + + // Init buffers for 1 request + auto const& choices = decodingConfig.getMedusaChoices().value_or(medusaModule->getMedusaChoices()); + runtime::utils::initTensorsFromChoices(*medusaModule, choices, mTopKs, medusaGenerationLengthsHost, + medusaPositionOffsetsHostSlice, medusaTreeIdsHostSlice, medusaPathsHostSlice, attentionPackedMaskHostSlice); + + auto scatterToBatch = [maxBatchSize, &manager](TensorPtr& data) + { + auto srcSlice = ITensor::slice(data, 0, 1); + // Populate data from the 1st request to the other requests in the batch + for (SizeType32 bi = 1; bi < maxBatchSize; ++bi) + { + auto dstSlice = ITensor::slice(data, bi, 1); + manager.copy(*srcSlice, *dstSlice); + } + }; + + scatterToBatch(medusaPositionOffsetsHost); + scatterToBatch(medusaTreeIdsHost); + scatterToBatch(medusaPathsHost); + scatterToBatch(attentionPackedMaskHost); + + // Copy buffers to device + // 1st dimension of packed mask is num_total_generation_tokens now (packed without paddings). + attentionPackedMaskHost->reshape(ITensor::makeShape({maxNumSequences * maxDecodingTokens, numPackedMasks})); + attentionPackedMaskDevice = manager.copyFrom(*attentionPackedMaskHost, runtime::MemoryType::kGPU); + medusaGenerationLengthsDevice = manager.copyFrom(*medusaGenerationLengthsHost, runtime::MemoryType::kGPU); + medusaPositionOffsetsDevice = manager.copyFrom(*medusaPositionOffsetsHost, runtime::MemoryType::kGPU); + medusaTreeIdsDevice = manager.copyFrom(*medusaTreeIdsHost, runtime::MemoryType::kGPU); + medusaPathsDevice = manager.copyFrom(*medusaPathsHost, runtime::MemoryType::kGPU); + + // use speculative decoding buffer + medusaUseSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + runtime::bufferCast<SizeType32>(*medusaUseSpecDecoding)[0] = 1; + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + void MedusaBuffers::reshape(SizeType32 /* numCtxSequences */, SizeType32 numGenSequences, SizeType32 tokensPerStep) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); diff --git a/cpp/tensorrt_llm/batch_manager/microBatchScheduler.cpp b/cpp/tensorrt_llm/batch_manager/microBatchScheduler.cpp index 2807f33463a1..3e8fca0be052 100644 --- a/cpp/tensorrt_llm/batch_manager/microBatchScheduler.cpp +++ b/cpp/tensorrt_llm/batch_manager/microBatchScheduler.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -230,9 +230,8 @@ void MicroBatchScheduler::setCtxRequestsChunkSize<MicroBatchScheduler::ContextCh // Assigns chunk sizes to context requests under the kFORCE_CHUNK policy. // -// Requests with expected snapshot points advance to their next absolute snapshot position. -// Otherwise, every request consumes its full remaining context. -// Capacity and context-length truncation are rounded down to a chunk-unit boundary. +// Every request is assigned exactly min(contextRemainingLength, chunkUnitSize) tokens. +// Requests whose chunk would push the running total past ctxTokensCapacity are zeroed. // // This policy is designed for linear attention state caching, so reusable KV-cache tokens are NOT // calculated because it's not supported yet. @@ -249,36 +248,16 @@ void MicroBatchScheduler::setCtxRequestsChunkSize<MicroBatchScheduler::ContextCh SizeType32 totalTokens{0}; for (auto& llmReq : contextsToBeChunked) { - SizeType32 chunkSize = llmReq->getContextRemainingLength(); - auto const& expectedSnapshotPoints = llmReq->getExpectedSnapshotPoints(); - if (!expectedSnapshotPoints.empty()) - { - auto const currentPosition = llmReq->getContextCurrentPosition(); - std::optional<SizeType32> nextSnapshotPoint; - for (auto const point : expectedSnapshotPoints) - { - if (point > currentPosition && (!nextSnapshotPoint || point < nextSnapshotPoint.value())) - { - nextSnapshotPoint = point; - } - } - chunkSize = nextSnapshotPoint - ? std::max<SizeType32>(0, std::min(nextSnapshotPoint.value(), llmReq->getPromptLen()) - currentPosition) - : llmReq->getContextRemainingLength(); - } - - if (maxContextLength && chunkSize > maxContextLength.value()) + SizeType32 const chunkSize = std::min(llmReq->getContextRemainingLength(), chunkUnitSize); + if (ctxTokensCapacity && totalTokens + chunkSize > ctxTokensCapacity.value()) { - chunkSize = maxContextLength.value() / chunkUnitSize * chunkUnitSize; + llmReq->setContextChunkSize(0); } - if (ctxTokensCapacity && totalTokens + chunkSize > ctxTokensCapacity.value()) + else { - auto const remainingCapacity = std::max<SizeType32>(0, ctxTokensCapacity.value() - totalTokens); - chunkSize = std::min(chunkSize, remainingCapacity) / chunkUnitSize * chunkUnitSize; + llmReq->setContextChunkSize(chunkSize); + totalTokens += llmReq->getContextChunkSize(); } - - llmReq->setContextChunkSize(chunkSize); - totalTokens += llmReq->getContextChunkSize(); } } @@ -288,9 +267,8 @@ void MicroBatchScheduler::setCtxRequestsChunkSize<MicroBatchScheduler::ContextCh // kEQUAL_PROGRESS — all requests advance together one chunkUnitSize at a time. // kFIRST_COME_FIRST_SERVED — requests are served greedily in order until the budget // is exhausted. -// kFORCE_CHUNK — requests advance to the next expected snapshot point, or consume -// the remaining context when none are configured; budget is charged -// at face value (no reuse discount). +// kFORCE_CHUNK — every request gets exactly min(remaining, chunkUnitSize) +// tokens; budget is charged at face value (no reuse discount). // // EQUAL_PROGRESS and FIRST_COME_FIRST_SERVED are compute-aware: tokens covered by the // reusable KV-cache prefix are not charged against ctxTokensCapacity. @@ -458,7 +436,7 @@ std::tuple<RequestVector, RequestVector> MicroBatchScheduler::operator()(Request allContextRequestsFit = false; } - // FORCE_CHUNK must always run boundary selection even when all contexts fit. + // For FORCE_CHUNK policy, always re-chunk regardless of whether all contexts fit. if (mCtxChunkConfig && mCtxChunkConfig.value().chunkingPolicy == ContextChunkingPolicy::kFORCE_CHUNK) { allContextRequestsFit = false; diff --git a/cpp/tensorrt_llm/batch_manager/mlaCacheFormatter.cpp b/cpp/tensorrt_llm/batch_manager/mlaCacheFormatter.cpp index 2631df9aa9a2..ad55455ad3c2 100644 --- a/cpp/tensorrt_llm/batch_manager/mlaCacheFormatter.cpp +++ b/cpp/tensorrt_llm/batch_manager/mlaCacheFormatter.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -124,13 +124,9 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses { NVTX3_SCOPED_RANGE(MLACacheFormatter_format); session.setTime(TransferSession::kTimeFormatter); - auto llmRequest = session.getLlmRequest(); - if (llmRequest.has_value()) - { - TLLM_LOG_DEBUG( - mpi::MpiComm::world().getRank(), "Start sending KV cache for request ID: %ld.", (*llmRequest)->mRequestId); - TLLM_CHECK_WITH_INFO((*llmRequest)->mSamplingConfig.beamWidth == 1, "Currently only supports beam width 1."); - } + auto const& llmRequest = session.getLlmRequest(); + TLLM_LOG_DEBUG( + mpi::MpiComm::world().getRank(), "Start sending KV cache for request ID: %ld.", llmRequest.mRequestId); auto const& selfConfig = session.getSelfState().getCacheState().value(); auto const& destConfig = session.getOtherState().getCacheState().value(); auto const selfIdx = session.getSelfState().getCommState().value().getSelfIdx(); @@ -138,6 +134,7 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses auto const& lastBlockKey = session.getLastBlockKey(); auto const& connections = session.getConnections(); auto& bufferManager = session.getBufferManager(); + TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently only supports beam width 1."); TLLM_CHECK(!connections.empty()); if (!needSendCache(selfConfig, destConfig, selfIdx)) { @@ -148,6 +145,7 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses auto targetNum = pickUpConnections.size(); if (targetNum == 0) { + TLLM_LOG_DEBUG("No targets to send KV cache to for request ID: %ld", llmRequest.mRequestId); return; } @@ -221,11 +219,8 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses } } - if (llmRequest.has_value()) - { - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", - (*llmRequest)->mRequestId); - } + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", + llmRequest.mRequestId); return; } @@ -258,9 +253,7 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses return bufferSizeForTarget; }; auto bufferEleSizes = getBufferSizeForTarget(); - auto const* sendCancelFlag - = common::getEnvDisaggEnableInflightCancel() ? &session.getDataContext().getTransferTerminate() : nullptr; - auto cacheBufferId = mCacheTransBufferManagers[transferIndexerKCache]->assignBufferIndexForSend(sendCancelFlag); + auto cacheBufferId = mCacheTransBufferManagers[transferIndexerKCache]->assignBufferIndexForSend(); BufferIndexHolder sendHolder( *mCacheTransBufferManagers[transferIndexerKCache], cacheBufferId, /*isRecv=*/false); auto result = mCacheTransBufferManagers[transferIndexerKCache]->getOrAllocateSendBuffers( @@ -353,84 +346,63 @@ void MLACacheFormatter::format(tensorrt_llm::batch_manager::TransferSession& ses session.appendMeasure(startTime, endTime, outputSplitCaches.at(cacheIdx)->getSizeInBytes()); }; - if (sendCancelFlag != nullptr && sendCancelFlag->load(std::memory_order_relaxed)) + if (pickUpConnections.size() > 1) { - TLLM_THROW("MLA cache transfer cancelled before NIXL submission"); - } - - try - { - if (pickUpConnections.size() > 1) + if (!common::getEnvEnableReceiveKVCacheParallel()) { - if (!common::getEnvEnableReceiveKVCacheParallel()) + TLLM_LOG_DEBUG("Disable parallel receiving of the KV cache."); + for (size_t i = 0; i < pickUpConnections.size(); i++) { - TLLM_LOG_DEBUG("Disable parallel receiving of the KV cache."); - for (size_t i = 0; i < pickUpConnections.size(); i++) - { - sendBufferFun(deviceId, pickUpConnections[i]); - } + sendBufferFun(deviceId, pickUpConnections[i]); } - else - { - // concurrency num - auto concurrencyNum - = std::min(std::max(static_cast<size_t>(1), bufferCoverTargetNum), pPDomainSize * cPDomainSize); + } + else + { + // concurrency num + auto concurrencyNum + = std::min(std::max(static_cast<size_t>(1), bufferCoverTargetNum), pPDomainSize * cPDomainSize); - auto remainSendNum = pickUpConnections.size(); + auto remainSendNum = pickUpConnections.size(); - while (remainSendNum > 0) + while (remainSendNum > 0) + { + auto sendConcurrencyNum = std::min(remainSendNum, concurrencyNum); + std::vector<std::future<void>> futures; + futures.reserve(sendConcurrencyNum); + for (size_t i = 0; i < sendConcurrencyNum; i++) { - auto sendConcurrencyNum = std::min(remainSendNum, concurrencyNum); - std::vector<std::future<void>> futures; - futures.reserve(sendConcurrencyNum); - for (size_t i = 0; i < sendConcurrencyNum; i++) - { - size_t idx = i + (pickUpConnections.size() - remainSendNum); - size_t connIdx = pickUpConnections[idx]; - TLLM_CHECK(idx < pickUpConnections.size()); - TLLM_CHECK(connIdx < session.getConnections().size()); - futures.push_back(std::async(std::launch::async, sendBufferFun, deviceId, connIdx)); - } - for (auto& future : futures) - { - future.get(); - } - remainSendNum -= sendConcurrencyNum; + size_t idx = i + (pickUpConnections.size() - remainSendNum); + size_t connIdx = pickUpConnections[idx]; + TLLM_CHECK(idx < pickUpConnections.size()); + TLLM_CHECK(connIdx < session.getConnections().size()); + futures.push_back(std::async(std::launch::async, sendBufferFun, deviceId, connIdx)); + } + for (auto& future : futures) + { + future.get(); } + remainSendNum -= sendConcurrencyNum; } } - else - { - sendBufferFun(deviceId, pickUpConnections[0]); - } } - catch (...) + else { - if (agentConnection != nullptr && common::getEnvDisaggEnableInflightCancel()) - { - sendHolder.poison(); - } - throw; + sendBufferFun(deviceId, pickUpConnections[0]); } sendHolder.release(); } session.setTime(TransferSession::kTimeTransmissions); session.setTime(TransferSession::kTimePostprocess); - if (llmRequest.has_value()) - { - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", - (*llmRequest)->mRequestId); - } + TLLM_LOG_DEBUG( + mpi::MpiComm::world().getRank(), "End the sending of KV cache for the request ID: %ld.", llmRequest.mRequestId); } void MLACacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& session) { NVTX3_SCOPED_RANGE(MLACacheFormatter_unformat); session.setTime(TransferSession::kTimeFormatter); - auto llmRequestOpt = session.getLlmRequest(); - TLLM_CHECK_WITH_INFO(llmRequestOpt.has_value(), "LlmRequest required for receiving KV cache"); - auto const& llmRequest = **llmRequestOpt; + auto const& llmRequest = session.getLlmRequest(); TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently only supports beam width 1."); auto const ctxReqId = llmRequest.getContextPhaseParams().value().getReqId(); TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), @@ -535,18 +507,13 @@ void MLACacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& s if (preAssignedId.has_value()) { cacheBufferId = static_cast<int>(*preAssignedId); - if (!session.hasReservedRecvBuffer(*mCacheTransBufferManagers[transferIndexerKCache])) - { - recvHolder = BufferIndexHolder( - *mCacheTransBufferManagers[transferIndexerKCache], cacheBufferId, /*isRecv=*/true); - } } else { cacheBufferId = mCacheTransBufferManagers[transferIndexerKCache]->assignBufferIndexForRecv(); - recvHolder = BufferIndexHolder( - *mCacheTransBufferManagers[transferIndexerKCache], cacheBufferId, /*isRecv=*/true); } + recvHolder + = BufferIndexHolder(*mCacheTransBufferManagers[transferIndexerKCache], cacheBufferId, /*isRecv=*/true); auto targetNum = pickUpConnections.size(); @@ -694,7 +661,6 @@ void MLACacheFormatter::unformat(tensorrt_llm::batch_manager::TransferSession& s bufferManager.getStream().synchronize(); } - (void) session.releaseReservedRecvBuffer(*mCacheTransBufferManagers[transferIndexerKCache]); recvHolder.release(); } session.setTime(TransferSession::kTimePostprocess); diff --git a/cpp/tensorrt_llm/batch_manager/peftCacheManager.cpp b/cpp/tensorrt_llm/batch_manager/peftCacheManager.cpp index 89cc475b82ee..0bf9a989fd65 100644 --- a/cpp/tensorrt_llm/batch_manager/peftCacheManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/peftCacheManager.cpp @@ -30,7 +30,7 @@ #include "tensorrt_llm/runtime/workerPool.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cstdint> #include <limits> @@ -52,8 +52,7 @@ PeftTaskNotCachedException::PeftTaskNotCachedException(std::string const& msg) PeftTaskNotCachedException::~PeftTaskNotCachedException() noexcept = default; std::pair<uint64_t, uint64_t> PeftCacheManager::getMaxNumSlots(PeftCacheManagerConfig const& config, - tensorrt_llm::DataType dataType, uint64_t pageWidth, uint64_t max1dModSize, - runtime::BufferManager const& bufferManager) + nvinfer1::DataType dataType, uint64_t pageWidth, uint64_t max1dModSize, runtime::BufferManager const& bufferManager) { TLLM_LOG_DEBUG("max1dModeSize=%llu", max1dModSize); TLLM_LOG_DEBUG("pageWidth=%llu", pageWidth); diff --git a/cpp/tensorrt_llm/batch_manager/promptTuningBuffers.cpp b/cpp/tensorrt_llm/batch_manager/promptTuningBuffers.cpp new file mode 100644 index 000000000000..1cf73a2c0d21 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/promptTuningBuffers.cpp @@ -0,0 +1,323 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/promptTuningBuffers.h" + +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/common/nvtxUtils.h" + +namespace tensorrt_llm::batch_manager +{ +using SizeType32 = tensorrt_llm::runtime::SizeType32; +using TensorPtr = runtime::ITensor::SharedPtr; + +PromptTuningBuffers::PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig) +{ + auto maxPromptEmbeddingTableSize = modelConfig.getMaxPromptEmbeddingTableSize(); + auto const hiddenSize = modelConfig.getHiddenSize() * worldConfig.getTensorParallelism(); + + // vocabSize and mMaxPromptVocabSize + mPromptTuningParams.vocabSize = manager.gpu(runtime::ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + mMaxPromptVocabSize = maxPromptEmbeddingTableSize / maxBatchSize; + + auto promptVocabSizeHost + = runtime::BufferManager::pinned(runtime::ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + auto promptVocabSizeHostData = runtime::bufferCast<SizeType32>(*promptVocabSizeHost); + promptVocabSizeHostData[0] = mMaxPromptVocabSize; + manager.copy(*promptVocabSizeHost, *mPromptTuningParams.vocabSize); + + // embeddingTable + mPromptTuningParams.embeddingTable = manager.gpu( + runtime::ITensor::makeShape({maxPromptEmbeddingTableSize, hiddenSize}), modelConfig.getDataType()); + + // tasks + mPromptTuningParams.tasks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); +} + +PromptTuningBuffers::PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, bool promptTableOffloading) +{ + auto maxPromptEmbeddingTableSize = modelConfig.getMaxPromptEmbeddingTableSize(); + auto const hiddenSize = modelConfig.getHiddenSize() * worldConfig.getTensorParallelism(); + + // vocabSize and mMaxPromptVocabSize + mPromptTuningParams.vocabSize = manager.gpu(runtime::ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + mMaxPromptVocabSize = maxPromptEmbeddingTableSize / maxBatchSize; + mPromptTableOffloading = promptTableOffloading; + + auto promptVocabSizeHost + = runtime::BufferManager::pinned(runtime::ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + auto promptVocabSizeHostData = runtime::bufferCast<SizeType32>(*promptVocabSizeHost); + promptVocabSizeHostData[0] = mMaxPromptVocabSize; + manager.copy(*promptVocabSizeHost, *mPromptTuningParams.vocabSize); + + // embeddingTable + mPromptTuningParams.embeddingTable = manager.gpu( + runtime::ITensor::makeShape({maxPromptEmbeddingTableSize, hiddenSize}), modelConfig.getDataType()); + + // tasks + mPromptTuningParams.tasks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); +} + +void PromptTuningBuffers::validate( + std::optional<TensorPtr> const& optReqPromptEmbeddingTable, std::optional<SizeType32> const& optReqPromptVocabSize) +{ + // Need to copy request embeddingTable to promptEmbeddingTable + if (optReqPromptEmbeddingTable.has_value()) + { + + auto reqPromptEmbeddingTable = optReqPromptEmbeddingTable.value(); + auto reqPromptVocabSize = optReqPromptVocabSize.value(); + + if (reqPromptVocabSize > mMaxPromptVocabSize) + { + std::string errStr = "Prompt vocab size" + std::to_string(reqPromptVocabSize) + + " is larger than max prompt vocab size of " + std::to_string(mMaxPromptVocabSize) + + ". Max prompt vocab size is computed from max_prompt_embedding_table_size / max_batch_size. "; + TLLM_LOG_ERROR(errStr); + throw std::runtime_error(errStr); + } + else + { + // Check that type matches model weights + if (reqPromptEmbeddingTable->getDataType() != mPromptTuningParams.embeddingTable->getDataType()) + { + std::string errStr = "Request embedding table data type doesn't match model weight data type."; + TLLM_LOG_ERROR(errStr); + throw std::runtime_error(errStr); + } + + if (reqPromptEmbeddingTable->getShape().d[1] != reqPromptVocabSize) + { + std::string errStr + = "First dimension of request embedding table is expected to be equal to prompt vocab size"; + TLLM_LOG_ERROR(errStr); + throw std::runtime_error(errStr); + } + } + } +} + +void PromptTuningBuffers::fill(RequestVector const& contextRequests, RequestVector const& genRequests, + runtime::BufferManager const& manager, bool packed) +{ + NVTX3_SCOPED_RANGE_WITH_NAME(range, "PromptTuningBuffers::fill"); + + auto const numContextRequests = static_cast<SizeType32>(contextRequests.size()); + + std::vector<SizeType32> reqBeamWidths; + std::vector<SizeType32> reqPromptLengths; + mPromptTuningParams.promptTuningEnabled.clear(); + + SizeType32 batchIdx{0}; + for (auto const& requests : {contextRequests, genRequests}) + { + for (auto const& llmReq : requests) + { + reqBeamWidths.push_back(llmReq->mSamplingConfig.beamWidth); + if (batchIdx < numContextRequests) + { + SizeType32 numContextTokens = 0; + auto const draftLength = llmReq->isLastContextChunk() ? llmReq->getNumDraftTokens() : 0; + auto const contextChunkSize = llmReq->getContextChunkSize(); + numContextTokens += contextChunkSize + draftLength; + reqPromptLengths.push_back(numContextTokens); + } + + std::optional<TensorPtr> optReqPromptEmbeddingTable = std::nullopt; + std::optional<SizeType32> optReqPromptVocabSize = std::nullopt; + + if (mPromptTableOffloading) + { + optReqPromptEmbeddingTable = getChunkPtableBuffer(getChunkPtableCurrentIndex()); + optReqPromptVocabSize = getChunkPtableBufferSliceSize(getChunkPtableCurrentIndex(), batchIdx); + } + else + { + optReqPromptEmbeddingTable = llmReq->getPromptEmbeddingTable(); + optReqPromptVocabSize = llmReq->getPromptVocabSize(); + } + + mPromptTuningParams.promptTuningEnabled.push_back(optReqPromptEmbeddingTable.has_value()); + + // If context request & has embedding table, validate it + if (optReqPromptEmbeddingTable.has_value()) + { + // If a context request, validate prompt tensors and move to GPU + if (batchIdx < numContextRequests) + { + if (mPromptTableOffloading) + { + // Need to slice the ptable since we don't need the entire buffer + // The size depends on optReqPromptVocabSize which stores how many fake prompts are in the chunk + auto slicedPtable = runtime::ITensor::slice( + optReqPromptEmbeddingTable.value(), 0, optReqPromptVocabSize.value()); + slicedPtable->unsqueeze(0); + optReqPromptEmbeddingTable = std::move(slicedPtable); + } + else + { + // Move to GPU + llmReq->movePromptEmbeddingTableToGpu(manager); + optReqPromptEmbeddingTable = llmReq->getPromptEmbeddingTable(); + } + + // Validate the table, prompt_vocab_size + validate(optReqPromptEmbeddingTable, optReqPromptVocabSize); + } + + auto const reqPromptEmbeddingTable = optReqPromptEmbeddingTable.value(); + auto const reqPromptVocabSize = optReqPromptVocabSize.value(); + + // TODO: Use invokeCopyBatch to avoid multiple bs1 copies + // Copy into large prompt embedding table + TensorPtr reqPromptEmbeddingTableView = runtime::ITensor::view(reqPromptEmbeddingTable); + reqPromptEmbeddingTableView->squeeze(0); + auto const promptEmbeddingTableSlice = runtime::ITensor::slice( + mPromptTuningParams.embeddingTable, batchIdx * mMaxPromptVocabSize, reqPromptVocabSize); + manager.copy(*reqPromptEmbeddingTable, *promptEmbeddingTableSlice); + // TODO: src: 2007040 (llmReq->getPromptEmbeddingTable()) != dst: 1003520 (reqPromptVocabSize) + // (original shape passed from + // python == 196 * 5120, fp16) + // VILA mode 1 , 2 images in one request + } + ++batchIdx; + } + } + + auto const batchSize = batchIdx; + std::vector<SizeType32> tasksHostVec(batchSize); + std::iota(tasksHostVec.begin(), tasksHostVec.end(), 0); + + // Create a tensor that wraps the vector and convert unique_ptr to shared_ptr + auto tasksHost = std::shared_ptr<runtime::ITensor>( + runtime::ITensor::wrap(tasksHostVec, runtime::ITensor::makeShape({batchSize})).release()); + + mPromptTuningParams.fillTasksTensor( + tasksHost, batchSize, numContextRequests, reqBeamWidths, reqPromptLengths, manager, packed); +} + +void PromptTuningBuffers::initializeChunkPtableBuffers(runtime::BufferManager const& manager, + runtime::ModelConfig const& modelConfig, SizeType32 contextChunkSize, std::shared_ptr<LlmRequest> const& llmReq) +{ + if (mChunkPtableInitialized) + { + return; + } + + std::array<TensorPtr, 2> buffers; + std::vector<std::vector<SizeType32>> startPositions(2); + for (int i = 0; i < 2; i++) + { + startPositions[i].emplace_back(0); + auto memType = llmReq->getPromptEmbeddingTable().value()->getDataType(); + buffers[i] = manager.gpu(runtime::ITensor::makeShape({contextChunkSize, modelConfig.getHiddenSize()}), memType); + } + + mChunkPtableBuffers = std::move(buffers); + mChunkPtableBufferStartPositions = std::move(startPositions); + + mChunkPtableCurrentIndex = 0; + mChunkPtableInitialized = true; +} + +void PromptTuningBuffers::switchChunkPtableBuffer() +{ + mChunkPtableCurrentIndex = 1 - mChunkPtableCurrentIndex; + clearBufferStartPositions(mChunkPtableCurrentIndex); +} + +size_t PromptTuningBuffers::getChunkPtableCurrentIndex() +{ + return mChunkPtableCurrentIndex; +} + +TensorPtr& PromptTuningBuffers::getChunkPtableBuffer(size_t index) +{ + if (!mChunkPtableBuffers.has_value()) + { + TLLM_THROW("Chunk ptable buffers not initialized"); + } + if (!mChunkPtableBuffers.value()[index]) + { + TLLM_THROW("Chunk ptable buffer at index %zu is null", index); + } + return mChunkPtableBuffers.value()[index]; +} + +SizeType32 PromptTuningBuffers::getChunkPtableBufferSliceSize(size_t index, size_t batchIdx) +{ + if (!mChunkPtableBufferStartPositions.has_value()) + { + return 0; + } + + if (batchIdx + 1 >= mChunkPtableBufferStartPositions.value()[index].size()) + { + TLLM_THROW("Batch index %zu + 1 out of bounds for buffer %zu (size: %zu)", batchIdx, index, + mChunkPtableBufferStartPositions.value()[index].size()); + } + + return mChunkPtableBufferStartPositions.value()[index][batchIdx + 1] + - mChunkPtableBufferStartPositions.value()[index][batchIdx]; +} + +SizeType32 PromptTuningBuffers::getChunkPtableBufferStartPosition(size_t index, size_t batchIdx) +{ + if (!mChunkPtableBufferStartPositions.has_value()) + { + return 0; + } + + if (batchIdx >= mChunkPtableBufferStartPositions.value()[index].size()) + { + TLLM_THROW("Batch index %zu out of bounds for buffer %zu (size: %zu)", batchIdx, index, + mChunkPtableBufferStartPositions.value()[index].size()); + } + + // For first batch, return the value directly + if (batchIdx == 0) + { + return mChunkPtableBufferStartPositions.value()[index][0]; + } + + // For other batches, return difference from previous position + return mChunkPtableBufferStartPositions.value()[index][batchIdx] + - mChunkPtableBufferStartPositions.value()[index][batchIdx - 1]; +} + +void PromptTuningBuffers::updateBufferStartPosition(size_t index, SizeType32 numRows) +{ + if (!mChunkPtableBufferStartPositions.has_value()) + { + return; + } + auto& positions = mChunkPtableBufferStartPositions.value()[index]; + positions.push_back(positions.back() + numRows); +} + +void PromptTuningBuffers::clearBufferStartPositions(size_t index) +{ + if (mChunkPtableBufferStartPositions.has_value()) + { + mChunkPtableBufferStartPositions.value()[index].clear(); + mChunkPtableBufferStartPositions.value()[index].emplace_back(0); + } +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/rnnCacheFormatter.cpp b/cpp/tensorrt_llm/batch_manager/rnnCacheFormatter.cpp index b7ceb727efa3..c5e4262e34ab 100644 --- a/cpp/tensorrt_llm/batch_manager/rnnCacheFormatter.cpp +++ b/cpp/tensorrt_llm/batch_manager/rnnCacheFormatter.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,6 @@ #include "tensorrt_llm/batch_manager/kvCacheUtils.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/nvtxUtils.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" @@ -47,9 +46,7 @@ void RnnCacheFormatter::format(TransferSession& session) NVTX3_SCOPED_RANGE(RnnCacheFormatter_formatUnifiedPool); session.setTime(TransferSession::kTimeFormatter); - auto llmRequestOpt = session.getLlmRequest(); - TLLM_CHECK_WITH_INFO(llmRequestOpt.has_value(), "LlmRequest required for RNN state transfer"); - auto const& llmRequest = **llmRequestOpt; + auto const& llmRequest = session.getLlmRequest(); TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "Start sending unified pool RNN state for request ID: %ld.", llmRequest.mRequestId); TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently, only beam width 1 is supported."); @@ -81,7 +78,7 @@ void RnnCacheFormatter::format(TransferSession& session) bool const recvSideHasCP = destConfig.getParallelConfig().mContextParallelism > 1; auto const indexFromEnd = session.getIndexFromEnd(); auto blockRange = kv_cache_manager::getBlockRangeForSending( - mKvCacheManager, llmRequestOpt, lastBlockKey, indexFromEnd, recvSideHasCP, ppSize); + mKvCacheManager, llmRequest, lastBlockKey, indexFromEnd, recvSideHasCP, ppSize); auto const& blockIdsPerWindow = blockRange.getBlockIdsPerWindow(); auto const allWindowSizes = blockRange.getWindowSizes(); @@ -172,26 +169,11 @@ void RnnCacheFormatter::format(TransferSession& session) bufferSizesPerTarget[t] = ssmBufBytes + convBufBytes; } - auto const* sendCancelFlag = common::getEnvDisaggEnableInflightCancel() - ? &session.getDataContext().getTransferTerminate() - : nullptr; - auto cacheBufferId = mRnnCacheTransBufferManager->assignBufferIndexForSend(sendCancelFlag); - BufferIndexHolder sendHolder(*mRnnCacheTransBufferManager, cacheBufferId, /*isRecv=*/false); + auto cacheBufferId = mRnnCacheTransBufferManager->assignBufferIndexForSend(); auto allocationResult = mRnnCacheTransBufferManager->getOrAllocateSendBuffers( cacheBufferId, static_cast<int>(bufferTargetNum), bufferSizesPerTarget, bufferManager); auto& outputBuffers = std::get<0>(allocationResult); auto& bufferCoverTargetNum = std::get<1>(allocationResult); - auto& onlyUseDynamicBuffer = std::get<2>(allocationResult); - TLLM_CHECK(cacheBufferId.has_value() || onlyUseDynamicBuffer); - auto const* agentConnection = rnnSendConns.empty() - ? nullptr - : dynamic_cast<executor::kv_cache::AgentConnection const*>(connections[rnnSendConns[0]]); - if (agentConnection != nullptr) - { - TLLM_CHECK_WITH_INFO(bufferCoverTargetNum == bufferTargetNum, - "Agent needs all unified-pool RNN send buffers pre-allocated"); - TLLM_CHECK(onlyUseDynamicBuffer == false); - } // Split each outputBuffer into SSM and conv portions. std::vector<runtime::ITensor::SharedPtr> ssmOutputBuffers(numTargets); @@ -221,26 +203,11 @@ void RnnCacheFormatter::format(TransferSession& session) int deviceId; TLLM_CUDA_CHECK(cudaGetDevice(&deviceId)); auto preAllocSendBuffer = mRnnCacheTransBufferManager->getSendBuffer(cacheBufferId); - if (sendCancelFlag != nullptr && sendCancelFlag->load(std::memory_order_relaxed)) - { - TLLM_THROW("Unified-pool RNN cache transfer cancelled before NIXL submission"); - } - try - { - sendAllBuffers(session, deviceId, outputBuffers, bufferCoverTargetNum, preAllocSendBuffer, - bufferManager, targetInfo, rnnSendConns); - } - catch (...) - { - if (agentConnection != nullptr && common::getEnvDisaggEnableInflightCancel()) - { - sendHolder.poison(); - } - throw; - } + sendAllBuffers(session, deviceId, outputBuffers, bufferCoverTargetNum, preAllocSendBuffer, bufferManager, + targetInfo, rnnSendConns); session.setTime(TransferSession::kTimeTransmissions); - sendHolder.release(); + mRnnCacheTransBufferManager->freeBufferIndexForSend(cacheBufferId); } } @@ -253,9 +220,7 @@ void RnnCacheFormatter::unformat(TransferSession& session) NVTX3_SCOPED_RANGE(RnnCacheFormatter_unformatUnifiedPool); session.setTime(TransferSession::kTimeFormatter); - auto llmRequestOpt = session.getLlmRequest(); - TLLM_CHECK_WITH_INFO(llmRequestOpt.has_value(), "LlmRequest required for RNN state transfer"); - auto const& llmRequest = **llmRequestOpt; + auto const& llmRequest = session.getLlmRequest(); TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "Start receiving unified pool RNN state for request ID: %ld.", llmRequest.mRequestId); TLLM_CHECK_WITH_INFO(llmRequest.mSamplingConfig.beamWidth == 1, "Currently, only beam width 1 is supported."); @@ -382,21 +347,15 @@ void RnnCacheFormatter::unformat(TransferSession& session) // Use pre-assigned buffer ID from NIXL connection if available. std::optional<int> cacheBufferId = std::nullopt; - BufferIndexHolder recvHolder; auto preAssignedRnnId = connections[rnnRecvConns[0]]->getPreAssignedBufferId(static_cast<uint8_t>(BufferKind::kRNN)); if (preAssignedRnnId.has_value()) { cacheBufferId = static_cast<int>(*preAssignedRnnId); - if (!session.hasReservedRecvBuffer(*mRnnCacheTransBufferManager)) - { - recvHolder = BufferIndexHolder(*mRnnCacheTransBufferManager, cacheBufferId, /*isRecv=*/true); - } } else { cacheBufferId = mRnnCacheTransBufferManager->assignBufferIndexForRecv(); - recvHolder = BufferIndexHolder(*mRnnCacheTransBufferManager, cacheBufferId, /*isRecv=*/true); } auto allocationResult = mRnnCacheTransBufferManager->getOrAllocateRecvBuffers( @@ -443,12 +402,10 @@ void RnnCacheFormatter::unformat(TransferSession& session) bufferManager.getStream().synchronize(); - recvHolder.release(); + mRnnCacheTransBufferManager->freeBufferIndexForRecv(cacheBufferId); } } - (void) session.releaseReservedRecvBuffer(*mRnnCacheTransBufferManager); - TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), "End receiving unified pool RNN state for request ID: %ld.", llmRequest.mRequestId); } diff --git a/cpp/tensorrt_llm/batch_manager/rnnCacheTransBuffer.cpp b/cpp/tensorrt_llm/batch_manager/rnnCacheTransBuffer.cpp index f9c04200e8d2..37af8e31baf9 100644 --- a/cpp/tensorrt_llm/batch_manager/rnnCacheTransBuffer.cpp +++ b/cpp/tensorrt_llm/batch_manager/rnnCacheTransBuffer.cpp @@ -21,7 +21,6 @@ #include "tensorrt_llm/common/dataType.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include <algorithm> @@ -96,7 +95,7 @@ size_t RnnCacheTransBufferManager::computeTransferBufferSizeFromPool( RnnCacheTransBufferManager::RnnCacheTransBufferManager(kv_cache_manager::BaseKVCacheManager* kvCacheManager, executor::kv_cache::CacheState const& cacheState, std::optional<size_t> maxNumTokens) : BaseTransBufferManager(computeTransferBufferSizeFromPool(kvCacheManager, cacheState, maxNumTokens), - tensorrt_llm::DataType::kUINT8, maxNumTokens) + nvinfer1::DataType::kUINT8, maxNumTokens) { TLLM_CHECK(kvCacheManager != nullptr); TLLM_LOG_INFO("RnnCacheTransBufferManager created for unified pool RNN cache"); diff --git a/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.cpp b/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.cpp new file mode 100644 index 000000000000..6fc7977ef8f1 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.cpp @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "rnnStateBuffers.h" + +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/rnnStateManager.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" + +using namespace tensorrt_llm::runtime; + +namespace tensorrt_llm::batch_manager +{ + +RnnStateBuffers::RnnStateBuffers(SizeType32 maxBatchSize, runtime::TllmRuntime const& runtime) +{ + auto const& manager = runtime.getBufferManager(); + + slotMappingHost = BufferManager::cpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + slotMappingDevice = manager.gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); +} + +void RnnStateBuffers::reshape(SizeType32 numSequences) +{ + slotMappingHost->reshape(ITensor::makeShape({numSequences})); + slotMappingDevice->reshape(ITensor::makeShape({numSequences})); +} + +void RnnStateBuffers::fillSlotMappings( + RequestVector const& contextRequests, rnn_state_manager::RnnStateManager* rnnStateManager) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(rnnStateBuffersFillSlotMappings); + + SizeType32 batchIdx{0}; + for (auto const& llmReq : contextRequests) + { + auto const seqSlot = llmReq->mSeqSlot.value(); + auto const reqBeamWidth = llmReq->mSamplingConfig.beamWidth; + rnnStateManager->fillSlotMapping(*slotMappingHost, batchIdx, seqSlot, reqBeamWidth); + ++batchIdx; + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void RnnStateBuffers::copySlotMappingH2D(runtime::TllmRuntime const& runtime) +{ + auto const& manager = runtime.getBufferManager(); + manager.copy(*slotMappingHost, *slotMappingDevice); +} + +void RnnStateBuffers::getBuffers(TensorMap& inputBuffers) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(rnnStateBuffersGetBuffers); + + inputBuffers.insert_or_assign("slot_mapping", slotMappingDevice); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.h b/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.h new file mode 100644 index 000000000000..e25df47382a1 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/rnnStateBuffers.h @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/runtime/iTensor.h" + +namespace tensorrt_llm::runtime +{ +class TllmRuntime; +} // namespace tensorrt_llm::runtime + +namespace tensorrt_llm::batch_manager +{ + +namespace rnn_state_manager +{ +class RnnStateManager; +} + +class RnnStateBuffers +{ +public: + using SizeType32 = tensorrt_llm::runtime::SizeType32; + using TensorPtr = runtime::ITensor::SharedPtr; + using TensorMap = runtime::StringPtrMap<runtime::ITensor>; + + // others should be in rnnStateManager, we only need slotMapping here. + TensorPtr slotMappingHost; // [batch_size] + TensorPtr slotMappingDevice; // [batch_size] + + RnnStateBuffers(SizeType32 maxBatchSize, runtime::TllmRuntime const& runtime); + + void reshape(SizeType32 numSequences); + + void fillSlotMappings(RequestVector const& contextRequests, rnn_state_manager::RnnStateManager* rnnStateManager); + + void copySlotMappingH2D(runtime::TllmRuntime const& runtime); + + void getBuffers(TensorMap& inputBuffers) const; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/rnnStateManager.cpp b/cpp/tensorrt_llm/batch_manager/rnnStateManager.cpp index 7d032a268fdd..7608079fb396 100644 --- a/cpp/tensorrt_llm/batch_manager/rnnStateManager.cpp +++ b/cpp/tensorrt_llm/batch_manager/rnnStateManager.cpp @@ -17,7 +17,6 @@ #include "tensorrt_llm/batch_manager/rnnStateManager.h" #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/utils/runtimeUtils.h" @@ -81,7 +80,7 @@ RnnStateManager::RnnStateManager(SizeType32 maxNumSequences, tensorrt_llm::runti {localNbLayers, mMaxNumSequences * mBeamSlotsPerSequence, convKernel - 1, rnnConvDimSize}); mDtype = dataType; - mSsmCacheDtype = tensorrt_llm::DataType::kFLOAT; + mSsmCacheDtype = nvinfer1::DataType::kFLOAT; // Store RNN model config for CacheTransceiver mDState = stateSize; @@ -118,7 +117,7 @@ RnnStateManager::RnnStateManager(SizeType32 maxNumSequences, tensorrt_llm::runti RnnStateManager::RnnStateManager(SizeType32 dState, SizeType32 dConv, SizeType32 numHeads, SizeType32 nGroups, SizeType32 headDim, SizeType32 maxBatchSize, WorldConfig const& worldConfig, int64_t stream, - tensorrt_llm::DataType dtype, tensorrt_llm::DataType ssmCacheDtype, std::vector<SizeType32> const& ppLayers, + nvinfer1::DataType dtype, nvinfer1::DataType ssmCacheDtype, std::vector<SizeType32> const& ppLayers, SizeType32 numLayers) : mMaxNumSequences(maxBatchSize) , mMaxBeamWidth{1} @@ -298,12 +297,12 @@ RnnStateManager::TensorPtr RnnStateManager::getSsmStates() const return pagedRnnStates; } -tensorrt_llm::DataType RnnStateManager::getConvStateDataType() const noexcept +nvinfer1::DataType RnnStateManager::getConvStateDataType() const noexcept { return mDtype; } -tensorrt_llm::DataType RnnStateManager::getSsmStateDataType() const noexcept +nvinfer1::DataType RnnStateManager::getSsmStateDataType() const noexcept { return mSsmCacheDtype; } diff --git a/cpp/tensorrt_llm/batch_manager/runtimeBuffers.cpp b/cpp/tensorrt_llm/batch_manager/runtimeBuffers.cpp new file mode 100644 index 000000000000..ea5b9b06a96e --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/runtimeBuffers.cpp @@ -0,0 +1,1029 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" + +#include "tensorrt_llm/batch_manager/encoderBuffers.h" +#include "tensorrt_llm/batch_manager/kvCacheManager.h" +#include "tensorrt_llm/batch_manager/loraBuffers.h" +#include "tensorrt_llm/batch_manager/medusaBuffers.h" +#include "tensorrt_llm/batch_manager/promptTuningBuffers.h" +#include "tensorrt_llm/batch_manager/rnnStateBuffers.h" +#include "tensorrt_llm/batch_manager/rnnStateManager.h" +#include "tensorrt_llm/batch_manager/transformerBuffers.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/stlUtils.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/decoderState.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/runtimeKernels.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" + +#include <algorithm> +#include <iterator> +#include <memory> +#include <numeric> +#include <vector> + +using namespace tensorrt_llm::runtime; + +namespace tensorrt_llm::batch_manager +{ + +RuntimeBuffers::RuntimeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, + std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, + TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + executor::DecodingConfig const& decodingConfig, bool gatherGenerationLogits, std::optional<SizeType32> maxNumTokens, + std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs, + bool promptTableOffloadingParam) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + promptTableOffloading = promptTableOffloadingParam; + + create(maxBatchSize, maxBeamWidth, maxAttentionWindowVec, maxAttentionWindow, sinkTokenLen, runtime, modelConfig, + worldConfig, decodingConfig, gatherGenerationLogits, additionalModelOutputs); + + // pre-allocate + setMaxBufferSizes(maxBatchSize, maxBeamWidth, modelConfig, maxNumTokens); + reshape(runtime, modelConfig, worldConfig, gatherGenerationLogits); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +RuntimeBuffers::~RuntimeBuffers() = default; + +void RuntimeBuffers::create(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, + std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, + TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + executor::DecodingConfig const& decodingConfig, bool gatherGenerationLogits, + std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const& manager = runtime.getBufferManager(); + auto const& engine = runtime.getEngine(); + + if (modelConfig.isTransformerBased()) + { + transformerBuffers = std::make_unique<TransformerBuffers>(maxBatchSize, maxBeamWidth, maxAttentionWindowVec, + maxAttentionWindow, sinkTokenLen, runtime, modelConfig, worldConfig); + } + if (modelConfig.isRnnBased()) + { + rnnStateBuffers = std::make_unique<RnnStateBuffers>(maxBatchSize, runtime); + } + + auto constexpr nvTokenIdType = TRTDataType<TokenIdType>::value; + inputsIds = manager.emptyTensor(MemoryType::kGPU, nvTokenIdType); + + mropeRotaryCosSin = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kFLOAT); + mropePositionDeltas = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + + if (worldConfig.isLastPipelineParallelRank()) + { + auto const logitsType = engine.getTensorDataType(batch_manager::RuntimeBuffers::kLogitsTensorName); + logits = manager.emptyTensor(MemoryType::kGPU, logitsType); + } + + // TODO: check which tensors can be allocated as pinned for max size + requestTypes = manager.emptyTensor(MemoryType::kCPU, TRTDataType<runtime::RequestType>::value); + + contextLengthsHost = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); + contextLengthsDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + sequenceLengthsHost = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); + sequenceLengthsDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + + lastTokenIdsHost = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); + lastTokenIdsDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + logitsIdsHost = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); + + inputsIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + + if (worldConfig.isPipelineParallel()) + { + hiddenStates = manager.emptyTensor(MemoryType::kGPU, modelConfig.getDataType()); + } + + auto const maxBatchSizeShape = ITensor::makeShape({maxBatchSize}); + seqSlots = tensorrt_llm::runtime::BufferManager::pinnedPool(maxBatchSizeShape, nvinfer1::DataType::kINT32); + seqSlotsDevice = manager.gpu(maxBatchSizeShape, nvinfer1::DataType::kINT32); + + cacheIndirDecoderIOBatchedCopySrcOffsets + = tensorrt_llm::runtime::BufferManager::pinnedPool(maxBatchSizeShape, nvinfer1::DataType::kINT64); + cacheIndirDecoderIOBatchedCopyDstOffsets + = tensorrt_llm::runtime::BufferManager::pinnedPool(maxBatchSizeShape, nvinfer1::DataType::kINT64); + cacheIndirDecoderIOBatchedCopySizes + = tensorrt_llm::runtime::BufferManager::pinnedPool(maxBatchSizeShape, nvinfer1::DataType::kINT64); + mCacheIndirDecoderIOBatchedCopySrcOffsetsSliceDevice = manager.gpu(maxBatchSizeShape, nvinfer1::DataType::kINT64); + mCacheIndirDecoderIOBatchedCopyDstOffsetsSliceDevice = manager.gpu(maxBatchSizeShape, nvinfer1::DataType::kINT64); + mCacheIndirDecoderIOBatchedCopyCopySizesDevice = manager.gpu(maxBatchSizeShape, nvinfer1::DataType::kINT64); + + // Pre-allocate buffer for saving generation logits for model w/o draft tokens + if (gatherGenerationLogits + && (modelConfig.getSpeculativeDecodingMode().isDraftTokensExternal() + || modelConfig.getSpeculativeDecodingMode().isNone()) + && worldConfig.isLastPipelineParallelRank()) + { + auto const vocabSizePadded = modelConfig.getVocabSizePadded(worldConfig.getSize()); + auto const logitsType = engine.getTensorDataType(batch_manager::RuntimeBuffers::kLogitsTensorName); + + generationLogitsCache.transposedLogits = manager.gpu( + ITensor::makeShape({maxBeamWidth, GenerationLogitsCache::kCACHE_LENGTH, vocabSizePadded}), logitsType); + generationLogitsCache.logits = manager.gpu( + ITensor::makeShape({GenerationLogitsCache::kCACHE_LENGTH, maxBatchSize * maxBeamWidth, vocabSizePadded}), + logitsType); + + generationLogitsCache.fragmentPointerDevice = manager.gpu( + ITensor::makeShape({maxBatchSize, GenerationLogitsCache::kCACHE_LENGTH}), nvinfer1::DataType::kINT64); + generationLogitsCache.fragmentPointerHost = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize, GenerationLogitsCache::kCACHE_LENGTH}), nvinfer1::DataType::kINT64); + } + + if (modelConfig.useCrossAttention()) + { + encoderBuffers = std::make_unique<EncoderBuffers>(); + encoderBuffers->create(maxBatchSize, modelConfig, runtime); + } + + if (modelConfig.usePromptTuning()) + { + promptTuningBuffers = std::make_unique<PromptTuningBuffers>( + maxBatchSize, manager, modelConfig, worldConfig, promptTableOffloading); + } + + if (modelConfig.useLoraPlugin()) + { + loraBuffers = std::make_unique<LoraBuffers>(maxBatchSize, maxBeamWidth, runtime, modelConfig, worldConfig); + } + + if (modelConfig.getSpeculativeDecodingMode().isMedusa()) + { + mMedusaBuffers = std::make_unique<MedusaBuffers>( + maxBatchSize, maxBeamWidth, manager, modelConfig, worldConfig, decodingConfig, runtime); + } + else if (modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) + { + mLookaheadBuffers = std::make_unique<runtime::LookaheadRuntimeBuffers>( + maxBatchSize, maxBeamWidth, manager, modelConfig, worldConfig, decodingConfig, runtime); + } + else if (modelConfig.getSpeculativeDecodingMode().isExplicitDraftTokens()) + { + mExplicitDraftTokensBuffers = std::make_unique<runtime::ExplicitDraftTokensBuffers>( + maxBatchSize, maxBeamWidth, manager, modelConfig, worldConfig); + } + else if (modelConfig.getSpeculativeDecodingMode().isEagle()) + { + mEagleBuffers = std::make_unique<runtime::EagleBuffers>( + maxBatchSize, maxBeamWidth, manager, modelConfig, worldConfig, decodingConfig); + } + + if (modelConfig.useLanguageAdapter()) + { + languageAdapterRoutings = manager.emptyTensor(MemoryType::kGPU, TRTDataType<SizeType32>::value); + } + + for (auto const& output : additionalModelOutputs.value_or(std::vector<executor::AdditionalModelOutput>{})) + { + auto const& engine = runtime.getEngine(); + auto const dataType = engine.getTensorDataType(output.name.c_str()); + mAdditionalOutputTensors.emplace(output.name, manager.emptyTensor(runtime::MemoryType::kGPU, dataType)); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void RuntimeBuffers::setMaxBufferSizes(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, + runtime::ModelConfig const& modelConfig, std::optional<SizeType32> maxNumRuntimeTokens) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // `maxNumSequences` is reached when all requests are in generation + numContextRequests = 0; + numGenRequests = maxBatchSize; + numGenSequences = maxBatchSize * maxBeamWidth; + + auto const maxDraftTokens = modelConfig.getMaxDecodingDraftTokens(); + // Draft-Tokens and Beam-Search are mutually exclusive + numLogits = maxBatchSize * std::max(1 + maxDraftTokens, maxBeamWidth); + auto const maxNumModelTokens = modelConfig.getMaxNumTokens(); + auto const maxNumContextTokens = maxBatchSize * modelConfig.getMaxInputLen(); + auto const maxNumGenTokens = numLogits; + // For pre-allocation + numContextTokens = 0; // Set in `setBufferSizes` rather than here for `computeContextLogits` + numGenTokens + = maxNumRuntimeTokens.value_or(maxNumModelTokens.value_or(std::max(maxNumContextTokens, maxNumGenTokens))); + + if (modelConfig.useCrossAttention()) + { + encoderBuffers->setMaxBufferSizes(maxBatchSize, modelConfig); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void RuntimeBuffers::setBufferSizes(RequestVector const& contextRequests, RequestVector const& genRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(runtimeBuffersSetBufferSizes); + + // set context sizes + numContextRequests = static_cast<SizeType32>(contextRequests.size()); + auto numContextLogits = numContextRequests; + numContextTokens = 0; + maxContextLength = 0; + for (auto const& llmReq : contextRequests) + { + auto const draftLength = llmReq->isLastContextChunk() ? llmReq->getNumDraftTokens() : 0; + numContextLogits += draftLength; + + auto const contextChunkSize = llmReq->getContextChunkSize(); + numContextTokens += contextChunkSize + draftLength; + if (maxContextLength < llmReq->mPromptLen) + { + maxContextLength = llmReq->mPromptLen; + } + } + + // set generation sizes + numGenRequests = static_cast<SizeType32>(genRequests.size()); + numGenSequences = 0; + numGenTokens = 0; + for (auto const& llmReq : genRequests) + { + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + numGenSequences += reqBeamWidth; + auto const draftLen = llmReq->getNumDraftTokens(); + numGenTokens += draftLen + reqBeamWidth; + } + + numLogits = numContextLogits + numGenTokens; + + if (encoderBuffers) + { + encoderBuffers->setBufferSizes(contextRequests, genRequests); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void RuntimeBuffers::reshape(TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + bool gatherGenerationLogits) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(runtimeBuffersReshape); + + if (worldConfig.isLastPipelineParallelRank()) + { + auto const vocabSizePadded = modelConfig.getVocabSizePadded(worldConfig.getSize()); + + if (modelConfig.computeContextLogits() && (numContextRequests > 0)) + { + // Only when need to return context logits, and there are new requests will execute context phase, + // logits buffer need to be re-allocated with size of [numContextTokens + numGenSequences, vocabSizePadded] + auto const& engine = runtime.getEngine(); + auto const& manager = runtime.getBufferManager(); + auto const logitsType = engine.getTensorDataType(kLogitsTensorName); + logits = manager.gpu(ITensor::makeShape({numContextTokens + numGenSequences, vocabSizePadded}), logitsType); + } + else if (gatherGenerationLogits && modelConfig.getSpeculativeDecodingMode().isNone()) + { + // If need to return generation logits, re-point the logit buffer to avoid overwrite, + // so we could write back GenerationLogitsCache::kCACHE_LENGTH steps' logits together + // logits shape: [1, maxBatchSize * maxBeamWidth, vocabSizePadded] + // which is large enough to cover both numContextRequests and numGenSequences + logits = ITensor::slice(generationLogitsCache.logits, generationLogitsCache.offset, 1); + generationLogitsCache.offset = (generationLogitsCache.offset + 1) % GenerationLogitsCache::kCACHE_LENGTH; + logits->squeeze(0); + } + else + { + logits->reshape(ITensor::makeShape({numLogits, vocabSizePadded})); + } + } + + auto const numSequences = getNumSequences(); + auto const numSequencesShape = ITensor::makeShape({numSequences}); + requestTypes->reshape(numSequencesShape); + contextLengthsHost->reshape(numSequencesShape); + contextLengthsDevice->reshape(numSequencesShape); + sequenceLengthsHost->reshape(numSequencesShape); + sequenceLengthsDevice->reshape(numSequencesShape); + + auto const numLogitsShape = ITensor::makeShape({numLogits}); + lastTokenIdsHost->reshape(numLogitsShape); + lastTokenIdsDevice->reshape(numLogitsShape); + logitsIdsHost->reshape(numLogitsShape); + + if (transformerBuffers) + { + transformerBuffers->reshape(numSequences, numContextTokens + numGenTokens); + } + + if (rnnStateBuffers) + { + rnnStateBuffers->reshape(numSequences); + } + + if (modelConfig.useCrossAttention()) + { + encoderBuffers->reshape(); + } + + if (modelConfig.useLoraPlugin()) + { + loraBuffers->reshape(numSequences); + } + + if (mMedusaBuffers) + { + mMedusaBuffers->reshape( + numContextRequests, numGenRequests, modelConfig.getSpeculativeDecodingModulePtr()->getMaxDecodingTokens()); + } + + if (mLookaheadBuffers && modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) + { + mLookaheadBuffers->reshape( + numContextRequests, numGenRequests, modelConfig.getSpeculativeDecodingModulePtr()->getMaxDecodingTokens()); + } + + if (mExplicitDraftTokensBuffers) + { + mExplicitDraftTokensBuffers->reshape(numContextRequests, numGenRequests, modelConfig); + } + + if (mEagleBuffers) + { + mEagleBuffers->reshape(numContextRequests, numGenRequests, modelConfig); + } + + auto const numRequests = getNumRequests(); + auto const numRequestsShape = ITensor::makeShape({numRequests}); + seqSlots->reshape(numRequestsShape); + seqSlotsDevice->reshape(numRequestsShape); + + auto const numTokens = getNumTokens(); + inputsIds->reshape(ITensor::makeShape({numTokens})); + + if (modelConfig.useMrope()) + { + auto const mropeRotaryCosSinSize = modelConfig.getMaxPositionEmbeddings() * modelConfig.getRotaryEmbeddingDim(); + mropeRotaryCosSin->reshape(ITensor::makeShape({numSequences, mropeRotaryCosSinSize})); + mropePositionDeltas->reshape(ITensor::makeShape({numSequences, 1})); + } + + if (worldConfig.isPipelineParallel()) + { + auto const hiddenSize = (!modelConfig.getPpReduceScatter() || worldConfig.isFirstPipelineParallelRank()) + ? modelConfig.getHiddenSize() * worldConfig.getTensorParallelism() + : modelConfig.getHiddenSize(); + + auto const hiddenStatesShape = ITensor::makeShape({numTokens, hiddenSize}); + hiddenStates->reshape(hiddenStatesShape); + } + + if (modelConfig.useLanguageAdapter()) + { + languageAdapterRoutings->reshape(ITensor::makeShape({numTokens, 1})); + } + + for (auto const& outputTensor : mAdditionalOutputTensors) + { + auto const& [name, tensor] = outputTensor; + auto const& engine = runtime.getEngine(); + auto shape = engine.getTensorShape(name.c_str()); + TLLM_CHECK_WITH_INFO( + shape.d[0] == -1, "First dimension of additional output tensor '%s' must be dynamic", name.c_str()); + shape.d[0] = numTokens; + tensor->reshape(shape); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void RuntimeBuffers::prepareBuffersForCudaGraph(SizeType32 maxSequenceLength) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(prepareBuffersForCudaGraph); + + TLLM_CHECK(numContextRequests == 0); + + if (transformerBuffers) + { + // Set pastKeyValueLength for graph capturing. This way we will capture graph with + // maxKvCacheLengthRounded rounded to the next kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE. + // MMHA will launch excessive amount of blocks and some of them will exit early during the actual launch. + // We can reuse the same graph for the next kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE iterations. + + // make sure the size does not overflow the max allowed pastKvCacheLength + auto const pastKvCacheLength = std::min(maxSequenceLength - 1, maxKvCacheLengthRounded); + + auto* pastKeyValueLengthsPtr = bufferCast<SizeType32>(*transformerBuffers->pastKeyValueLengths); + std::fill_n(pastKeyValueLengthsPtr, getNumSequences(), pastKvCacheLength); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void RuntimeBuffers::setFromInputs(RequestVector const& contextRequests, RequestVector const& genRequests, + SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, runtime::decoder::DecoderState const& decoderState, + kv_cache_manager::BaseKVCacheManager* kvCacheManagerPtr, + kv_cache_manager::BaseKVCacheManager* crossKvCacheManagerPtr, + rnn_state_manager::RnnStateManager* rnnStateManagerPtr, PeftTable const& peftTable, + runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig, bool trtOverlap, OptionalRef<runtime::ITensor const> newOutputTokens) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(runtimeBuffersSetFromInputs); + + auto const& manager = runtime.getBufferManager(); + auto const& stream = runtime.getStream(); + + // Fill requestTypes + { + auto* hostRequestTypes = bufferCast<runtime::RequestType>(*requestTypes); + std::fill_n(hostRequestTypes, numContextRequests, runtime::RequestType::kCONTEXT); + std::fill_n(hostRequestTypes + numContextRequests, numGenSequences, runtime::RequestType::kGENERATION); + } + + SizeType32 totalInputSize = 0; + std::vector<TokenIdType> inputHost; + std::vector<SizeType32> positionIdsHost; + std::vector<SizeType32> positionIdsHostRow2; + std::vector<SizeType32> mropePositionDeltasHost; + std::vector<SizeType32> languageAdapterRoutingsHost; + + auto* contextLengthsHostPtr = bufferCast<SizeType32>(*contextLengthsHost); + auto* sequenceLengthsHostPtr = bufferCast<SizeType32>(*sequenceLengthsHost); + auto* pastKeyValueLengthsPtr + = transformerBuffers ? bufferCast<SizeType32>(*transformerBuffers->pastKeyValueLengths) : nullptr; + SizeType32 totalNumLogits{0}; + auto* logitsIdsHostPtr = bufferCast<SizeType32>(*logitsIdsHost); + bool const isChatGlm = modelConfig.getModelVariant() == ModelConfig::ModelVariant::kChatGlm; + bool const isGlm = modelConfig.getModelVariant() == ModelConfig::ModelVariant::kGlm; + auto const mropeRotaryCosSinSize = modelConfig.getMaxPositionEmbeddings() * modelConfig.getRotaryEmbeddingDim(); + + { + NVTX3_SCOPED_RANGE(seqSlotsLoop); + auto* seqSlotIndices = bufferCast<SizeType32>(*seqSlots); + + SizeType32 batchIdx{0}; + for (auto const& requests : {contextRequests, genRequests}) + { + for (auto const& llmReq : requests) + { + // Get position of the current sequence in the decoder + auto const seqSlot = llmReq->mSeqSlot.value(); + seqSlotIndices[batchIdx] = seqSlot; + ++batchIdx; + } + } + + TLLM_CHECK(seqSlots->getSize() == static_cast<std::size_t>(batchIdx)); + manager.copy(*seqSlots, *seqSlotsDevice); + } + + // context preparation loop + if (!contextRequests.empty()) + { + NVTX3_SCOPED_RANGE(contextPrepareLoop); + numContextLogits.resize(contextRequests.size()); + + SizeType32 batchIdx{0}; + for (auto const& llmReq : contextRequests) + { + TLLM_CHECK_WITH_INFO(llmReq->isContextInitState() || llmReq->isDisaggGenerationTransmissionComplete(), + "The request should be in context phase or disaggregated generation tranmissionComplete phase."); + TLLM_CHECK_WITH_INFO( + llmReq->getMaxNumGeneratedTokens() == 0, "Context request should not have generated tokens."); + + auto const& reqTokens = llmReq->getTokens(0); + auto const& draftTokens = llmReq->getDraftTokens(); + auto const draftLength = llmReq->getNumDraftTokens(); + auto const& positionIds = llmReq->getPositionIds(); + + auto const contextChunkSize = llmReq->getContextChunkSize(); + auto const beginCompute = llmReq->getContextCurrentPosition(); + auto const endCompute = beginCompute + contextChunkSize; + inputHost.insert(inputHost.end(), reqTokens.begin() + beginCompute, reqTokens.begin() + endCompute); + + logitsIdsHostPtr[totalNumLogits++] = contextChunkSize; + numContextLogits.at(batchIdx) = modelConfig.computeContextLogits() ? contextChunkSize : 1; + + if (llmReq->isLastContextChunk()) + { + inputHost.insert(inputHost.end(), draftTokens->begin(), draftTokens->end()); + std::fill_n(logitsIdsHostPtr + totalNumLogits, draftLength, 1); + totalNumLogits += draftLength; + } + auto const inputLength = contextChunkSize + (llmReq->isLastContextChunk() ? draftLength : 0); + contextLengthsHostPtr[batchIdx] = inputLength; + auto const sequenceLen = inputLength + llmReq->getContextCurrentPosition(); + sequenceLengthsHostPtr[batchIdx] = sequenceLen; + + if (static_cast<bool>(pastKeyValueLengthsPtr)) + { + pastKeyValueLengthsPtr[batchIdx] = beginCompute + inputLength; + } + + if (positionIds.has_value()) + { + TLLM_CHECK_WITH_INFO(!(isChatGlm || isGlm), "ChatGLM-6B and Glm only use the default initialization"); + positionIdsHost.insert(positionIdsHost.end(), positionIds.value()->begin() + beginCompute, + positionIds.value()->begin() + endCompute); + } + else + { + if (isChatGlm) + { + // Specialize for ChatGLM-6B with 2D-Position-Embedding + positionIdsHost.resize(totalInputSize + inputLength); + std::iota(std::begin(positionIdsHost) + totalInputSize, std::end(positionIdsHost), 0); + positionIdsHost.back() = positionIdsHost.back() - 1; + + positionIdsHostRow2.resize(totalInputSize + inputLength); + positionIdsHostRow2.back() = 1; + } + else if (isGlm) + { + // Specialize for GLM-10B with 2D-Position-Embedding and special value of the mask id position + auto start = inputHost.begin() + totalInputSize; + auto end = start + inputLength; + auto it = std::find_if( + start, end, [](SizeType32 id) { return id == 50260 || id == 50263 || id == 50264; }); + llmReq->mMaskPosition = (it != end) ? std::distance(start, it) : maxContextLength; + + positionIdsHost.resize(totalInputSize + inputLength); + std::iota(std::begin(positionIdsHost) + totalInputSize, std::end(positionIdsHost), 0); + positionIdsHost.back() = llmReq->mMaskPosition; + + positionIdsHostRow2.resize(totalInputSize + inputLength); + positionIdsHostRow2.back() = 1; + } + else + { + // Other models + positionIdsHost.resize(totalInputSize + inputLength); + std::iota(std::begin(positionIdsHost) + totalInputSize, + std::begin(positionIdsHost) + totalInputSize + inputLength, beginCompute); + } + } + if (modelConfig.useMrope()) + { + auto optMropeRotaryCosSin = llmReq->getMropeRotaryCosSin().value(); + TLLM_CHECK_WITH_INFO(optMropeRotaryCosSin->getShape().d[0] == mropeRotaryCosSinSize, + "Provided MropeRotarySinCos is %ld and expected is %d.\n", optMropeRotaryCosSin->getShape().d[0], + int(mropeRotaryCosSinSize)); + + auto const mropeRotaryCosSinCtx = ITensor::slice(mropeRotaryCosSin, batchIdx, 1); + manager.copy(*optMropeRotaryCosSin, *mropeRotaryCosSinCtx); + } + + if (modelConfig.useLanguageAdapter()) + { + auto const languageAdapterRouting = llmReq->getLanguageAdapterRouting( + modelConfig.getNumLanguages().value(), endCompute - beginCompute); + languageAdapterRoutingsHost.insert(languageAdapterRoutingsHost.end(), + std::begin(languageAdapterRouting), std::end(languageAdapterRouting)); + } + totalInputSize += inputLength; + ++batchIdx; + } + + if (rnnStateBuffers) + { + rnnStateBuffers->fillSlotMappings(contextRequests, rnnStateManagerPtr); + } + } + + // generation preparation loop + if (!genRequests.empty()) + { + NVTX3_SCOPED_RANGE(genPrepareLoop); + + auto const numContextRequests = static_cast<SizeType32>(contextRequests.size()); + auto numSequences = numContextRequests; + for (auto const& llmReq : genRequests) + { + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + auto const draftLength = llmReq->getNumDraftTokens(); + auto const& draftTokens = llmReq->getDraftTokens(); + auto const numLogits = draftLength + reqBeamWidth; + TLLM_CHECK(draftLength == 0 || reqBeamWidth == 1); + + auto const promptLen = llmReq->mPromptLen; + auto const sequenceLen + = promptLen + llmReq->getMaxNumGeneratedTokens() + static_cast<SizeType32>(trtOverlap); + auto const& positionIds = llmReq->getPositionIds(); + for (int beam = 0; beam < reqBeamWidth; ++beam) + { + auto const numTokens = llmReq->getNumTokens(beam) + static_cast<SizeType32>(trtOverlap); + // TODO: can this be removed completely? + if (!trtOverlap) + { + auto const lastToken = llmReq->getLastTokens(beam); + inputHost.push_back(lastToken); + if (draftLength > 0) + { + inputHost.insert(inputHost.end(), draftTokens->begin(), draftTokens->end()); + } + } + + // If model updates generation position ids do not append them here. + if (!modelConfig.getSpeculativeDecodingMode().updatesPositionIds()) + { + if (positionIds.has_value()) + { + TLLM_CHECK_WITH_INFO( + !(isChatGlm || isGlm), "ChatGLM-6B and Glm only use the default initialization"); + auto last_context_position_id = positionIds.value()->back(); + positionIdsHost.push_back( + static_cast<SizeType32>(last_context_position_id + sequenceLen - promptLen)); + } + else + { + if (isChatGlm) // ChatGLM-6B + { + positionIdsHost.push_back(static_cast<SizeType32>(promptLen - 2)); + positionIdsHostRow2.push_back(static_cast<SizeType32>(sequenceLen - promptLen + 1)); + } + else if (isGlm) + { + positionIdsHost.push_back(llmReq->mMaskPosition); + positionIdsHostRow2.push_back(static_cast<SizeType32>(sequenceLen - promptLen + 1)); + } + else // GPT / ChatGLM2-6B / ChatGLM3-6B / BART + { + // positionIds is just the size of tokens -1 + positionIdsHost.push_back(numTokens - 1); + } + } + } + + if (modelConfig.useMrope()) + { + auto optMropePositionDeltas = llmReq->getMropePositionDeltas().value(); + mropePositionDeltasHost.push_back(optMropePositionDeltas); + } + + if (modelConfig.useLanguageAdapter()) + { + // Generation requests only have one token per sequence + auto const languageAdapterRouting + = llmReq->getLanguageAdapterRouting(modelConfig.getNumLanguages().value(), 1); + languageAdapterRoutingsHost.insert(languageAdapterRoutingsHost.end(), + std::begin(languageAdapterRouting), std::end(languageAdapterRouting)); + } + } + + if (static_cast<bool>(pastKeyValueLengthsPtr)) + { + SizeType32 pastKeyValueLength = sequenceLen - 1; + std::fill_n(pastKeyValueLengthsPtr + numSequences, reqBeamWidth, pastKeyValueLength); + } + totalInputSize += numLogits; + + std::fill_n(logitsIdsHostPtr + totalNumLogits, numLogits, 1); + + totalNumLogits += numLogits; + + if (rnnStateBuffers) + { + auto const seqSlot = llmReq->mSeqSlot.value(); + auto& rnnStateManager = *rnnStateManagerPtr; + rnnStateManager.fillSlotMapping(*rnnStateBuffers->slotMappingHost, numSequences, seqSlot, reqBeamWidth); + } + numSequences += reqBeamWidth; + } + + if (transformerBuffers && maxBeamWidth > 1) + { + transformerBuffers->copyCacheIndirection(genRequests, decoderState.getCacheIndirectionOutput(), stream); + } + + numSequences = numContextRequests; + for (auto const& llmReq : genRequests) + { + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + auto const draftLength = llmReq->getNumDraftTokens(); + + auto const contextQLength = llmReq->mPromptLen + draftLength; + auto const sequenceLen + = contextQLength + llmReq->getMaxNumGeneratedTokens() + static_cast<SizeType32>(trtOverlap); + + std::fill_n(contextLengthsHostPtr + numSequences, reqBeamWidth, contextQLength); + std::fill_n(sequenceLengthsHostPtr + numSequences, reqBeamWidth, sequenceLen); + numSequences += reqBeamWidth; + } + if (modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) + { + // copy from lookahead decoding buffer + mLookaheadBuffers->setFromInputs(numContextRequests, numGenRequests, *requestTypes, *seqSlots, + decoderState.getLookaheadBuffers(), runtime, modelConfig, worldConfig); + } + } + + // check skipCrossAttnBlocks + if (transformerBuffers && modelConfig.skipCrossAttnBlocks()) + { + bool isSkipCrossAttn = true; + for (auto const& requests : {contextRequests, genRequests}) + { + for (auto const& llmReq : requests) + { + bool tmpValue = false; + if (llmReq->getSkipCrossAttnBlocks() != nullptr) + { + manager.copy(*llmReq->getSkipCrossAttnBlocks(), &tmpValue); + } + isSkipCrossAttn &= tmpValue; + } + } + transformerBuffers->copySkipCrossAttnBlocks(isSkipCrossAttn, runtime); + } + + if (isChatGlm || isGlm) + { + positionIdsHost.reserve(totalInputSize * 2); + positionIdsHost.insert(positionIdsHost.end(), positionIdsHostRow2.begin(), positionIdsHostRow2.end()); + } + + if (modelConfig.useCrossAttention()) + { + encoderBuffers->fill(contextRequests, genRequests, manager); + } + if (modelConfig.usePromptTuning()) + { + promptTuningBuffers->fill(contextRequests, genRequests, manager, modelConfig.usePackedInput()); + } + if (modelConfig.useLoraPlugin()) + { + loraBuffers->fill(contextRequests, genRequests, peftTable, manager, modelConfig, worldConfig); + } + if (modelConfig.useMrope()) + { + if (!mropePositionDeltasHost.empty()) + { + auto mropePositionDeltasGen = ITensor::slice(mropePositionDeltas, 0, numGenSequences); + manager.copy(mropePositionDeltasHost.data(), *mropePositionDeltasGen); + } + } + + { + NVTX3_SCOPED_RANGE(bufferCopies); + if (trtOverlap) + { + auto contextInputsIds = ITensor::slice(inputsIds, 0, numContextTokens); + manager.copy(inputHost.data(), *contextInputsIds); + + if (!genRequests.empty()) + { + auto generationInputsIds = ITensor::slice(inputsIds, numContextTokens); + auto seqSlotsDeviceSlice = ITensor::slice(seqSlotsDevice, numContextRequests); + runtime::kernels::invokeGatherBatch( + *generationInputsIds, *newOutputTokens, *seqSlotsDeviceSlice, maxBeamWidth, stream); + } + } + else + { + manager.copy(inputHost.data(), *inputsIds); + } + // In generation phase, device ptr of context lengths need to be tiled. + manager.copy(*contextLengthsHost, *contextLengthsDevice); + manager.copy(*sequenceLengthsHost, *sequenceLengthsDevice); + auto const logitsIdsHostRange = BufferRange<SizeType32>(*logitsIdsHost); + auto lastTokenIdsHostRange = BufferRange<SizeType32>(*lastTokenIdsHost); + common::stl_utils::inclusiveScan( + logitsIdsHostRange.begin(), logitsIdsHostRange.end(), lastTokenIdsHostRange.begin()); + manager.copy(*lastTokenIdsHost, *lastTokenIdsDevice); + if (transformerBuffers) + { + TensorPtr decoderPositionIds = modelConfig.getSpeculativeDecodingMode().isLookaheadDecoding() + ? mLookaheadBuffers->positionIdsDevice + : nullptr; + transformerBuffers->copyPositionIds(runtime, positionIdsHost, isChatGlm || isGlm, decoderPositionIds); + } + if (rnnStateBuffers) + { + rnnStateBuffers->copySlotMappingH2D(runtime); + } + if (modelConfig.useLanguageAdapter()) + { + manager.copy(languageAdapterRoutingsHost.data(), *languageAdapterRoutings); + } + } + + if (transformerBuffers && static_cast<bool>(kvCacheManagerPtr)) + { + transformerBuffers->copyKvBlockOffsets( + contextRequests, genRequests, kvCacheManagerPtr, crossKvCacheManagerPtr, manager); + } + + if (modelConfig.useCrossAttention()) + { + transformerBuffers->copyCrossAttentionMasks(contextRequests, genRequests, contextLengthsDevice, + encoderBuffers->inputLengths, maxContextLength, encoderBuffers->getMaxInputLengthInBatch(), runtime); + } + + maxKvCacheLengthRounded = 0; + if (static_cast<bool>(pastKeyValueLengthsPtr)) + { + auto const maxKvCacheLength + = *std::max_element(pastKeyValueLengthsPtr, pastKeyValueLengthsPtr + getNumSequences()); + // Round up kv cache length + maxKvCacheLengthRounded = common::ceilDiv(maxKvCacheLength, kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE) + * kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE; + } + + if (modelConfig.getSpeculativeDecodingMode().needsDecoderPrologue()) + { + if (modelConfig.getSpeculativeDecodingMode().isExplicitDraftTokens()) + { + prepareExplicitDraftTokenBuffers( + decoderState.getExplicitDraftTokensBuffers(), runtime, modelConfig, worldConfig); + } + if (modelConfig.getSpeculativeDecodingMode().isEagle()) + { + prepareEagleBuffers( + contextRequests, genRequests, decoderState.getEagleBuffers(), runtime, modelConfig, worldConfig); + } + } + + sync_check_cuda_error(stream.get()); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void RuntimeBuffers::prepareExplicitDraftTokenBuffers( + runtime::ExplicitDraftTokensBuffers::Inputs const& explicitDraftTokensBuffers, TllmRuntime const& runtime, + ModelConfig const& modelConfig, WorldConfig const& worldConfig) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK(mExplicitDraftTokensBuffers); + + mExplicitDraftTokensBuffers->setFromInputs(numContextRequests, numGenRequests, *requestTypes, *seqSlots, + explicitDraftTokensBuffers, *transformerBuffers->positionIds, modelConfig, worldConfig, + runtime.getBufferManager(), runtime.getStream()); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void RuntimeBuffers::prepareEagleBuffers(RequestVector const& contextRequests, RequestVector const& genRequests, + runtime::EagleBuffers::Inputs const& eagleBuffers, TllmRuntime const& runtime, ModelConfig const& modelConfig, + WorldConfig const& worldConfig) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK(mEagleBuffers); + + mEagleBuffers->setFromInputs(contextRequests, genRequests, *requestTypes, *seqSlots, eagleBuffers, + runtime.getBufferManager(), modelConfig, worldConfig); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +std::tuple<SizeType32, RuntimeBuffers::TensorMap const&, RuntimeBuffers::TensorMap&> RuntimeBuffers::prepareStep( + RequestVector const& contextRequests, RequestVector const& genRequests, SizeType32 maxBeamWidth, + SizeType32 maxAttentionWindow, runtime::decoder::DecoderState const& decoderState, + kv_cache_manager::BaseKVCacheManager* kvCacheManager, kv_cache_manager::BaseKVCacheManager* crossKvCacheManager, + rnn_state_manager::RnnStateManager* rnnStateManager, PeftTable const& peftTable, TllmRuntime const& runtime, + ModelConfig const& modelConfig, WorldConfig const& worldConfig, bool gatherGenerationLogits, bool trtOverlap, + OptionalRef<runtime::ITensor const> newOutputTokens) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(runtimeBuffersPrepareStep); + + setBufferSizes(contextRequests, genRequests); + reshape(runtime, modelConfig, worldConfig, gatherGenerationLogits); + + setFromInputs(contextRequests, genRequests, maxBeamWidth, maxAttentionWindow, decoderState, kvCacheManager, + crossKvCacheManager, rnnStateManager, peftTable, runtime, modelConfig, worldConfig, trtOverlap, + newOutputTokens); + + fillIOMaps(modelConfig, worldConfig); + + auto const numTokens = getNumTokens(); + auto const optProfileId = runtime.getOptProfileId(numTokens, ModelConfig::getOptProfilesSplitPoints()); + setContextIndex(optProfileId); + TLLM_LOG_DEBUG("numTokens: %d, optProfileId: %d", numTokens, optProfileId); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return {optProfileId, inputMap, outputMap}; +} + +void RuntimeBuffers::fillIOMaps(ModelConfig const& modelConfig, WorldConfig const& worldConfig) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(runtimeBuffersFillIOMaps); + + inputMap.clear(); + outputMap.clear(); + + if (transformerBuffers) + { + transformerBuffers->getBuffers(inputMap, outputMap, modelConfig); + } + if (rnnStateBuffers) + { + rnnStateBuffers->getBuffers(inputMap); + } + + if (worldConfig.isLastPipelineParallelRank()) + { + // feed a view to TensorRT runtime so reshaping does not change logits buffer + outputMap.insert_or_assign(kLogitsTensorName, ITensor::view(logits)); + } + else + { + outputMap.insert_or_assign(kHiddenStatesOutputTensorName, hiddenStates); + } + + if (worldConfig.isFirstPipelineParallelRank()) + { + inputMap.insert_or_assign(kInputIdsTensorName, inputsIds); + } + else + { + inputMap.insert_or_assign(kHiddenStatesInputTensorName, hiddenStates); + } + + inputMap.insert_or_assign(kLastTokenIdsTensorName, lastTokenIdsDevice); + + inputMap.insert_or_assign(kHostRequestTypesTensorName, requestTypes); + // In the generation phase, we still pass context lengths. + inputMap.insert_or_assign(kContextLengthsTensorName, contextLengthsDevice); + inputMap.insert_or_assign(kHostContextLengthsTensorName, contextLengthsHost); + inputMap.insert_or_assign(kSequenceLengthsTensorName, sequenceLengthsDevice); + + if (modelConfig.useCrossAttention()) + { + encoderBuffers->insertInputTensors(inputMap); + } + if (modelConfig.usePromptTuning()) + { + auto const& promptTuningParams = promptTuningBuffers->mPromptTuningParams; + inputMap.insert_or_assign(kPromptEmbeddingTableTensorName, promptTuningParams.embeddingTable); + inputMap.insert_or_assign(kTasksTensorName, promptTuningParams.tasks); + inputMap.insert_or_assign(kPromptVocabSizeTensorName, promptTuningParams.vocabSize); + } + if (modelConfig.useMrope()) + { + + inputMap.insert_or_assign(kMRopeRotaryCosSinTensorName, mropeRotaryCosSin); + inputMap.insert_or_assign(kMRopePositionDeltasTensorName, mropePositionDeltas); + } + if (modelConfig.useLoraPlugin()) + { + loraBuffers->insertInputTensors(inputMap, loraBuffers->mLoraWeightsPointersHost, + loraBuffers->mLoraAdapterSizesHost, modelConfig, worldConfig); + } + if (modelConfig.useLanguageAdapter()) + { + inputMap.insert_or_assign("language_adapter_routings", languageAdapterRoutings); + } + + if (mMedusaBuffers) + { + mMedusaBuffers->insertInputTensors(inputMap, outputMap, worldConfig); + } + if (mLookaheadBuffers) + { + mLookaheadBuffers->insertInputTensors(inputMap, outputMap, worldConfig); + } + if (mExplicitDraftTokensBuffers) + { + mExplicitDraftTokensBuffers->insertInputTensors(inputMap, outputMap, worldConfig); + } + if (mEagleBuffers) + { + mEagleBuffers->insertInputTensors(inputMap, outputMap, worldConfig); + } + + for (auto const& outputTensor : mAdditionalOutputTensors) + { + outputMap.insert_or_assign(outputTensor.first, outputTensor.second); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/transformerBuffers.cpp b/cpp/tensorrt_llm/batch_manager/transformerBuffers.cpp new file mode 100644 index 000000000000..4f81c8926682 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/transformerBuffers.cpp @@ -0,0 +1,679 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/transformerBuffers.h" + +#include "tensorrt_llm/batch_manager/kvCacheManager.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/kernels/attentionMask.h" +#include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmhaPackedMask.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/runtimeKernels.h" +#include "tensorrt_llm/runtime/tllmBuffers.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" +#include <cstdint> + +using namespace tensorrt_llm::runtime; +namespace tk = tensorrt_llm::kernels; + +namespace tensorrt_llm::batch_manager +{ + +TransformerBuffers::TransformerBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, + std::vector<SizeType32> const& maxAttentionWindowVec, SizeType32 maxAttentionWindow, SizeType32 sinkTokenLen, + runtime::TllmRuntime const& runtime, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig) + : maxInputLen(modelConfig.getMaxInputLen()) + , maxEncoderOutputLen(modelConfig.getMaxEncoderLen()) +{ + auto const& manager = runtime.getBufferManager(); + auto const& engine = runtime.getEngine(); + + positionIds = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + + auto const localNbAttnLayers + = modelConfig.getNbAttentionLayers(worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); + // find the index of the first attention layer in the current rank + auto const firstLayerId = modelConfig.countLowerRankLayers(runtime::ModelConfig::LayerType::kATTENTION, + worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); + + cacheIndirection + = manager.gpu(ITensor::makeShape({maxBatchSize, maxBeamWidth, maxAttentionWindow}), nvinfer1::DataType::kINT32); + + if (!modelConfig.getMaxNumTokens().has_value()) + { + TLLM_THROW("Model must configure a max number of tokens."); + } + maxNumTokens = modelConfig.getMaxNumTokens().value(); + + if (modelConfig.isKVCacheEnabled()) + { + auto const kvCacheBlockOffsetsType = engine.getTensorDataType("kv_cache_block_offsets"); + kvCacheBlockOffsetsHost = manager.emptyTensor(MemoryType::kPINNEDPOOL, kvCacheBlockOffsetsType); + kvCacheBlockOffsetsDevice = manager.emptyTensor(MemoryType::kGPU, kvCacheBlockOffsetsType); + + if (modelConfig.useCrossAttention()) + { + crossKvCacheBlockOffsetsHost = manager.emptyTensor(MemoryType::kPINNEDPOOL, kvCacheBlockOffsetsType); + crossKvCacheBlockOffsetsDevice = manager.emptyTensor(MemoryType::kGPU, kvCacheBlockOffsetsType); + crossAttentionMaskDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kBOOL); + crossAttentionMaskPinnedHost = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxNumTokens, maxEncoderOutputLen}), nvinfer1::DataType::kBOOL); + crossAttentionPackedMaskDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + crossAttentionCuQSeqLensDevice = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + crossAttentionPackedMaskCuMaskRowsDevice + = manager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); + + // Pinned memory for batch copy of attention masks. + // There will be paddings in the dim1, so copy it by tokens. + crossAttentionMaskCopySrcOffsets = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxNumTokens}), nvinfer1::DataType::kINT64); + crossAttentionMaskCopyDstOffsets = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxNumTokens}), nvinfer1::DataType::kINT64); + crossAttentionMaskCopySizes = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxNumTokens}), nvinfer1::DataType::kINT64); + } + } + + fillValuesAlt = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + fillValuesAltDevice = manager.gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + seqSlotsAlt = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + seqSlotsAltDevice = manager.gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + + cacheIndirBatchedCopySrcOffsets = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT64); + cacheIndirBatchedCopyDstOffsets = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT64); + cacheIndirBatchedCopySizes = tensorrt_llm::runtime::BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT64); + skipCrossAttnBlocks + = tensorrt_llm::runtime::BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kBOOL); + + pastKeyValueLengths = manager.emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kINT32); + + maxAttentionWindows = BufferManager::cpu(ITensor::makeShape({localNbAttnLayers}), nvinfer1::DataType::kINT32); + auto* maxAttentionWindowsPtr = bufferCast<SizeType32>(*maxAttentionWindows); + auto const attentionWindowLength = maxAttentionWindowVec.size(); + for (SizeType32 i = 0; i < localNbAttnLayers; ++i) + { + maxAttentionWindowsPtr[i] = maxAttentionWindowVec[(firstLayerId + i) % attentionWindowLength]; + } + + sinkTokenLengths = BufferManager::cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + bufferCast<SizeType32>(*sinkTokenLengths)[0] = sinkTokenLen; + + contextProgressHost = BufferManager::cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT64); + bufferCast<int64_t>(*contextProgressHost)[0] = 0; + + if (modelConfig.useGemmAllReducePlugin() && worldConfig.isTensorParallel()) + { + nvinfer1::DataType ARType = modelConfig.getGemmAllReduceDtype(); + + auto hiddenSize = modelConfig.getHiddenSize() * worldConfig.getTensorParallelism(); + + auto tpGroup = worldConfig.getTensorParallelGroup(); + std::set<int> tpGroupSet(tpGroup.begin(), tpGroup.end()); + + auto outputDims = ITensor::makeShape({modelConfig.getMaxNumTokens().value() * hiddenSize}); + + gemmAllReduceOutput = std::make_shared<MulticastTensor>(outputDims, ARType, tpGroupSet); + } +} + +void TransformerBuffers::reshape(SizeType32 numSequences, SizeType32 numInputTokens) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + pastKeyValueLengths->reshape(ITensor::makeShape({numSequences})); + + if (kvCacheBlockOffsetsHost) + { + auto cacheBlockOffsetsShape = kvCacheBlockOffsetsHost->getShape(); + if (cacheBlockOffsetsShape.nbDims > 0) + { + cacheBlockOffsetsShape.d[1] = numSequences; + kvCacheBlockOffsetsHost->reshape(cacheBlockOffsetsShape); + kvCacheBlockOffsetsDevice->reshape(cacheBlockOffsetsShape); + } + else + { + TLLM_LOG_DEBUG("kvCacheBlockOffsets not allocated yet"); + } + } + + if (crossKvCacheBlockOffsetsHost) + { + TLLM_CHECK_WITH_INFO( + crossKvCacheBlockOffsetsDevice, "crossKvCacheBlockOffsetsDevice is empty for model with cross attention!"); + auto crossCacheBlockOffsetsShape = crossKvCacheBlockOffsetsHost->getShape(); + if (crossCacheBlockOffsetsShape.nbDims > 0) + { + crossCacheBlockOffsetsShape.d[1] = numSequences; + crossKvCacheBlockOffsetsHost->reshape(crossCacheBlockOffsetsShape); + crossKvCacheBlockOffsetsDevice->reshape(crossCacheBlockOffsetsShape); + } + else + { + TLLM_LOG_DEBUG("crossKvCacheBlockOffsets not allocated yet"); + } + } + + if (crossAttentionMaskDevice) + { + auto crossAttentionMaskShape = crossAttentionMaskDevice->getShape(); + if (crossAttentionMaskShape.nbDims > 0) + { + crossAttentionMaskShape.d[0] = numInputTokens; + crossAttentionMaskDevice->reshape(crossAttentionMaskShape); + crossAttentionMaskPinnedHost->reshape(crossAttentionMaskShape); + crossAttentionMaskCopySrcOffsets->reshape(ITensor::makeShape({numInputTokens})); + crossAttentionMaskCopyDstOffsets->reshape(ITensor::makeShape({numInputTokens})); + crossAttentionMaskCopySizes->reshape(ITensor::makeShape({numInputTokens})); + } + else + { + TLLM_LOG_DEBUG("crossAttentionMaskDevice not allocated yet"); + } + } + + if (crossAttentionPackedMaskDevice) + { + auto crossAttentionMaskPackedShape = crossAttentionPackedMaskDevice->getShape(); + if (crossAttentionMaskPackedShape.nbDims > 0) + { + crossAttentionMaskPackedShape.d[0] = numInputTokens; + crossAttentionPackedMaskDevice->reshape(crossAttentionMaskPackedShape); + } + else + { + TLLM_LOG_DEBUG("crossAttentionPackedMaskDevice not allocated yet"); + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TransformerBuffers::reshapeKvTensors(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, SizeType32 maxBlocksPerSeq, + kv_cache_manager::CacheType kvCacheType, SizeType32 numPools, BufferManager const& manager) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // allocate with max shape during init + if (kvCacheType == kv_cache_manager::CacheType::kSELF) + { + auto const cacheBlockOffsetsShape + = ITensor::makeShape({numPools, maxBatchSize * maxBeamWidth, 2, maxBlocksPerSeq}); + + kvCacheBlockOffsetsHost->reshape(cacheBlockOffsetsShape); + manager.setZero(*kvCacheBlockOffsetsHost); + + kvCacheBlockOffsetsDevice->reshape(cacheBlockOffsetsShape); + manager.setZero(*kvCacheBlockOffsetsDevice); + } + else if (kvCacheType == kv_cache_manager::CacheType::kCROSS) + { + auto const crossCacheBlockOffsetsShape + = ITensor::makeShape({numPools, maxBatchSize * maxBeamWidth, 2, maxBlocksPerSeq}); + + crossKvCacheBlockOffsetsHost->reshape(crossCacheBlockOffsetsShape); + manager.setZero(*crossKvCacheBlockOffsetsHost); + + crossKvCacheBlockOffsetsDevice->reshape(crossCacheBlockOffsetsShape); + manager.setZero(*crossKvCacheBlockOffsetsDevice); + + crossAttentionMaskDevice->reshape(ITensor::makeShape({maxNumTokens, maxEncoderOutputLen})); + manager.setZero(*crossAttentionMaskDevice); + manager.setZero(*crossAttentionMaskPinnedHost); + + // Only context attention needs this, so allocate it by shape [maxBatchSize, maxInputLen, maxEncoderOutputLen]. + auto [packedMaskM, packedMaskN] = tk::roundUpPackedMaskMNDims(maxInputLen, maxEncoderOutputLen); + crossAttentionPackedMaskDevice->reshape(ITensor::makeShape({maxBatchSize * packedMaskM, packedMaskN})); + manager.setZero(*crossAttentionPackedMaskDevice); + + crossAttentionCuQSeqLensDevice->reshape(ITensor::makeShape({maxBatchSize + 1})); + manager.setZero(*crossAttentionCuQSeqLensDevice); + + crossAttentionPackedMaskCuMaskRowsDevice->reshape(ITensor::makeShape({maxBatchSize + 1})); + manager.setZero(*crossAttentionPackedMaskCuMaskRowsDevice); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TransformerBuffers::getBuffers( + TensorMap& inputBuffers, TensorMap& outputBuffers, runtime::ModelConfig const& modelConfig) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(transformerBuffersGetBuffers); + + inputBuffers.insert_or_assign(kPositionIdsTensorName, positionIds); + inputBuffers.insert_or_assign(kHostPastKeyValueLengthsTensorName, pastKeyValueLengths); + inputBuffers.insert_or_assign(kCacheIndirectionsTensorName, cacheIndirection); + inputBuffers.insert_or_assign(kHostSinkTokenLengthTensorName, sinkTokenLengths); + + inputBuffers.insert_or_assign(kHostMaxAttentionWindowSizesTensorName, maxAttentionWindows); + inputBuffers.insert_or_assign(kKvCacheBlockOffsetsTensorName, kvCacheBlockOffsetsDevice); + inputBuffers.insert_or_assign(kHostKvCacheBlockOffsetsTensorName, kvCacheBlockOffsetsHost); + inputBuffers.insert_or_assign(kHostContextProgressTensorName, contextProgressHost); + + if (crossKvCacheBlockOffsetsHost) + { + inputBuffers.insert_or_assign(kCrossKvCacheBlockOffsetsTensorName, crossKvCacheBlockOffsetsDevice); + inputBuffers.insert_or_assign(kHostCrossKvCacheBlockOffsetsTensorName, crossKvCacheBlockOffsetsHost); + inputBuffers.insert_or_assign(kHostCrossKvCachePoolPointersTensorName, crossKvCacheBlockPoolPointers); + inputBuffers.insert_or_assign(kHostCrossKvCachePoolMappingTensorName, crossKvCacheBlockPoolMapping); + inputBuffers.insert_or_assign(kCrossAttentionMaskTensorName, crossAttentionMaskDevice); + inputBuffers.insert_or_assign(kCrossAttentionPackedMaskTensorName, crossAttentionPackedMaskDevice); + } + + if (skipCrossAttnBlocks) + { + inputBuffers.insert_or_assign(kSkipCrossAttentionBlocksTensorName, skipCrossAttnBlocks); + } + + if (modelConfig.useGemmAllReducePlugin()) + { + for (int idx = 0; idx < modelConfig.getNbAttentionLayers() * 2; ++idx) + { + // XXX (xsimmons): this is a bit hacky as it assumes + // 2x RowLinear layers per attention block. + // This will be fixed soon when I remove coupling between model + // and runtime. + auto gemmARViewUC = gemmAllReduceOutput->getTensorView(MulticastTensorView::ViewType::kUNICAST); + auto gemmARViewMC = gemmAllReduceOutput->getTensorView(MulticastTensorView::ViewType::kMULTICAST); + auto gemmARViewIpc = gemmAllReduceOutput->getTensorView(MulticastTensorView::ViewType::kIPC_LIST); + + outputBuffers.insert_or_assign("gemm_allreduce_uc_out_" + std::to_string(idx), gemmARViewUC); + outputBuffers.insert_or_assign("gemm_allreduce_mc_out_" + std::to_string(idx), gemmARViewMC); + outputBuffers.insert_or_assign("gemm_allreduce_ipc_out_" + std::to_string(idx), gemmARViewIpc); + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TransformerBuffers::copyPositionIds(runtime::TllmRuntime const& runtime, + std::vector<SizeType32> const& positionIdsHost, bool isChatGlm, TensorPtr const& decoderPositionIds) +{ + auto const& manager = runtime.getBufferManager(); + if (isChatGlm) + { + positionIds->reshape(ITensor::makeShape({2, static_cast<int>(positionIdsHost.size()) / 2})); + manager.copy(positionIdsHost.data(), *positionIds); + } + else if (decoderPositionIds == nullptr) + { + positionIds->reshape(ITensor::makeShape({static_cast<int>(positionIdsHost.size())})); + manager.copy(positionIdsHost.data(), *positionIds); + } + else + { + // concat context phase and generation phase positionIds. + auto const contextPositionIdsLen = static_cast<ITensor::DimType64>(positionIdsHost.size()); + auto const generationPositionIdsLen = ITensor::volume(decoderPositionIds->getShape()); + positionIds->reshape(ITensor::makeShape({contextPositionIdsLen + generationPositionIdsLen})); + manager.copy(positionIdsHost.data(), *ITensor::slice(positionIds, 0, contextPositionIdsLen)); + manager.copy(*decoderPositionIds, *ITensor::slice(positionIds, contextPositionIdsLen)); + } +} + +void TransformerBuffers::copyKvBlockOffsets(RequestVector const& contextRequests, RequestVector const& genRequests, + kv_cache_manager::BaseKVCacheManager const* kvCacheManager, + kv_cache_manager::BaseKVCacheManager const* crossKvCacheManager, BufferManager const& manager) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(copyKvBlockOffsets); + + auto const& cudaStream = manager.getStream(); + + SizeType32 constexpr contextBeamWidth{1}; + SizeType32 numSequences{0}; + SizeType32 maxBlockCount{0}; + SizeType32 maxCrossBlockCount{0}; + for (auto const& requests : {contextRequests, genRequests}) + { + for (auto const& llmReq : requests) + { + auto const requestId = llmReq->mRequestId; + auto const isContextRequest = llmReq->isContextInitState(); + auto const beamWidth = isContextRequest ? contextBeamWidth : llmReq->getBeamWidthByIter(); + auto const maxBeamBlockCount + = kvCacheManager->copyBlockOffsets(*kvCacheBlockOffsetsHost, numSequences, requestId); + maxBlockCount = std::max(maxBlockCount, maxBeamBlockCount); + if (crossKvCacheBlockOffsetsHost) + { + auto const maxCrossBeamBlockCount + = crossKvCacheManager->copyBlockOffsets(*crossKvCacheBlockOffsetsHost, numSequences, requestId); + maxCrossBlockCount = std::max(maxCrossBlockCount, maxCrossBeamBlockCount); + } + numSequences += beamWidth; + } + } + + // requests' block offsets collected as [totalNumSequences, 2, maxBlocksPerSeq], copy to device + auto copyOffsetsToDevice = [&cudaStream](TensorPtr& offsetsHost, TensorPtr& offsetsDevice, SizeType32 maxBlockCount) + { + // shape should be [totalNumSequences, 2, maxBlocksPerSeq] + auto const& offsetsShape = offsetsHost->getShape(); + auto const maxBlocksPerSeq = offsetsShape.d[3]; + auto const offsetsTypeSize = tensorrt_llm::common::getDTypeSize(offsetsHost->getDataType()); + auto const copyPitch = maxBlocksPerSeq * offsetsTypeSize; + auto const copyHeight = offsetsShape.d[0] * offsetsShape.d[1] * offsetsShape.d[2]; + auto const copyWidth = maxBlockCount * offsetsTypeSize; + auto* srcPtr = bufferCast<tk::KVCacheIndex>(*offsetsHost); + auto* dstPtr = bufferCast<tk::KVCacheIndex>(*offsetsDevice); + + TLLM_CUDA_CHECK(cudaMemcpy2DAsync( + dstPtr, copyPitch, srcPtr, copyPitch, copyWidth, copyHeight, cudaMemcpyHostToDevice, cudaStream.get())); + }; + + copyOffsetsToDevice(kvCacheBlockOffsetsHost, kvCacheBlockOffsetsDevice, maxBlockCount); + if (crossKvCacheBlockOffsetsHost) + { + copyOffsetsToDevice(crossKvCacheBlockOffsetsHost, crossKvCacheBlockOffsetsDevice, maxCrossBlockCount); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TransformerBuffers::copyCacheIndirection( + RequestVector const& genRequests, TensorPtr const& decoderCacheIndirectionOutput, CudaStream const& stream) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(copyCacheIndirection); + + auto const numGenerationRequests = genRequests.size(); + + auto batchedCopySrcOffsets = BufferRange<SizeType64>(*cacheIndirBatchedCopySrcOffsets); + auto batchedCopyDstOffsets = BufferRange<SizeType64>(*cacheIndirBatchedCopyDstOffsets); + auto batchedCopySizes = BufferRange<SizeType64>(*cacheIndirBatchedCopySizes); + + auto cacheIndirShape = decoderCacheIndirectionOutput->getShape(); + + // At present, all requests of a batch must have the same beam width in one generation step (or they will not + // be batched together). So, the beam width of the first request is taken here to reshape the buffer. + // Corresponding changes must be done if Diverse-Beam-Width-Search (DBWS, requests with diverse beam width in + // a batch in one generation step) is supported in the future. + auto reqBeamWidth = genRequests[0]->getBeamWidthByIter(); + + // Get size of copying from shape of `CacheIndirectionOutput` + cacheIndirShape.d[0] = 1; + cacheIndirShape.d[1] = reqBeamWidth; // Use beam width of current step rather than max beam width as dst offset + auto const copySize = static_cast<SizeType64>(ITensor::volume(cacheIndirShape)); + + std::transform(genRequests.begin(), genRequests.end(), batchedCopySrcOffsets.begin(), + [copySize](auto const& llmReq) { return llmReq->mSeqSlot.value() * copySize; }); + std::generate_n( + batchedCopyDstOffsets.begin(), numGenerationRequests, [copySize, i = 0]() mutable { return (i++) * copySize; }); + std::fill_n(batchedCopySizes.begin(), numGenerationRequests, copySize); + + auto const batchedCopySrcOffsetsSlice = ITensor::slice(cacheIndirBatchedCopySrcOffsets, 0, numGenerationRequests); + auto const batchedCopyDstOffsetsSlice = ITensor::slice(cacheIndirBatchedCopyDstOffsets, 0, numGenerationRequests); + auto const batchedCopySizesSlice = ITensor::slice(cacheIndirBatchedCopySizes, 0, numGenerationRequests); + runtime::kernels::invokeCopyBatch(*decoderCacheIndirectionOutput, *cacheIndirection, *batchedCopySrcOffsetsSlice, + *batchedCopyDstOffsetsSlice, *batchedCopySizesSlice, copySize, stream); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TransformerBuffers::copyCrossAttentionMasks(RequestVector const& contextRequests, RequestVector const& genRequests, + TensorPtr const& decoderContextLengthsDevice, TensorPtr const& encoderInputLengths, + SizeType32 maxDecoderContextLength, SizeType32 maxEncoderInputLengthInBatch, TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + auto const& manager = runtime.getBufferManager(); + + // Reshape the tensor to make sure the dim1 matches maxEncoderInputLengthInBatch. + auto crossAttentionMaskShape = crossAttentionMaskDevice->getShape(); + crossAttentionMaskShape.d[1] = maxEncoderInputLengthInBatch; + crossAttentionMaskDevice->reshape(crossAttentionMaskShape); + // Set crossAttentionMask to true by default if it is not provided. + manager.setMem(*crossAttentionMaskDevice, 1); + + // Check if all context requests have cross attention mask. + bool allContextCrossAttentionMaskProvided = true; + for (auto const& llmReq : contextRequests) + { + auto const& crossAttentionMaskRequest = llmReq->getCrossAttentionMask(); + if (bufferCastOrNull<bool>(crossAttentionMaskRequest) == nullptr) + { + allContextCrossAttentionMaskProvided = false; + break; + } + } + // If not all requests have cross attention mask, let us create the default ones. + auto const& stream = runtime.getStream(); + if (!allContextCrossAttentionMaskProvided) + { + TLLM_LOG_WARNING("Default padding attention mask will be used as not all requests have cross attention mask."); + tk::AttentionMaskParams<bool> attentionMaskParams; + memset((void*) &attentionMaskParams, 0, sizeof(attentionMaskParams)); + // Set parameters. + attentionMaskParams.mask = bufferCastOrNull<bool>(crossAttentionMaskDevice); + attentionMaskParams.cuQSeqLens = bufferCastOrNull<SizeType32>(crossAttentionCuQSeqLensDevice); + attentionMaskParams.actualQSeqLens = bufferCastOrNull<SizeType32>(decoderContextLengthsDevice); + attentionMaskParams.actualKvSeqLens = bufferCastOrNull<SizeType32>(encoderInputLengths); + attentionMaskParams.attentionMaskType = tk::AttentionMaskType::PADDING; + attentionMaskParams.batchSize = static_cast<SizeType32>(contextRequests.size()); + attentionMaskParams.maxQSeqLen = maxDecoderContextLength; + attentionMaskParams.maxKvSeqLen = maxEncoderInputLengthInBatch; + // Launch the kernel. + tk::invokeBuildAttentionMask(attentionMaskParams, stream.get()); + sync_check_cuda_error(stream.get()); + } + // Use the first request's cross attention mask tensor's pointer address as the primary source pointer. + auto const& attentionMaskSrc = !contextRequests.empty() ? contextRequests[0]->getCrossAttentionMask() + : genRequests[0]->getCrossAttentionMask(); + bool const* primarySrcPtr = bufferCastOrNull<bool>(attentionMaskSrc); + + // Pinned-memory buffer preparation for batch copy. + auto batchedCopySrcOffsets = BufferRange<SizeType64>(*crossAttentionMaskCopySrcOffsets); + auto batchedCopyDstOffsets = BufferRange<SizeType64>(*crossAttentionMaskCopyDstOffsets); + auto batchedCopySizes = BufferRange<SizeType64>(*crossAttentionMaskCopySizes); + // Requests with cross-attention-mask don't need to copy. + manager.setZero(*crossAttentionMaskCopySizes); + sync_check_cuda_error(stream.get()); + + SizeType32 numTokens = 0; + SizeType32 numCopiedTokens = 0; + bool* pinnedMemPtr = bufferCastOrNull<bool>(crossAttentionMaskPinnedHost); + for (auto const& llmReq : contextRequests) + { + auto const& crossAttentionMaskRequest = llmReq->getCrossAttentionMask(); + auto const position = llmReq->getContextCurrentPosition(); + auto const size = llmReq->getContextChunkSize(); + if (bufferCastOrNull<bool>(crossAttentionMaskRequest) != nullptr) + { + auto memType = crossAttentionMaskRequest->getMemoryType(); + auto const crossAttentionMaskRequestDim0 + = static_cast<SizeType64>(crossAttentionMaskRequest->getShape().d[0]); + auto const crossAttentionMaskRequestDim1 + = static_cast<SizeType64>(crossAttentionMaskRequest->getShape().d[1]); + TLLM_LOG_DEBUG("copyCrossAttentionMasks (shape [%d, %d]) from contextRequests position %d chunkSize %d", + crossAttentionMaskRequestDim0, crossAttentionMaskRequestDim1, position, size); + if ((position + size - 1) >= crossAttentionMaskRequestDim0) + { + TLLM_LOG_WARNING( + "The provided crossAttentionMask input is not complete for context phases, the last row " + "will be " + "used by default."); + } + // copy it to pinned memory if it is a cpu tensor. + if (memType == MemoryType::kCPU) + { + TLLM_LOG_DEBUG("CrossAttentionMask tensor is on CPU."); + auto const copiedPosition + = std::min(crossAttentionMaskRequestDim0 - 1, static_cast<SizeType64>(position)); + auto const copiedSize + = std::min(crossAttentionMaskRequestDim0 - copiedPosition, static_cast<SizeType64>(size)); + SizeType64 inputMaskOffset = (copiedPosition * crossAttentionMaskRequestDim1); + SizeType64 inputMaskSize = (copiedSize * crossAttentionMaskRequestDim1); + std::memcpy( + pinnedMemPtr, bufferCastOrNull<bool>(crossAttentionMaskRequest) + inputMaskOffset, inputMaskSize); + pinnedMemPtr += inputMaskSize; + for (SizeType32 tokenId = position; tokenId < position + size; tokenId++) + { + SizeType64 tokenIdInPinnedMem + = std::min(copiedSize - 1, static_cast<SizeType64>(tokenId - position)); + batchedCopySrcOffsets.begin()[numCopiedTokens] + = (pinnedMemPtr - primarySrcPtr) + tokenIdInPinnedMem * crossAttentionMaskRequestDim1; + batchedCopyDstOffsets.begin()[numCopiedTokens] + = numTokens * static_cast<SizeType64>(maxEncoderInputLengthInBatch); + batchedCopySizes.begin()[numCopiedTokens] = crossAttentionMaskRequestDim1; + numCopiedTokens++; + numTokens++; + } + } + else + { + TLLM_LOG_DEBUG("CrossAttentionMask tensor is on GPU."); + for (SizeType32 tokenId = position; tokenId < position + size; tokenId++) + { + batchedCopySrcOffsets.begin()[numCopiedTokens] + = static_cast<SizeType64>(bufferCastOrNull<bool>(crossAttentionMaskRequest) - primarySrcPtr) + + std::min(crossAttentionMaskRequestDim0 - 1, static_cast<SizeType64>(tokenId)) + * crossAttentionMaskRequestDim1; + batchedCopyDstOffsets.begin()[numCopiedTokens] + = numTokens * static_cast<SizeType64>(maxEncoderInputLengthInBatch); + batchedCopySizes.begin()[numCopiedTokens] = crossAttentionMaskRequestDim1; + numCopiedTokens++; + numTokens++; + } + } + } + else + { + numTokens += size; + TLLM_LOG_WARNING( + "CrossAttentionMask is not provided for the request. Default padding attention mask will be " + "created."); + } + } + sync_check_cuda_error(stream.get()); + + for (auto const& llmReq : genRequests) + { + auto const promptLen = llmReq->mPromptLen; + auto const decodingIter = llmReq->getDecodingIter(); + auto const& crossAttentionMaskRequest = llmReq->getCrossAttentionMask(); + if (bufferCastOrNull<bool>(crossAttentionMaskRequest) != nullptr) + { + auto const memType = crossAttentionMaskRequest->getMemoryType(); + auto const crossAttentionMaskRequestDim0 + = static_cast<SizeType64>(crossAttentionMaskRequest->getShape().d[0]); + auto const crossAttentionMaskRequestDim1 + = static_cast<SizeType64>(crossAttentionMaskRequest->getShape().d[1]); + TLLM_LOG_DEBUG("copyCrossAttentionMasks (shape [%d, %d]) from genRequests decodingIter %d", + crossAttentionMaskRequestDim0, crossAttentionMaskRequestDim1, decodingIter); + if (promptLen + decodingIter - 1 >= crossAttentionMaskRequestDim0) + { + TLLM_LOG_WARNING( + "The provided crossAttentionMask input is not complete for generation phases, the last row " + "will be " + "used by default."); + } + // copy it to pinned memory if it is a cpu tensor. + if (memType == MemoryType::kCPU) + { + TLLM_LOG_DEBUG("CrossAttentionMask tensor is on CPU."); + SizeType64 copiedPosition = std::min( + crossAttentionMaskRequestDim0 - 1, static_cast<SizeType64>(promptLen + decodingIter - 1)); + SizeType64 inputMaskOffset = (copiedPosition * crossAttentionMaskRequestDim1); + SizeType64 inputMaskSize = crossAttentionMaskRequestDim1; + std::memcpy( + pinnedMemPtr, bufferCastOrNull<bool>(crossAttentionMaskRequest) + inputMaskOffset, inputMaskSize); + pinnedMemPtr += inputMaskSize; + batchedCopySrcOffsets.begin()[numCopiedTokens] = static_cast<SizeType64>(pinnedMemPtr - primarySrcPtr); + batchedCopyDstOffsets.begin()[numCopiedTokens] + = numTokens * static_cast<SizeType64>(maxEncoderInputLengthInBatch); + batchedCopySizes.begin()[numCopiedTokens] = crossAttentionMaskRequestDim1; + } + else + { + TLLM_LOG_DEBUG("CrossAttentionMask tensor is on GPU."); + batchedCopySrcOffsets.begin()[numCopiedTokens] + = static_cast<SizeType64>(bufferCastOrNull<bool>(crossAttentionMaskRequest) - primarySrcPtr) + + std::min(crossAttentionMaskRequestDim0 - 1, static_cast<SizeType64>(promptLen + decodingIter - 1)) + * crossAttentionMaskRequestDim1; + batchedCopyDstOffsets.begin()[numCopiedTokens] + = numTokens * static_cast<SizeType64>(maxEncoderInputLengthInBatch); + batchedCopySizes.begin()[numCopiedTokens] = crossAttentionMaskRequestDim1; + } + numCopiedTokens++; + numTokens++; + } + else + { + numTokens++; + TLLM_LOG_WARNING( + "CrossAttentionMask is not provided for the generation request. Full valid attentionMask will " + "be used " + "by default."); + } + } + sync_check_cuda_error(stream.get()); + + // Copy all requests' attention mask in one kernel. + if (attentionMaskSrc != nullptr) + { + crossAttentionMaskCopySrcOffsets->reshape(ITensor::makeShape({numCopiedTokens})); + crossAttentionMaskCopyDstOffsets->reshape(ITensor::makeShape({numCopiedTokens})); + crossAttentionMaskCopySizes->reshape(ITensor::makeShape({numCopiedTokens})); + runtime::kernels::invokeCopyBatch(*attentionMaskSrc, *crossAttentionMaskDevice, + *crossAttentionMaskCopySrcOffsets, *crossAttentionMaskCopyDstOffsets, *crossAttentionMaskCopySizes, + maxEncoderInputLengthInBatch, stream); + } + sync_check_cuda_error(stream.get()); + + // The packed mask is only needed by context requests now. + if (!contextRequests.empty()) + { + // Set the parameters for creating packed mask for context FMHA. + tk::PackedMaskParams<bool> maskParams{}; + maskParams.maskInput = bufferCastOrNull<bool>(crossAttentionMaskDevice); + maskParams.cuQSeqLens = bufferCastOrNull<SizeType32>(crossAttentionCuQSeqLensDevice); + maskParams.packedMask = bufferCastOrNull<uint32_t>(crossAttentionPackedMaskDevice); + maskParams.cuMaskRows = bufferCastOrNull<SizeType32>(crossAttentionPackedMaskCuMaskRowsDevice); + maskParams.actualQSeqLens = bufferCastOrNull<SizeType32>(decoderContextLengthsDevice); + maskParams.actualKvSeqLens = bufferCastOrNull<SizeType32>(encoderInputLengths); + maskParams.batchSize = contextRequests.size(); + maskParams.maxQSeqLen = maxDecoderContextLength; + maskParams.maxKvSeqLen = maxEncoderInputLengthInBatch; + maskParams.attentionMaskType = tk::ContextAttentionMaskType::CUSTOM_MASK; + maskParams.validPosVal = true; + + // Launch the pack mask kernel. + tk::invokeBuildPackedMask(maskParams, stream.get()); + sync_check_cuda_error(stream.get()); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TransformerBuffers::copySkipCrossAttnBlocks(bool const& _skipCrossAttnBlocks, runtime::TllmRuntime const& runtime) +{ + auto const& manager = runtime.getBufferManager(); + manager.copy(&_skipCrossAttnBlocks, *skipCrossAttnBlocks); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtEncoderModel.cpp b/cpp/tensorrt_llm/batch_manager/trtEncoderModel.cpp new file mode 100644 index 000000000000..0d7dbfde42e6 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/trtEncoderModel.cpp @@ -0,0 +1,618 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "trtEncoderModel.h" +#include "encoderBuffers.h" +#include "tensorrt_llm/batch_manager/capacityScheduler.h" +#include "tensorrt_llm/batch_manager/microBatchScheduler.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" +#include "tensorrt_llm/runtime/utils/runtimeUtils.h" + +#include <algorithm> +#include <cstddef> +#include <vector> + +using namespace tensorrt_llm::runtime; +using namespace tensorrt_llm::mpi; + +namespace tensorrt_llm::batch_manager +{ + +TrtEncoderModel::TrtEncoderModel(runtime::ModelConfig const& modelConfig, WorldConfig const& worldConfig, + runtime::RawEngine const& rawEngine, std::shared_ptr<nvinfer1::ILogger> logger, + executor::ExecutorConfig const& executorConfig) + : TrtGptModel(modelConfig, worldConfig, executorConfig) + , mModelConfig{modelConfig} + , mWorldConfig{worldConfig} + , mDevice{runtime::utils::initDevice(worldConfig)} + , mLogger{logger ? std::move(logger) : std::make_shared<TllmLogger>()} + , mRuntime{std::make_shared<TllmRuntime>( + rawEngine, mLogger.get(), executorConfig.getUseGpuDirectStorage(), executorConfig.getGpuWeightsPercent())} + , mNumMicroBatches{1} + , mNumBuffers{mNumMicroBatches} + , mCopyBufferManager{std::make_shared<CudaStream>()} +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK_WITH_INFO( + !mWorldConfig.isPipelineParallel(), "Pipeline parallelism is currently not supported for encoder models."); + + createRuntimeContexts(); + + createBuffers(); + + if (mWorldConfig.isPipelineParallel()) + { + auto const& commSession = COMM_SESSION; + mMpiCommPipelinePara = std::make_shared<tensorrt_llm::mpi::MpiComm>( + commSession.split(mWorldConfig.getTensorParallelRank(), mWorldConfig.getPipelineParallelRank())); + } + + mMicroBatchScheduledRequests.resize(mNumMicroBatches); + // mEncoderWaitEvents.resize(mNumMicroBatches); + + // set noScheduleUntilState to LlmRequestState::kENCODER_INIT for encoder model + // when null kv cache manager is given, request scheduler will use MaxRequests as capacity scheduler, i.e. no + // handling of maximizing utilization or pause/evict + // TODO: finer control on encoder requests scheduling + mCapacityScheduler = std::make_unique<tensorrt_llm::batch_manager::CapacityScheduler>( + getMaxBatchSize() * mNumMicroBatches, executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy(), + /*hasKvCacheManager=*/false, /*twoStepsLookAhead=*/false, + /*noScheduleUntilState=*/LlmRequestState::kENCODER_INIT, + /*noScheduleAfterState=*/LlmRequestState::kCONTEXT_INIT, + /*enablePrefixAwareScheduling=*/executorConfig.getSchedulerConfig().getEnablePrefixAwareScheduling()); + + mMicroBatchScheduler = std::make_unique<tensorrt_llm::batch_manager::MicroBatchScheduler>( + std::nullopt, mModelConfig.getMaxInputLen(), LlmRequestState::kENCODER_INIT, LlmRequestState::kCONTEXT_INIT); + + mHiddenSize = modelConfig.getHiddenSize(); + + mMaxInputLen = mModelConfig.getMaxInputLen(); + TLLM_LOG_INFO("TRTEncoderModel mMaxInputLen: reset to %d from build config.", mMaxInputLen); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +BufferManager const& TrtEncoderModel::getBufferManager() const +{ + return mRuntime->getBufferManager(); +} + +BufferManager::CudaStreamPtr TrtEncoderModel::getRuntimeStreamPtr() const +{ + return mRuntime->getStreamPtr(); +} + +nvinfer1::DataType TrtEncoderModel::getTensorDataType(std::string const& name) const +{ + auto const& engine = mRuntime->getEngine(); + return engine.getTensorDataType(name.c_str()); +} + +nvinfer1::Dims TrtEncoderModel::getTensorShape(std::string const& name) const +{ + auto const& engine = mRuntime->getEngine(); + return engine.getTensorShape(name.c_str()); +} + +void TrtEncoderModel::getCurrentIterationStats(executor::IterationStats& stats) const +{ + stats.iter = mIterCounter; +} + +void TrtEncoderModel::getCurrentRequestStats(executor::RequestStatsPerIteration& stats) const +{ + stats.iter = mIterCounter; +} + +executor::DebugTensorsPerIteration TrtEncoderModel::getCurrentDebugTensors() const +{ + executor::DebugTensorsPerIteration debugTensors; + debugTensors.iter = mIterCounter; + + TLLM_LOG_WARNING("TrtEncoderModel doesn't support getting debug tensors."); + + return debugTensors; +} + +void TrtEncoderModel::setLayerProfiler() +{ + TLLM_CHECK(mRuntime); + mRuntime->setLayerProfiler(); +} + +std::string TrtEncoderModel::getLayerProfileInfo() const +{ + TLLM_CHECK(mRuntime); + return mRuntime->getLayerProfileInfo(); +} + +void TrtEncoderModel::createRuntimeContexts() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + mRuntime->clearContexts(); + auto const numProfiles = mRuntime->getNbProfiles(); + TLLM_CHECK_WITH_INFO(numProfiles == 1, "Encoder only expects one optimization profile"); + for (auto i = 0; i < numProfiles; ++i) + { + mRuntime->addContext(i); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::executeContext(SizeType32 runtimeContextId) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(executeContext); + auto enqueueSuccessful = mRuntime->executeContext(runtimeContextId); + if (!enqueueSuccessful) + { + throw std::runtime_error("Executing TRT engine failed!"); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::createBuffers() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + for (SizeType32 i = 0; i < mNumBuffers; ++i) + { + mBuffers.emplace_back( + std::make_shared<EncoderBuffers>(getMaxBatchSize(), mModelConfig, mWorldConfig, *mRuntime)); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::executeBatch(ScheduledRequests const& scheduledRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(executeBatch); + + // encoder model only have one optimization profile for now, so no optimization profile switch + SizeType32 optProfileIndex = 0; + auto const bufferId = getBufferId(); + if (!scheduledRequests.contextRequests.empty()) + { + // engine I/O + auto [inputMap, outputMap] + = mBuffers[bufferId]->prepareIO(scheduledRequests.contextRequests, mModelConfig, mWorldConfig, *mRuntime); + mRuntime->setInputTensors(optProfileIndex, inputMap); + mRuntime->setOutputTensors(optProfileIndex, outputMap); + + // engine run + executeContext(optProfileIndex); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::rearrangeOutputs(ScheduledRequests const& scheduledRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(rearrangeOutputs); + + auto const bufferId = getBufferId(); + if (!scheduledRequests.contextRequests.empty()) + { + mBuffers[bufferId]->rearrangeOutputs(scheduledRequests.contextRequests, mModelConfig, mWorldConfig, *mRuntime); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::forwardSync() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE_WITH_NAME(range, "TrtEncoderModel::forwardSync"); + + auto const device = mWorldConfig.getDevice(); + TLLM_CUDA_CHECK(cudaSetDevice(device)); + + auto& currRequests = mMicroBatchScheduledRequests.at(mMicroBatchId); + // auto& encoderWaitEvent = mEncoderWaitEvents.at(mMicroBatchId); + + if (!currRequests.empty()) + { + if (!mWorldConfig.isPipelineParallel() || !mWorldConfig.isLastPipelineParallelRank()) + { + // TLLM_CHECK_WITH_INFO(mEncStepAsyncSndHdl.get() == nullptr, "encoderSync handle must be nullptr."); + // // Wait for encoding for requests in flight for the current micro batch + // mEncStepAsyncSndHdl = encoderSync(currRequests, encoderWaitEvent); + } + else + { + } + + NVTX3_SCOPED_RANGE(pauseFlaggedCurrRequests); + for (auto const& requests : {currRequests.contextRequests}) + { + for (auto const& llmReq : requests) + { + auto const reqId = llmReq->mRequestId; + mInflightReqIds.erase(reqId); + TLLM_LOG_DEBUG("request ID %u removed from ENCODER inflight set", reqId); + + // If a request in encoder phase had been flagged to be paused, pause it right away + if (mReqIdsToPause.find(reqId) != mReqIdsToPause.end()) + { + terminateRequest(llmReq, true); + mReqIdsToPause.erase(reqId); + } + } + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::forwardAsync(RequestList const& activeRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE_WITH_NAME(range, "TrtEncoderModel::ForwardAsync"); + auto const device = mWorldConfig.getDevice(); + TLLM_CUDA_CHECK(cudaSetDevice(device)); + + try + { + auto& currRequests = mMicroBatchScheduledRequests.at(mMicroBatchId); + // auto& encoderWaitEvent = mEncoderWaitEvents.at(mMicroBatchId); + + // Get a new set of requests for encoder + // The scheduler will not include any requests that are already in flight for encoder models + // TODO: add pause handling logic + TLLM_LOG_DEBUG("Running ENCODER request scheduler"); + + auto [fittingRequests, fittingDisaggeGenInitReuqests, requestsToPause] = (*mCapacityScheduler)(activeRequests); + + TLLM_CHECK_WITH_INFO( + fittingDisaggeGenInitReuqests.empty(), "Disaggregated servering is not support by encoder model."); + + std::tie(currRequests.contextRequests, std::ignore) = (*mMicroBatchScheduler)( + fittingRequests, mInflightReqIds, getMaxBatchSize(), mModelConfig.getMaxNumTokens()); + + { + NVTX3_SCOPED_RANGE(pauseRequestsFlaggedByScheduler); + // Loop over requests flagged to be paused, and if not in flight pause it right away + for (auto const& llmReq : requestsToPause) + { + auto const reqId = llmReq->mRequestId; + if (mInflightReqIds.find(reqId) == mInflightReqIds.end()) + { + // Not in flight, can terminate right away + terminateRequest(llmReq, true); + } + else + { + // In flight, add to set for pausing later + mReqIdsToPause.insert(reqId); + } + } + } + + TLLM_CHECK(currRequests.size() <= static_cast<size_t>(getMaxBatchSize())); + + if (!currRequests.empty()) + { + TLLM_LOG_DEBUG("Running ENCODER model with batch size: %u", currRequests.size()); + { + NVTX3_SCOPED_RANGE(updateInflightReqIds); + // Add to set of requests in flight + for (auto const& requests : {currRequests.contextRequests}) + { + for (auto const& llmReq : requests) + { + TLLM_LOG_DEBUG("request ID %u added to ENCODER inflight set", llmReq->mRequestId); + mInflightReqIds.insert(llmReq->mRequestId); + } + } + } + + executeBatch(currRequests); + + sync_check_cuda_error(mRuntime->getStream().get()); + + rearrangeOutputs(currRequests); + + sync_check_cuda_error(mRuntime->getStream().get()); + + // encoderWaitEvent = encoderStepAsync(currRequests); + + for (auto const& requests : {currRequests.contextRequests}) + { + for (auto const& llmReq : requests) + { + if (llmReq->isEncoderInitState()) + { + llmReq->setState(LlmRequestState::kCONTEXT_INIT); + TLLM_LOG_DEBUG("request ID: %u finishes encoder phase", llmReq->mRequestId); + } + } + } + } + + // TODO: PP handling + if (!currRequests.empty()) + { + if (mWorldConfig.isPipelineParallel() && mWorldConfig.isLastPipelineParallelRank()) + { + // TLLM_CHECK_WITH_INFO(mEncStepAsyncSndHdl.get() == nullptr, "decoderSync handle must be nullptr."); + // Wait for encoding for requests in flight for the current micro batch + // mEncStepAsyncSndHdl = encoderSync(currRequests, encoderWaitEvent); + } + } + + // Update the micro batch ID + mMicroBatchId = (mMicroBatchId + 1) % mNumMicroBatches; + } + // In case of error, we need to free the batch slot associated with those requests + catch (std::exception const& e) + { + for (auto const& llmReq : activeRequests) + { + terminateRequest(llmReq); + } + throw; + } + + ++mIterCounter; + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::terminateRequest(std::shared_ptr<LlmRequest> const& llmReq, bool pause) +{ + // For encoder-only models, just change req state here. might need to do more when using an asynced forward + // For enc-dec models, only remove cross kv cache after decoder + // genenration has finished + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + if (llmReq->isEncoderInitState()) + { + llmReq->setState(LlmRequestState::kCONTEXT_INIT); + } + else + { + TLLM_LOG_DEBUG("Non-encoder request terminated in encoder model: id %lu", llmReq->mRequestId); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::terminateRequestSync( + std::shared_ptr<LlmRequest> const& llmReq, executor::FinishReason finishReason) +{ + terminateRequest(llmReq, false); + llmReq->finishByReason(finishReason); + llmReq->clearGeneratedTokens(); +} + +void TrtEncoderModel::fillEncoderOutputSync(RequestVector const& requestList, TensorMap outputTensors) +{ + auto const totalTokensNb = outputTensors["encoder_output"]->getShape().d[0]; + auto const encoderOutputDtype = mRuntime->getEngine().getTensorDataType("encoder_output"); + SizeType32 const bytesPerValue = (encoderOutputDtype == nvinfer1::DataType::kFLOAT) ? 4 : 2; + std::vector<std::byte> encoderOutputHost( + totalTokensNb * mHiddenSize * bytesPerValue * mWorldConfig.getTensorParallelism()); + TLLM_CHECK_WITH_INFO(encoderOutputHost.size() > 0, "Encoder output size is 0!"); + getBufferManager().copy(*(outputTensors["encoder_output"]), reinterpret_cast<void*>(encoderOutputHost.data())); + getBufferManager().getStream().synchronize(); // TODO: change engine call to async to improve perf. Also + // need to store output buffers, cuda events, etc. + + auto encoderOutputHostPtr = encoderOutputHost.data(); + for (auto const& llmReq : requestList) + { + SizeType32 const seqLen = llmReq->getEncoderOutputLen(); + TensorPtr currentEncoderOutput + = mCopyBufferManager.copyFrom(reinterpret_cast<half const*>(encoderOutputHostPtr), + ITensor::makeShape({seqLen, mHiddenSize * mWorldConfig.getTensorParallelism()}), MemoryType::kCPU); + llmReq->setEncoderOutputHost(currentEncoderOutput); + encoderOutputHostPtr += seqLen * mHiddenSize * bytesPerValue * mWorldConfig.getTensorParallelism(); + + if (llmReq->isEncoderInitState()) + { + llmReq->setState(LlmRequestState::kCONTEXT_INIT); + } + else + { + TLLM_LOG_DEBUG("Non-encoder request terminated in encoder model: id %lu", llmReq->mRequestId); + } + } +} + +void TrtEncoderModel::executeBatch(RequestVector const& requestList) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(executeBatch); + + auto const modelName = mModelConfig.getModelName(); + TLLM_CHECK_WITH_INFO(modelName == "EncoderModel" || modelName == "WhisperEncoder", "Model not supported."); + TensorMap inputTensors; + TensorMap outputTensors; + TensorPtr rankOutput; + + std::vector<TokenIdType> inputIdsHost; + std::vector<SizeType32> positionIdsHost; + SizeType32 totalOutputLength = 0; + SizeType32 totalInputLength = 0; + std::vector<SizeType32> inputLengthsHost; + std::vector<std::byte> inputFeaturesHost; + + inputLengthsHost.reserve(requestList.size()); + SizeType32 maxInputLengthHost = 0; + + for (auto const& llmReq : requestList) + { + SizeType32 length = 0; + if (mModelConfig.getModelName() == "EncoderModel") + { + auto const& reqTokens = *(llmReq->getEncoderTokens().value()); + length = reqTokens.size(); + + inputIdsHost.insert(inputIdsHost.end(), reqTokens.begin(), reqTokens.end()); + maxInputLengthHost = std::max(maxInputLengthHost, static_cast<SizeType32>(length)); + } + else if (mModelConfig.getModelName() == "WhisperEncoder") + { + auto const& reqFeatures = llmReq->getEncoderInputFeatures(); // [length, featureDim] + length = reqFeatures->getShape().d[0]; + + auto const curFeatureBytes = reqFeatures->getSizeInBytes(); + auto const srcPtr = reinterpret_cast<std::byte*>(reqFeatures->data()); + inputFeaturesHost.insert(inputFeaturesHost.end(), srcPtr, srcPtr + curFeatureBytes); + } + positionIdsHost.reserve(positionIdsHost.size() + length); + auto const newReqPosBegin = positionIdsHost.end(); + positionIdsHost.resize(positionIdsHost.size() + length); + std::iota(newReqPosBegin, positionIdsHost.end(), 0); + + totalOutputLength += llmReq->getEncoderOutputLen(); + totalInputLength += length; + inputLengthsHost.push_back(length); + } + + TensorPtr hiddenStatesInput; + TensorPtr inputLengths = getBufferManager().copyFrom( + inputLengthsHost, ITensor::makeShape({static_cast<SizeType32>(inputLengthsHost.size())}), MemoryType::kGPU); + inputTensors.emplace("input_lengths", inputLengths); + + if (mModelConfig.getModelName() == "EncoderModel") + { + // use shape of maxInputLength to indicates max length, content is not important + TensorPtr maxInputLength + = getBufferManager().gpu(ITensor::makeShape({maxInputLengthHost}), nvinfer1::DataType::kINT32); + inputTensors.emplace("max_input_length", maxInputLength); + } + + // engine outputs + rankOutput = getBufferManager().gpu( + ITensor::makeShape({totalOutputLength, mHiddenSize * mWorldConfig.getTensorParallelism()}), + mModelConfig.getDataType()); + + if (mWorldConfig.isFirstPipelineParallelRank()) + { + if (mModelConfig.getModelName() == "EncoderModel") + { + // Engine inputs + TensorPtr inputIds + = getBufferManager().copyFrom(inputIdsHost, ITensor::makeShape({totalInputLength}), MemoryType::kGPU); + TensorPtr positionIds = getBufferManager().copyFrom( + positionIdsHost, ITensor::makeShape({totalInputLength}), MemoryType::kGPU); + inputTensors.emplace("input_ids", inputIds); + inputTensors.emplace("position_ids", positionIds); + } + else if (mModelConfig.getModelName() == "WhisperEncoder") + { + auto inputFeaturesHostPtr = inputFeaturesHost.data(); + auto const featureDim = requestList.front()->getEncoderInputFeatures()->getShape().d[1]; + auto const dtype = requestList.front()->getEncoderInputFeatures()->getDataType(); + TensorPtr inputFeatures = getBufferManager().gpu(ITensor::makeShape({totalInputLength, featureDim}), dtype); + getBufferManager().copy( + reinterpret_cast<void const*>(inputFeaturesHostPtr), *inputFeatures, runtime::MemoryType::kCPU); + TensorPtr positionIds = getBufferManager().copyFrom( + positionIdsHost, ITensor::makeShape({totalOutputLength}), MemoryType::kGPU); + inputTensors.emplace("input_features", inputFeatures); + inputTensors.emplace("position_ids", positionIds); + } + } + else + { + SizeType32 length = mModelConfig.getModelName() == "WhisperEncoder" ? totalOutputLength : totalInputLength; + hiddenStatesInput + = getBufferManager().gpu(ITensor::makeShape({length, mHiddenSize * mWorldConfig.getTensorParallelism()}), + mModelConfig.getDataType()); + + inputTensors.emplace("hidden_states_input", hiddenStatesInput); + } + + auto const outputName = mWorldConfig.isLastPipelineParallelRank() ? "encoder_output" : "hidden_states_output"; + outputTensors.emplace(outputName, rankOutput); + + // Set input / output tensors to context, encoder model only have one context + mRuntime->setInputTensors(0, inputTensors); + mRuntime->setOutputTensors(0, outputTensors); + + executeContext(0); + + // copy encoder output to llmRequest, if last PP rank + // dispatch result to each llmReq, only needed by the last PP rank + // TODO: more dtypes support + if (mWorldConfig.isLastPipelineParallelRank()) + { + fillEncoderOutputSync(requestList, outputTensors); + } + else + { + getBufferManager().getStream().synchronize(); + } + + // Update the micro batch ID for next microbatches + mMicroBatchId = (mMicroBatchId + 1) % mWorldConfig.getPipelineParallelism(); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::forward(RequestVector& activeRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + auto const device = mWorldConfig.getDevice(); + TLLM_CUDA_CHECK(cudaSetDevice(device)); + + try + { + if (activeRequests.empty()) + { + return; + } + + executeBatch(activeRequests); + } + catch (std::exception const& e) + { + for (auto& req : activeRequests) + { + terminateRequest(req); + } + throw; + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtEncoderModel::setLogitsPostProcessorBatched( + std::optional<LogitsPostProcessorBatched> logitsPostProcessorBatched) +{ + TLLM_CHECK_WITH_INFO(!logitsPostProcessorBatched.has_value(), "TrtEncoderModel does not use logits processor."); +} + +void TrtEncoderModel::setReplicateLogitsPostProcessor(bool replicateLogitsPostProcessor) +{ + TLLM_THROW("TrtEncoderModel does not use logits processor."); +} + +bool TrtEncoderModel::getReplicateLogitsPostProcessor() const +{ + TLLM_THROW("TrtEncoderModel does not use logits processor."); +} + +TrtEncoderModel::~TrtEncoderModel() = default; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtEncoderModel.h b/cpp/tensorrt_llm/batch_manager/trtEncoderModel.h new file mode 100644 index 000000000000..31f7d3d0c89b --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/trtEncoderModel.h @@ -0,0 +1,205 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "trtGptModel.h" + +#include <NvInferRuntime.h> + +namespace tensorrt_llm::runtime +{ +class TllmRuntime; +class NcclCommunicator; +} // namespace tensorrt_llm::runtime + +namespace tensorrt_llm::batch_manager +{ +class CapacityScheduler; +class MicroBatchScheduler; +class EncoderBuffers; + +class TrtEncoderModel : public TrtGptModel +{ +public: + using SizeType32 = tensorrt_llm::runtime::SizeType32; + using TokenIdType = tensorrt_llm::runtime::TokenIdType; + using BufferManager = tensorrt_llm::runtime::BufferManager; + using TensorMap = runtime::StringPtrMap<runtime::ITensor>; + using TensorPtr = runtime::ITensor::SharedPtr; + + TrtEncoderModel(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + runtime::RawEngine const& rawEngine, std::shared_ptr<nvinfer1::ILogger> logger, + executor::ExecutorConfig const& executorConfig); + + ~TrtEncoderModel() override; + + void terminateRequest(std::shared_ptr<LlmRequest> const& llmRequest, bool pause = false) override; + void terminateRequestSync( + std::shared_ptr<LlmRequest> const& llmRequest, executor::FinishReason finishReason) override; + + void forward(RequestVector& activeRequests); + + void forwardSync() override; + + void forwardAsync(RequestList const& activeRequests) override; + + [[nodiscard]] runtime::BufferManager const& getBufferManager() const override; + [[nodiscard]] runtime::BufferManager::CudaStreamPtr getRuntimeStreamPtr() const override; + + runtime::ModelConfig const& getModelConfig() const override + { + return mModelConfig; + } + + [[nodiscard]] bool getGatherGenerationLogits() const override + { + return getModelConfig().computeGenerationLogits(); + } + + runtime::WorldConfig const& getWorldConfig() const override + { + return mWorldConfig; + } + + [[nodiscard]] SizeType32 getHiddenSize() const override + { + return mHiddenSize; + } + + [[nodiscard]] SizeType32 getMaxInputLen() const override + { + return mMaxInputLen; + } + + [[nodiscard]] SizeType32 getNumMicroBatches() const override + { + return mNumMicroBatches; + } + + [[nodiscard]] nvinfer1::DataType getLogitDataType() const override + { + return getModelConfig().getDataType(); + } + + nvinfer1::DataType getTensorDataType(std::string const& name) const override; + nvinfer1::Dims getTensorShape(std::string const& name) const override; + + [[nodiscard]] TrtGptModelType getModelType() const override + { + throw std::runtime_error("TrtEncoderModel does not have model type."); // FIXME: + } + + [[nodiscard]] executor::IterationType getIterCounter() const noexcept override + { + return mIterCounter; + } + + void updatePeftCache(std::shared_ptr<LlmRequest> const& /*llmRequest*/) override + { + throw std::runtime_error("TrtEncoderModel does not have Peft Cache."); + } + + void getCurrentIterationStats(executor::IterationStats& stats) const override; + void getCurrentRequestStats(executor::RequestStatsPerIteration& stats) const override; + [[nodiscard]] executor::DebugTensorsPerIteration getCurrentDebugTensors() const override; + + void setLayerProfiler() override; + std::string getLayerProfileInfo() const override; + + void setLogitsPostProcessorBatched(std::optional<LogitsPostProcessorBatched> logitsPostProcessorBatched) override; + void setReplicateLogitsPostProcessor(bool replicateLogitsPostProcessor) override; + [[nodiscard]] bool getReplicateLogitsPostProcessor() const override; + + void resetIterationStats() override {} + + [[nodiscard]] SizeType32 getMaxCapacityBatchSize(SizeType32 inputLength, SizeType32 outputLength) const override + { + return 0; + }; + +protected: + std::shared_ptr<kv_cache_manager::BaseKVCacheManager> getKVCacheManager() override + { + throw std::runtime_error("TrtEncoderModel does not have KVCache."); + } + + [[nodiscard]] std::shared_ptr<kv_cache_manager::BaseKVCacheManager const> getKVCacheManager() const override + { + throw std::runtime_error("TrtEncoderModel does not have KVCache."); + } + + [[nodiscard]] std::shared_ptr<BasePeftCacheManager> getPeftCacheManager() override + { + throw std::runtime_error("TrtEncoderModel does not use PEFT."); + } + + [[nodiscard]] std::shared_ptr<BasePeftCacheManager const> getPeftCacheManager() const override + { + throw std::runtime_error("TrtEncoderModel does not use PEFT."); + } + +private: + [[nodiscard]] SizeType32 getBufferId() const + { + return mMicroBatchId; + } + + void createRuntimeContexts(); + void executeContext(SizeType32 runtimeContextId); + void createBuffers(); + void executeBatch(RequestVector const& requestList); + void executeBatch(ScheduledRequests const& scheduledRequests); + void rearrangeOutputs(ScheduledRequests const& scheduledRequests); + void createCustomAllReduceWorkspace(); + void fillEncoderOutputSync(RequestVector const& requestList, TensorMap outputTensors); + + runtime::ModelConfig const mModelConfig; + runtime::WorldConfig const mWorldConfig; + int mDevice{-1}; + std::shared_ptr<tensorrt_llm::mpi::MpiComm> mMpiCommPipelinePara; + + std::shared_ptr<nvinfer1::ILogger> mLogger; + std::shared_ptr<runtime::TllmRuntime> mRuntime; + + SizeType32 mMicroBatchId{0}; + + // TODO: Add runtime buffers for async PP + std::vector<std::shared_ptr<EncoderBuffers>> mBuffers; + + SizeType32 mNumMicroBatches; + SizeType32 mNumBuffers; + + std::vector<ScheduledRequests> mMicroBatchScheduledRequests; + ReqIdsSet mInflightReqIds; + ReqIdsSet mReqIdsToPause; + + std::unique_ptr<tensorrt_llm::batch_manager::CapacityScheduler const> mCapacityScheduler; + std::unique_ptr<tensorrt_llm::batch_manager::MicroBatchScheduler const> mMicroBatchScheduler; + + SizeType32 mHiddenSize; // already divided by Tensor Parallelism + SizeType32 mMaxInputLen; // WAR for max_input_len == max_seq_len at all circumstances + + runtime::BufferManager mCopyBufferManager; + + // Iteration counter used to distinguish debug output + executor::IterationType mIterCounter{0}; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtGptModel.h b/cpp/tensorrt_llm/batch_manager/trtGptModel.h new file mode 100644 index 000000000000..54ad36b13895 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/trtGptModel.h @@ -0,0 +1,339 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/peftCacheManager.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/stlUtils.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/model.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +#include <memory> + +namespace tc = tensorrt_llm::common; + +namespace tensorrt_llm::batch_manager +{ +enum class TrtGptModelType +{ + InflightBatching, + InflightFusedBatching +}; + +class LlmRequest; + +namespace kv_cache_manager +{ +class BaseKVCacheManager; +} // namespace kv_cache_manager + +class TrtGptModel : public executor::Model +{ +public: + using SizeType32 = tensorrt_llm::runtime::SizeType32; + + TrtGptModel(runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + executor::ExecutorConfig const& executorConfig) + : mMaxBatchSize{executorConfig.getMaxBatchSize().value_or(modelConfig.getMaxBatchSize())} + , mMaxBeamWidth{executorConfig.getMaxBeamWidth()} + , mMaxSequenceLen{modelConfig.getMaxSequenceLen()} + , mMaxDraftLen{modelConfig.getMaxDecodingDraftTokens()} + , mVocabSizePadded{modelConfig.getVocabSizePadded(worldConfig.getSize())} + , mNormalizeLogProbs{executorConfig.getNormalizeLogProbs()} + , mEnableTrtOverlap{executorConfig.getEnableTrtOverlap()} + , mCudaGraphMode{executorConfig.getExtendedRuntimePerfKnobConfig().getCudaGraphMode()} + { + TLLM_CHECK_WITH_INFO(mMaxBeamWidth <= modelConfig.getMaxBeamWidth(), + "Runtime configured max beam width (%d) must not exceed engine max beam width (%d)", mMaxBeamWidth, + modelConfig.getMaxBeamWidth()); + TLLM_CHECK_WITH_INFO(mMaxBatchSize <= modelConfig.getMaxBatchSize(), + "Runtime configured max batch size (%d) must not exceed engine max batch size (%d)", mMaxBatchSize, + modelConfig.getMaxBatchSize()); + if (executorConfig.getEnableTrtOverlap()) + { + if (mMaxBeamWidth > 1) + { + mEnableTrtOverlap = false; + TLLM_LOG_WARNING( + "TRT overlap is not supported with beam search (maxBeamWidth is set to %d) and will be disabled.", + mMaxBeamWidth); + } + if (!modelConfig.getSpeculativeDecodingMode().isNone()) + { + mEnableTrtOverlap = false; + TLLM_LOG_WARNING("TRT overlap is not supported with speculative decoding and will be disabled."); + } + } + + mMaxAttentionWindow = 0; + if (executorConfig.getKvCacheConfig().getMaxAttentionWindowVec().has_value()) + { + bool warning = false; + auto const& maxAttentionWindowVec = executorConfig.getKvCacheConfig().getMaxAttentionWindowVec(); + for (int maxAttenWin : maxAttentionWindowVec.value()) + { + mMaxAttentionWindowVec.push_back(std::min(maxAttenWin, mMaxSequenceLen)); + mMaxAttentionWindow = std::max(mMaxAttentionWindow, mMaxAttentionWindowVec.back()); + if (maxAttenWin > mMaxSequenceLen) + { + warning = true; + } + TLLM_CHECK_WITH_INFO(mMaxAttentionWindowVec.back() > 0, + "Attention window sizes (elements in maxAttentionWindowVec) must be > 0"); + } + if (warning) + { + TLLM_LOG_WARNING( + "The value of maxAttentionWindow cannot exceed mMaxSequenceLen. " + "Therefore, it has been adjusted to match the value of mMaxSequenceLen."); + } + } + else + { + mMaxAttentionWindowVec.push_back(mMaxSequenceLen); + mMaxAttentionWindow = mMaxSequenceLen; + } + + mSinkTokenLen = executorConfig.getKvCacheConfig().getSinkTokenLength().has_value() + ? executorConfig.getKvCacheConfig().getSinkTokenLength().value() + : 0; + + mMaxNumSequences = mMaxBatchSize * worldConfig.getPipelineParallelism(); + + auto const numTotalAttenLayers = modelConfig.getNbAttentionLayers(); + auto const numRepeatsAttenWindow = numTotalAttenLayers / mMaxAttentionWindowVec.size(); + auto const numRemainsAttenWindow = numTotalAttenLayers % mMaxAttentionWindowVec.size(); + std::string attenWindowRemainInfo = numRemainsAttenWindow > 0 + ? " + " + tc::arr2str(mMaxAttentionWindowVec.data(), numRemainsAttenWindow) + : ""; + + TLLM_LOG_INFO("TRTGptModel maxNumSequences: %d", mMaxNumSequences); + TLLM_LOG_INFO("TRTGptModel maxBatchSize: %d", mMaxBatchSize); + TLLM_LOG_INFO("TRTGptModel maxBeamWidth: %d", mMaxBeamWidth); + TLLM_LOG_INFO("TRTGptModel maxSequenceLen: %d", mMaxSequenceLen); + TLLM_LOG_INFO("TRTGptModel maxDraftLen: %d", mMaxDraftLen); + TLLM_LOG_INFO("TRTGptModel mMaxAttentionWindowSize: %s * %d%s", tc::vec2str(mMaxAttentionWindowVec).c_str(), + numRepeatsAttenWindow, attenWindowRemainInfo.c_str()); + TLLM_LOG_INFO("TRTGptModel enableTrtOverlap: %d", mEnableTrtOverlap); + TLLM_LOG_INFO("TRTGptModel normalizeLogProbs: %d", mNormalizeLogProbs); + + mMaxNumTokens = modelConfig.getMaxNumTokens(); + if (executorConfig.getMaxNumTokens().has_value() && mMaxNumTokens) + { + if (executorConfig.getMaxNumTokens().value() > mMaxNumTokens.value()) + { + TLLM_LOG_WARNING( + "Runtime configured max num tokens (%d) is larger than model max num tokens (%d) and will be " + "ignored.", + executorConfig.getMaxNumTokens().value(), mMaxNumTokens.value()); + } + else + { + mMaxNumTokens = executorConfig.getMaxNumTokens(); + } + } + if (mMaxNumTokens) + { + TLLM_LOG_INFO("TRTGptModel maxNumTokens: %d", mMaxNumTokens.value()); + } + + if (executorConfig.getEnableChunkedContext()) + { + mMaxInputLen = mMaxSequenceLen - 1; + TLLM_LOG_INFO( + "TRTGptModel maxInputLen: %d = maxSequenceLen - 1 since chunked context is enabled", mMaxInputLen); + TLLM_LOG_INFO( + "TRTGptModel If model type is encoder, maxInputLen would be reset in trtEncoderModel to maxInputLen: " + "%d = maxSequenceLen.", + mMaxSequenceLen); + } + else if (modelConfig.getContextFMHA() && modelConfig.usePackedInput()) + { + TLLM_CHECK_WITH_INFO( + mMaxNumTokens, "Max number of tokens has to be set for context FMHA and usePackedInput case."); + mMaxInputLen = std::min(mMaxSequenceLen - 1, mMaxNumTokens.value()); + TLLM_LOG_INFO( + "TRTGptModel maxInputLen: %d = min(maxSequenceLen - 1, maxNumTokens) since context FMHA " + "and usePackedInput are enabled", + mMaxInputLen); + TLLM_LOG_INFO( + "TRTGptModel If model type is encoder, maxInputLen would be reset in trtEncoderModel to maxInputLen: " + "min(maxSequenceLen, maxNumTokens)."); + } + else + { + mMaxInputLen = modelConfig.getMaxInputLen(); + TLLM_LOG_INFO("TRTGptModel maxInputLen: %d = max_input_len (in trtllm-build args)", mMaxInputLen); + } + + using tensorrt_llm::common::stl_utils::toString; + + TLLM_LOG_INFO("Capacity Scheduler Policy: %s", + toString(executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy()).c_str()); + TLLM_LOG_INFO("Context Chunking Scheduler Policy: %s", + toString(executorConfig.getSchedulerConfig().getContextChunkingPolicy()).c_str()); + } + + [[nodiscard]] std::optional<SizeType32> getMaxNumTokens() const + { + return mMaxNumTokens; + } + + [[nodiscard]] SizeType32 getMaxNumSequences() const override + { + return mMaxNumSequences; + } + + [[nodiscard]] SizeType32 getMaxBatchSize() const + { + return mMaxBatchSize; + } + + [[nodiscard]] SizeType32 getMaxInputLen() const override + { + return mMaxInputLen; + } + + [[nodiscard]] SizeType32 getHiddenSize() const override + { + return getModelConfig().getHiddenSize(); + }; + + [[nodiscard]] SizeType32 getMaxSequenceLen() const override + { + return mMaxSequenceLen; + } + + [[nodiscard]] virtual TrtGptModelType getModelType() const = 0; + + [[nodiscard]] SizeType32 getVocabSizePadded() const override + { + return mVocabSizePadded; + } + + [[nodiscard]] SizeType32 getMaxDraftLen() const override + { + return mMaxDraftLen; + } + + [[nodiscard]] SizeType32 getOperatingBeamWidth() const override + { + return mMaxBeamWidth; + } + + [[nodiscard]] bool hasSpeculativeDecodingFastLogits() const noexcept override + { + return false; + } + + [[nodiscard]] bool hasGuidedDecoder() const noexcept override + { + return false; + } + + virtual void setLayerProfiler() = 0; + [[nodiscard]] virtual std::string getLayerProfileInfo() const = 0; + + [[nodiscard]] bool hasKVCacheManager() const + { + return getKVCacheManager() != nullptr; + } + +protected: + [[nodiscard]] SizeType32 getMaxBeamWidth() const + { + return mMaxBeamWidth; + } + + [[nodiscard]] std::vector<SizeType32> getMaxAttentionWindowVec() const + { + return mMaxAttentionWindowVec; + } + + [[nodiscard]] SizeType32 getMaxAttentionWindow() const + { + return mMaxAttentionWindow; + } + + [[nodiscard]] SizeType32 getSinkTokenLen() const + { + return mSinkTokenLen; + } + + [[nodiscard]] bool isNormalizeLogProbs() const + { + return mNormalizeLogProbs; + } + + [[nodiscard]] bool isTrtOverlap() const + { + return mEnableTrtOverlap; + } + + [[nodiscard]] bool isCudaGraphMode() const + { + return mCudaGraphMode; + } + + void setMaxAttentionWindowVec(std::vector<SizeType32> const& maxAttentionWindowVec) + { + TLLM_CHECK_WITH_INFO(maxAttentionWindowVec.size() == mMaxAttentionWindowVec.size(), + "The size of maxAttentionWindowVec must match the size of mMaxAttentionWindowVec"); + mMaxAttentionWindowVec = maxAttentionWindowVec; + mMaxAttentionWindow = *std::max_element(std::begin(mMaxAttentionWindowVec), std::end(mMaxAttentionWindowVec)); + } + + void setMaxSequenceLen(SizeType32 maxSequenceLen) + { + mMaxSequenceLen = maxSequenceLen; + } + + void setMaxInputLen(SizeType32 maxInputLen) + { + mMaxInputLen = maxInputLen; + } + + [[nodiscard]] std::shared_ptr<kv_cache_manager::BaseKVCacheManager> getKVCacheManager() override = 0; + [[nodiscard]] std::shared_ptr<kv_cache_manager::BaseKVCacheManager const> getKVCacheManager() const override = 0; + + [[nodiscard]] virtual std::shared_ptr<BasePeftCacheManager> getPeftCacheManager() = 0; + [[nodiscard]] virtual std::shared_ptr<BasePeftCacheManager const> getPeftCacheManager() const = 0; + +private: + std::optional<SizeType32> mMaxNumTokens; + SizeType32 mMaxNumSequences; + SizeType32 mMaxBatchSize; + SizeType32 mMaxBeamWidth; + SizeType32 mMaxInputLen; + SizeType32 mMaxSequenceLen; + SizeType32 mMaxDraftLen; + + SizeType32 mVocabSizePadded; + std::vector<SizeType32> mMaxAttentionWindowVec; + SizeType32 mMaxAttentionWindow; + SizeType32 mSinkTokenLen; + + bool mNormalizeLogProbs; + bool mEnableTrtOverlap; + bool mCudaGraphMode; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtGptModelFactory.h b/cpp/tensorrt_llm/batch_manager/trtGptModelFactory.h new file mode 100644 index 000000000000..bd4d7c767378 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/trtGptModelFactory.h @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/trtGptModel.h" +#include "tensorrt_llm/batch_manager/trtGptModelInflightBatching.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/runtime/gptJsonConfig.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +#include <NvInferPlugin.h> + +#include <memory> +#include <optional> + +namespace tensorrt_llm::batch_manager +{ + +class TrtGptModelFactory +{ +public: + using SizeType32 = tensorrt_llm::runtime::SizeType32; + + static std::shared_ptr<TrtGptModel> create(std::filesystem::path const& trtEnginePath, TrtGptModelType modelType, + executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode) + { + auto const jsonConfig = runtime::GptJsonConfig::parse(trtEnginePath / "config.json"); + auto const& deviceIds = executorConfig.getParallelConfig().value_or(executor::ParallelConfig()).getDeviceIds(); + auto const worldConfig = getWorldConfig(jsonConfig, deviceIds); + auto const enginePath = trtEnginePath / jsonConfig.engineFilename(worldConfig); + + auto const& modelConfig = jsonConfig.getModelConfig(); + return create( + runtime::RawEngine(enginePath), modelConfig, worldConfig, modelType, executorConfig, isLeaderInOrchMode); + } + + static std::shared_ptr<TrtGptModel> create(std::filesystem::path const& trtEnginePath, TrtGptModelType modelType, + runtime::GptJsonConfig const& jsonConfig, runtime::WorldConfig const& worldConfig, + executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode) + { + auto const enginePath = trtEnginePath / jsonConfig.engineFilename(worldConfig); + auto const& modelConfig = jsonConfig.getModelConfig(); + return create( + runtime::RawEngine(enginePath), modelConfig, worldConfig, modelType, executorConfig, isLeaderInOrchMode); + } + + static std::shared_ptr<TrtGptModel> create(runtime::RawEngine const& rawEngine, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, TrtGptModelType modelType, + executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode) + { + auto logger = std::make_shared<runtime::TllmLogger>(); + auto const device = worldConfig.getDevice(); + auto const rank = worldConfig.getRank(); + TLLM_LOG_INFO("Rank %d is using GPU %d", rank, device); + TLLM_CUDA_CHECK(cudaSetDevice(device)); + + if ((modelType == TrtGptModelType::InflightBatching) || (modelType == TrtGptModelType::InflightFusedBatching)) + { + executor::ExecutorConfig const& fixedExecutorConfig + = TrtGptModelInflightBatching::executorConfigIsValid(modelConfig, executorConfig) + ? executorConfig + : TrtGptModelInflightBatching::fixExecutorConfig(modelConfig, executorConfig); + bool const ctxGenFusion = modelType == TrtGptModelType::InflightFusedBatching; + return std::make_shared<TrtGptModelInflightBatching>( + logger, modelConfig, worldConfig, rawEngine, ctxGenFusion, fixedExecutorConfig, isLeaderInOrchMode); + } + + throw std::runtime_error("Invalid modelType in trtGptModelFactory"); + } + +private: + static runtime::WorldConfig getWorldConfig( + runtime::GptJsonConfig const& json, std::optional<std::vector<SizeType32>> const& deviceIds) + { + return runtime::WorldConfig::mpi(json.getGpusPerNode(), json.getTensorParallelism(), + json.getPipelineParallelism(), json.getContextParallelism(), deviceIds); + } +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.cpp b/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.cpp new file mode 100644 index 000000000000..7a0d78beb8a0 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.cpp @@ -0,0 +1,3136 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "trtGptModelInflightBatching.h" + +#include "tensorrt_llm/batch_manager/allocateKvCache.h" +#include "tensorrt_llm/batch_manager/assignReqSeqSlots.h" +#include "tensorrt_llm/batch_manager/cacheTransceiver.h" +#include "tensorrt_llm/batch_manager/capacityScheduler.h" +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/batch_manager/contextProgress.h" +#include "tensorrt_llm/batch_manager/createNewDecoderRequests.h" +#include "tensorrt_llm/batch_manager/decoderBuffers.h" +#include "tensorrt_llm/batch_manager/disaggTransferAdmissionController.h" +#include "tensorrt_llm/batch_manager/guidedDecoder.h" +#include "tensorrt_llm/batch_manager/handleContextLogits.h" +#include "tensorrt_llm/batch_manager/handleGenerationLogits.h" +#include "tensorrt_llm/batch_manager/kvCacheEventManager.h" +#include "tensorrt_llm/batch_manager/kvCacheManager.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/logitsPostProcessor.h" +#include "tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h" +#include "tensorrt_llm/batch_manager/microBatchScheduler.h" +#include "tensorrt_llm/batch_manager/pauseRequests.h" +#include "tensorrt_llm/batch_manager/peftCacheManager.h" +#include "tensorrt_llm/batch_manager/promptTuningBuffers.h" +#include "tensorrt_llm/batch_manager/rnnStateManager.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" +#include "tensorrt_llm/batch_manager/sequenceSlotManager.h" +#include "tensorrt_llm/batch_manager/transformerBuffers.h" +#include "tensorrt_llm/batch_manager/updateDecoderBuffers.h" +#include "tensorrt_llm/batch_manager/utils/debugUtils.h" +#include "tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h" +#include "tensorrt_llm/batch_manager/utils/logitsThread.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/timestampUtils.h" +#include "tensorrt_llm/kernels/decodingCommon.h" +#include "tensorrt_llm/layers/defaultDecodingParams.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/gptDecoderBatched.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/ipcUtils.h" +#include "tensorrt_llm/runtime/lookaheadModule.h" +#include "tensorrt_llm/runtime/memoryCounters.h" +#include "tensorrt_llm/runtime/runtimeKernels.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tensorrt_llm/runtime/utils/runtimeUtils.h" + +#include <algorithm> +#include <cstddef> +#include <cstring> +#include <memory> +#include <numeric> +#include <optional> +#include <stdexcept> +#include <thread> +#include <utility> +#include <vector> + +using namespace tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; +namespace tk = tensorrt_llm::kernels; + +using tensorrt_llm::batch_manager::CacheTransceiverFactory; + +namespace tensorrt_llm::batch_manager +{ + +std::map<SizeType32, SizeType32> TrtGptModelInflightBatching::calculateCacheSizePerTokenForDisagg( + ModelConfig const& modelConfig, WorldConfig const& worldConfig, + std::vector<SizeType32> const& maxAttentionWindowVec, bool isCrossAttention, SizeType32 kvFactor) +{ + // These are the number of attention layers on this PP rank. + auto const numLocalAttnLayers + = modelConfig.getNbAttentionLayers(worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); + // These are the number of attention layers on all previous PP ranks. + auto const numLowerRankAttnLayers = modelConfig.countLowerRankLayers(ModelConfig::LayerType::kATTENTION, + worldConfig.getPipelineParallelism(), worldConfig.getPipelineParallelRank()); + // Use global ranks of attention layers to lookup from maxAttentionWindowVec. + auto const startAttnLayerId = numLowerRankAttnLayers; + auto const endAttnLayerId = numLowerRankAttnLayers + numLocalAttnLayers; + auto const numNonUniqueWindowSizes = static_cast<SizeType32>(maxAttentionWindowVec.size()); + std::map<SizeType32, std::vector<SizeType32>> uniqueWindowSizeToLayers; + for (SizeType32 layerIdx = startAttnLayerId; layerIdx < endAttnLayerId; layerIdx++) + { + // maxAttentionWindowVec may or may not be stretched to the length of numLayers yet. + // If not stretched yet, we cycle through the window sizes. + auto const windowSize = maxAttentionWindowVec.at(layerIdx % numNonUniqueWindowSizes); + uniqueWindowSizeToLayers[windowSize].push_back(layerIdx); + } + std::map<SizeType32, SizeType32> cacheSizeBytesPerTokenPerWindow; + for (auto const& [windowSize, globalLayerIds] : uniqueWindowSizeToLayers) + { + auto const nkvh = modelConfig.getNumKvHeadsForGivenLayers(globalLayerIds, isCrossAttention); + auto const sumLocalHeads = std::reduce(nkvh.cbegin(), nkvh.cend()); + auto const cacheSizePerToken = sumLocalHeads * kvFactor * modelConfig.getSizePerHead(); + auto const cacheSizeBytesPerToken = cacheSizePerToken * BufferDataType(modelConfig.getKvDataType()).getSize(); + cacheSizeBytesPerTokenPerWindow[windowSize] = cacheSizeBytesPerToken; + } + + return cacheSizeBytesPerTokenPerWindow; +}; + +bool TrtGptModelInflightBatching::executorConfigIsValid( + ModelConfig const& modelConfig, executor::ExecutorConfig const& executorConfig) +{ + // Make sure logic in this function matches fixExecutorConfig + if (executorConfig.getKvCacheConfig().getEnableBlockReuse()) + { + if (!modelConfig.getPagedContextFMHA()) + { + return false; + } + // Context logits cannot be returned for reused tokens, so disable reuse + if (modelConfig.computeContextLogits()) + { + return false; + } + } + return true; +} + +executor::ExecutorConfig TrtGptModelInflightBatching::fixExecutorConfig( + ModelConfig const& modelConfig, executor::ExecutorConfig const& executorConfig) +{ + // Make sure logic in this function matches executorConfigIsValid + if (executorConfig.getKvCacheConfig().getEnableBlockReuse()) + { + auto kvCacheConfig = executorConfig.getKvCacheConfig(); + + if (!modelConfig.getPagedContextFMHA()) + { + TLLM_LOG_WARNING( + "Fixing executorConfig: KV cache reuse disabled because model was not built with paged context FMHA " + "support"); + kvCacheConfig.setEnableBlockReuse(false); + } + if (modelConfig.computeContextLogits()) + { + TLLM_LOG_WARNING( + "Fixing executorConfig: KV cache reuse disabled because model was built to return context logits"); + kvCacheConfig.setEnableBlockReuse(false); + } + + auto fixedExecutorConfig = executor::ExecutorConfig(executorConfig); + fixedExecutorConfig.setKvCacheConfig(kvCacheConfig); + return fixedExecutorConfig; + } + return executorConfig; +} + +TrtGptModelInflightBatching::TrtGptModelInflightBatching(std::shared_ptr<nvinfer1::ILogger> logger, + ModelConfig const& modelConfig, WorldConfig const& worldConfig, RawEngine const& rawEngine, bool ctxGenFusion, + executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode) + : TrtGptModel(modelConfig, worldConfig, executorConfig) + , mModelConfig(modelConfig) + , mWorldConfig(worldConfig) + , mDevice{runtime::utils::initDevice(worldConfig)} + , mDecodingConfig{executorConfig.getDecodingConfig().value_or(executor::DecodingConfig{})} + , mExtendedRuntimePerfKnobConfig{executorConfig.getExtendedRuntimePerfKnobConfig()} + , mDebugConfig{executorConfig.getDebugConfig()} + , mAdditionalModelOutputs{worldConfig.isLastPipelineParallelRank() ? executorConfig.getAdditionalModelOutputs() + : std::nullopt} + , mLogger{logger ? std::move(logger) : std::make_shared<TllmLogger>()} + , mRuntime{std::make_unique<TllmRuntime>(rawEngine, mLogger.get(), executorConfig.getUseGpuDirectStorage(), + executorConfig.getGpuWeightsPercent(), modelConfig.useShapeInference())} + , mCopyBufferManager{std::make_shared<CudaStream>()} + , mCtxGenFusion(ctxGenFusion) + , mOperatingBeamWidth{getMaxBeamWidth()} + , mGatherGenerationLogits{executorConfig.getGatherGenerationLogits()} + , mPromptTableOffloading{executorConfig.getPromptTableOffloading()} + , mIsLeaderInOrchMode{isLeaderInOrchMode} +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_LOG_INFO("gatherContextLogits: %d", mModelConfig.computeContextLogits()); + TLLM_LOG_INFO("gatherGenerationLogits: %d", getGatherGenerationLogits()); + + if (!(mModelConfig.supportsInflightBatching())) + { + throw std::runtime_error( + "TrtGptModelInflightBatching requires GPT attention/Mamba Conv 1d plugin with " + "packed input and paged KV cache."); + } + if (mWorldConfig.isTensorParallel()) + { + mRuntime->initializeUserBuffer(mWorldConfig, mModelConfig.getMaxBatchSize(), mModelConfig.getMaxBeamWidth(), + mModelConfig.getMaxSequenceLen(), mModelConfig.getHiddenSize(), getMaxNumTokens()); + } + if (mWorldConfig.isPipelineParallel()) + { + mNumMicroBatches = mWorldConfig.getPipelineParallelism(); + } + else + { + mNumMicroBatches = isTrtOverlap() ? 2 : 1; + } + + mNumBuffers = (mCtxGenFusion ? 1 : 2) * mNumMicroBatches; + + auto const& kvCacheConfig = executorConfig.getKvCacheConfig(); + + if (mModelConfig.getSpeculativeDecodingMode().isDraftTokensExternal()) + { + TLLM_CHECK_WITH_INFO(kvCacheConfig.getEnableBlockReuse(), + "KV cache block reuse must be enabled for speculative decoding target model"); + } + + if (mCtxGenFusion) + { + TLLM_CHECK_WITH_INFO(!mModelConfig.isRnnBased(), "RNN based model doesn't support context generation fusion."); + TLLM_CHECK_WITH_INFO( + mModelConfig.isTransformerBased(), "Only transformer based model support context generation fusion now."); + } + + if (mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) + { + mSeamlessLADMaxDraftLen = modelConfig.getMaxDecodingDraftTokens(); + // TODO: enable it when speculativeDecodingMode is None and run with '--lookahead_config' + mUseSeamlessLookahead = false; + } + + setupSpeculativeDecodingModule(mDecodingConfig); + + if (mWorldConfig.isLastPipelineParallelRank() && executorConfig.getGuidedDecodingConfig()) + { + mGuidedDecoder = std::make_unique<GuidedDecoder>(executorConfig.getGuidedDecodingConfig().value(), + getMaxNumSequences(), mModelConfig.getVocabSizePadded(mWorldConfig.getSize()), + mModelConfig.getLogitsDtype(), mRuntime->getBufferManager()); + } + + createRuntimeContexts(); + + if (mWorldConfig.isTensorParallel()) + { + createCustomAllReduceWorkspace(); + } + + if (mModelConfig.isTransformerBased()) + { + createRuntimePerfKnobsTensor(mExtendedRuntimePerfKnobConfig); + } + + auto& memCounter = MemoryCounters::getInstance(); + auto const gpuUsage1 = memCounter.getGpu(); + createBuffers(mDecodingConfig, mAdditionalModelOutputs); + auto const gpuUsage2 = memCounter.getGpu(); + TLLM_LOG_INFO("[MemUsageChange] Allocated %s GPU memory for runtime buffers.", + memCounter.bytesToString(gpuUsage2 - gpuUsage1).c_str()); + + createDecoder(mDecodingConfig.getDecodingMode()); + auto const gpuUsage3 = memCounter.getGpu(); + TLLM_LOG_INFO("[MemUsageChange] Allocated %s GPU memory for decoder.", + memCounter.bytesToString(gpuUsage3 - gpuUsage2).c_str()); + + if (modelConfig.getManageWeightsType() != ModelConfig::ManageWeightsType::kDisabled) + { + mRuntime->loadManagedWeights(rawEngine, worldConfig.getLocalRank()); + } + + if (mModelConfig.useLoraPlugin()) + { + auto const peftCacheManagerConfig + = PeftCacheManagerConfig(executorConfig.getPeftCacheConfig().value_or(executor::PeftCacheConfig())); + mPeftCacheManager = std::make_shared<PeftCacheManager>( + peftCacheManagerConfig, mModelConfig, mWorldConfig, mRuntime->getBufferManager()); + } + else + { + mPeftCacheManager = std::make_shared<NoOpPeftCacheManager>(); + } + + if (mModelConfig.isRnnBased()) + { + createRnnStateManager(); + } + if (mModelConfig.isTransformerBased() && modelConfig.isKVCacheEnabled()) + { + auto cacheTransceiverConfig + = executorConfig.getCacheTransceiverConfig().value_or(executor::CacheTransceiverConfig()); + + auto const cacheSizeBytesPerTokenPerWindow = calculateCacheSizePerTokenForDisagg( + mModelConfig, mWorldConfig, getMaxAttentionWindowVec(), mModelConfig.useCrossAttention(), 2); + auto cacheTransPreAllocaSize = kv_cache_manager::CacheTransBufferManager::preAllocBufferSize( + cacheSizeBytesPerTokenPerWindow, mModelConfig.getTokensPerBlock(), cacheTransceiverConfig); + + auto const [freePrimaryMemBytes, freeSecondaryMemBytes] + = BaseKVCacheManager::calculateFreeMemBytes(mRuntime->getBufferManager(), kvCacheConfig); + if (mModelConfig.useCrossAttention()) + { + TLLM_CHECK_WITH_INFO(kvCacheConfig.getCrossKvCacheFraction().has_value(), + "Must set crossKvCacheFraction for encoder-decoder model"); + auto const crossKvCacheFraction = kvCacheConfig.getCrossKvCacheFraction().value(); + mKvCacheManager = createKvCacheManager(kvCacheConfig, KvCacheType::kSELF, + freePrimaryMemBytes * (1.0f - crossKvCacheFraction), + freeSecondaryMemBytes * (1.0f - crossKvCacheFraction), cacheTransPreAllocaSize, + executorConfig.getFailFastOnAttentionWindowTooLarge()); + mCrossKvCacheManager = createKvCacheManager(kvCacheConfig, KvCacheType::kCROSS, + freePrimaryMemBytes * crossKvCacheFraction, freeSecondaryMemBytes * crossKvCacheFraction, + cacheTransPreAllocaSize, executorConfig.getFailFastOnAttentionWindowTooLarge()); + TLLM_LOG_INFO("This is an Encoder-Decoder model, set %0.1f cross KV cache fraction based on the config.", + crossKvCacheFraction); + } + else + { + TLLM_CHECK_WITH_INFO(!kvCacheConfig.getCrossKvCacheFraction().has_value(), + "Do not set crossKvCacheFraction for decoder-only model"); + mKvCacheManager = createKvCacheManager(kvCacheConfig, KvCacheType::kSELF, freePrimaryMemBytes, + freeSecondaryMemBytes, cacheTransPreAllocaSize, executorConfig.getFailFastOnAttentionWindowTooLarge()); + } + + mCacheTransceiver + = CacheTransceiverFactory::createCacheTransceiver(mKvCacheManager.get(), mModelConfig, mWorldConfig, + executor::kv_cache::CacheState::AttentionType::kDEFAULT, executorConfig.getCacheTransceiverConfig()); + mDisaggTransferAdmissionController = std::make_unique<DisaggTransferAdmissionController>( + cacheTransceiverConfig.getMaxTokensInBuffer(), mModelConfig.getTokensPerBlock()); + } + + if (mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind()) + { + TLLM_CHECK_WITH_INFO( + mModelConfig.isKVCacheEnabled(), "When needsKVCacheRewind() returns true, KV cache needs to be enabled."); + auto const& blockManager = mKvCacheManager->getBlockManager(); + + TLLM_CHECK_WITH_INFO(blockManager.getNumPools() == 1, + "Rewinding KV cache blocks for models with multiple pools is not supported"); + + // Two "redundant" checks given the pool size check above, but those below don't rely on an implementation + // detail I guess. + TLLM_CHECK_WITH_INFO( + !blockManager.isVariableWindow(), "Rewinding KV cache blocks for variable SWA models isn't supported"); + auto const maxBlocksPerSeq = blockManager.getMaxBlockPerSeqWhenSingleWindowSize(); + + // TODO(oargov): VGQA is not supported, assume all layers have the same num_kv_heads + TLLM_CHECK_WITH_INFO( + !blockManager.isVariableGQA(), "Rewinding KV cache blocks for variable GQA models isn't supported"); + auto const numKvHeads = mModelConfig.getNbKvHeads(0); + + mRewindInputs = RewindInputs{maxBlocksPerSeq, /*isUseOneMoreBlock*/ false, numKvHeads}; + } + + if (mWorldConfig.isPipelineParallel()) + { + mAsyncSendWaitThread = std::make_unique<tensorrt_llm::mpi::MpiWaitThread>( + "asyncSendWaitThread", + [this]() + { + mDecStepAsyncSndHdls.clear(); + mDecSlotAsyncSndHdls.clear(); + }, + [this]() { TLLM_CUDA_CHECK(cudaSetDevice(mWorldConfig.getDevice())); }); + + auto const& commSession = COMM_SESSION; + mMpiCommPipelinePara = std::make_unique<tensorrt_llm::mpi::MpiComm>( + commSession.split(mWorldConfig.getTensorParallelRank(), mWorldConfig.getPipelineParallelRank())); + mDecSlotAsyncSndHdls.reserve(getMaxBatchSize()); + } + if (mWorldConfig.isTensorParallel()) + { + auto const& commSession = COMM_SESSION; + mMpiCommTensorPara = std::make_unique<tensorrt_llm::mpi::MpiComm>( + commSession.split(mWorldConfig.getPipelineParallelRank(), mWorldConfig.getTensorParallelRank())); + } + + mSeqSlotManager + = std::make_shared<SequenceSlotManager>(getMaxNumSequences(), executorConfig.getMaxSeqIdleMicroseconds()); + + mMicroBatchScheduledRequests.resize(mNumMicroBatches); + mDecoderFinishedEvents.resize(mNumMicroBatches); + mPeftTables.resize(mNumMicroBatches); + + if (modelConfig.isRnnBased()) + { + TLLM_CHECK_WITH_INFO(modelConfig.getMaxBeamWidth() == 1, "RNN based model doesn't support beam search now."); + TLLM_CHECK_WITH_INFO( + !executorConfig.getEnableChunkedContext(), "RNN based model doesn't support Chunked Context now."); + TLLM_CHECK_WITH_INFO( + modelConfig.getSpeculativeDecodingMode().isNone(), "RNN based model doesn't support speculative decoding."); + } + + std::optional<batch_scheduler::ContextChunkingConfig> ctxChunkConfig; + if (executorConfig.getEnableChunkedContext()) + { + TLLM_CHECK_WITH_INFO(modelConfig.isKVCacheEnabled() && mModelConfig.getPagedContextFMHA(), + "Chunked context requires context FMHA, paged kv_cache and paged context FMHA all enabled at the same " + "time."); + SizeType32 chunkUnitSize = mKvCacheManager->getTokensPerBlock(); + // If sliding window attention is used, then make sure the unit size aligns with the paged context fmha's kv + // step size. + if (getMaxInputLen() > getMaxAttentionWindow()) // TODO(nhaber): minAttentionWindow + { + chunkUnitSize = std::max(/* maxKvStepSizeInFmha */ 256, chunkUnitSize); + TLLM_LOG_INFO("ChunkUnitSize is set to %d as sliding window attention is used.", chunkUnitSize); + } + ctxChunkConfig = batch_scheduler::ContextChunkingConfig{ + executorConfig.getSchedulerConfig().getContextChunkingPolicy().value_or( + executor::ContextChunkingPolicy::kFIRST_COME_FIRST_SERVED), + chunkUnitSize}; + } + + auto maxNumTokens = getMaxNumTokens(); + TLLM_CHECK_WITH_INFO(maxNumTokens, "Max number of tokens is not set in model config."); + + // Max context size is limited by `max_num_tokens` for chunked-context or context-FMHA, + // or by `max_input_len` of the model. + auto const maxContextLength = (executorConfig.getEnableChunkedContext() || mModelConfig.getContextFMHA()) + ? maxNumTokens + : std::make_optional<SizeType32>(mModelConfig.getMaxInputLen()); + + mMaxBatchSizeTunerRecommended = 0; + mMaxBatchSizeRuntime = getMaxBatchSize(); + mMaxNumTokensStatic = maxNumTokens; + mMaxNumTokensTunerRecommended = 0; + mMaxNumTokensRuntime = maxNumTokens; + + if (mKvCacheManager && ctxChunkConfig) + { + TLLM_CHECK_WITH_INFO(ctxChunkConfig.value().chunkUnitSize % mKvCacheManager->getTokensPerBlock() == 0, + "To prevent cache fragmentation, the context chunk unit size (%d) should be divisible by the number of " + "tokens per kv-cache block (%d).", + ctxChunkConfig.value().chunkUnitSize, mKvCacheManager->getTokensPerBlock()); + } + + mCapacityScheduler = std::make_unique<CapacityScheduler>(getMaxNumSequences(), + executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy(), mKvCacheManager != nullptr, + /*twoStepsLookAhead=*/mWorldConfig.isPipelineParallel(), + /*noScheduleUntilState=*/LlmRequestState::kCONTEXT_INIT, + /*noScheduleAfterState=*/LlmRequestState::kGENERATION_COMPLETE, + /*enablePrefixAwareScheduling=*/executorConfig.getSchedulerConfig().getEnablePrefixAwareScheduling()); + + mMicroBatchScheduler = std::make_unique<MicroBatchScheduler>(ctxChunkConfig, maxContextLength); + + if (ctxChunkConfig) + { + if (maxContextLength) + { + ctxChunkConfig.value().chunkUnitSize + = std::min(ctxChunkConfig.value().chunkUnitSize, maxContextLength.value()); + } + TLLM_CHECK_WITH_INFO(ctxChunkConfig.value().chunkUnitSize > 0, + "Context chunk size (%d) must be a positive integer.", maxContextLength.value()); + } + else + { + if (maxContextLength && maxNumTokens) + { + TLLM_CHECK_WITH_INFO(maxContextLength.value() <= maxNumTokens.value(), + "Without enabling chunked context, the max context length (%d) needs to be less than or equal to the " + "max number of tokens (%d).", + maxContextLength.value(), maxNumTokens.value()); + } + } + + mPauseRequests = std::make_unique<PauseRequests>(getMaxInputLen()); + mAssignReqSeqSlots = std::make_unique<AssignReqSeqSlots>(); + mAllocateKvCache = std::make_unique<AllocateKvCache>(); + + if (isCudaGraphMode()) + { + // Limit cuda graph cache size. Depending on the model one graph is 4-10MB of GPU memory. + SizeType32 cudaGraphCacheSize + = std::min(getMaxBatchSize(), std::max(mExtendedRuntimePerfKnobConfig.getCudaGraphCacheSize(), 1)); + // We can't have common cache for all microbatches as cuda graph is tied to the memory pointers of the runtime + // buffers. + mCudaGraphExecutorCaches.resize(mNumBuffers, utils::CudaGraphExecutorCache(cudaGraphCacheSize)); + } + + mSpeculativeDecodingFastLogits + = executorConfig.getSpecDecConfig().has_value() && executorConfig.getSpecDecConfig()->fastLogits; + if (mSpeculativeDecodingFastLogits && modelConfig.getSpeculativeDecodingMode().isNone() && mIsLeaderInOrchMode) + { + mDraftModelSendLogitsThread + = std::make_unique<std::thread>(&utils::draftModelSendLogitsThread, mDevice, &mDraftModelThreadShouldExit, + &mDraftRequestsWaitingToSendLogits, &mDraftRequestsDoneSendingLogits, &mDraftRequestsMtx); + } + + mCreateNewDecoderRequests = std::make_unique<CreateNewDecoderRequests>( + mSpeculativeDecodingFastLogits, mIsLeaderInOrchMode, isNormalizeLogProbs()); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +TrtGptModelInflightBatching::~TrtGptModelInflightBatching() +{ + if (mCacheTransceiver) + { + mCacheTransceiver->checkContextTransferStatus(1, true); + TLLM_CHECK_WITH_INFO(mCacheTransceiver->checkGenTransferComplete(), "Generation transfer not complete"); + } + if (mAsyncSendWaitThread) + { + mAsyncSendWaitThread.reset(nullptr); + } + if (mDraftModelSendLogitsThread) + { + mDraftModelThreadShouldExit = true; + mDraftModelSendLogitsThread->join(); + mDraftModelSendLogitsThread.reset(nullptr); + } +} + +void TrtGptModelInflightBatching::setupSpeculativeDecodingModule(executor::DecodingConfig const& decodingConfig) +{ + if (mModelConfig.getSpeculativeDecodingMode().isExplicitDraftTokens() + || mModelConfig.getSpeculativeDecodingMode().isEagle()) + { + TLLM_CHECK_WITH_INFO(mCtxGenFusion, "Current speculative decoding mode requires context-gen fusion IFB"); + } + + if (mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding() && decodingConfig.getLookaheadDecodingConfig()) + { + // FIXME choose defaults + auto maxLookaheadConfig = decodingConfig.getLookaheadDecodingConfig().value(); + + SizeType32 maxDraftTokens{0}; + SizeType32 maxDraftPathLen{0}; + std::tie(std::ignore, std::ignore, maxDraftTokens, maxDraftPathLen) + = maxLookaheadConfig.calculateSpeculativeResource(); + TLLM_CHECK(maxDraftTokens <= mModelConfig.getMaxDecodingDraftTokens()); + mModelConfig.getSpeculativeDecodingModulePtr()->setMaxDraftTokens(maxDraftTokens); + mModelConfig.getSpeculativeDecodingModulePtr()->setMaxDraftPathLen(maxDraftPathLen); + + auto lookaheadModulePtr + = std::dynamic_pointer_cast<runtime::LookaheadModule>(mModelConfig.getSpeculativeDecodingModulePtr()); + lookaheadModulePtr->setExecutionConfig(maxLookaheadConfig); + } +} + +void TrtGptModelInflightBatching::reshapeKvTensors(OffsetTableDimensions const& dims) +{ + TLLM_CHECK(mBuffers.size() == static_cast<size_t>(mNumBuffers)); + auto const& manager = mRuntime->getBufferManager(); + for (auto& buffers : mBuffers) + { + TLLM_CHECK(buffers->transformerBuffers); + // any method that operates on transformerBuffers must distinguish between self and cross cache, because + // transformerBuffers is not managed by KVCacheManager same rule applies to kv pool pointers below + buffers->transformerBuffers->reshapeKvTensors( + getMaxBatchSize(), mOperatingBeamWidth, dims.maxBlocksPerSeq, dims.cacheType, dims.numPools, manager); + } +} + +using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; + +std::pair<BlocksPerWindow, std::vector<SizeType32>> +TrtGptModelInflightBatching::clampWindowSizesToFitAtLeastOneSequence( + BlocksPerWindow const& blocksPerWindow, bool const failFastOnAttentionWindowTooLarge) +{ + // At this point, we can only validate that the cheapest sequence in terms of kv-cache resources still fits. More + // validation is needed on a per-request basis, once the prompt / output lengths and the actual beam width are + // known. + auto const promptLength = getMaxInputLen(); + auto const outputLength + = getMaxSequenceLen() - promptLength; // This makes it the best case scenario, as context tokens are 'cheaper' + // in terms of kv-cache resources on average. + auto const sinkTokenLength = getSinkTokenLen(); + auto const maxBeamWidth = getMaxBeamWidth(); + auto const tokensPerBlock = mModelConfig.getTokensPerBlock(); + auto const& oldMaxAttentionWindowVec = getMaxAttentionWindowVec(); + std::vector<SizeType32> newMaxAttentionWindowVec; + BlocksPerWindow newBlocksPerWindow; + + newMaxAttentionWindowVec.reserve(oldMaxAttentionWindowVec.size()); + for (auto const windowSize : oldMaxAttentionWindowVec) + { + auto const bestCaseBlockRequirements = kv_cache_manager::KVCacheManager::calculateMaxBlockRequirements( + promptLength, outputLength, sinkTokenLength, windowSize, maxBeamWidth, tokensPerBlock); + auto const [numPrimaryBlocks, numSecondaryBlocks] = blocksPerWindow.at(windowSize); + if (bestCaseBlockRequirements > numPrimaryBlocks) + { + auto const newMaxAttentionWindow = KVCacheManager::calculateMaxAttentionWindow( + promptLength, outputLength, sinkTokenLength, numPrimaryBlocks, maxBeamWidth, tokensPerBlock); + newMaxAttentionWindowVec.push_back(newMaxAttentionWindow); + newBlocksPerWindow[newMaxAttentionWindow] = std::make_tuple(numPrimaryBlocks, numSecondaryBlocks); + } + else + { + newMaxAttentionWindowVec.push_back(windowSize); + newBlocksPerWindow[windowSize] = std::make_tuple(numPrimaryBlocks, numSecondaryBlocks); + } + } + if (newMaxAttentionWindowVec == getMaxAttentionWindowVec()) + { + return {blocksPerWindow, newMaxAttentionWindowVec}; + } + TLLM_LOG_WARNING("maxAttentionWindowVec too large to fit at least one sequence in kvCache. Old: %s, New: %s", + common::vec2str(getMaxAttentionWindowVec()).c_str(), common::vec2str(newMaxAttentionWindowVec).c_str()); + + if (failFastOnAttentionWindowTooLarge) + { + throw std::runtime_error( + "Attention window too large to fit even a single sequence in the KV cache. Failing fast rather than " + "attempting an adjustment of the window sizes. " + "Old: " + + common::vec2str(getMaxAttentionWindowVec()) + ", New: " + common::vec2str(newMaxAttentionWindowVec)); + } + + setMaxAttentionWindowVec(newMaxAttentionWindowVec); + if (getMaxSequenceLen() > getMaxAttentionWindow()) + { + TLLM_LOG_WARNING("maxSequenceLen is reduced to maxAttentionWindow: %d", getMaxAttentionWindow()); + setMaxSequenceLen(getMaxAttentionWindow()); + if (getMaxInputLen() > getMaxSequenceLen() - 1) + { + setMaxInputLen(getMaxSequenceLen() - 1); + TLLM_LOG_WARNING("maxInputLen is reduced to %d", getMaxInputLen()); + } + } + // createBuffers depends on: + // maxAttentionWindow; maxAttentionWindowVec; maxSequenceLen; + // TODO: This is problematic, as createBuffers edits the state of trtGptModelInflightBatching, but + // what if there are different window values for cross+self etc. in encoder+decoder scenario... + createBuffers(mDecodingConfig, mAdditionalModelOutputs); + createDecoder(mDecodingConfig.getDecodingMode()); + return {newBlocksPerWindow, newMaxAttentionWindowVec}; +} + +std::unique_ptr<kv_cache_manager::KVCacheManager> TrtGptModelInflightBatching::createKvCacheManager( + KvCacheConfig const& kvCacheConfig, KvCacheType kvCacheType, uint64_t freePrimaryMemBytes, + uint64_t freeSecondaryMemBytes, size_t extraCostMemory, bool const failFastOnAttentionWindowTooLarge) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + bool isCrossAttention = kvCacheType == KvCacheType::kCROSS; + TLLM_CHECK_WITH_INFO( + mModelConfig.isTransformerBased(), "KvCacheManager is only needed by transformer based model."); + + auto const tokensPerBlock = mModelConfig.getTokensPerBlock(); + auto const kvDtype = mModelConfig.getKvDataType(); + + // init KV cache block manager + auto [numKvHeadsPerLayerBegin, numKvHeadsPerLayerEnd] = mModelConfig.getNumKvHeadsPerLayerLocalRange( + mWorldConfig.getPipelineParallelism(), mWorldConfig.getPipelineParallelRank(), isCrossAttention); + auto numKvHeadsPerLayer = std::vector<SizeType32>(numKvHeadsPerLayerBegin, numKvHeadsPerLayerEnd); + + auto maxAttentionWindowVec = getMaxAttentionWindowVec(); + if (kvCacheType != KvCacheType::kSELF) // TODO(nhaber): more foolproof way of initing cross-kvcache-manager + { + maxAttentionWindowVec = std::vector<SizeType32>{mModelConfig.getMaxEncoderLen()}; + } + + auto const numLayers = static_cast<SizeType32>(numKvHeadsPerLayer.size()); + auto const windowSizeToLayers = KVCacheManager::groupLayersByWindowSize(maxAttentionWindowVec, numLayers); + auto const sizePerHead = mModelConfig.getSizePerHead(); + auto blocksPerWindow = KVCacheManager::calculateMaxNumBlocks(kvCacheConfig, kvDtype, numKvHeadsPerLayer, + sizePerHead, tokensPerBlock, mWorldConfig, windowSizeToLayers, freePrimaryMemBytes, freeSecondaryMemBytes, + extraCostMemory, 2, getMaxBatchSize()); + + // now we check if any of the window sizes is too large for at least one sequence to fit in kvCache + // this can happen if e.g. maxSeqLen is deduced from the model and is too large + // and user also didn't provide maxAttentionWindow, which leads it to be equal to maxSeqLen + if (kvCacheType == KvCacheType::kSELF) + { + std::tie(blocksPerWindow, maxAttentionWindowVec) + = clampWindowSizesToFitAtLeastOneSequence(blocksPerWindow, failFastOnAttentionWindowTooLarge); + } + + if (kvCacheType == KvCacheType::kCROSS && kvCacheConfig.getEnableBlockReuse()) + { + TLLM_LOG_INFO( + "Cross KV cache does not support reuse because cross attention depends on encoder and decoder input ids. " + "Thus, KV cache reuse is disabled for cross KV cache."); + } + auto const enableBlockReuse = kvCacheType == KvCacheType::kSELF ? kvCacheConfig.getEnableBlockReuse() : false; + + auto kvCacheManager = std::make_unique<KVCacheManager>(numKvHeadsPerLayer, sizePerHead, tokensPerBlock, + blocksPerWindow, getMaxNumSequences(), getMaxBeamWidth(), maxAttentionWindowVec, kvDtype, getSinkTokenLen(), + mRuntime->getStreamPtr(), + kvCacheType == KvCacheType::kCROSS ? mModelConfig.getMaxEncoderLen() : getMaxSequenceLen(), + getMaxNumTokens().value(), enableBlockReuse, kvCacheType, kvCacheConfig.getSecondaryOffloadMinPriority(), + kvCacheConfig.getEventBufferMaxSize() > 0 + ? std::make_unique<kv_cache_manager::KVCacheEventManager>(kvCacheConfig.getEventBufferMaxSize()) + : nullptr, + kvCacheConfig.getEnablePartialReuse(), kvCacheConfig.getCopyOnPartialReuse()); + + reshapeKvTensors(kvCacheManager->getOffsetTableDimensions()); + + kvCacheManager->allocatePools(kvCacheConfig.getUseUvm()); + + TensorMap inputBuffers; + TensorPtr poolPointers = kvCacheManager->getBlockPoolPointers(); + TensorPtr poolMapping = kvCacheManager->getLayerToPoolMapping(); + + if (kvCacheType == KvCacheType::kSELF) + { + inputBuffers.insert_or_assign("host_kv_cache_pool_pointers", std::move(poolPointers)); + inputBuffers.insert_or_assign("host_kv_cache_pool_mapping", std::move(poolMapping)); + } + else + { + inputBuffers.insert_or_assign("host_cross_kv_cache_pool_pointers", std::move(poolPointers)); + inputBuffers.insert_or_assign("host_cross_kv_cache_pool_mapping", std::move(poolMapping)); + } + mRuntime->setStaticInputTensors(inputBuffers); + + // Emit the `created` event + kvCacheManager->flushIterationEvents(); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return kvCacheManager; +} + +void TrtGptModelInflightBatching::createRnnStateManager() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK_WITH_INFO(mModelConfig.isRnnBased(), "RnnStateManager is only needed by RNN based model."); + + mRnnStateManager = std::make_unique<RnnStateManager>( + getMaxNumSequences(), mModelConfig, mWorldConfig, mRuntime->getBufferManager()); + + TensorMap inputBuffers; + mRnnStateManager->getPtrBuffers(inputBuffers, mModelConfig, mWorldConfig); + mRuntime->setStaticInputTensors(inputBuffers); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::createCustomAllReduceWorkspace() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + TLLM_CHECK(mWorldConfig.isTensorParallel()); + + auto const& manager = mRuntime->getBufferManager(); + auto const hiddenSize = mModelConfig.getHiddenSize(); + + mAllReduceBuffers = std::make_unique<AllReduceBuffers>(getMaxBatchSize(), getMaxBeamWidth(), getMaxSequenceLen(), + hiddenSize, manager, mWorldConfig, mRuntime->isUserBufferEnabled()); + + TensorMap inputBuffers; + inputBuffers.insert_or_assign("all_reduce_workspace", mAllReduceBuffers->mAllReduceCommPtrs); + mRuntime->setStaticInputTensors(inputBuffers); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::createRuntimePerfKnobsTensor( + executor::ExtendedRuntimePerfKnobConfig const& extendedRuntimePerfKnobConfig) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + SizeType32 constexpr perfKnobSize{16}; + mExtendedRuntimePerfKnobsHost = BufferManager::cpu(ITensor::makeShape({perfKnobSize}), nvinfer1::DataType::kINT64); + auto* runtimePerfKnobsHostPtr = bufferCast<int64_t>(*mExtendedRuntimePerfKnobsHost); + std::fill_n(runtimePerfKnobsHostPtr, perfKnobSize, -1); + SizeType32 multiBlockModeVal = extendedRuntimePerfKnobConfig.getMultiBlockMode() ? 1 : 0; + SizeType32 enableContextFMHAFP32AccVal = extendedRuntimePerfKnobConfig.getEnableContextFMHAFP32Acc() ? 1 : 0; + runtimePerfKnobsHostPtr[0] = multiBlockModeVal; + runtimePerfKnobsHostPtr[1] = enableContextFMHAFP32AccVal; + + TensorMap inputBuffers; + inputBuffers.insert_or_assign("host_runtime_perf_knobs", mExtendedRuntimePerfKnobsHost); + mRuntime->setStaticInputTensors(inputBuffers); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::terminateRequest(LlmRequestPtr const& llmReq, bool pause) +{ + utils::terminateRequest( + *mSeqSlotManager, *llmReq, getMaxInputLen(), mKvCacheManager, mCrossKvCacheManager, mPeftCacheManager, pause); +} + +void TrtGptModelInflightBatching::terminateRequestSync( + LlmRequestPtr const& llmRequest, executor::FinishReason finishReason) +{ + TLLM_LOG_DEBUG("Registering termination for request %lu with finish reason %d", llmRequest->mRequestId, + static_cast<int>(finishReason)); + mReqIdsToTerminate.try_emplace(llmRequest->mRequestId, finishReason); +} + +TrtGptModelInflightBatching::IterationStatsIFB TrtGptModelInflightBatching::fillIterationStats( + ScheduledRequests const& scheduledRequests, RequestVector const& requestsToPause) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(fillIterationStats); + + IterationStatsIFB iterationStatsIfb{mMicroBatchId}; + iterationStatsIfb.numCtxRequests = scheduledRequests.contextRequests.size(); + iterationStatsIfb.numGenRequests = scheduledRequests.generationRequests.size(); + iterationStatsIfb.avgNumDecodedTokensPerIter = 0; + + auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); + auto const& buffers = mBuffers.at(contextBufferId); + iterationStatsIfb.numCtxTokens = buffers->getNumContextTokens(); + + for (auto const& llmReq : scheduledRequests.contextRequests) + { + iterationStatsIfb.scheduledRequests.insert(llmReq->mRequestId); + } + for (auto const& llmReq : scheduledRequests.generationRequests) + { + iterationStatsIfb.scheduledRequests.insert(llmReq->mRequestId); + iterationStatsIfb.avgNumDecodedTokensPerIter += llmReq->getAvgDecodedTokensPerIter(); + } + if (iterationStatsIfb.numGenRequests > 0) + { + iterationStatsIfb.avgNumDecodedTokensPerIter /= iterationStatsIfb.numGenRequests; + TLLM_LOG_DEBUG( + "iterationStatsIfb.avgNumDecodedTokensPerIter = %.2f", iterationStatsIfb.avgNumDecodedTokensPerIter); + } + for (auto const& llmReq : requestsToPause) + { + iterationStatsIfb.pausedRequests.insert(llmReq->mRequestId); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return iterationStatsIfb; +} + +void TrtGptModelInflightBatching::forwardSync() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE_WITH_NAME(range, "TrtGptModelInflightBatching::forwardSync"); + + TLLM_CUDA_CHECK(cudaSetDevice(mWorldConfig.getDevice())); + + if (!mWorldConfig.isLastPipelineParallelRank()) + { + mAsyncSendWaitThread->waitStop(); + } + + auto& currRequests = mMicroBatchScheduledRequests.at(mMicroBatchId); + + if (!currRequests.empty()) + { + if (!mWorldConfig.isPipelineParallel() || !mWorldConfig.isLastPipelineParallelRank()) + { + for (auto& hdl : mDecStepAsyncSndHdls) + { + TLLM_CHECK_WITH_INFO(hdl.get() == nullptr, "decoderSync handle must be nullptr."); + } + // Wait for decoding for requests in flight for the current micro batch + auto& decoderWaitEvent = mDecoderFinishedEvents.at(mMicroBatchId); + mDecStepAsyncSndHdls = decoderSync(currRequests, decoderWaitEvent); + decoderWaitEvent.reset(); + + if (!mWorldConfig.isLastPipelineParallelRank()) + { + mAsyncSendWaitThread->notifyStart(); + } + } + else + { + for (auto const& requests : {currRequests.contextRequests, currRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + for (SizeType32 beam = 0; beam < llmReq->mSamplingConfig.beamWidth; ++beam) + { + llmReq->setNumPreDecodedTokens(0, beam); + } + if (llmReq->isGenerationToCompleteState()) + { + llmReq->setState(LlmRequestState::kGENERATION_COMPLETE); + terminateRequest(llmReq); + } + } + } + } + + (*mPauseRequests)(currRequests.generationRequests, mInflightReqIds, mReqIdsToPause, true, *mSeqSlotManager, + mKvCacheManager, mCrossKvCacheManager, mPeftCacheManager); + + if (!mReqIdsToTerminate.empty()) + { + for (auto const& requests : {currRequests.contextRequests, currRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + if (mReqIdsToTerminate.count(llmReq->mRequestId) != 0U) + { + if (!llmReq->isGenerationCompleteState()) + { + TLLM_LOG_DEBUG("Terminating request %lu with finish reason %d", llmReq->mRequestId, + static_cast<int>(mReqIdsToTerminate[llmReq->mRequestId])); + terminateRequest(llmReq); + llmReq->finishByReason(mReqIdsToTerminate[llmReq->mRequestId]); + llmReq->clearGeneratedTokens(); + } + mReqIdsToTerminate.erase(llmReq->mRequestId); + } + } + } + } + + // Terminate draft requests whose logits have been sent by the background thread. + { + RequestVector doneSending; + { + std::lock_guard<std::mutex> lk(mDraftRequestsMtx); + doneSending.swap(mDraftRequestsDoneSendingLogits); + } + for (auto const& llmReq : doneSending) + { + terminateRequest(llmReq); + } + } + + // Finished context requests have been moved to generationRequests by moveFinishedContextRequestsToGeneration + for (auto const& llmReq : currRequests.generationRequests) + { + // If a context-only request is finished, send its KV cache and mark it. + if (llmReq->isContextOnlyRequest() && llmReq->isContextFinished()) + { + // TODO: skip if sending layer-wise + { + TLLM_CHECK_WITH_INFO(mCacheTransceiver, + "Disaggregated serving is not enabled, please check the configuration of " + "cacheTransceiverConfig."); + mCacheTransceiver->respondAndSendAsync(llmReq); + } + mSeqSlotManager->freeSequenceSlot(llmReq->mRequestId); + } + } + } + // report profile data + auto const bufferId = getFusedBufferId(); + auto const contextId = mBuffers[bufferId]->getContextIndex(); + if (mRuntime->hasLayerProfiler(contextId)) + { + mRuntime->reportToProfiler(contextId); + } + if (mCacheTransceiver) + { + mCacheTransceiver->checkContextTransferStatus(0, true); + } + ++mIterCounter; + + if (mKvCacheManager) + { + mKvCacheManager->flushIterationEvents(); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::storeContextBlocks(std::shared_ptr<LlmRequest> const& llmReq) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // TMJ - Note + // Make context blocks reusable immediately after context phase finishes. + // For chunked contexts, this occurs in step that processes last context chunk. + // isLastContextChunk() is always true for non-chunked contexts. + // This check is made in code that calls storeContextBlocks, so omitted here. + if (mKvCacheManager) + { + mKvCacheManager->storeContextBlocks(*llmReq); + } + if (mCrossKvCacheManager) + { + mCrossKvCacheManager->storeContextBlocks(*llmReq); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::storeNewBlock(std::shared_ptr<LlmRequest> const& llmReq) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // TMJ - Note + // Make context blocks reusable immediately after each generation step. + + if (mKvCacheManager) + { + mKvCacheManager->storeNewBlock(*llmReq); + } + if (mCrossKvCacheManager) + { + mCrossKvCacheManager->storeNewBlock(*llmReq); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::resetIterationStats() +{ + mLastIterationStatsIFB = IterationStatsIFB{mMicroBatchId}; +} + +void TrtGptModelInflightBatching::forwardAsync(RequestList const& activeRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE_WITH_NAME(range, "TrtGptModelInflightBatching::forwardAsync"); + + TLLM_CUDA_CHECK(cudaSetDevice(mWorldConfig.getDevice())); + + try + { + verifyRequests(activeRequests); + if (mModelConfig.isTransformerBased() && getKVCacheManager() && mCacheTransceiver) + { + checkDisaggGenTransferStatus(activeRequests); + } + auto& currRequests = mMicroBatchScheduledRequests.at(mMicroBatchId); + + // Get a new set of requests for that context + // The scheduler will not include any requests that are (i) still in encoder state if encoder-decoder models OR + // (ii) already in flight for decoder models + TLLM_LOG_DEBUG("Running DECODER request scheduler"); + auto [fittingRequests, fittingDisaggGenInitRequests, requestsToPause] + = (*mCapacityScheduler)(activeRequests, mKvCacheManager, mPeftCacheManager, mCrossKvCacheManager); + // Remove from fitting requests the requests that cannot be scheduled due to disagg KV cache transfer + bool waitForDisaggGenTransferProgress = false; + if (mModelConfig.isTransformerBased() && getKVCacheManager() && mCacheTransceiver) + { + if (mDisaggTransferAdmissionController && mDisaggTransferAdmissionController->enabled() + && !fittingDisaggGenInitRequests.empty()) + { + auto admissionResult + = mDisaggTransferAdmissionController->select(activeRequests, fittingDisaggGenInitRequests); + waitForDisaggGenTransferProgress = admissionResult.isBlockedByActiveTransfers(); + if (admissionResult.deferredRequestCount > 0) + { + TLLM_LOG_DEBUG( + "Disagg transfer admission deferred %zu requests; active transfer blocks=%zu, admitted " + "transfer blocks=%zu, budget=%zu", + admissionResult.deferredRequestCount, admissionResult.activeTransferBlocks, + admissionResult.admittedTransferBlocks, + mDisaggTransferAdmissionController->getMaxTransferBlocks().value_or(0)); + } + fittingDisaggGenInitRequests = std::move(admissionResult.admittedRequests); + } + prepareDisaggGenInitRequests(activeRequests, fittingDisaggGenInitRequests); + } + if (fittingRequests.empty() && fittingDisaggGenInitRequests.empty()) + { + TLLM_LOG_WARNING( + "CapacityScheduler didn't schedule any requests in iteration %lu, " + "probably because of insufficient resources such as KV cache, " + "will try wait for KV cache transfer to complete", + mIterCounter); + if (mCacheTransceiver) + { + if (waitForDisaggGenTransferProgress) + { + TLLM_LOG_DEBUG("Waiting for generation KV cache transfer progress to free disagg admission budget"); + mCacheTransceiver->checkGenTransferStatus(1); + } + else + { + mCacheTransceiver->checkContextTransferStatus(1, true); + // will free kvCache in next iteration. + } + } + } + std::tie(currRequests.contextRequests, currRequests.generationRequests) + = (*mMicroBatchScheduler)(fittingRequests, mInflightReqIds, mMaxBatchSizeRuntime, mMaxNumTokensRuntime); + TLLM_CHECK(currRequests.size() <= static_cast<size_t>(getMaxBatchSize())); + + (*mPauseRequests)(requestsToPause, mInflightReqIds, mReqIdsToPause, false, *mSeqSlotManager, mKvCacheManager, + mCrossKvCacheManager, mPeftCacheManager); + + if (mUseSeamlessLookahead) + { + changeSpecDecMode(currRequests); + } + + if (!currRequests.empty()) + { + TLLM_LOG_DEBUG("Running DECODER model with batch size: %lu", currRequests.size()); + // For overlap don't store inflight requests, so they are not skipped in scheduler + if (!isTrtOverlap()) + { + NVTX3_SCOPED_RANGE(updateInflightReqIds); + // Add requests to in-flight set, so they can be skipped in other micro batches + for (auto const& llmReq : currRequests.contextRequests) + { + // Context requests that are chunking are not added to inflight set, so they are scheduled in the + // next micro batch. + if (llmReq->isLastContextChunk()) + { + TLLM_LOG_DEBUG( + "Context request with ID %lu added to DECODER model inflight set", llmReq->mRequestId); + mInflightReqIds.insert(llmReq->mRequestId); + } + } + for (auto const& llmReq : currRequests.generationRequests) + { + TLLM_LOG_DEBUG( + "Generation request with ID %lu added to DECODER model inflight set", llmReq->mRequestId); + mInflightReqIds.insert(llmReq->mRequestId); + } + } + + (*mAssignReqSeqSlots)(*mSeqSlotManager, currRequests.contextRequests, currRequests.generationRequests); + + if (mKvCacheManager) + { + (*mAllocateKvCache)(*mKvCacheManager, currRequests.contextRequests, currRequests.generationRequests, + mModelConfig, mCrossKvCacheManager); + } + + mPeftTables.at(mMicroBatchId) + = mPeftCacheManager->ensureBatch(currRequests.contextRequests, currRequests.generationRequests, true); + + // Do decoder setup before context phase if model needs to setup buffers for the context phase. + if (mModelConfig.getSpeculativeDecodingMode().needsDecoderPrologue()) + { + auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); + setupDecoderStep(currRequests.contextRequests, *mBuffers.at(contextBufferId), + mDecoderInputBuffers.at(getFusedBufferId())); + // WAR: Sync to ensure that the decoder setup is complete before the context phase starts. + // Without this, there may be a race condition between the decoder setup and the context phase + // which also leads to spurious test failure in trtGptModelRealDecoderTest. + mRuntime->getStream().synchronize(); + } + else + { + prepareDistGenBufferAndDecoder(currRequests.generationRequests); + } + sync_check_cuda_error(mRuntime->getStream().get()); + + executeBatch(currRequests); + if (mWorldConfig.isLastPipelineParallelRank() && mGuidedDecoder) + { + // XGrammar: build maskcache for context requests and perform maskgen for all requests + // These need to be overlapped with the kernel execution of forward step + mGuidedDecoder->build(currRequests); + } + + sync_check_cuda_error(mRuntime->getStream().get()); + + // Postpone decoder setup if model does not need to setup buffers for the context phase. + if (!mModelConfig.getSpeculativeDecodingMode().needsDecoderPrologue()) + { + auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); + setupDecoderStep(currRequests.contextRequests, *mBuffers.at(contextBufferId), + mDecoderInputBuffers.at(getFusedBufferId())); + } + + sync_check_cuda_error(mRuntime->getStream().get()); + + if (isTrtOverlap()) + { + // WAR: Because the decoder is not stateless (yet) a sync is needed between + // decoder execution and next decoder step preparation. + auto const prevMicroBatchId = getPrevMicroBatchId(mMicroBatchId); + auto& prevDecoderFinishedEvent = mDecoderFinishedEvents.at(prevMicroBatchId); + if (prevDecoderFinishedEvent) + { + prevDecoderFinishedEvent->synchronize(); + } + } + + auto& decoderFinishedEvent = mDecoderFinishedEvents.at(mMicroBatchId); + TLLM_CHECK_WITH_INFO(!decoderFinishedEvent.has_value(), "decoderFinishedEvent must be nullopt."); + decoderFinishedEvent = mWorldConfig.isLastPipelineParallelRank() + ? std::make_optional(decoderStepAsync(currRequests)) + : std::nullopt; + + sync_check_cuda_error(mRuntime->getStream().get()); + + mLastIterationStatsIFB = fillIterationStats(currRequests, requestsToPause); + for (auto const& requests : {currRequests.contextRequests, currRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + if (llmReq->isContextInitState()) + { + llmReq->moveToNextContextChunk(); + if (llmReq->getContextRemainingLength() == 0) + { + TLLM_LOG_DEBUG("[RANK %d] request with ID %lu finishes decoder ctx phase", + COMM_SESSION.getRank(), llmReq->mRequestId); + + llmReq->setState(LlmRequestState::kGENERATION_IN_PROGRESS); + + // for encoder-decoder models, free encoder output buffers after decoder context phase is + // completed + if (llmReq->getEncoderTokens().has_value()) + { + llmReq->freeEncoderOutputBuffers(); + } + storeContextBlocks(llmReq); + + if (isTrtOverlap() && llmReq->willCompleteNextIteration()) + { + // This prohibits the request from being scheduled for another iteration if only one + // iteration is expected. + llmReq->setState(LlmRequestState::kGENERATION_TO_COMPLETE); + } + } + } + else if (llmReq->isGenerationInProgressState()) + { + storeNewBlock(llmReq); + TLLM_LOG_DEBUG("request with ID %lu forwards a step in decoder gen phase", llmReq->mRequestId); + } + } + } + + utils::moveFinishedContextRequestsToGeneration(currRequests); + } + else + { + mLastIterationStatsIFB = IterationStatsIFB{mMicroBatchId}; + } + + if (mWorldConfig.isPipelineParallel() && mWorldConfig.isLastPipelineParallelRank()) + { + mAsyncSendWaitThread->waitStop(); + if (!currRequests.empty()) + { + for (auto& hdl : mDecStepAsyncSndHdls) + { + TLLM_CHECK_WITH_INFO(hdl.get() == nullptr, "decoderSync handle must be nullptr."); + } + // Wait for decoding for requests in flight for the current micro batch + auto& decoderFinishedEvent = mDecoderFinishedEvents.at(mMicroBatchId); + mDecStepAsyncSndHdls = decoderSync(currRequests, decoderFinishedEvent); + decoderFinishedEvent.reset(); + + mAsyncSendWaitThread->notifyStart(); + } + } + + // Update the micro batch ID + mMicroBatchId = getNextMicroBatchId(mMicroBatchId); + } + // In case of error, we need to free the batch slot associated with those requests + catch (std::exception const&) + { + try + { + for (auto const& llmReq : activeRequests) + { + // Remove from mInflightReqIds so changeBeamWidth can proceed on the next iteration. + // terminateRequest frees seqSlot/KV cache but does not clean up mInflightReqIds. + mInflightReqIds.erase(llmReq->mRequestId); + terminateRequest(llmReq); + } + // Force buffer/decoder reset to clean up any partial state from the aborted batch + // (e.g. partially-filled cross-KV block offsets from mid-context-chunk processing). + // Guard on mInflightReqIds.empty(): in pipeline-parallel multi-micro-batch mode, + // other micro-batches may still have requests tracked here; changeBeamWidth asserts + // emptiness so we skip the reset and let the next successful forwardAsync iteration + // perform it when the set is clear. + if (mWorldConfig.isLastPipelineParallelRank() && mInflightReqIds.empty()) + { + changeBeamWidth(mOperatingBeamWidth); + } + } + catch (std::exception const& e) + { + TLLM_LOG_ERROR("forwardAsync catch-all catch block that runs `terminateRequest` has failed with:"); + TLLM_LOG_EXCEPTION(e); + TLLM_LOG_ERROR("Rethrowing *outer* exception:"); + } + throw; + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::setRuntimeBatchSize(SizeType32 runtimeMaxBatchSize) +{ + mMaxBatchSizeTunerRecommended = runtimeMaxBatchSize; + mMaxBatchSizeRuntime = std::min(getMaxBatchSize(), runtimeMaxBatchSize); +} + +SizeType32 TrtGptModelInflightBatching::getRuntimeBatchSize() const +{ + return mMaxBatchSizeRuntime; +} + +void TrtGptModelInflightBatching::setRuntimeMaxNumTokens(SizeType32 runtimeMaxNumTokens) +{ + mMaxNumTokensTunerRecommended = runtimeMaxNumTokens; + mMaxNumTokensRuntime + = (mMaxNumTokensStatic) ? std::min(mMaxNumTokensStatic.value(), runtimeMaxNumTokens) : runtimeMaxNumTokens; +} + +void TrtGptModelInflightBatching::updatePeftCache(std::shared_ptr<LlmRequest> const& llmRequest) +{ + mPeftCacheManager->addRequestPeft(llmRequest, true); +} + +runtime::BufferManager const& TrtGptModelInflightBatching::getBufferManager() const +{ + return mRuntime->getBufferManager(); +} + +BufferManager::CudaStreamPtr TrtGptModelInflightBatching::getRuntimeStreamPtr() const +{ + return mRuntime->getStreamPtr(); +} + +void TrtGptModelInflightBatching::executeContext(SizeType32 runtimeContextId, SizeType32 bufferId) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(executeContext); + + auto const& currBatchState = mBuffers[bufferId]->getBatchState(); + + bool hasCudaGraph = false; + // If batch state is context only, do not capture/launch graph and execute the engine as is. + if (isCudaGraphMode() && !currBatchState.isAnyContext()) + { + auto cudaGraphOpt = mCudaGraphExecutorCaches[bufferId].get(currBatchState); + // If graph exists for current batch state, launch it. + if (cudaGraphOpt.has_value()) + { + hasCudaGraph = true; + } + } + + // If there is no graph for current state, execute the engine. + if (!hasCudaGraph) + { + auto enqueueSuccessful = mRuntime->executeContext(runtimeContextId); + if (!enqueueSuccessful) + { + throw std::runtime_error("Executing TRT engine failed!"); + } + } + else + { + // Launch graph. + auto cudaGraphOpt = mCudaGraphExecutorCaches[bufferId].get(currBatchState); + cudaGraphOpt.value()->launch(mRuntime->getStream()); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::setLayerProfiler() +{ + mRuntime->setLayerProfiler(); +} + +std::string TrtGptModelInflightBatching::getLayerProfileInfo() const +{ + return mRuntime->getLayerProfileInfo(); +} + +void TrtGptModelInflightBatching::verifyRequests(RequestList const& activeRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(verifyRequests); + + if (activeRequests.empty()) + { + return; + } + + auto const& firstRequest = activeRequests.front(); + auto const firstRequestId = firstRequest->mRequestId; + auto const firstBeamWidth = firstRequest->mSamplingConfig.beamWidth; + + for (auto const& llmReq : activeRequests) + { + auto const beamWidth = llmReq->mSamplingConfig.beamWidth; + auto const draftLength = llmReq->getNumDraftTokens(); + auto const maxDraftLength = mModelConfig.getMaxDecodingDraftTokens(); + + TLLM_CHECK_WITH_INFO(beamWidth == 1 || draftLength == 0, "Can't use speculative decoding with beam search."); + TLLM_CHECK_WITH_INFO(draftLength <= maxDraftLength, + "Number of draft tokens (%d) is larger than maximum number of draft tokens (%d)", draftLength, + maxDraftLength); + + // FIXME: Remove this check when varying beam width is supported + { + TLLM_CHECK_WITH_INFO(beamWidth == firstBeamWidth, + "All active requests must have same beam width, " + "but request %lu with beam width %d differs from first request %lu with beam width %d", + llmReq->mRequestId, beamWidth, firstRequestId, firstBeamWidth); + } + } + + if (firstBeamWidth != mOperatingBeamWidth) + { + changeBeamWidth(firstBeamWidth); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::executeBatch(ScheduledRequests const& scheduledRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(executeBatch); + + if (!mCtxGenFusion) + { + if (!scheduledRequests.contextRequests.empty()) + { + auto const bufferId = getContextBufferId(); + executeStep(scheduledRequests.contextRequests, {}, bufferId); + } + if (!scheduledRequests.generationRequests.empty()) + { + auto const bufferId = getGenerationBufferId(); + executeStep({}, scheduledRequests.generationRequests, bufferId); + } + } + else + { + auto const bufferId = getFusedBufferId(); + executeStep(scheduledRequests.contextRequests, scheduledRequests.generationRequests, bufferId); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::createRuntimeContexts() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + mRuntime->clearContexts(); + auto const numProfiles = mRuntime->getNbProfiles(); + for (auto i = 0; i < numProfiles; ++i) + { + mRuntime->addContext(i); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +namespace +{ +// TODO: move this somewhere else? +/** + * This function logic is also implemented in tensorrt_llm/_torch/pyexecutor/_util.py get_decoding_mode(). + */ +executor::DecodingMode getDecodingMode(SpeculativeDecodingMode specDecodingMode, + std::optional<executor::DecodingMode> const& decodingModeOpt, runtime::SizeType32 const beamWidth) +{ + auto getDefaultDecodingMode = [beamWidth](std::optional<executor::DecodingMode> const& decodingModeOpt) + { + if (decodingModeOpt.has_value() && !decodingModeOpt->isAuto()) + { + return decodingModeOpt.value(); + } + return (beamWidth == 1) ? executor::DecodingMode::TopKTopP() : executor::DecodingMode::BeamSearch(); + }; + + auto decodingMode = getDefaultDecodingMode(decodingModeOpt); + // Variable-Beam-Width-Search (special mode of Beam-Search) is enabled. + if (decodingMode.isBeamSearch() && decodingMode.isUseVariableBeamWidthSearch()) + { + TLLM_LOG_INFO("Variable-Beam-Width-Search is enabled"); + } + // Overwrite decoding mode when beam width is one. + if (beamWidth == 1 && decodingMode.isBeamSearch()) + { + TLLM_LOG_WARNING( + "Beam width is set to 1, but decoding mode is BeamSearch. Overwriting decoding mode to TopKTopP."); + decodingMode = executor::DecodingMode::TopKTopP(); + } + // Overwrite decoding mode when Medusa is used. + if (specDecodingMode.isMedusa() && !decodingMode.isMedusa()) + { + TLLM_LOG_WARNING("Model is Medusa, but decoding mode is not Medusa. Overwriting decoding mode to Medusa."); + decodingMode = executor::DecodingMode::Medusa(); + } + // Overwrite decoding mode when Medusa is not used. + if (!specDecodingMode.isMedusa() && decodingMode.isMedusa()) + { + TLLM_LOG_WARNING("Model is not Medusa, but decoding mode is Medusa. Overwriting decoding mode."); + decodingMode = getDefaultDecodingMode(decodingModeOpt); + } + // Overwrite decoding mode when lookahead decoding is used. + if (specDecodingMode.isLookaheadDecoding() && !decodingMode.isLookahead()) + { + TLLM_LOG_WARNING( + "Model is Lookahead, but decoding mode is not Lookahead. Overwriting decoding mode to Lookahead."); + decodingMode = executor::DecodingMode::Lookahead(); + } + // Overwrite decoding mode when lookahead decoding is not used. + if (!specDecodingMode.isLookaheadDecoding() && decodingMode.isLookahead()) + { + TLLM_LOG_WARNING( + "Model is not built with Lookahead decoding, but decoding mode is Lookahead. Overwriting decoding " + "mode."); + decodingMode = getDefaultDecodingMode(decodingModeOpt); + } + // Overwrite decoding mode when 'explicit draft tokens' is used. + if (specDecodingMode.isExplicitDraftTokens() && !decodingMode.isExplicitDraftTokens()) + { + TLLM_LOG_WARNING( + "Model is built with 'explicit draft tokens' decoding, but decoding mode is something else. Overwriting " + "decoding mode."); + decodingMode = executor::DecodingMode::ExplicitDraftTokens(); + } + // Overwrite decoding mode when 'explicit draft tokens' is not used. + if (!specDecodingMode.isExplicitDraftTokens() && decodingMode.isExplicitDraftTokens()) + { + TLLM_LOG_WARNING( + "Model is not built with 'explicit draft tokens' decoding, but decoding mode is set to it. Overwriting " + "decoding " + "mode to default."); + decodingMode = getDefaultDecodingMode(decodingModeOpt); + } + // Overwrite decoding mode when EAGLE is used. + if (specDecodingMode.isEagle() && !decodingMode.isEagle()) + { + TLLM_LOG_WARNING("Model is Eagle, but decoding mode is not Eagle. Overwriting decoding mode to Eagle."); + decodingMode = executor::DecodingMode::Eagle(); + } + // Overwrite decoding mode when Eagle is not used. + if (!specDecodingMode.isEagle() && decodingMode.isEagle()) + { + TLLM_LOG_WARNING("Model is not Eagle, but decoding mode is Eagle. Overwriting decoding mode."); + decodingMode = getDefaultDecodingMode(decodingModeOpt); + } + if (specDecodingMode.isDraftTokensExternal()) + { + TLLM_LOG_WARNING("Overwriting decoding mode to external draft token"); + decodingMode = executor::DecodingMode::ExternalDraftTokens(); + } + TLLM_LOG_DEBUG("DecodingMode: %s", decodingMode.getName()); + return decodingMode; +} +} // namespace + +void TrtGptModelInflightBatching::createDecoder(std::optional<executor::DecodingMode> const& decodingModeOpt) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + mDecoderState = std::make_unique<runtime::decoder::DecoderState>(); + + if (mWorldConfig.isLastPipelineParallelRank()) + { + auto decoderType = mRuntime->getEngine().getTensorDataType("logits"); + + auto const decodingMode + = getDecodingMode(mModelConfig.getSpeculativeDecodingMode(), decodingModeOpt, mOperatingBeamWidth); + + if (decodingMode.isExplicitDraftTokens()) + { + // There are no logits in Explicit draft tokens model. + decoderType = mModelConfig.getDataType(); + // Decoder is not instantiated for bf16. We use half to get the same data size + // and explicitly pass dtype to redrafter that has bf16 kernels. + if (decoderType == nvinfer1::DataType::kBF16) + { + decoderType = nvinfer1::DataType::kHALF; + } + } + + mDecoder = std::make_unique<runtime::GptDecoderBatched>(mRuntime->getStreamPtr()); + mDecoder->setup( + decodingMode, getMaxNumSequences(), mOperatingBeamWidth, decoderType, mModelConfig, mWorldConfig); + + mDecoderState->setup(getMaxNumSequences(), mOperatingBeamWidth, getMaxAttentionWindow(), getSinkTokenLen(), + getMaxSequenceLen(), decoderType, mModelConfig, mWorldConfig, mRuntime->getBufferManager()); + + if (!mModelConfig.getSpeculativeDecodingMode().isNone()) + { + mDecoderState->setupSpeculativeDecoding(mModelConfig.getSpeculativeDecodingMode(), + mModelConfig.getMaxDecodingTokens(), decoderType, mModelConfig, mWorldConfig, + mRuntime->getBufferManager()); + } + } + else + { + mDecoderState->setupCacheIndirection( + getMaxNumSequences(), mOperatingBeamWidth, getMaxAttentionWindow(), mRuntime->getBufferManager()); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::createBuffers(executor::DecodingConfig const& decodingConfig, + std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + mBuffers.clear(); + for (SizeType32 i = 0; i < mNumBuffers; ++i) + { + mBuffers.emplace_back( + std::make_unique<RuntimeBuffers>(getMaxBatchSize(), mOperatingBeamWidth, getMaxAttentionWindowVec(), + getMaxAttentionWindow(), getSinkTokenLen(), *mRuntime, mModelConfig, mWorldConfig, decodingConfig, + getGatherGenerationLogits(), getMaxNumTokens(), additionalModelOutputs, mPromptTableOffloading)); + } + + mDecoderInputBuffers.clear(); + mDecoderOutputBuffers.clear(); + for (SizeType32 i = 0; i < mNumMicroBatches; ++i) + { + mDecoderInputBuffers.emplace_back( + getMaxBatchSize(), mModelConfig.getMaxDecodingTokens(), mRuntime->getBufferManager()); + mDecoderInputBuffers.back().setupMedusaLogits(getMaxNumSequences(), mModelConfig); + mDecoderOutputBuffers.emplace_back(getMaxNumSequences(), mOperatingBeamWidth, getMaxSequenceLen(), + mModelConfig.getMaxDecodingTokens(), mRuntime->getBufferManager()); + mDecoderOutputBuffers.back().setupSpeculativeDecoding( + getMaxNumSequences(), mModelConfig.getMaxDecodingTokens(), mModelConfig); + } + + mSlotDecoderBuffers.clear(); + for (SizeType32 i = 0; i < getMaxNumSequences(); ++i) + { + mSlotDecoderBuffers.emplace_back(std::make_unique<SlotDecoderBuffers>( + mOperatingBeamWidth, getMaxSequenceLen(), mRuntime->getBufferManager())); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::prepareDisaggGenInitRequests( + RequestList const& activeRequests, RequestVector& newGenReqs) +{ + NVTX3_SCOPED_RANGE(prepareDisaggGenInitRequests); + + // Allocate KV cache by treating them as context requests + (*mAllocateKvCache)(*mKvCacheManager, newGenReqs, {}, mModelConfig, mCrossKvCacheManager); + + // Initiate KV cache transfer + auto timeStart = std::chrono::steady_clock::now(); + + if (tc::getEnvDisaggBenchmarkGenOnly()) + { + TLLM_LOG_DEBUG("Disaggregated generation only benchmark mode is enabled"); + for (auto& req : newGenReqs) + { + req->setState(LlmRequestState::kDISAGG_GENERATION_TRANS_COMPLETE); + } + return; + } + + auto const genInitReqNum = std::count_if(activeRequests.begin(), activeRequests.end(), + [](auto const& req) { return req->isDisaggGenerationInitState(); }); + + // Loop over the new disagg gen requests and trigger receive of KV cache + for (auto& newGenReq : newGenReqs) + { + TLLM_CHECK_WITH_INFO( + mCacheTransceiver, "Disaggregated serving is not enabled, please check the configuration."); + if (common::getEnvDisableKVCacheTransferOverlap()) + { + mCacheTransceiver->requestAndReceiveSync(newGenReq); + } + else + { + mCacheTransceiver->requestAndReceiveAsync(newGenReq); + } + } + if (!common::getEnvDisableKVCacheTransferOverlap()) + { + auto const blockTransfer = std::all_of(activeRequests.begin(), activeRequests.end(), + [](auto const& req) { return req->isDisaggGenerationTransmissionInProgress(); }); + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), + "newGenReqs.size():%ld requests, activeRequests.size():%ld allTransferInProgress:%d original " + "gen_only_requests_num:%ld", + newGenReqs.size(), activeRequests.size(), blockTransfer, genInitReqNum); + mCacheTransceiver->checkGenTransferStatus(0); + auto timeEnd = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration<float, std::milli>(timeEnd - timeStart).count(); + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), + "receiveDisaggGenCache time:%f ms, " + "blockTransfer:%d,genInitReqNum:%ld,newGenReqs.size():%ld,activeRequests.size():%ld", + duration, blockTransfer, genInitReqNum, newGenReqs.size(), activeRequests.size()); + } + + return; +} + +void TrtGptModelInflightBatching::checkDisaggGenTransferStatus(RequestList const& activeRequests) +{ + NVTX3_SCOPED_RANGE(checkDisaggGenTransferStatus); + + if (common::getEnvDisableKVCacheTransferOverlap()) + { + return; + } + + auto timeStart = std::chrono::steady_clock::now(); + + // TODO: + auto const needCheck = std::any_of(activeRequests.begin(), activeRequests.end(), + [](auto const& req) { return req->isDisaggGenerationTransmissionInProgress(); }); + + if (needCheck) + { + mCacheTransceiver->checkGenTransferStatus(0); + + auto timeEnd = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration<float, std::milli>(timeEnd - timeStart).count(); + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), + "no Prepare checkDisaggGenTransferStatus time:%f ms, " + "needCheck:%d,activeRequests.size():%ld", + duration, needCheck, activeRequests.size()); + } +} + +void TrtGptModelInflightBatching::prepareDistGenBufferAndDecoder(RequestVector const& generationRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // set decoderStep for disagg_generation + RequestVector cacheTransCompleteRequests; + for (auto const& request : generationRequests) + { + if (request->isDisaggGenerationTransmissionComplete()) + { + cacheTransCompleteRequests.push_back((request)); + } + } + if (!cacheTransCompleteRequests.empty()) + { + auto timeStart = std::chrono::steady_clock::now(); + auto const bufferId = getFusedBufferId(); + auto& runtimeBuffers = *mBuffers[bufferId]; + runtimeBuffers.prepareStep(cacheTransCompleteRequests, {}, getMaxBeamWidth(), getMaxAttentionWindow(), + *mDecoderState, mKvCacheManager.get(), mCrossKvCacheManager.get(), mRnnStateManager.get(), + mPeftTables[mMicroBatchId], *mRuntime, mModelConfig, mWorldConfig, getGatherGenerationLogits(), + isTrtOverlap()); + auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); + setupDecoderStep( + cacheTransCompleteRequests, *mBuffers.at(contextBufferId), mDecoderInputBuffers.at(getFusedBufferId())); + sync_check_cuda_error(mRuntime->getStream().get()); + auto timeEnd = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration<float, std::milli>(timeEnd - timeStart).count(); + TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(), + "prepareDistGenBufferAndDecoder time:%f ms , cacheTransCompleteRequests.size():%ld", duration, + cacheTransCompleteRequests.size()); + } + for (auto& request : cacheTransCompleteRequests) + { + request->setState(LlmRequestState::kGENERATION_IN_PROGRESS); + request->setContextCurrentPosition(request->mPromptLen); + request->setDecodingIter(1); + auto const reqBeamWidth = request->mSamplingConfig.beamWidth; + auto firstGenTokens = request->getContextPhaseParams().value().getFirstGenTokens(); + for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) + { + request->addNewToken(firstGenTokens.at(beam), beam); + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::debugIOTensors(RequestVector const& contextRequests, + RequestVector const& generationRequests, TensorMap const& inputMap, TensorMap const& outputMap) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + TLLM_CHECK(mDebugConfig); + + auto const& manager = mRuntime->getBufferManager(); + auto requestIds = utils::collectRequestIds(contextRequests, generationRequests); + + if (mDebugConfig->getDebugTensorsMaxIterations() > 0) + { + mLastIterationDebugTensors.clear(); + mLastIterationDebugTensors = utils::storeIOTensors(*mDebugConfig, requestIds, inputMap, outputMap, manager); + } + else + { + utils::dumpIOTensors(*mDebugConfig, mIterCounter, requestIds, inputMap, outputMap, mWorldConfig, manager); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +std::tuple<SizeType32, runtime::StringPtrMap<runtime::ITensor> const&, runtime::StringPtrMap<runtime::ITensor>&> +TrtGptModelInflightBatching::prepareBuffers( + RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(prepareBuffers); + + auto& runtimeBuffers = *mBuffers.at(bufferId); + + auto allNewTokens = mWorldConfig.isLastPipelineParallelRank() + ? RuntimeBuffers::OptionalRef<runtime::ITensor const>(mDecoderState->getAllNewTokens()) + : std::nullopt; + + auto [optProfileId, inputMap, outputMap] = runtimeBuffers.prepareStep(contextRequests, generationRequests, + mOperatingBeamWidth, getMaxAttentionWindow(), *mDecoderState, mKvCacheManager.get(), mCrossKvCacheManager.get(), + mRnnStateManager.get(), mPeftTables[bufferId], *mRuntime, mModelConfig, mWorldConfig, + getGatherGenerationLogits(), isTrtOverlap(), allNewTokens); + + // For Variable-Beam-Width-Search + mRuntime->setCurrentBeamWidths( + tensorrt_llm::batch_manager::utils::getRequestBeamWidths(contextRequests, generationRequests)); + + mRuntime->setInputTensors(optProfileId, inputMap); + mRuntime->setOutputTensors(optProfileId, outputMap); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return {optProfileId, inputMap, outputMap}; +} + +void TrtGptModelInflightBatching::prepareGraph(SizeType32 bufferId, SizeType32 optProfileId) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(prepareGraph); + + auto const nextBatchState = mBuffers[bufferId]->getBatchState(); + auto cudaGraphOpt = mCudaGraphExecutorCaches[bufferId].get(nextBatchState); + // If graph is not found in the cache, capture it. + if (!cudaGraphOpt.has_value()) + { + // We need to prepare some tensors once again to properly set values for graph capture. + // Graph capture requires setting some tensors (e.g. past_kv_len) + // to the round_up(max_kv_cache_len, kKV_CACHE_LEN_CUDA_GRAPH_ROUND_SIZE) + // in order to capture the kernels with the large enough grid. + mBuffers[bufferId]->prepareBuffersForCudaGraph(getMaxSequenceLen()); + + auto cudaGraph = std::make_shared<utils::CudaGraphExecutor>(); + cudaGraph->prepareNextGraph(mRuntime, optProfileId); + mCudaGraphExecutorCaches[bufferId].put(nextBatchState, cudaGraph); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::executeStep( + RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE_WITH_NAME(range, + "executeStep: " + std::to_string(contextRequests.size()) + " ctx reqs, " + + std::to_string(generationRequests.size()) + " gen reqs"); + + if (mPromptTableOffloading) + { + prefetchNextPromptTableChunk(contextRequests, /* isFirstChunk */ true, bufferId); + } + + auto [optProfileId, inputMap, outputMap] = prepareBuffers(contextRequests, generationRequests, bufferId); + + if (mBuffers[bufferId]->transformerBuffers) + { + // Creation of context progress, or remains nullptr if not needed + std::shared_ptr<ContextProgress> progress = nullptr; + RequestVector layerWiseRequests; + if (common::getEnvDisaggLayerwise()) + { + for (auto const& request : contextRequests) + { + bool const enableLayerWise = request->isContextOnlyRequest() && request->isLastContextChunk(); + if (enableLayerWise) + { + layerWiseRequests.push_back(request); + } + } + } + // TODO: support layer-wise cross kv cache in encoder-decoder models + if (!layerWiseRequests.empty() && !mModelConfig.useCrossAttention()) + { + int const numLayers = mModelConfig.getNbAttentionLayers( + mWorldConfig.getPipelineParallelism(), mWorldConfig.getPipelineParallelRank()); + progress = std::make_shared<ContextProgress>(numLayers); + } + bufferCast<void*>(*mBuffers[bufferId]->transformerBuffers->contextProgressHost)[0] = progress.get(); + if (progress) + { + TLLM_CHECK_WITH_INFO(mCacheTransceiver, + "Disaggregated serving is not enabled, please check the configuration of cacheTransceiverConfig."); + mCacheTransceiver->respondAndSendLayerWise(layerWiseRequests, progress); + } + } + + if (mPromptTableOffloading) + { + prefetchNextPromptTableChunk(contextRequests, /* isFirstChunk */ false, bufferId); + } + + executeContext(optProfileId, bufferId); + + // If batch state has any context request, do not capture this graph. + if (isCudaGraphMode() && contextRequests.empty()) + { + // Capture graph of current batch state during engine execution. + // This is based on the assumptions that + // a) We can hide CPU graph capture behind the GPU engine execution. + // b) Batch size in the next iterations won't change and we can reuse the graph multiple times. + prepareGraph(bufferId, optProfileId); + } + + if (mDebugConfig) + { + debugIOTensors(contextRequests, generationRequests, inputMap, outputMap); + } + + if (mAdditionalModelOutputs.has_value() && !mAdditionalModelOutputs.value().empty()) + { + utils::copyAdditionalOutputs( + mAdditionalModelOutputs.value(), contextRequests, generationRequests, outputMap, getBufferManager()); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::setupDecoderStep( + RequestVector const& contextRequests, RuntimeBuffers const& buffers, DecoderInputBuffers& inputBuffers) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(setupDecoderStep); + + if (mWorldConfig.isLastPipelineParallelRank() && !contextRequests.empty()) + { + auto const logitsType = mRuntime->getEngine().getTensorDataType("logits"); + + auto [batchSlots, samplingConfigs, lookaheadPrompt, lookaheadAlgoConfigs] + = (*mCreateNewDecoderRequests)(mModelConfig, mWorldConfig, mDecodingConfig, contextRequests, logitsType, + inputBuffers, *mDecoderState, mRuntime->getStream(), *mDecoder->getDecoderStream(), getMaxSequenceLen(), + mOperatingBeamWidth, buffers.mMedusaBuffers); + + auto const localBatchSize = batchSlots->getSize(); + if (localBatchSize > 0) + { + auto samplingConfig = SamplingConfig(samplingConfigs); + mDecoder->getUnderlyingDecoder().setup(samplingConfig, localBatchSize, batchSlots, + {mDecoderState->getJointDecodingOutput()}, mModelConfig.getDataType(), lookaheadPrompt, + lookaheadAlgoConfigs); + + auto const& stream = mDecoder->getDecoderStream(); + CudaEvent event{}; + stream->record(event); + mRuntime->getStreamPtr()->wait(event); + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::postProcessRequest( + LlmRequest& llmReq, std::vector<SizeType32> const& numDroppedTokens) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + auto const seqSlot = llmReq.mSeqSlot.value(); + auto const reqBeamWidth = llmReq.getBeamWidthByIter(true); + auto const& bufferManager = getBufferManager(); + + if (llmReq.getReturnGenerationLogits() && !llmReq.getGenerationLogitsFragments().empty()) + { + TLLM_CHECK(!llmReq.isStreaming()); + auto const genBufferId = mCtxGenFusion ? getFusedBufferId() : getGenerationBufferId(); + auto& genRuntimeBuffers = *mBuffers.at(genBufferId); + + auto constexpr beforeDecoder = false; + utils::copyGenerationLogits( + genRuntimeBuffers.generationLogitsCache, bufferManager, llmReq, beforeDecoder, numDroppedTokens); + + bufferManager.getStream().synchronize(); + } + + if (reqBeamWidth == 1) + { + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return; + } + + // Update mDecoderBuffers->slotOutputIdsHost and synchronize + getDecoderSlotHostOutputs(seqSlot, llmReq.returnLogProbs(), llmReq.mSamplingConfig, llmReq.isStreaming()); + + auto const* outputIdsHostData = bufferCast<TokenIdType>(*mSlotDecoderBuffers[seqSlot]->outputIdsHost); + auto const* sequenceLengthsHostData = bufferCast<SizeType32>(*mSlotDecoderBuffers[seqSlot]->sequenceLengthsHost); + auto const* cumLogProbsHostData = bufferCast<float>(*mSlotDecoderBuffers[seqSlot]->cumLogProbsHost); + auto logProbsHost = mSlotDecoderBuffers[seqSlot]->logProbsHost; + auto const* logProbsHostData = bufferCast<float>(*logProbsHost); + + auto const& outputIdsShape = mSlotDecoderBuffers[seqSlot]->outputIdsHost->getShape(); + auto const maxSeqLength = outputIdsShape.d[1]; + + std::vector<std::vector<TokenIdType>> generatedTokens(reqBeamWidth); + for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) + { + auto const* const begin = outputIdsHostData + tc::flat_index2(beam, llmReq.mPromptLen, maxSeqLength); + auto const generatedLength = sequenceLengthsHostData[beam] - llmReq.mPromptLen; + auto const* const end = begin + generatedLength; + generatedTokens[beam].assign(begin, end); + + if (llmReq.returnLogProbs()) + { + llmReq.setCumLogProb(cumLogProbsHostData[beam], beam); + + auto const beginLogProbsOffset = reqBeamWidth == 1 ? llmReq.mPromptLen : 0; + auto const* const begin = logProbsHostData + beam * logProbsHost->getShape().d[1] + beginLogProbsOffset; + auto const* const end = begin + generatedLength; + LlmRequest::VecLogProbs logProbs(begin, end); + llmReq.setLogProbs(logProbs, beam); + } + } + + // store the generated tokens into the mTokensGathered buffer + llmReq.setGeneratedTokens(generatedTokens); + + if (llmReq.getReturnGenerationLogits() && llmReq.getGenerationLogitsHost() + && mWorldConfig.isLastPipelineParallelRank()) + { + reorderGenerationLogitsForBeamSearch( + llmReq, seqSlot, reqBeamWidth, maxSeqLength, outputIdsHostData, sequenceLengthsHostData); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::reorderGenerationLogitsForBeamSearch(LlmRequest& llmReq, SizeType32 seqSlot, + SizeType32 reqBeamWidth, SizeType32 maxSeqLength, TokenIdType const* outputIdsHostData, + SizeType32 const* sequenceLengthsHostData) +{ + // Reorder generation logits to match the gathered (finalized) beam ordering. + // During generation, logits are stored indexed by beam SLOT position. After beam search + // finalization (gatherTree), output_ids are reordered by tracing parentIds to reconstruct + // the correct beam paths. However, generation_logits are NOT reordered by gatherTree. + // We fix this here by tracing parentIds on the host to build the beam-slot mapping, + // then reindexing the logits accordingly. + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const promptLen = llmReq.mPromptLen; + + // Copy parentIds and ids (ungathered step IDs) from GPU to temporary host buffers. + // parentIds[slot][t] = the parent slot of beam slot `slot` at position t. + // ids[slot][t] = the token in beam slot `slot` at position t (before gather). + auto parentIdsDevice = ITensor::at(mDecoderState->getParentIds(), {seqSlot}); + auto idsDevice = mDecoderState->getIds(seqSlot); + + auto parentIdsHost = runtime::BufferManager::pinnedPool(parentIdsDevice->getShape(), nvinfer1::DataType::kINT32); + auto idsHost = runtime::BufferManager::pinnedPool(idsDevice->getShape(), nvinfer1::DataType::kINT32); + + mCopyBufferManager.copy(*parentIdsDevice, *parentIdsHost); + mCopyBufferManager.copy(*idsDevice, *idsHost); + mCopyBufferManager.getStream().synchronize(); + + auto const* parentIdsData = bufferCast<TokenIdType>(*parentIdsHost); + auto const* idsData = bufferCast<TokenIdType>(*idsHost); + + // For each final beam b, find the beam slot at the last generated step, then + // trace back through parentIds to build the slot trace for every generation step. + // slotTrace[beam][genStep] = the beam slot that produced the logits at that step. + auto const generationLogitsHost = llmReq.getGenerationLogitsHost(); + auto const& logitsShape = generationLogitsHost->getShape(); + // Non-streaming shape: [beamWidth, maxNewTokens, vocabSizePadded] + TLLM_CHECK_WITH_INFO(logitsShape.d[0] == reqBeamWidth, + "Generation logits beam dimension (%ld) does not match beam width (%d).", logitsShape.d[0], reqBeamWidth); + auto const maxNewTokens = logitsShape.d[1]; + auto const vocabSizePadded = logitsShape.d[2]; + + std::vector<std::vector<SizeType32>> slotTrace(reqBeamWidth, std::vector<SizeType32>(maxNewTokens, 0)); + bool anyReorderNeeded = false; + + for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) + { + auto const seqLen = sequenceLengthsHostData[beam]; + auto const genLen = seqLen - promptLen; + if (genLen <= 0) + { + continue; + } + + // Find the starting beam slot at the last generated step by matching the + // backtracked token sequence against the gathered (finalized) output. + SizeType32 startSlot = -1; + for (SizeType32 s = 0; s < reqBeamWidth; ++s) + { + SizeType32 slot = s; + bool matches = true; + for (SizeType32 t = seqLen - 1; t >= promptLen; --t) + { + if (idsData[slot * maxSeqLength + t] != outputIdsHostData[beam * maxSeqLength + t]) + { + matches = false; + break; + } + if (t > promptLen) + { + slot = parentIdsData[slot * maxSeqLength + t]; + } + } + if (matches) + { + startSlot = s; + break; + } + } + + TLLM_CHECK_WITH_INFO(startSlot >= 0, + "Could not determine beam slot mapping for beam %d during generation logits reordering.", beam); + + // Build the slot trace: slotTrace[beam][g] = the pre-reassignment slot whose + // logits correspond to generation step g of this beam. + // + // The model runs BEFORE beam search reassigns beams to slots, so + // generationLogits[slot][g] was produced by the pre-reassignment slot — + // i.e. the slot the beam occupied in the *previous* step. + // parentIds[postSlot][promptLen+g] gives exactly that pre-reassignment slot, + // so taking the parentIds lookup before storing (rather than after) yields + // the correct source slot in a single pass. + SizeType32 slot = startSlot; + for (SizeType32 t = seqLen - 1; t >= promptLen; --t) + { + slot = parentIdsData[slot * maxSeqLength + t]; + slotTrace[beam][t - promptLen] = slot; + } + + // Check if any reordering is actually needed for this beam + auto& slotTraceIds = slotTrace[beam]; + anyReorderNeeded |= std::any_of( + slotTraceIds.begin(), slotTraceIds.begin() + genLen, [beam](SizeType32 s) { return s != beam; }); + } + + // Reorder the generation logits in-place using a per-step temporary buffer. + if (anyReorderNeeded) + { + auto const logitsDataType = generationLogitsHost->getDataType(); + auto const elemSize = runtime::BufferDataType(logitsDataType).getSize(); + auto const stepSize = static_cast<size_t>(vocabSizePadded) * elemSize; + + // Temp buffer for one generation step across all beams: [beamWidth, vocabSizePadded] + auto tempLogits + = runtime::BufferManager::pinnedPool(ITensor::makeShape({reqBeamWidth, vocabSizePadded}), logitsDataType); + + auto* logitsPtr = static_cast<uint8_t*>(generationLogitsHost->data()); + auto* tempPtr = static_cast<uint8_t*>(tempLogits->data()); + + std::vector<SizeType32> genLens(reqBeamWidth); + SizeType32 maxGenLen = 0; + for (SizeType32 b = 0; b < reqBeamWidth; ++b) + { + genLens[b] = std::max(SizeType32{0}, sequenceLengthsHostData[b] - promptLen); + maxGenLen = std::max(maxGenLen, genLens[b]); + } + + for (SizeType32 g = 0; g < maxGenLen; ++g) + { + // Check if any beam that generated this step needs reordering + bool stepNeedsReorder = false; + for (SizeType32 b = 0; b < reqBeamWidth; ++b) + { + if (g < genLens[b] && slotTrace[b][g] != b) + { + stepNeedsReorder = true; + break; + } + } + if (!stepNeedsReorder) + { + continue; + } + + // Copy all beams' logits at this step to the temp buffer + for (SizeType32 b = 0; b < reqBeamWidth; ++b) + { + // logits layout: [beamWidth, maxNewTokens, vocabSizePadded] + auto const offset = (static_cast<size_t>(b) * maxNewTokens + g) * stepSize; + std::memcpy(tempPtr + static_cast<size_t>(b) * stepSize, logitsPtr + offset, stepSize); + } + + // Reorder: logits[b][g] = temp[slotTrace[b][g]] + for (SizeType32 b = 0; b < reqBeamWidth; ++b) + { + if (g >= genLens[b]) + { + continue; + } + auto const dstOffset = (static_cast<size_t>(b) * maxNewTokens + g) * stepSize; + auto const srcSlot = slotTrace[b][g]; + std::memcpy(logitsPtr + dstOffset, tempPtr + static_cast<size_t>(srcSlot) * stepSize, stepSize); + } + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::getDecoderSlotHostOutputs( + SizeType32 seqSlot, bool returnLogProbs, SamplingConfig const& samplingConfig, bool streaming) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + if (mWorldConfig.isLastPipelineParallelRank()) + { + auto event = mDecoder->finalize(*mDecoderState, seqSlot, samplingConfig, streaming); + // Make sure that postprocessing is done before copying outputIds + mCopyBufferManager.getStream().wait(event.get()); + + auto sequenceLengths = mDecoderState->getSequenceLengths(seqSlot); + auto outputIds = mDecoderState->getGatheredIds(seqSlot); + auto cumLogProbs = mDecoderState->getCumLogProbs(seqSlot); + auto logProbs = mDecoderState->getLogProbs(seqSlot); + + mCopyBufferManager.copy(*sequenceLengths, *mSlotDecoderBuffers[seqSlot]->sequenceLengths); + mCopyBufferManager.copy(*outputIds, *mSlotDecoderBuffers[seqSlot]->outputIds); + if (returnLogProbs) + { + mCopyBufferManager.copy(*cumLogProbs, *mSlotDecoderBuffers[seqSlot]->cumLogProbs); + mCopyBufferManager.copy(*logProbs, *mSlotDecoderBuffers[seqSlot]->logProbs); + } + + if (mWorldConfig.isPipelineParallel()) + { + // Make sure that postprocessing is done before sending outputIds + event.synchronize(); + + auto const peerSend = 0; + mDecSlotAsyncSndHdls.emplace_back(std::make_unique<DecoderSlotAsyncSend>( + outputIds, sequenceLengths, cumLogProbs, logProbs, returnLogProbs, *mMpiCommPipelinePara, peerSend)); + } + } + else + { + auto const peerRecv = mWorldConfig.getPipelineParallelRank() == 0 ? mWorldConfig.getPipelineParallelism() - 1 + : mWorldConfig.getPipelineParallelRank() - 1; + DecoderSlotAsyncSend::recv(*mSlotDecoderBuffers[seqSlot], returnLogProbs, *mMpiCommPipelinePara, peerRecv); + + auto const peerSend = mWorldConfig.getPipelineParallelRank() + 1; + if (peerSend != mWorldConfig.getPipelineParallelism() - 1) + { + mDecSlotAsyncSndHdls.emplace_back(std::make_unique<DecoderSlotAsyncSend>( + *mSlotDecoderBuffers[seqSlot], returnLogProbs, *mMpiCommPipelinePara, peerSend)); + } + } + sync_check_cuda_error(mRuntime->getStream().get()); + + // Here copy stream is synchronized after receiving decoderSlotOutputIdsView either by copy or by receive + // before copying to host on copy stream + runtime::CudaEvent beforeEvent{}; + mRuntime->getStreamPtr()->record(beforeEvent); + mCopyBufferManager.getStream().wait(beforeEvent); + mCopyBufferManager.copy(*mSlotDecoderBuffers[seqSlot]->outputIds, *mSlotDecoderBuffers[seqSlot]->outputIdsHost); + mCopyBufferManager.copy( + *mSlotDecoderBuffers[seqSlot]->sequenceLengths, *mSlotDecoderBuffers[seqSlot]->sequenceLengthsHost); + + if (returnLogProbs) + { + mCopyBufferManager.copy( + *mSlotDecoderBuffers[seqSlot]->cumLogProbs, *mSlotDecoderBuffers[seqSlot]->cumLogProbsHost); + mCopyBufferManager.copy(*mSlotDecoderBuffers[seqSlot]->logProbs, *mSlotDecoderBuffers[seqSlot]->logProbsHost); + } + + // Make sure copy is done before continuing on host + mCopyBufferManager.getStream().synchronize(); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +namespace +{ +// Check if one of the request needs log probs, need to get from decoder and communicate +bool batchReturnLogProbs(ScheduledRequests const& scheduledRequests) +{ + auto pred = [](auto const& llmReq) { return llmReq->returnLogProbs(); }; + return std::any_of(scheduledRequests.contextRequests.begin(), scheduledRequests.contextRequests.end(), pred) + || std::any_of(scheduledRequests.generationRequests.begin(), scheduledRequests.generationRequests.end(), pred); +} +} // namespace + +runtime::CudaEvent TrtGptModelInflightBatching::decoderStepAsync(ScheduledRequests const& scheduledRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(decoderStepAsync); + + auto& decoderInputBuffers = mDecoderInputBuffers.at(getFusedBufferId()); + + auto const contextBufferId = mCtxGenFusion ? getFusedBufferId() : getContextBufferId(); + auto& contextRuntimeBuffers = mBuffers.at(contextBufferId); + auto const logitsIndex = (*mHandleContextLogits)(decoderInputBuffers, scheduledRequests.contextRequests, + contextRuntimeBuffers->logits, contextRuntimeBuffers->numContextLogits, mModelConfig, + mRuntime->getBufferManager(), contextRuntimeBuffers->mMedusaBuffers); + + auto const genLogitsIndex = mCtxGenFusion ? logitsIndex : 0; + auto const genBufferId = mCtxGenFusion ? getFusedBufferId() : getGenerationBufferId(); + auto& genRuntimeBuffers = mBuffers.at(genBufferId); + (*mHandleGenerationLogits)(decoderInputBuffers, scheduledRequests.generationRequests, genRuntimeBuffers->logits, + genLogitsIndex, mModelConfig, mRuntime->getBufferManager(), *genRuntimeBuffers, + genRuntimeBuffers->mMedusaBuffers); + + if (mOperatingBeamWidth > 1) + { + copyCacheIndirectionFromOutputsToInputs(scheduledRequests, genBufferId); + } + + mLogitsPostProcessorIsApplied = (*mLogitsPostProcessor)(decoderInputBuffers, mReplicateLogitsPostProcessor, + mWorldConfig, mRuntime->getStreamPtr(), mLogitsPostProcessorBatched); + + if (mGuidedDecoder) + { + mGuidedDecoder->execute(decoderInputBuffers, mRuntime->getBufferManager()); + } + + auto const fusedBufferId = getFusedBufferId(); + auto& fusedRuntimeBuffers = mBuffers.at(fusedBufferId); + + (*mMakeDecodingBatchInputOutput)(decoderInputBuffers, *mDecoderState, mModelConfig, *fusedRuntimeBuffers); + + auto decoderFinishEvent = mDecoder->forwardAsync(*mDecoderState, decoderInputBuffers); + + auto const returnLogProbs = batchReturnLogProbs(scheduledRequests); + auto updateDecoderBuffersEvent = (*mUpdateDecoderBuffers)(mModelConfig, mDecoderOutputBuffers.at(fusedBufferId), + mRuntime->getBufferManager(), *mDecoderState, returnLogProbs, decoderFinishEvent); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return updateDecoderBuffersEvent; +} + +void TrtGptModelInflightBatching::copyCacheIndirectionFromOutputsToInputs( + ScheduledRequests const& scheduledRequests, SizeType32 genBufferId) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(copyCacheIndirectionFromOutputsToInputs); + + auto& genRuntimeBuffers = *mBuffers.at(genBufferId); + auto* srcOffsetsPtr = bufferCast<SizeType64>(*genRuntimeBuffers.cacheIndirDecoderIOBatchedCopySrcOffsets); + auto* dstOffsetsPtr = bufferCast<SizeType64>(*genRuntimeBuffers.cacheIndirDecoderIOBatchedCopyDstOffsets); + auto* copySizesPtr = bufferCast<SizeType64>(*genRuntimeBuffers.cacheIndirDecoderIOBatchedCopySizes); + + // Only `cacheIndirShape.d[2]` is used + auto const& cacheIndirShape = mDecoderState->getCacheIndirectionOutput()->getShape(); + auto const maxBeamWidth = cacheIndirShape.d[1]; + auto const maxAttentionWindow = cacheIndirShape.d[2]; + auto const slotOffset = maxBeamWidth * maxAttentionWindow; + + SizeType32 batchIdx{0}; + SizeType64 maxCopySize{0}; + auto& manager = mRuntime->getBufferManager(); + for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + auto const seqSlot = llmReq->mSeqSlot.value(); + auto const copySize = reqBeamWidth * maxAttentionWindow; + srcOffsetsPtr[batchIdx] = seqSlot * slotOffset; + dstOffsetsPtr[batchIdx] = seqSlot * slotOffset; + copySizesPtr[batchIdx] = copySize; + maxCopySize = std::max(maxCopySize, copySize); + batchIdx++; + } + } + if (batchIdx != 0) + { + auto const srcOffsetsSlice + = ITensor::slice(genRuntimeBuffers.cacheIndirDecoderIOBatchedCopySrcOffsets, 0, batchIdx); + auto const srcOffsetsSliceDeviceSlice + = ITensor::slice(genRuntimeBuffers.mCacheIndirDecoderIOBatchedCopySrcOffsetsSliceDevice, 0, batchIdx); + manager.copy(srcOffsetsSlice->data(), *srcOffsetsSliceDeviceSlice, + runtime::MemoryType::kGPU); // Explicitly move to device for faster access. + auto const dstOffsetsSlice + = ITensor::slice(genRuntimeBuffers.cacheIndirDecoderIOBatchedCopyDstOffsets, 0, batchIdx); + auto const dstOffsetsSliceDeviceSlice + = ITensor::slice(genRuntimeBuffers.mCacheIndirDecoderIOBatchedCopyDstOffsetsSliceDevice, 0, batchIdx); + manager.copy(dstOffsetsSlice->data(), *dstOffsetsSliceDeviceSlice, + runtime::MemoryType::kGPU); // Explicitly move to device for faster access. + auto const sizesSlice = ITensor::slice(genRuntimeBuffers.cacheIndirDecoderIOBatchedCopySizes, 0, batchIdx); + auto const copySizesDeviceSlice + = ITensor::slice(genRuntimeBuffers.mCacheIndirDecoderIOBatchedCopyCopySizesDevice, 0, batchIdx); + manager.copy(sizesSlice->data(), *copySizesDeviceSlice); // Explicitly move to device for faster access. + runtime::kernels::invokeCopyBatch(*mDecoderState->getCacheIndirectionOutput(), + *mDecoderState->getCacheIndirectionInput(), *srcOffsetsSliceDeviceSlice, *dstOffsetsSliceDeviceSlice, + *copySizesDeviceSlice, maxCopySize, manager.getStream()); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +std::vector<std::unique_ptr<DecoderStepAsyncSend>> TrtGptModelInflightBatching::communicateDecoderBuffers( + bool returnLogProbs) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(communicateDecoderBuffers); + + auto& decoderOutputBuffers = mDecoderOutputBuffers.at(getFusedBufferId()); + + std::vector<std::unique_ptr<DecoderStepAsyncSend>> asyncHandles; + if (mWorldConfig.isLastPipelineParallelRank()) + { + if (broadcastPostDecoder()) + { + DecoderStepAsyncSend::bcast(decoderOutputBuffers, *mDecoderState, returnLogProbs, mOperatingBeamWidth, + mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind(), *mMpiCommTensorPara, 0); + } + + if (mWorldConfig.isPipelineParallel()) + { + auto const peerSend = 0; + asyncHandles.emplace_back(std::make_unique<DecoderStepAsyncSend>(decoderOutputBuffers, *mDecoderState, + returnLogProbs, mOperatingBeamWidth, mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind(), + *mMpiCommPipelinePara, peerSend)); + } + } + else + { + auto const peerRecv = mWorldConfig.isFirstPipelineParallelRank() ? mWorldConfig.getPipelineParallelism() - 1 + : mWorldConfig.getPipelineParallelRank() - 1; + DecoderStepAsyncSend::recv(decoderOutputBuffers, *mDecoderState, returnLogProbs, mOperatingBeamWidth, + mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind(), *mMpiCommPipelinePara, peerRecv); + auto const peerSend = mWorldConfig.getPipelineParallelRank() + 1; + if (peerSend != mWorldConfig.getPipelineParallelism() - 1) + { + asyncHandles.emplace_back(std::make_unique<DecoderStepAsyncSend>(decoderOutputBuffers, *mDecoderState, + returnLogProbs, mOperatingBeamWidth, mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind(), + *mMpiCommPipelinePara, peerSend)); + } + } + TLLM_CHECK_WITH_INFO(asyncHandles.size() <= 2, "Up to two decoder step async handles expected"); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return asyncHandles; +} + +void TrtGptModelInflightBatching::updateRequests(ScheduledRequests const& scheduledRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(updateRequests); + + auto const& decoderOutputBuffers = mDecoderOutputBuffers.at(getFusedBufferId()); + + auto const hostNewOutputTokensShape = decoderOutputBuffers.newOutputTokensHost->getShape(); + auto const* const hostNewOutputTokensData + = bufferCast<TokenIdType const>(*decoderOutputBuffers.newOutputTokensHost); + auto const* const sequenceLengthsHostData = bufferCast<SizeType32 const>(*decoderOutputBuffers.sequenceLengthsHost); + auto const* const decoderFinishedSumPtr = bufferCast<SizeType32 const>(*decoderOutputBuffers.finishedSumHost); + auto const* const cumLogProbsPtr = bufferCast<float const>(*decoderOutputBuffers.cumLogProbsHost); + auto const* const logProbsPtr = bufferCast<float const>(*decoderOutputBuffers.logProbsHost); + auto const* const finishReasonsHostData + = bufferCast<kernels::FinishedState>(*decoderOutputBuffers.finishReasonsHost); + + // Update only requests that ran through the decoder + for (auto const& llmReq : scheduledRequests.generationRequests) + { + if (llmReq->isGenerationCompleteState()) + { + continue; + } + auto const reqBeamWidth = llmReq->getBeamWidthByIter(true); + auto const seqSlot = llmReq->mSeqSlot.value(); + auto const currentNumOfTokens = llmReq->getMaxBeamNumTokens(); + + // Save the accepted token logits from target model + if (mModelConfig.getSpeculativeDecodingMode().isDraftTokensExternal() && llmReq->getReturnGenerationLogits() + && llmReq->hasDraftTokens()) + { + TLLM_CHECK_WITH_INFO(reqBeamWidth == 1, "Speculative decoding only works for beam width == 1"); + + SizeType32 numAcceptedTokens + = sequenceLengthsHostData[seqSlot * mOperatingBeamWidth + 0] - llmReq->getMaxBeamNumTokens(); + + auto const& generationLogitsHost = llmReq->getGenerationLogitsHost(); + auto shape = generationLogitsHost->getShape(); + shape.d[1] = numAcceptedTokens; + generationLogitsHost->reshape(shape); + } + + std::vector<SizeType32> numNewTokens(reqBeamWidth); + std::vector<SizeType32> numDroppedTokens(reqBeamWidth); + + // numGeneratedTokens is the number of tokens generated by the decoder. + // Some tokens might be dropped due to end token or rejected draft tokens. + auto const numGeneratedTokens = llmReq->getNumDraftTokens() + 1; + + for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) + { + // Sequence length is only advanced for accepted tokens. + auto const seqLen = sequenceLengthsHostData[seqSlot * mOperatingBeamWidth + beam]; + // Actual number of tokens that should be added to the request. + auto const numNewOutputTokens = seqLen - llmReq->getNumTokens(beam); + if (reqBeamWidth == 1) + { + TLLM_CHECK_WITH_INFO(numGeneratedTokens >= numNewOutputTokens, + "numNewOutputTokens must not be greater than numGeneratedTokens: " + "numGeneratedTokens %d < numNewOutputTokens %d", + numGeneratedTokens, numNewOutputTokens); + } + numNewTokens[beam] = std::min(numGeneratedTokens, numNewOutputTokens); + numDroppedTokens[beam] = numGeneratedTokens - numNewTokens[beam]; + for (SizeType32 step = 0; step < numNewTokens[beam]; ++step) + { + auto const newTokenIdx = tc::flat_index(hostNewOutputTokensShape.d, step, seqSlot, beam); + auto const newToken = hostNewOutputTokensData[newTokenIdx]; + llmReq->addNewToken(newToken, beam); + TLLM_LOG_DEBUG("request ID %ld beam %d newToken %d", llmReq->mRequestId, beam, newToken); + + if (llmReq->returnLogProbs()) + { + auto const cumLogProb = cumLogProbsPtr[seqSlot * mOperatingBeamWidth + beam]; + llmReq->setCumLogProb(cumLogProb, beam); + + auto const beginLogProbsOffset = reqBeamWidth == 1 ? llmReq->mPromptLen : 0; + SizeType32 offset + = (seqSlot * mOperatingBeamWidth + beam) * getMaxSequenceLen() + beginLogProbsOffset; + auto const generatedLength = seqLen - llmReq->mPromptLen; + std::vector<float> logProbs(logProbsPtr + offset, logProbsPtr + offset + generatedLength); + llmReq->setLogProbs(logProbs, beam); + } + } + + auto const finishReason = finishReasonsHostData[seqSlot * mOperatingBeamWidth + beam]; + llmReq->setFinishedReason(finishReason.toFinishReason(), beam); + + TLLM_LOG_DEBUG("[RANK %d] decoderSync: request ID %lu beam %d tokens %s finished %d", + COMM_SESSION.getRank(), llmReq->mRequestId, beam, common::vec2str(llmReq->getTokens(beam)).c_str(), + static_cast<int>(finishReason.toFinishReason())); + } + + // Set number of tokens predicted per runtime iteration. Will be > 1 for speculative decoding. + llmReq->updateNumTokensPerIteration(llmReq->getMaxBeamNumTokens() - currentNumOfTokens, mModelConfig); + + // Fill new draft tokens for the next step + if (decoderFinishedSumPtr[seqSlot] != reqBeamWidth + && (mModelConfig.getSpeculativeDecodingMode().predictsDraftTokens() + || mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind())) + { + auto const maxDraftTokensLen = mModelConfig.getMaxDecodingDraftTokens(); + auto prevDraftTokensLen = llmReq->getNumDraftTokens(); + + // We overallocate KV cache for EAGLE to the maxDecodingTokens + maxPathLen in order to fit both + // Base model verification (needs up to maxDecodingTokens) and + // Drafter (needs up to maxPathLen of accepted tokens and maxDecodingDraftTokens for new draft tokens). + if (mModelConfig.getSpeculativeDecodingMode().isEagle()) + { + prevDraftTokensLen = mModelConfig.getSpeculativeDecodingModule().getMaxDecodingTokens() + + mModelConfig.getSpeculativeDecodingModule().getMaxPathLen() - 1; + } + + auto nextDraftTokensLen = mModelConfig.getSpeculativeDecodingModule().getMaxDecodingDraftTokens(); + if (mModelConfig.getSpeculativeDecodingMode().variableDraftLength()) + { + auto const* const nextDraftTokensLengthsHostData + = bufferCast<SizeType32 const>(*decoderOutputBuffers.nextDraftTokensLengthsHost); + nextDraftTokensLen = nextDraftTokensLengthsHostData[seqSlot]; + } + TLLM_CHECK(nextDraftTokensLen <= maxDraftTokensLen); + + auto const* const nextDraftTokensHostData + = bufferCast<TokenIdType const>(*decoderOutputBuffers.nextDraftTokensHost); + auto draftTokensShared + = std::make_shared<std::vector<TokenIdType>>(nextDraftTokensHostData + seqSlot * maxDraftTokensLen, + nextDraftTokensHostData + seqSlot * maxDraftTokensLen + nextDraftTokensLen); + + llmReq->setDraftTokens(draftTokensShared); + + // For all phases except context that does not have draft tokens + if (!llmReq->isGenerationCompleteState() && prevDraftTokensLen != 0 + && mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind()) + { + // -1 here is for current 'main' token + auto const acceptedTokensLen = llmReq->getMaxBeamNumTokens() - currentNumOfTokens - 1; + auto const rewindLength = prevDraftTokensLen - acceptedTokensLen; + + TLLM_LOG_DEBUG("request ID %lu (seqSlot %d): accepted %d of %d draft tokens, rewind %d tokens", + llmReq->mRequestId, seqSlot, acceptedTokensLen, prevDraftTokensLen, rewindLength); + TLLM_CHECK(0 <= acceptedTokensLen && acceptedTokensLen <= prevDraftTokensLen); + + // At this point, KV cache rows are already gathered and moved to the right location. + // We can safely rewind (draft - accepted) tokens + mKvCacheManager->rewindKVCache(llmReq->mRequestId, rewindLength); + } + } + + // Terminate if request has finished or if it is speculative decoding target model + if (decoderFinishedSumPtr[seqSlot] == reqBeamWidth + || (mModelConfig.getSpeculativeDecodingMode().isDraftTokensExternal() && llmReq->hasDraftTokens())) + { + postProcessRequest(*llmReq, numDroppedTokens); + + if (!mWorldConfig.isPipelineParallel() || !mWorldConfig.isLastPipelineParallelRank()) + { + if (llmReq->getReturnGenerationLogits() && mSpeculativeDecodingFastLogits && mIsLeaderInOrchMode) + { + std::lock_guard<std::mutex> lk(mDraftRequestsMtx); + mDraftRequestsWaitingToSendLogits.push_back(llmReq); + } + else + { + terminateRequest(llmReq); + } + llmReq->setState(LlmRequestState::kGENERATION_COMPLETE); + } + else + { + llmReq->setState(LlmRequestState::kGENERATION_TO_COMPLETE); + } + } + else + { + // gather tokens in the case of streaming and beam search + if (llmReq->isStreaming() && llmReq->mSamplingConfig.beamWidth > 1) + { + postProcessRequest(*llmReq, numDroppedTokens); + } + if (llmReq->isContextInitState()) + { + llmReq->setState(LlmRequestState::kGENERATION_IN_PROGRESS); + } + + if (isTrtOverlap() && llmReq->willCompleteNextIteration()) + { + // This state prohibits the request from being scheduled for another iteration. It assumes that the next + // iteration has already been scheduled and the request can finish in the next call to updateRequests(). + llmReq->setState(LlmRequestState::kGENERATION_TO_COMPLETE); + } + } + + if (llmReq->getReturnPerfMetrics()) + { + llmReq->updatePerfMetrics(mIterCounter); + } + + llmReq->advanceDecodingIter(); + + if (mWorldConfig.isPipelineParallel() && mWorldConfig.isLastPipelineParallelRank()) + { + for (SizeType32 beam = 0; beam < reqBeamWidth; ++beam) + { + llmReq->setNumPreDecodedTokens(numNewTokens[beam], beam); + } + } + } + + if (mModelConfig.getSpeculativeDecodingMode().needsKVCacheRewind()) + { + SizeType32 numSequences{0}; + for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + auto const reqBeamWidth = llmReq->mSamplingConfig.beamWidth; + numSequences += reqBeamWidth; + } + } + + TLLM_CHECK_WITH_INFO(mCtxGenFusion, "Current speculative decoding mode requires context-gen fusion IFB"); + rewindKVCacheBlocks(numSequences); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +std::vector<std::unique_ptr<DecoderStepAsyncSend>> TrtGptModelInflightBatching::decoderSync( + ScheduledRequests const& scheduledRequests, std::optional<runtime::CudaEvent> const& decoderFinishEvent) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(decoderSync); + + if (mWorldConfig.isLastPipelineParallelRank()) + { + decoderFinishEvent->synchronize(); + } + + auto const returnLogProbs = batchReturnLogProbs(scheduledRequests); + auto asyncHandles = communicateDecoderBuffers(returnLogProbs); + + updateRequests(scheduledRequests); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return asyncHandles; +} + +void TrtGptModelInflightBatching::rewindKVCacheBlocks(SizeType32 numSequences) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + auto const bufferId = getFusedBufferId(); + auto& runtimeBuffers = *mBuffers.at(bufferId); + auto& decoderOutputBuffers = mDecoderOutputBuffers.at(bufferId); + + auto localNbLayers = mModelConfig.getNbAttentionLayers( + mWorldConfig.getPipelineParallelism(), mWorldConfig.getPipelineParallelRank()); + if (mWorldConfig.isLastPipelineParallelRank() && mModelConfig.getSpeculativeDecodingMode().isEagle()) + { + // Do not correct the last kv caches, which are for EagleNet drafter. Those KV caches are managed separately. + auto eagleModulePtr + = std::dynamic_pointer_cast<runtime::EagleModule>(mModelConfig.getSpeculativeDecodingModulePtr()); + localNbLayers -= eagleModulePtr->getNumTransformerLayers(); + } + + auto const tokensPerBlock = mModelConfig.getTokensPerBlock(); + auto const elemSize = BufferDataType(mModelConfig.getKvDataType()).getSize(); + auto const sizeInBytesPerKVHead = mModelConfig.getSizePerHead() * elemSize; + + auto const poolPointers = mKvCacheManager->getBlockPoolPointers(); + auto* const* pointerArrayPtr = bufferCast<void*>(*poolPointers); + auto const* offsetArrayPtr + = bufferCast<tk::KVCacheIndex>(*runtimeBuffers.transformerBuffers->kvCacheBlockOffsetsDevice); + + auto commonRewindLen = mModelConfig.getSpeculativeDecodingModule().getMaxDecodingDraftTokens(); + SizeType32 const* rewindLens = nullptr; + if (mModelConfig.getSpeculativeDecodingMode().variableDraftLength()) + { + commonRewindLen = 0; + rewindLens = bufferCast<SizeType32 const>(*decoderOutputBuffers.prevDraftTokensLengthsHost); + } + + tensorrt_llm::runtime::kernels::invokeUpdateKVBlockArrayDraftTokenLocation( + *mDecoderState->getAcceptedLengthsCumSum(), *mDecoderState->getAcceptedPackedPaths(), + *runtimeBuffers.sequenceLengthsDevice, pointerArrayPtr, offsetArrayPtr, localNbLayers, numSequences, + mRewindInputs.numKvHeads, sizeInBytesPerKVHead, commonRewindLen, rewindLens, *runtimeBuffers.seqSlots, + getMaxAttentionWindow(), mRewindInputs.maxBlocksPerSeq, tokensPerBlock, mRewindInputs.isUseOneMoreBlock, + mRuntime->getStreamPtr()->get()); + + sync_check_cuda_error(mRuntime->getStream().get()); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +nvinfer1::DataType TrtGptModelInflightBatching::getLogitDataType() const +{ + return mModelConfig.getLogitsDtype(); +} + +TrtGptModelInflightBatching::SizeType32 TrtGptModelInflightBatching::numCachedCudaGraphs() const +{ + return std::accumulate(mCudaGraphExecutorCaches.begin(), mCudaGraphExecutorCaches.end(), SizeType32{0}, + [](SizeType32 sum, auto const& cache) { return sum + cache.size(); }); +} + +void TrtGptModelInflightBatching::changeBeamWidth(SizeType32 beamWidth) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK(mInflightReqIds.empty()); + + TLLM_CHECK_WITH_INFO(beamWidth <= getMaxBeamWidth(), + "Requested beam width %d is larger than configured max beam width %d", beamWidth, getMaxBeamWidth()); + TLLM_LOG_DEBUG("Changing operating beam width from %d to %d", mOperatingBeamWidth, beamWidth); + mOperatingBeamWidth = beamWidth; + + if (isCudaGraphMode()) + { + for (auto& cache : mCudaGraphExecutorCaches) + { + cache.clear(); + } + } + createBuffers(mDecodingConfig, mAdditionalModelOutputs); + createDecoder(mDecodingConfig.getDecodingMode()); + + if (static_cast<bool>(mKvCacheManager)) + { + auto const dims = mKvCacheManager->getOffsetTableDimensions(); + reshapeKvTensors(dims); + } + if (static_cast<bool>(mCrossKvCacheManager)) + { + auto const dims = mCrossKvCacheManager->getOffsetTableDimensions(); + reshapeKvTensors(dims); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::changeSpecDecMode(ScheduledRequests const& scheduledRequests) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + if ((!mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding() + && !mModelConfig.getSpeculativeDecodingMode().isNone()) + || scheduledRequests.empty() || mSeamlessLADMaxDraftLen == 0 || getGatherGenerationLogits() + || mModelConfig.isRnnBased()) + { + return; + } + + bool canUseLookahead = false; + auto maxNumRequestForLad = mDecodingConfig.getLookaheadDecodingMaxNumRequest(); + SizeType32 numRequests = scheduledRequests.contextRequests.size() + scheduledRequests.generationRequests.size(); + if (numRequests > maxNumRequestForLad) + { + if (mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding()) + { + canUseLookahead = false; + } + else + { + return; + } + } + { + bool useTopKTopP = false; + bool useBanWords = false; + bool useTempAccVocabPenalties = false; // use temperature and penalties that need to accumulate #vocab. + SizeType32 beamWidth = 1; + for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + useTopKTopP |= !(llmReq->mSamplingConfig.useDefaultValues( + llmReq->mSamplingConfig.topK, layers::DefaultDecodingParams::getTopK()) + || llmReq->mSamplingConfig.useDefaultValues(llmReq->mSamplingConfig.topK, 1)); + useTopKTopP |= !llmReq->mSamplingConfig.useDefaultValues( + llmReq->mSamplingConfig.topP, layers::DefaultDecodingParams::getTopP()); + useBanWords |= llmReq->getBadWordsList().has_value(); + useBanWords |= !llmReq->mSamplingConfig.useDefaultValues( + llmReq->mSamplingConfig.noRepeatNgramSize, layers::DefaultDecodingParams::getNoRepeatNgramSize()); + useTempAccVocabPenalties |= !llmReq->mSamplingConfig.useDefaultValues( + llmReq->mSamplingConfig.temperature, layers::DefaultDecodingParams::getTemperature()); + useTempAccVocabPenalties |= !llmReq->mSamplingConfig.useDefaultValues( + llmReq->mSamplingConfig.repetitionPenalty, layers::DefaultDecodingParams::getRepetitionPenalty()); + useTempAccVocabPenalties |= !llmReq->mSamplingConfig.useDefaultValues( + llmReq->mSamplingConfig.presencePenalty, layers::DefaultDecodingParams::getPresencePenalty()); + useTempAccVocabPenalties |= !llmReq->mSamplingConfig.useDefaultValues( + llmReq->mSamplingConfig.frequencyPenalty, layers::DefaultDecodingParams::getFrequencyPenalty()); + beamWidth = llmReq->mSamplingConfig.beamWidth; + if (useTopKTopP || useBanWords || useTempAccVocabPenalties || beamWidth > 1) + { + break; + } + } + canUseLookahead = !(useTopKTopP || useBanWords || useTempAccVocabPenalties || beamWidth > 1); + } + } + + // Change speculative decoding mode + auto const bufferId = mCtxGenFusion + ? getFusedBufferId() + : (!scheduledRequests.contextRequests.empty() ? getContextBufferId() : getGenerationBufferId()); + // TODO: enable lookahead for generation requests. + bool canChangeToLookahead = scheduledRequests.generationRequests.empty(); + if (mModelConfig.getSpeculativeDecodingMode().isNone() && canUseLookahead && canChangeToLookahead) + { + // None -> Lookahead + mModelConfig.enableSeamlessLookaheadDecoding(mSeamlessLADMaxDraftLen); + mDecodingConfig.enableSeamlessLookaheadDecoding(); + setupSpeculativeDecodingModule(mDecodingConfig); + mBuffers.at(bufferId)->mLookaheadBuffers->enableLookaheadDecoding( + getMaxBatchSize(), mModelConfig.getMaxDecodingTokens()); + mDecoderOutputBuffers.at(getFusedBufferId()) + .enableLookaheadDecoding(getMaxNumSequences(), mModelConfig.getMaxDecodingTokens()); + createDecoder(mDecodingConfig.getDecodingMode()); + } + else if (mModelConfig.getSpeculativeDecodingMode().isLookaheadDecoding() + && (!canUseLookahead || numRequests > maxNumRequestForLad)) + { + // Lookahead -> None + mModelConfig.disableSeamlessLookaheadDecoding(); + mDecodingConfig.setDecodingMode(executor::DecodingMode::Auto()); + mBuffers.at(bufferId)->mLookaheadBuffers->disableLookaheadDecoding(); + mDecoderOutputBuffers.at(getFusedBufferId()).disableLookaheadDecoding(getMaxNumSequences()); + mDecoder->disableLookahead( + scheduledRequests.generationRequests, mDecoderInputBuffers.at(getFusedBufferId()).setupBatchSlots); + mDecoderState->disableLookahead(scheduledRequests.generationRequests); + for (auto const& llmReq : scheduledRequests.generationRequests) + { + if (llmReq->getNumDraftTokens() > 0) + { + llmReq->discardDraftTokens(llmReq->getNumDraftTokens()); + } + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TrtGptModelInflightBatching::getCurrentIterationStats(executor::IterationStats& stats) const +{ + stats.iter = mIterCounter; + + // Max batch size and max num tokens can be tuned at runtime + stats.maxBatchSizeStatic = getMaxBatchSize(); + stats.maxBatchSizeTunerRecommended = mMaxBatchSizeTunerRecommended; + stats.maxBatchSizeRuntime = mMaxBatchSizeRuntime; + stats.maxNumTokensStatic = mMaxNumTokensStatic.value_or(0); + stats.maxNumTokensTunerRecommended = mMaxNumTokensTunerRecommended; + stats.maxNumTokensRuntime = mMaxNumTokensRuntime.value_or(0); + + // KVCacheManager statistics + auto const& kvCacheManager = getKVCacheManager(); + if (kvCacheManager) + { + executor::KvCacheStats kvStats{}; + auto kvCacheStats = kvCacheManager->getKvCacheStats(); + kvStats.maxNumBlocks = kvCacheStats.maxNumBlocks; + kvStats.freeNumBlocks = kvCacheStats.freeNumBlocks; + kvStats.usedNumBlocks = kvCacheStats.usedNumBlocks; + kvStats.tokensPerBlock = kvCacheStats.toksPerBlock; + kvStats.allocTotalBlocks = kvCacheStats.allocTotalBlocks; + kvStats.allocNewBlocks = kvCacheStats.allocNewBlocks; + kvStats.reusedBlocks = kvCacheStats.reusedBlocks; + kvStats.missedBlocks = kvCacheStats.missedBlocks; + kvStats.cacheHitRate = kvCacheStats.cacheHitRate; + stats.kvCacheStats = kvStats; + } + auto const& crossKvCacheManager = getCrossKVCacheManager(); + if (crossKvCacheManager) + { + executor::KvCacheStats kvStats{}; + auto kvCacheStats = crossKvCacheManager->getKvCacheStats(); + kvStats.maxNumBlocks = kvCacheStats.maxNumBlocks; + kvStats.freeNumBlocks = kvCacheStats.freeNumBlocks; + kvStats.usedNumBlocks = kvCacheStats.usedNumBlocks; + kvStats.tokensPerBlock = kvCacheStats.toksPerBlock; + kvStats.allocTotalBlocks = kvCacheStats.allocTotalBlocks; + kvStats.allocNewBlocks = kvCacheStats.allocNewBlocks; + kvStats.reusedBlocks = kvCacheStats.reusedBlocks; + kvStats.missedBlocks = kvCacheStats.missedBlocks; + kvStats.cacheHitRate = kvCacheStats.cacheHitRate; + stats.crossKvCacheStats = kvStats; + } + executor::InflightBatchingStats modelStats{}; + modelStats.numScheduledRequests = mLastIterationStatsIFB.scheduledRequests.size(); + modelStats.numContextRequests = mLastIterationStatsIFB.numCtxRequests; + modelStats.numGenRequests = mLastIterationStatsIFB.numGenRequests; + modelStats.numPausedRequests = mLastIterationStatsIFB.pausedRequests.size(); + modelStats.avgNumDecodedTokensPerIter = mLastIterationStatsIFB.avgNumDecodedTokensPerIter; + modelStats.numCtxTokens = mLastIterationStatsIFB.numCtxTokens; + modelStats.microBatchId = mLastIterationStatsIFB.microBatchId; + stats.inflightBatchingStats = modelStats; +} + +void TrtGptModelInflightBatching::getCurrentRequestStats(executor::RequestStatsPerIteration& stats) const +{ + stats.iter = mIterCounter; + for (auto& requestStat : stats.requestStats) + { + requestStat.scheduled + = mLastIterationStatsIFB.scheduledRequests.count(static_cast<RequestIdType>(requestStat.id)); + requestStat.paused = mLastIterationStatsIFB.pausedRequests.count(static_cast<RequestIdType>(requestStat.id)); + } +} + +executor::DebugTensorsPerIteration TrtGptModelInflightBatching::getCurrentDebugTensors() const +{ + executor::DebugTensorsPerIteration debugTensors; + debugTensors.iter = mIterCounter; + + for (auto const& [name, tensor] : mLastIterationDebugTensors) + { + debugTensors.debugTensors.emplace(name, executor::detail::ofITensor(tensor)); + } + + return debugTensors; +} + +nvinfer1::DataType TrtGptModelInflightBatching::getTensorDataType(std::string const& name) const +{ + auto const& engine = mRuntime->getEngine(); + return engine.getTensorDataType(name.c_str()); +} + +nvinfer1::Dims TrtGptModelInflightBatching::getTensorShape(std::string const& name) const +{ + auto const& engine = mRuntime->getEngine(); + return engine.getTensorShape(name.c_str()); +} + +SizeType32 TrtGptModelInflightBatching::getMaxCapacityBatchSize(SizeType32 inputLength, SizeType32 outputLength) const +{ + return mKvCacheManager->getMaxCapacityBatchSize(inputLength, outputLength); +} + +/* + * Manages prefetching of prompt table chunks using a double-buffer strategy + * + * Function Flow: + * 1. First Chunk Processing (isFirstChunk == true): + * - Uses blocking prefetch on main runtime stream + * - Ensures initial data is ready before computation starts + * + * 2. Subsequent Chunks (isFirstChunk == false): + * - Uses non-blocking prefetch on separate copy stream + * - Overlaps data transfer with computation + * + * Synchronization: + * - First prefetch: No wait needed (fresh start) + * - Later prefetches: Wait for previous copy to complete + * - Uses mPtableCopyDoneEvent to track completion + * + * Key Functions: + * 1. prefetchNextPromptTableChunk: + * - Calls the correct function based on position in code (before or after prepareBuffers()) + * - Waits for previous copy to complete if not the first chunk + * + * 2. remapInputTokensForPromptTable: + * - Identifies tokens that need prompt table embeddings (tokens that are greater than vocabSize) + * - Remaps IDs to match chunked prompt table layout + * + * 3. copyPromptTableToGpuInChunk: + * - Handles actual transfer from CPU pinned memory to GPU + * - Uses appropriate buffer manager based on isFirstChunk + */ +void TrtGptModelInflightBatching::prefetchNextPromptTableChunk( + RequestVector const& contextRequests, bool isFirstChunk, SizeType32 bufferId) +{ + auto& promptTuningBuffers = mBuffers[bufferId]->promptTuningBuffers; + + if (!isFirstChunk) + { + // Only switch buffer after prepareBuffer() + promptTuningBuffers->switchChunkPtableBuffer(); + } + + SizeType32 contextId = 0; + for (auto const& llmReq : contextRequests) + { + if (llmReq->isFirstContextChunk() && isFirstChunk) + { + // For first chunk: Blocking prefetch on runtime stream to ensure data is ready + remapInputTokensForPromptTable(llmReq, true, bufferId, contextId); + } + else if (!isFirstChunk) // prefetching for subsequent chunks + { + // For the first prefetch chunk, don't need to wait for previous prefetch to complete + // For subsequent chunks: Need to wait for previous prefetch to complete + if (!llmReq->isFirstContextChunk()) + { + mRuntime->getBufferManager().getStream().wait(mPtableCopyDoneEvent); + } + + // Non-blocking prefetch on copy stream to prepare next chunk in pong buffer + if (llmReq->getContextRemainingLength() > 0) + { + remapInputTokensForPromptTable(llmReq, false, bufferId, contextId); + } + } + + ++contextId; + } +} + +void TrtGptModelInflightBatching::remapInputTokensForPromptTable( + std::shared_ptr<LlmRequest> const& llmReq, bool isFirstChunk, SizeType32 bufferId, SizeType32 contextId) +{ + NVTX3_SCOPED_RANGE_WITH_NAME(range, "remapInputTokensForPromptTable"); + auto& promptTuningBuffers = mBuffers[bufferId]->promptTuningBuffers; + auto const chunkSize = llmReq->getContextChunkSize(); + auto& inputTokensMutable = llmReq->getTokensMutable(0); + auto vocabSize = mModelConfig.getVocabSize(); + + if (isFirstChunk) + { + promptTuningBuffers->initializeChunkPtableBuffers( + mRuntime->getBufferManager(), mModelConfig, chunkSize, llmReq); + } + + size_t processChunkSize; + size_t beginPos; + + if (!isFirstChunk) + { + processChunkSize = std::min(chunkSize, llmReq->getContextRemainingLength() - chunkSize); + } + else + { + processChunkSize = std::min(chunkSize, llmReq->getContextRemainingLength()); + } + + if (!isFirstChunk) + { + // For prefetching next chunk + if (llmReq->getContextRemainingLength() - chunkSize <= 0) + { + promptTuningBuffers->updateBufferStartPosition(promptTuningBuffers->getChunkPtableCurrentIndex(), 0); + return; // No more chunks to prefetch + } + beginPos = llmReq->getContextCurrentPosition() + chunkSize; + } + else + { + // For current chunk + beginPos = llmReq->getContextCurrentPosition(); + } + + TLLM_CHECK_WITH_INFO(beginPos + processChunkSize <= inputTokensMutable.size(), + "Invalid chunk access: beginPos(%zu) + processChunkSize(%zu) > totalSize(%zu)", beginPos, processChunkSize, + inputTokensMutable.size()); + + auto inputTokensChunk = inputTokensMutable.begin() + beginPos; + std::vector<SizeType32> outOfVocabTokens; + SizeType32 ptableTokenId = vocabSize; + for (size_t i = 0; i < processChunkSize; i++) + { + if (inputTokensChunk[i] >= vocabSize) + { + outOfVocabTokens.push_back(inputTokensChunk[i]); + inputTokensChunk[i] = ptableTokenId++; + } + } + + copyPromptTableToGpuInChunk(llmReq, outOfVocabTokens, isFirstChunk, bufferId, contextId); +} + +void TrtGptModelInflightBatching::copyPromptTableToGpuInChunk(std::shared_ptr<LlmRequest> const& llmReq, + std::vector<int32_t> const& outOfVocabTokens, bool isFirstChunk, SizeType32 bufferId, SizeType32 contextId) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE_WITH_NAME(range, "copyPromptTableToGpuInChunk"); + auto& promptTuningBuffers = mBuffers[bufferId]->promptTuningBuffers; + + if (outOfVocabTokens.empty()) + { + return; + } + + auto const& promptTable = llmReq->getPromptEmbeddingTable(); + TLLM_CHECK_WITH_INFO(promptTable.has_value(), "promptTable is empty but there's fake_prompt"); + TLLM_CHECK_WITH_INFO(promptTable.value() != nullptr, "promptTable value is null but there's fake_prompt"); + + auto currentBufferManager = isFirstChunk ? mRuntime->getBufferManager() : mCopyBufferManager; + auto const hiddenSize = mModelConfig.getHiddenSize(); + auto numRows = outOfVocabTokens.size(); + std::size_t sliceSize = static_cast<size_t>(numRows * hiddenSize); + auto currentIndex = promptTuningBuffers->getChunkPtableCurrentIndex(); + + // Calculate the offset based on current position + size_t srcOffset = llmReq->mPtableCurrentPosition * hiddenSize; + size_t dstOffset = promptTuningBuffers->getChunkPtableBufferStartPosition(currentIndex, contextId); + + auto gpuBuffer = promptTuningBuffers->getChunkPtableBuffer(currentIndex); + + // First view as 1D tensor of elements + auto totalElements = promptTable.value()->getSize(); + auto table1D = runtime::ITensor::view( + promptTable.value(), runtime::ITensor::makeShape({static_cast<int64_t>(totalElements)})); + + TLLM_CHECK_WITH_INFO(srcOffset + sliceSize <= totalElements, + "Buffer bounds violation: Trying to access up to %zu elements but buffer only has %zu elements (offset: %zu, " + "slice size: %zu)", + srcOffset + sliceSize, totalElements, srcOffset, sliceSize); + + auto table1DShared = runtime::ITensor::SharedPtr(table1D.release()); + auto pTableView = runtime::ITensor::slice(table1DShared, srcOffset, sliceSize); + + auto gpuBufferSlice = runtime::ITensor::slice(gpuBuffer, dstOffset, numRows); + + currentBufferManager.copy(*pTableView, *gpuBufferSlice); + + promptTuningBuffers->updateBufferStartPosition(currentIndex, outOfVocabTokens.size()); + + llmReq->mPtableCurrentPosition += outOfVocabTokens.size(); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.h b/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.h new file mode 100644 index 000000000000..d6550281a758 --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/trtGptModelInflightBatching.h @@ -0,0 +1,639 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/batch_manager/kvCacheType.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tensorrt_llm/runtime/worldConfig.h" +#include "trtGptModel.h" + +#include <NvInferRuntime.h> + +namespace tensorrt_llm::runtime +{ +class TllmRuntime; +class GptDecoderBatched; +class AllReduceBuffers; +class NcclCommunicator; +class SpeculativeDecodingMode; + +namespace decoder +{ +class DecoderState; +} // namespace decoder + +namespace decoder_batch +{ +class Input; +class Output; +} // namespace decoder_batch + +} // namespace tensorrt_llm::runtime + +namespace tensorrt_llm::mpi +{ +class MpiWaitThread; +} // namespace tensorrt_llm::mpi + +namespace tensorrt_llm::batch_manager +{ +class BaseCacheTransceiver; +} + +namespace tensorrt_llm::batch_manager +{ + +namespace kv_cache_manager +{ +class KVCacheManager; +struct OffsetTableDimensions; +} // namespace kv_cache_manager + +namespace rnn_state_manager +{ +class RnnStateManager; +} // namespace rnn_state_manager + +class SequenceSlotManager; +class DecoderStepAsyncSend; +class DecoderSlotAsyncSend; +class DecoderInputBuffers; +class DecoderOutputBuffers; +class SlotDecoderBuffers; +class LlmRequest; +class RuntimeBuffers; +class BasePeftCacheManager; +class GuidedDecoder; +class TrtGptModelTest; + +// Algorithms +class CapacityScheduler; +class DisaggTransferAdmissionController; +class MicroBatchScheduler; +class PauseRequests; +class AssignReqSeqSlots; +class AllocateKvCache; +class HandleContextLogits; +class HandleGenerationLogits; +class GenerateRequestOptions; +class LogitsPostProcessor; +class MakeDecodingBatchInputOutput; +class CreateNewDecoderRequests; +class UpdateDecoderBuffers; + +namespace utils +{ +class CudaGraphExecutorCache; +} // namespace utils + +struct RewindInputs +{ + SizeType32 maxBlocksPerSeq; + bool isUseOneMoreBlock; + SizeType32 numKvHeads; +}; + +class TrtGptModelInflightBatching : public TrtGptModel +{ + using BaseKVCacheManager = kv_cache_manager::BaseKVCacheManager; + using OffsetTableDimensions = kv_cache_manager::OffsetTableDimensions; + using KVCacheManager = kv_cache_manager::KVCacheManager; + using KvCacheType = kv_cache_manager::CacheType; + using KvCacheConfig = executor::KvCacheConfig; + using RnnStateManager = rnn_state_manager::RnnStateManager; + using LlmRequestPtr = std::shared_ptr<batch_manager::LlmRequest>; + +public: + class IterationStatsIFB + { + public: + explicit IterationStatsIFB(SizeType32 microBatchId) + : microBatchId{microBatchId} + { + } + + SizeType32 microBatchId; + SizeType32 numCtxRequests{}; + SizeType32 numGenRequests{}; + SizeType32 numCtxTokens{}; + float avgNumDecodedTokensPerIter{}; + ReqIdsSet scheduledRequests; + ReqIdsSet pausedRequests; + }; + + using SizeType32 = tensorrt_llm::runtime::SizeType32; + using TokenIdType = tensorrt_llm::runtime::TokenIdType; + using BufferManager = tensorrt_llm::runtime::BufferManager; + using PeftTable = PeftCacheManager::PeftTable; + using TensorMap = runtime::StringPtrMap<runtime::ITensor>; + using TensorPtr = runtime::ITensor::SharedPtr; + + TrtGptModelInflightBatching(std::shared_ptr<nvinfer1::ILogger> logger, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig, runtime::RawEngine const& rawEngine, bool ctxGenFusion, + executor::ExecutorConfig const& executorConfig, bool isLeaderInOrchMode); + + ~TrtGptModelInflightBatching() override; + + /// @brief Calculate the cache size per token for the disaggregated serving. + /// @param modelConfig Model configuration. + /// @param worldConfig World configuration. + /// @param maxAttentionWindowVec Maximum attention window vector. (may have fewer elements than numLayers, in which + /// case it cycles) + /// @param isCrossAttention Whether the attention is cross attention. + /// @param kvFactor KV factor. + /// @return Cache size per token for the disaggregated layers. Note that window size is not included in the result + /// here. + [[nodiscard]] static std::map<SizeType32, SizeType32> calculateCacheSizePerTokenForDisagg( + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + std::vector<SizeType32> const& maxAttentionWindowVec, bool isCrossAttention, SizeType32 kvFactor); + + void terminateRequest(LlmRequestPtr const& llmRequest, bool pause = false) override; + + /// @brief Terminate request in the next forwardSync call that includes the request. + /// @details This function does not terminate requests immediately. It will add the requests to the + /// mReqIdsToTerminate set. The requests will be terminated in the next forwardSync call that + /// includes the request in the batch. + void terminateRequestSync(LlmRequestPtr const& llmRequest, executor::FinishReason finishReason) override; + + /// @brief Function that waits for the decoding of requests in flight. + /// When the requests have finished or using speculative decoding, the state of requests + /// will become LlmRequestState::kGENERATION_COMPLETE. Else, it will be set to + /// LlmRequestState::kGENERATION_IN_PROGRESS. + void forwardSync() override; + + /// @brief Function that tries to advance the active requests. + /// Depending on resources available, it's possible that not all requests will get advanced. + /// Requests that may be in state LlmRequestState::kCONTEXT_INIT become + /// LlmRequestState::kGENERATION_IN_PROGRESS or LlmRequestState::kGENERATION_TO_COMPLETE. + /// @param activeRequests The list of request to try to advance. + void forwardAsync(RequestList const& activeRequests) override; + + /// @brief Override the runtime batch size for the model + void setRuntimeBatchSize(SizeType32 runtimeMaxBatchSize) override; + + /// @brief Get the runtime batch size for the model + [[nodiscard]] SizeType32 getRuntimeBatchSize() const override; + + /// @brief Override the runtime max num tokens for the model + void setRuntimeMaxNumTokens(SizeType32 runtimeMaxNumTokens) override; + + void updatePeftCache(std::shared_ptr<LlmRequest> const& llmRequest) override; + + [[nodiscard]] IterationStatsIFB getLastIterationStats() const + { + return mLastIterationStatsIFB; + } + + [[nodiscard]] TrtGptModelType getModelType() const override + { + return mCtxGenFusion ? TrtGptModelType::InflightFusedBatching : TrtGptModelType::InflightBatching; + }; + + [[nodiscard]] runtime::BufferManager const& getBufferManager() const override; + [[nodiscard]] runtime::BufferManager::CudaStreamPtr getRuntimeStreamPtr() const override; + + void getCurrentIterationStats(executor::IterationStats& stats) const override; + void getCurrentRequestStats(executor::RequestStatsPerIteration& stats) const override; + [[nodiscard]] executor::DebugTensorsPerIteration getCurrentDebugTensors() const override; + + [[nodiscard]] executor::IterationType getIterCounter() const noexcept override + { + return mIterCounter; + } + + [[nodiscard]] static bool executorConfigIsValid( + runtime::ModelConfig const& modelConfig, executor::ExecutorConfig const& executorConfig); + [[nodiscard]] static executor::ExecutorConfig fixExecutorConfig( + runtime::ModelConfig const& modelConfig, executor::ExecutorConfig const& executorConfig); + + void prepareDisaggGenInitRequests(RequestList const& activeRequests, RequestVector& newGenReques); + void checkDisaggGenTransferStatus(RequestList const& activeRequests); + void prepareDistGenBufferAndDecoder(RequestVector const& generationRequests); + + void resetIterationStats() override; + + runtime::SpeculativeDecodingMode getSpeculativeDecodingMode() const noexcept + { + return mModelConfig.getSpeculativeDecodingMode(); + } + + [[nodiscard]] SizeType32 numCachedCudaGraphs() const; + +private: + friend class TrtGptModelTest; + + [[nodiscard]] SizeType32 getContextBufferId() const + { + return mMicroBatchId; + } + + [[nodiscard]] SizeType32 getGenerationBufferId() const + { + return mNumMicroBatches + mMicroBatchId; + } + + [[nodiscard]] SizeType32 getFusedBufferId() const + { + return mMicroBatchId; + } + + [[nodiscard]] SizeType32 getNextMicroBatchId(SizeType32 bufferId) const + { + return (bufferId + 1) % mNumMicroBatches; + } + + [[nodiscard]] SizeType32 getPrevMicroBatchId(SizeType32 bufferId) const + { + return (bufferId + mNumMicroBatches - 1) % mNumMicroBatches; + } + + //! @brief Store full kv cache blocks contributed by req. + //! These blocks become reusable from next step. + void storeContextBlocks(std::shared_ptr<LlmRequest> const& req); + + //! @brief Store newest kv cache block for reuse. + //! The block become reusable from next step. + void storeNewBlock(std::shared_ptr<LlmRequest> const& req); + + //! @brief Set LayerProfiler to collect performance per layer. + void setLayerProfiler() override; + + //! @brief Print profile information per layer. + std::string getLayerProfileInfo() const override; + + std::tuple<SizeType32, TensorMap const&, TensorMap&> prepareBuffers( + RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId); + + //! @brief Capture graph of current batch state during engine execution. + //! This is based on the assumptions that + //! a) We can hide CPU graph capture behind the GPU engine execution. + //! b) Batch size in the next iterations won't change and we can reuse the graph multiple times. + void prepareGraph(SizeType32 bufferId, SizeType32 optProfileId); + + void executeContext(SizeType32 runtimeContextId, SizeType32 bufferId); + void executeBatch(ScheduledRequests const& scheduledRequests); + void executeStep( + RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId); + + void debugIOTensors(RequestVector const& contextRequests, RequestVector const& generationRequests, + TensorMap const& inputMap, TensorMap const& outputMap); + + void createRuntimeContexts(); + void createDecoder(std::optional<executor::DecodingMode> const& decodingModeOpt); + void createBuffers(executor::DecodingConfig const& decodingConfig, + std::optional<std::vector<executor::AdditionalModelOutput>> const& additionalModelOutputs); + std::unique_ptr<KVCacheManager> createKvCacheManager(KvCacheConfig const& kvCacheConfig, KvCacheType kvCacheType, + uint64_t freePrimaryMemBytes, uint64_t freeSecondaryMemBytes, size_t extraCostMemory, + bool const failFastOnAttentionWindowTooLarge = false); + void createRnnStateManager(); + void createCustomAllReduceWorkspace(); + void createRuntimePerfKnobsTensor(executor::ExtendedRuntimePerfKnobConfig const& extendedRuntimePerfKnobConfig); + + /// @brief Verify draft token length and beam width of all active requests. + /// May change operating beam width if all requests agree on same beam width. + void verifyRequests(RequestList const& activeRequests); + + /// @brief Change the operating beam width. + /// Only possible if no requests are currently in-flight. + /// @param beamWidth New operating beam width. Must be smaller than initial maxBeamWidth. + void changeBeamWidth(SizeType32 beamWidth); + + SizeType32 getOperatingBeamWidth() const override + { + return mOperatingBeamWidth; + } + + /// @details Should be called after setting up the current batch in executeBatch to get the correct number of + /// context tokens. + IterationStatsIFB fillIterationStats( + ScheduledRequests const& scheduledRequests, RequestVector const& requestsToPause); + + /// @brief Function that sets up the TensorRT execution context that is going to be used for execution. If multiple + /// TensorRT optimization profiles are built in the engine, it selects the corresponding context that is going to be + /// used, and prepares the input and output tensors so that both buffers and the context is ready for the execution. + /// @return The TensorRT execution context index that has been setup. + void setupContext( + RequestVector const& contextRequests, RequestVector const& generationRequests, SizeType32 bufferId); + + void setupDecoderStep( + RequestVector const& contextRequests, RuntimeBuffers const& buffers, DecoderInputBuffers& inputBuffers); + runtime::CudaEvent decoderStepAsync(ScheduledRequests const& scheduledRequests); + std::vector<std::unique_ptr<DecoderStepAsyncSend>> decoderSync( + ScheduledRequests const& scheduledRequests, std::optional<runtime::CudaEvent> const& decoderFinishEvent); + + std::vector<std::unique_ptr<DecoderStepAsyncSend>> communicateDecoderBuffers(bool returnLogProbs); + void updateRequests(ScheduledRequests const& scheduledRequests); + + /// @brief It gathers the logits if they need to be returned, calls getDecoderSlotHostOutputs, + /// and overwrites the llmRequest tokens buffer. + /// Called either on request finishing, or at every step when doing beam search and streaming. + void postProcessRequest(LlmRequest& llmReq, std::vector<SizeType32> const& numDroppedTokens); + /// @brief Reorders generation logits to match finalized beam paths after gatherTree. + /// During beam search, logits are stored by beam slot. After finalization, output_ids are + /// reordered by parentIds, but logits are not. This method traces parentIds on the host + /// to build the slot mapping and reindexes the logits accordingly. + void reorderGenerationLogitsForBeamSearch(LlmRequest& llmReq, SizeType32 seqSlot, SizeType32 reqBeamWidth, + SizeType32 maxSeqLength, TokenIdType const* outputIdsHostData, SizeType32 const* sequenceLengthsHostData); + /// @brief Calls gatherTree (via finalize) and transmits the received data across ranks if PP>1 + void getDecoderSlotHostOutputs( + SizeType32 seqSlot, bool returnLogProbs, runtime::SamplingConfig const& samplingConfig, bool streaming); + void rewindKVCacheBlocks(SizeType32 numSequences); + void setupSpeculativeDecodingModule(executor::DecodingConfig const& decodingConfig); + + /// @brief Copies the content of the cache indirection outputs to the cache indirection inputs. + /// @param[in] scheduledRequests The requests to copy the cache indirections for. + /// @param[in] genBufferId The id of the generation buffers for those requests. + void copyCacheIndirectionFromOutputsToInputs(ScheduledRequests const& scheduledRequests, SizeType32 genBufferId); + + [[nodiscard]] bool getGatherGenerationLogits() const override + { + return getModelConfig().computeGenerationLogits() || mGatherGenerationLogits; + } + + [[nodiscard]] runtime::ModelConfig const& getModelConfig() const override + { + return mModelConfig; + } + + [[nodiscard]] runtime::WorldConfig const& getWorldConfig() const override + { + return mWorldConfig; + } + + [[nodiscard]] SizeType32 getNumMicroBatches() const override + { + return mNumMicroBatches; + } + + [[nodiscard]] nvinfer1::DataType getLogitDataType() const override; + + [[nodiscard]] nvinfer1::DataType getTensorDataType(std::string const& name) const override; + + [[nodiscard]] nvinfer1::Dims getTensorShape(std::string const& name) const override; + + void reshapeKvTensors(OffsetTableDimensions const& dims); + + [[nodiscard]] bool hasSpeculativeDecodingFastLogits() const noexcept override + { + return mSpeculativeDecodingFastLogits; + } + + [[nodiscard]] bool hasGuidedDecoder() const noexcept override + { + return static_cast<bool>(mGuidedDecoder); + } + + using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; + /// @brief Based on the KV-cache manager's capacity and configuration, we adjust the maximum supported attention + /// window. + /// + /// @param blocksPerWindow map of window size to number of blocks. + /// @param failFastOnAttentionWindowTooLarge if true, the function will report a runtime error if the attention + /// window is too large to fit even a single sequence in the KV cache. + /// @return pair of new blocks per window and new maxAttentionWindowVec + [[nodiscard]] std::pair<BlocksPerWindow, std::vector<SizeType32>> clampWindowSizesToFitAtLeastOneSequence( + BlocksPerWindow const& blocksPerWindow, bool const failFastOnAttentionWindowTooLarge = false); + + /// @brief Change the speculative decoding mode. + void changeSpecDecMode(ScheduledRequests const& scheduledRequests); + + void prefetchNextPromptTableChunk(RequestVector const& contextRequests, bool isFirstChunk, SizeType32 bufferId); + + void remapInputTokensForPromptTable( + std::shared_ptr<LlmRequest> const& llmReq, bool isCurrentChunk, SizeType32 bufferId, SizeType32 contextId); + + void copyPromptTableToGpuInChunk(std::shared_ptr<LlmRequest> const& llmReq, + std::vector<int32_t> const& outOfVocabTokens, bool useCurrentBuffer, SizeType32 bufferId, SizeType32 contextId); + +protected: + std::shared_ptr<BaseKVCacheManager> getKVCacheManager() override + { + return mKvCacheManager; + } + + [[nodiscard]] std::shared_ptr<BaseKVCacheManager const> getKVCacheManager() const override + { + return mKvCacheManager; + } + + std::shared_ptr<BaseKVCacheManager> getCrossKVCacheManager() + { + return mCrossKvCacheManager; + } + + [[nodiscard]] std::shared_ptr<BaseKVCacheManager const> getCrossKVCacheManager() const + { + return mCrossKvCacheManager; + } + + [[nodiscard]] std::shared_ptr<BasePeftCacheManager> getPeftCacheManager() override + { + return mPeftCacheManager; + } + + [[nodiscard]] std::shared_ptr<BasePeftCacheManager const> getPeftCacheManager() const override + { + return mPeftCacheManager; + } + + void setLogitsPostProcessorBatched(std::optional<LogitsPostProcessorBatched> logitsPostProcessorBatched) override + { + mLogitsPostProcessorBatched = logitsPostProcessorBatched; + } + + void setReplicateLogitsPostProcessor(bool replicateLogitsPostProcessor) override + { + mReplicateLogitsPostProcessor = replicateLogitsPostProcessor; + } + + [[nodiscard]] bool getReplicateLogitsPostProcessor() const override + { + return mReplicateLogitsPostProcessor; + } + + SizeType32 getMaxCapacityBatchSize(SizeType32 inputLength, SizeType32 outputLength) const override; + +private: + /******************** Configs ********************/ + // Parameters of the model (TRT engine) + runtime::ModelConfig mModelConfig; + // Parameters of the execution environment + runtime::WorldConfig mWorldConfig; + // Device ID of this instance + int mDevice{-1}; + // Config for (speculative) decoding + executor::DecodingConfig mDecodingConfig; + // Performance knobs for the engine. + executor::ExtendedRuntimePerfKnobConfig mExtendedRuntimePerfKnobConfig; + TensorPtr mExtendedRuntimePerfKnobsHost; + // Config for debugging output + std::optional<executor::DebugConfig> mDebugConfig; + // List of additional outputs for each request + std::optional<std::vector<executor::AdditionalModelOutput>> mAdditionalModelOutputs; + + /******************** Components ********************/ + std::shared_ptr<nvinfer1::ILogger> mLogger; + // Runner for the TRT engine. The engine produces logits. + std::unique_ptr<runtime::TllmRuntime> mRuntime; + // Decoder that generates new tokens from the logits. + std::unique_ptr<runtime::GptDecoderBatched> mDecoder; + // Decoder state for all requests + std::unique_ptr<runtime::decoder::DecoderState> mDecoderState; + // Synchronization handles for decoder + std::vector<std::optional<runtime::CudaEvent>> mDecoderFinishedEvents; + + // Manager that maps requests to slots + std::shared_ptr<SequenceSlotManager> mSeqSlotManager; + // KV cache manager for attention layers (optional) + std::shared_ptr<BaseKVCacheManager> mKvCacheManager; + // KV cache manager for cross attention in enc-dec models (optional) + std::shared_ptr<BaseKVCacheManager> mCrossKvCacheManager = nullptr; + // RNN state manager for recurrent layers (optional) + std::unique_ptr<RnnStateManager> mRnnStateManager; + // PEFT cache manager for LoRA tasks (optional) + std::shared_ptr<BasePeftCacheManager> mPeftCacheManager; + // BufferManager using a separate stream for async copy operations. + runtime::BufferManager mCopyBufferManager; + // Event for async data transfers + runtime::CudaEvent mPtableCopyDoneEvent; + + /******************** Logits Post-Processor ********************/ + std::optional<LogitsPostProcessorBatched> mLogitsPostProcessorBatched; + bool mReplicateLogitsPostProcessor{true}; + // Set if any request invoked a logits processor in current step + bool mLogitsPostProcessorIsApplied{false}; + + constexpr bool broadcastPostDecoder() + { + return mWorldConfig.isTensorParallel() && !mReplicateLogitsPostProcessor && mLogitsPostProcessorIsApplied; + } + + std::unique_ptr<tensorrt_llm::batch_manager::GuidedDecoder> mGuidedDecoder; + + /******************** Pipeline parallelism ********************/ + std::unique_ptr<tensorrt_llm::mpi::MpiComm> mMpiCommPipelinePara; + std::vector<std::unique_ptr<DecoderStepAsyncSend>> mDecStepAsyncSndHdls; + std::vector<std::unique_ptr<DecoderSlotAsyncSend>> mDecSlotAsyncSndHdls; + std::unique_ptr<tensorrt_llm::mpi::MpiWaitThread> mAsyncSendWaitThread; + + /******************** Tensor parallelism ********************/ + std::unique_ptr<tensorrt_llm::mpi::MpiComm> mMpiCommTensorPara; + std::unique_ptr<runtime::AllReduceBuffers> mAllReduceBuffers; + + /******************** Runtime parameters ********************/ + // Flag to select fused or unfused context+generation execution + bool mCtxGenFusion; + // ID of current micro batch, changes after each iteration + SizeType32 mMicroBatchId{0}; + // Number of micro batches. Multiple batches are used for overlapping setup and execution, + // and in pipeline parallelism. + SizeType32 mNumMicroBatches; + // Number of buffers to be added to mBuffers. + SizeType32 mNumBuffers; + // Current operating beam width. Can be changed with changeBeamWidth function. + SizeType32 mOperatingBeamWidth; + // Runtime batch size optimized during execution for microBatchScheduler: + /// The max batch size recommended by the dynamic tuner + SizeType32 mMaxBatchSizeTunerRecommended; + /// The min of mMaxBatchSize and mMaxBatchSizeTunerRecommended + SizeType32 mMaxBatchSizeRuntime; + // Runtime max num tokens optimized during execution for microBatchScheduler: + /// Build time max num tokens + std::optional<SizeType32> mMaxNumTokensStatic; + /// The max num tokens recommended by the dynamic tuner + SizeType32 mMaxNumTokensTunerRecommended; + /// The min of mMaxNumTokens and mMaxNumTokensTunerRecommended + std::optional<SizeType32> mMaxNumTokensRuntime; + // Controls if generation logits should be gathered, so that returnGenerationLogits can be requested. + bool mGatherGenerationLogits{false}; + // offloading and prefetching the prompt tuning table (only effective in chunked prefill mode) + bool mPromptTableOffloading; + + /******************** Buffers ********************/ + // Buffers for each micro batch. Unfused path (mCtxGenFusion==false) uses two times the buffers. + std::vector<std::unique_ptr<RuntimeBuffers>> mBuffers; + // Decoder input buffers for each micro batch. + std::vector<DecoderInputBuffers> mDecoderInputBuffers; + // Decoder output buffers for each micro batch. + std::vector<DecoderOutputBuffers> mDecoderOutputBuffers; + // Buffers for each slot in the decoder + std::vector<std::unique_ptr<SlotDecoderBuffers>> mSlotDecoderBuffers; + // PEFT table for each micro batch + std::vector<PeftTable> mPeftTables; + + /******************** Book keeping ********************/ + // List of requests in each micro batch + std::vector<ScheduledRequests> mMicroBatchScheduledRequests; + // Set of in-flight requests of *all* micro batches + ReqIdsSet mInflightReqIds; + // Requests that should be terminated (requested from outside the model) + std::unordered_map<RequestIdType, executor::FinishReason> mReqIdsToTerminate; + // Requests that the scheduler selected to be paused + ReqIdsSet mReqIdsToPause; + // Stats collected in last iteration + IterationStatsIFB mLastIterationStatsIFB{-1}; + // Iteration counter used to distinguish debug output + executor::IterationType mIterCounter{0}; + // Debug tensors of last itreation + TensorMap mLastIterationDebugTensors; + // Cuda graph instances for each microbatch. + std::vector<utils::CudaGraphExecutorCache> mCudaGraphExecutorCaches; + + /******************** Cache transceiver ********************/ + std::unique_ptr<BaseCacheTransceiver> mCacheTransceiver; + std::unique_ptr<DisaggTransferAdmissionController> mDisaggTransferAdmissionController; + + /******************** Spec dec ***********************/ + std::unique_ptr<std::thread> mDraftModelSendLogitsThread; + bool mSpeculativeDecodingFastLogits; + std::atomic<bool> mDraftModelThreadShouldExit{false}; + bool mIsLeaderInOrchMode{false}; + // List of completed draft requests which logits will need to be sent to the target model. + // Guarded by mDraftRequestsMtx (shared with the background logits sender thread). + RequestVector mDraftRequestsWaitingToSendLogits; + // Draft requests whose logits have been sent — pending termination by main thread. + // Guarded by mDraftRequestsMtx. + RequestVector mDraftRequestsDoneSendingLogits; + std::mutex mDraftRequestsMtx; + SizeType32 mSeamlessLADMaxDraftLen{0}; + bool mUseSeamlessLookahead{false}; + RewindInputs mRewindInputs; + + /******************** Algorithms ********************/ + // Algorithms are reentrant, they are assigned a state at + // construction time and it is not modified through execution, hence they are const. + // Schedulers that select which requests to run in each iteration + std::unique_ptr<tensorrt_llm::batch_manager::CapacityScheduler const> mCapacityScheduler; + std::unique_ptr<tensorrt_llm::batch_manager::MicroBatchScheduler const> mMicroBatchScheduler; + std::unique_ptr<tensorrt_llm::batch_manager::PauseRequests const> mPauseRequests; + std::unique_ptr<tensorrt_llm::batch_manager::AssignReqSeqSlots const> mAssignReqSeqSlots; + std::unique_ptr<tensorrt_llm::batch_manager::AllocateKvCache const> mAllocateKvCache; + std::unique_ptr<tensorrt_llm::batch_manager::HandleContextLogits const> mHandleContextLogits; + std::unique_ptr<tensorrt_llm::batch_manager::HandleGenerationLogits const> mHandleGenerationLogits; + std::unique_ptr<tensorrt_llm::batch_manager::LogitsPostProcessor const> mLogitsPostProcessor; + std::unique_ptr<tensorrt_llm::batch_manager::MakeDecodingBatchInputOutput const> mMakeDecodingBatchInputOutput; + std::unique_ptr<tensorrt_llm::batch_manager::CreateNewDecoderRequests const> mCreateNewDecoderRequests; + std::unique_ptr<tensorrt_llm::batch_manager::UpdateDecoderBuffers const> mUpdateDecoderBuffers; +}; + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/updateDecoderBuffers.cpp b/cpp/tensorrt_llm/batch_manager/updateDecoderBuffers.cpp new file mode 100644 index 000000000000..ead120135f3a --- /dev/null +++ b/cpp/tensorrt_llm/batch_manager/updateDecoderBuffers.cpp @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/updateDecoderBuffers.h" +#include "tensorrt_llm/batch_manager/decoderBuffers.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/runtime/decoderState.h" +#include "tensorrt_llm/runtime/iTensor.h" + +namespace tensorrt_llm::batch_manager +{ + +using BufferManager = tensorrt_llm::runtime::BufferManager; +using TensorPtr = runtime::ITensor::SharedPtr; +using ITensor = runtime::ITensor; +using SizeType32 = tensorrt_llm::runtime::SizeType32; + +runtime::CudaEvent UpdateDecoderBuffers::operator()(runtime::ModelConfig const& modelConfig, + DecoderOutputBuffers& decoderOutputBuffers, runtime::BufferManager const& copyBufferManager, + runtime::decoder::DecoderState const& decoderState, bool returnLogProbs, + runtime::CudaEvent const& decoderFinishEvent) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(updateDecoderBuffers); + + // Chain copy after decoder event, using a different stream + copyBufferManager.getStream().wait(decoderFinishEvent); + + copyBufferManager.copy(*decoderState.getAllNewTokens(), *decoderOutputBuffers.newOutputTokensHost); + copyBufferManager.copy(*decoderState.getSequenceLengths(), *decoderOutputBuffers.sequenceLengthsHost); + + auto const finishedSumDevice = decoderState.getFinishedSum(); + copyBufferManager.copy(*finishedSumDevice, *decoderOutputBuffers.finishedSumHost); + auto const finishReasonsDevice = decoderState.getFinishReasons(); + copyBufferManager.copy(*finishReasonsDevice, *decoderOutputBuffers.finishReasonsHost); + + if (returnLogProbs) + { + copyBufferManager.copy(*decoderState.getCumLogProbs(), *decoderOutputBuffers.cumLogProbsHost); + copyBufferManager.copy(*decoderState.getLogProbs(), *decoderOutputBuffers.logProbsHost); + } + + if (modelConfig.getSpeculativeDecodingMode().predictsDraftTokens()) + { + // TODO: keep data on device for next iteration + copyBufferManager.copy(*decoderState.getNextDraftTokens(), *decoderOutputBuffers.nextDraftTokensHost); + + if (modelConfig.getSpeculativeDecodingMode().variableDraftLength()) + { + copyBufferManager.copy( + *decoderState.getNextDraftTokensLengths(), *decoderOutputBuffers.nextDraftTokensLengthsHost); + copyBufferManager.copy( + *decoderState.getPrevDraftTokensLengths(), *decoderOutputBuffers.prevDraftTokensLengthsHost); + } + } + + runtime::CudaEvent copyEvent{}; + copyBufferManager.getStream().record(copyEvent); + // Store the event for later sync. Sync stream before calling next decoder. Sync host before updating requests. + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return copyEvent; +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tensorrt_llm/batch_manager/utils/debugUtils.h b/cpp/tensorrt_llm/batch_manager/utils/debugUtils.h index c041c7a71de8..e4732a75f649 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/debugUtils.h +++ b/cpp/tensorrt_llm/batch_manager/utils/debugUtils.h @@ -23,6 +23,11 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/worldConfig.h" +namespace tensorrt_llm::runtime +{ +class TllmRuntime; +} // namespace tensorrt_llm::runtime + namespace tensorrt_llm::batch_manager::utils { diff --git a/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.cpp b/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.cpp index a3e54a6b0f9b..416235f347b8 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.cpp +++ b/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.cpp @@ -16,6 +16,7 @@ */ #include "inflightBatchingUtils.h" +#include "tensorrt_llm/runtime/runtimeKernels.h" namespace tensorrt_llm::batch_manager::utils { @@ -87,6 +88,170 @@ void moveFinishedContextRequestsToGeneration(ScheduledRequests& scheduledRequest TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); } +void copyGenerationLogits(RuntimeBuffers::GenerationLogitsCache& generationLogitsCache, + runtime::BufferManager const& bufferManager, LlmRequest& llmReq, bool beforeDecoder, + std::vector<SizeType32> const& numDroppedTokens) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK_WITH_INFO( + !beforeDecoder || numDroppedTokens.empty(), "numDroppedTokens are only possible after decoder."); + + auto const reqBeamWidth = llmReq.getBeamWidthByIter(); + TLLM_CHECK_WITH_INFO(numDroppedTokens.empty() || numDroppedTokens.size() == static_cast<size_t>(reqBeamWidth), + "Dropped tokens have to be defined for all beams."); + + auto const fragmentSize = llmReq.getGenerationLogitsFragmentsSize(); + + // Merge logits fragments on device. getFragmentPointerSlot() returns the matching host and + // device rows for the current workIdx and advances the index atomically, so concurrent flushes + // for different requests in the same batch never clobber each other's pointer arrays. + auto const& transposeBufferPtr = generationLogitsCache.transposedLogits; + auto [cachePointerHost, cachePointerDevice] = generationLogitsCache.getFragmentPointerSlot(); + tensorrt_llm::runtime::kernels::mergeLogitsFragments(bufferManager, *transposeBufferPtr, + llmReq.getGenerationLogitsFragments(), *cachePointerDevice, *cachePointerHost, 0, 1, reqBeamWidth, + bufferManager.getStream(), 0); + llmReq.clearGenerationLogitsFragments(); + + // Copy logits to host + for (SizeType32 beam = 0; beam < reqBeamWidth; beam++) + { + auto const droppedSize = !numDroppedTokens.empty() ? numDroppedTokens.at(beam) : 0; + // Ignore logits of dropped tokens + auto const beamFragmentSize = fragmentSize - droppedSize; + // If this function is called before the decoder, the request does not contain the generated token of the + // current iteration, so we add 1 to the number of tokens. + auto const numGenerationToken + = static_cast<SizeType32>(beforeDecoder) + llmReq.getNumTokens(beam) - llmReq.mPromptLen; + auto const hostOffset = numGenerationToken - beamFragmentSize; + + // [beamWidth, GENERATION_LOGITS_BUFFER_LENGTH, vocabSizePadded] -> [beamFragmentSize, vocabSizePadded] + auto beamDeviceTensorPtr = ITensor::slice(transposeBufferPtr, {beam, 0}, beamFragmentSize); + // [beamWidth, mMaxNewTokens, vocabSizePadded] -> [beamFragmentSize, vocabSizePadded] + auto beamHostTensorPtr = ITensor::slice(llmReq.getGenerationLogitsHost(), {beam, hostOffset}, beamFragmentSize); + bufferManager.copy(*beamDeviceTensorPtr, *beamHostTensorPtr); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +namespace +{ + +std::pair<TensorPtr const&, bool> findOutputTensor(std::string const& outputTensorName, + std::vector<executor::AdditionalModelOutput> const& additionalModelOutputs, + RuntimeBuffers::TensorMap const& outputMap, bool isContext) +{ + auto const aoIter = std::find_if(additionalModelOutputs.cbegin(), additionalModelOutputs.cend(), + [&outputTensorName](auto const& ao) { return ao.name == outputTensorName; }); + TLLM_CHECK_WITH_INFO(aoIter != additionalModelOutputs.cend(), "Additional %s output tensor not found: %s", + isContext ? "context" : "generation", outputTensorName.c_str()); + + auto const gatherContext = aoIter->gatherContext; + if (isContext) + { + TLLM_CHECK_WITH_INFO( + gatherContext, "Additional context output tensor not gathered: %s", outputTensorName.c_str()); + } + + auto const tensorIt = outputMap.find(outputTensorName); + TLLM_CHECK_WITH_INFO(tensorIt != outputMap.end(), "Additional %s output tensor not found: %s", + isContext ? "context" : "generation", outputTensorName.c_str()); + + return {tensorIt->second, gatherContext}; +} + +} // namespace + +void copyAdditionalOutputs(std::vector<executor::AdditionalModelOutput> const& additionalModelOutputs, + RequestVector const& contextRequests, RequestVector const& generationRequests, + RuntimeBuffers::TensorMap const& outputMap, runtime::BufferManager const& manager) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // One index shared across all output tensors that have gatherContext + SizeType32 srcTensorIndexWithContext{0}; + // One index shared across all output tensors that do not have gatherContext + SizeType32 srcTensorIndexWithoutContext{0}; + + for (auto const& llmReq : contextRequests) + { + auto numContextTokens = llmReq->getContextChunkSize(); + for (auto const& outputTensor : llmReq->getAdditionalContextOutputs()) + { + auto const& [tensor, gatherContext] + = findOutputTensor(outputTensor.first, additionalModelOutputs, outputMap, true); + + auto const srcTensorIndex = srcTensorIndexWithContext; + auto srcView = ITensor::slice(tensor, srcTensorIndex, numContextTokens); + auto dstView = ITensor::slice(outputTensor.second, llmReq->getContextCurrentPosition(), numContextTokens); + manager.copy(*srcView, *dstView); + } + srcTensorIndexWithContext += numContextTokens; + srcTensorIndexWithoutContext += 1; + + // Copy output of last token to generation outputs + if (llmReq->isLastContextChunk()) + { + for (auto const& outputTensor : llmReq->getAdditionalGenerationOutputs()) + { + auto const& [tensor, gatherContext] + = findOutputTensor(outputTensor.first, additionalModelOutputs, outputMap, false); + + auto const srcTensorIndex = gatherContext ? srcTensorIndexWithContext : srcTensorIndexWithoutContext; + auto srcView = ITensor::slice(tensor, srcTensorIndex - 1, 1); + for (SizeType32 beam = 0; beam < llmReq->getBeamWidthByIter(); beam++) + { + auto dstView = ITensor::slice(outputTensor.second, {beam, 0}, 1); + manager.copy(*srcView, *dstView); + } + } + } + } + + for (auto const& llmReq : generationRequests) + { + auto const reqBeamWidth = llmReq->getBeamWidthByIter(); + for (auto const& outputTensor : llmReq->getAdditionalGenerationOutputs()) + { + auto const& [tensor, gatherContext] + = findOutputTensor(outputTensor.first, additionalModelOutputs, outputMap, false); + + auto const srcTensorIndex = gatherContext ? srcTensorIndexWithContext : srcTensorIndexWithoutContext; + for (SizeType32 beam = 0; beam < reqBeamWidth; beam++) + { + auto const generatedLength = llmReq->getNumTokens(beam) - llmReq->getPromptLen(); + TLLM_CHECK(generatedLength >= 1); + auto srcView = ITensor::slice(tensor, srcTensorIndex + beam, 1); + auto dstView = ITensor::slice(outputTensor.second, {beam, generatedLength}, 1); + manager.copy(*srcView, *dstView); + } + } + srcTensorIndexWithContext += reqBeamWidth; + srcTensorIndexWithoutContext += reqBeamWidth; + } + + // Check final indices + for (auto const& outputTensor : additionalModelOutputs) + { + auto const& outputTensorName = outputTensor.name; + auto const gatherContext = outputTensor.gatherContext; + + auto const tensorIt = outputMap.find(outputTensorName); + TLLM_CHECK_WITH_INFO( + tensorIt != outputMap.end(), "Additional output tensor not found: %s", outputTensorName.c_str()); + + auto const& outputShape = tensorIt->second->getShape(); + auto const outputSize = outputShape.d[0]; + auto const finalIndex = gatherContext ? srcTensorIndexWithContext : srcTensorIndexWithoutContext; + + TLLM_CHECK_WITH_INFO(finalIndex == outputSize, "Additional %s output tensor final index mismatch %d != %ld: %s", + gatherContext ? "context" : "generation", finalIndex, outputSize, outputTensorName.c_str()); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + void terminateRequest(SequenceSlotManager& seqSlotManager, LlmRequest& llmReq, SizeType32 maxInputLen, OptionalRef<kv_cache_manager::BaseKVCacheManager> kvCacheManager, OptionalRef<kv_cache_manager::BaseKVCacheManager> crossKvCacheManager, @@ -135,4 +300,109 @@ std::vector<SizeType32> getRequestBeamWidths( return beamWidths; } +void CudaGraphExecutor::create(cudaGraph_t const& graph) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + assert(mInstance == nullptr); + TLLM_CUDA_CHECK(cudaGraphInstantiate(&mInstance, graph, nullptr, nullptr, 0)); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void CudaGraphExecutor::uploadToStream(runtime::CudaStream const& stream) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + assert(hasInstance()); + TLLM_CUDA_CHECK(cudaGraphUpload(mInstance, stream.get())); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void CudaGraphExecutor::launch(runtime::CudaStream const& stream) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + TLLM_CUDA_CHECK(cudaGraphLaunch(mInstance, stream.get())); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +bool CudaGraphExecutor::update(cudaGraph_t const& graph) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + return cudaGraphExecUpdate(mInstance, graph, nullptr) != cudaSuccess; +} + +void CudaGraphExecutor::clear() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + if (mInstance != nullptr) + { + TLLM_CUDA_CHECK(cudaGraphExecDestroy(mInstance)); + mInstance = nullptr; + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void CudaGraphExecutor::prepareNextGraph(std::unique_ptr<runtime::TllmRuntime>& runtime, SizeType32 nextContextId) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + auto& stream = runtime->getStream(); + + cudaGraph_t nextGraph; + TLLM_CUDA_CHECK(cudaStreamBeginCapture(stream.get(), cudaStreamCaptureModeThreadLocal)); + runtime->executeContext(nextContextId); + TLLM_CUDA_CHECK(cudaStreamEndCapture(stream.get(), &nextGraph)); + + if (hasInstance()) + { + if (update(nextGraph)) + { + clear(); + create(nextGraph); + } + } + else + { + create(nextGraph); + } + + TLLM_CUDA_CHECK(cudaGraphDestroy(nextGraph)); + uploadToStream(stream); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +std::optional<std::shared_ptr<CudaGraphExecutor>> CudaGraphExecutorCache::get(BatchState const& state) +{ + auto it = mMap.find(state); + if (it == mMap.end()) + { + return std::nullopt; + } + mCache.splice(mCache.begin(), mCache, it->second); + return it->second->second; +} + +void CudaGraphExecutorCache::put(BatchState const& state, std::shared_ptr<CudaGraphExecutor> const& value) +{ + auto it = mMap.find(state); + if (it != mMap.end()) + { + mCache.erase(it->second); + } + mCache.emplace_front(state, value); + mMap[state] = mCache.begin(); + + if (static_cast<runtime::SizeType32>(mMap.size()) > mCapacity) + { + auto lastState = mCache.back().first; + mCache.pop_back(); + mMap.erase(lastState); + } +} + +void CudaGraphExecutorCache::clear() +{ + // Releasing the shared_ptrs runs ~CudaGraphExecutor, which calls + // cudaGraphExecDestroy on each cached instance. + mMap.clear(); + mCache.clear(); +} + } // namespace tensorrt_llm::batch_manager::utils diff --git a/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h b/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h index 374ae398f781..fe0c4e505218 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h +++ b/cpp/tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h @@ -20,6 +20,7 @@ #include "tensorrt_llm/batch_manager/common.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/peftCacheManager.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" #include "tensorrt_llm/batch_manager/sequenceSlotManager.h" #include "tensorrt_llm/common/optionalRef.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -49,6 +50,17 @@ void sortRequests(RequestVector& contextRequests, RequestVector& generationReque //! @param scheduledRequests The scheduled context and generation requests. void moveFinishedContextRequestsToGeneration(ScheduledRequests& scheduledRequests); +//! @param beforeDecoder Whether the function is called before the decoder. If it is true, correct the output offset. +//! @param numDroppedTokens The number of dropped tokens for each beam (e.g. when the requests finished early). +//! Generation logits for dropped tokens are ignored. +void copyGenerationLogits(RuntimeBuffers::GenerationLogitsCache& generationLogitsCache, + runtime::BufferManager const& bufferManager, LlmRequest& llmReq, bool beforeDecoder, + std::vector<SizeType32> const& numDroppedTokens = {}); + +void copyAdditionalOutputs(std::vector<executor::AdditionalModelOutput> const& additionalModelOutputs, + RequestVector const& contextRequests, RequestVector const& generationRequests, + RuntimeBuffers::TensorMap const& outputMap, runtime::BufferManager const& manager); + void terminateRequest(SequenceSlotManager& seqSlotManager, LlmRequest& llmRequest, SizeType32 maxInputLen, OptionalRef<kv_cache_manager::BaseKVCacheManager> kvCacheManager = std::nullopt, OptionalRef<kv_cache_manager::BaseKVCacheManager> crossKvCacheManager = std::nullopt, @@ -56,4 +68,66 @@ void terminateRequest(SequenceSlotManager& seqSlotManager, LlmRequest& llmReques std::vector<SizeType32> getRequestBeamWidths( RequestVector const& contextRequests, RequestVector const& generationRequests); + +class CudaGraphExecutor +{ +public: + CudaGraphExecutor() = default; + + ~CudaGraphExecutor() + { + try + { + clear(); + } + catch (std::exception& e) + { + TLLM_LOG_EXCEPTION(e); + } + } + + bool hasInstance() const + { + return mInstance != nullptr; + } + + void clear(); + void prepareNextGraph(std::unique_ptr<runtime::TllmRuntime>& runtime, SizeType32 nextContextId); + void launch(runtime::CudaStream const& stream); + +private: + void create(cudaGraph_t const& graph); + bool update(cudaGraph_t const& graph); + void uploadToStream(runtime::CudaStream const& stream); + + cudaGraphExec_t mInstance = nullptr; +}; + +class CudaGraphExecutorCache +{ + /// @brief LRU cache to store cuda graph instances. +public: + explicit CudaGraphExecutorCache(runtime::SizeType32 capacity) + : mCapacity(capacity) + { + } + + std::optional<std::shared_ptr<CudaGraphExecutor>> get(BatchState const& state); + + void put(BatchState const& state, std::shared_ptr<CudaGraphExecutor> const& value); + + void clear(); + + [[nodiscard]] runtime::SizeType32 size() const noexcept + { + return static_cast<runtime::SizeType32>(mCache.size()); + } + +private: + using BatchStateGraphExecutorPair = std::pair<BatchState, std::shared_ptr<CudaGraphExecutor>>; + using GraphExecutorLruCache = std::list<BatchStateGraphExecutorPair>; + SizeType32 mCapacity; + GraphExecutorLruCache mCache; + std::unordered_map<BatchState, GraphExecutorLruCache::iterator, BatchStateHash> mMap; +}; } // namespace tensorrt_llm::batch_manager::utils diff --git a/cpp/tensorrt_llm/batch_manager/utils/logitsThread.cpp b/cpp/tensorrt_llm/batch_manager/utils/logitsThread.cpp index 4f978a302187..941c1b655073 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/logitsThread.cpp +++ b/cpp/tensorrt_llm/batch_manager/utils/logitsThread.cpp @@ -19,7 +19,6 @@ #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/utils/mpiTags.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" @@ -125,7 +124,7 @@ void draftModelSendLogitsThread(int device, std::atomic<bool>* draftModelThreadS } void targetModelReceiveLogits(runtime::ITensor::SharedPtr& draftLogitsHost, - executor::SpeculativeDecodingFastLogitsInfo const& fastLogitsInfo, tensorrt_llm::DataType logitsDtype) + executor::SpeculativeDecodingFastLogitsInfo const& fastLogitsInfo, nvinfer1::DataType logitsDtype) { #if ENABLE_MULTI_DEVICE auto const& worldComm = tensorrt_llm::mpi::MpiComm::world(); diff --git a/cpp/tensorrt_llm/batch_manager/utils/logitsThread.h b/cpp/tensorrt_llm/batch_manager/utils/logitsThread.h index 7af3b1762d20..637f8a850610 100644 --- a/cpp/tensorrt_llm/batch_manager/utils/logitsThread.h +++ b/cpp/tensorrt_llm/batch_manager/utils/logitsThread.h @@ -18,7 +18,6 @@ #pragma once #include "tensorrt_llm/batch_manager/common.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -47,6 +46,6 @@ void draftModelSendLogitsThread(int device, std::atomic<bool>* draftModelThreadS std::mutex* draftRequestsMtx); void targetModelReceiveLogits(runtime::ITensor::SharedPtr& draftLogitsHost, - executor::SpeculativeDecodingFastLogitsInfo const& fastLogitsInfo, tensorrt_llm::DataType logitsDtype); + executor::SpeculativeDecodingFastLogitsInfo const& fastLogitsInfo, nvinfer1::DataType logitsDtype); } // namespace tensorrt_llm::batch_manager::utils diff --git a/cpp/tensorrt_llm/common/attentionOp.cpp b/cpp/tensorrt_llm/common/attentionOp.cpp index 762ed7b54164..b91c0ef98df6 100644 --- a/cpp/tensorrt_llm/common/attentionOp.cpp +++ b/cpp/tensorrt_llm/common/attentionOp.cpp @@ -22,7 +22,6 @@ #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/sageQuant.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention.h" #include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention/cascadeAttentionKernel.h" #include "tensorrt_llm/kernels/flashMLA/flash_mla.h" @@ -217,7 +216,6 @@ bool AttentionOp::convertMMHAParamsToXQAParams(tensorrt_llm::kernels::XQAParams& // Medusa mode will have multiple query tokens. xqaParams.multi_query_tokens = mIsSpecDecodingEnabled && mUseSpecDecoding; xqaParams.is_spec_dec_tree = mIsSpecDecTree; - xqaParams.force_prepare_spec_dec_tree_mask = mForcePrepareSpecDecTreeMask; xqaParams.layer_idx = generationsParams.layer_idx; if (mKVCacheQuantMode.hasInt8KvCache()) @@ -760,8 +758,8 @@ size_t AttentionOp::getFmhaMultiCtasKvScratchSize() const noexcept return partialStatsSize + partialOSize; } -size_t AttentionOp::getWorkspaceSizeForContext(tensorrt_llm::DataType type, int32_t max_num_seq, - int32_t input_seq_length, int32_t cross_kv_length, int32_t max_num_tokens, int32_t total_kv_len) const noexcept +size_t AttentionOp::getWorkspaceSizeForContext(nvinfer1::DataType type, int32_t max_num_seq, int32_t input_seq_length, + int32_t cross_kv_length, int32_t max_num_tokens, int32_t total_kv_len) const noexcept { if (max_num_tokens == 0) { @@ -913,7 +911,7 @@ size_t AttentionOp::getWorkspaceSizeForContext(tensorrt_llm::DataType type, int3 return context_workspace_size; } -size_t AttentionOp::getWorkspaceSizeForGeneration(tensorrt_llm::DataType type, int32_t max_num_seq, +size_t AttentionOp::getWorkspaceSizeForGeneration(nvinfer1::DataType type, int32_t max_num_seq, int32_t max_attention_window_size, int32_t max_num_tokens, int32_t max_blocks_per_sequence) const noexcept { if (max_num_tokens == 0) @@ -2820,7 +2818,7 @@ int AttentionOp::initialize() noexcept if (mEnableContextFMHA) { mEnableContextFMHA = false; - if (!(mType == tensorrt_llm::DataType::kHALF || mType == tensorrt_llm::DataType::kBF16)) + if (!(mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16)) { TLLM_LOG_WARNING("Fall back to unfused MHA because of unsupported data type."); } @@ -2865,7 +2863,7 @@ int AttentionOp::initialize() noexcept "mFP8ContextFMHA must enable if FP4 KV cache is enabled"); TLLM_CHECK(isRoPE() == (mRotaryEmbeddingDim != 0)); - TLLM_CHECK_WITH_INFO((mSM >= 80) || (mType != tensorrt_llm::DataType::kBF16), + TLLM_CHECK_WITH_INFO((mSM >= 80) || (mType != nvinfer1::DataType::kBF16), "Unsupported data type, pre SM 80 GPUs do not support bfloat16"); // Pre-check whether the head size is supported by MMHA. @@ -2917,11 +2915,11 @@ int AttentionOp::initialize() noexcept // Pre-checked during constructing. Data_type data_type, data_type_kv; - if (mType == tensorrt_llm::DataType::kHALF) + if (mType == nvinfer1::DataType::kHALF) { data_type = DATA_TYPE_FP16; } - else if (mType == tensorrt_llm::DataType::kBF16) + else if (mType == nvinfer1::DataType::kBF16) { data_type = DATA_TYPE_BF16; } @@ -3081,13 +3079,13 @@ int AttentionOp::initialize() noexcept Data_type kvDataType = DATA_TYPE_FP32; Data_type outputDataType = DATA_TYPE_FP32; - if (mType == tensorrt_llm::DataType::kHALF) + if (mType == nvinfer1::DataType::kHALF) { qDataType = DATA_TYPE_FP16; kvDataType = DATA_TYPE_FP16; outputDataType = DATA_TYPE_FP16; } - else if (mType == tensorrt_llm::DataType::kBF16) + else if (mType == nvinfer1::DataType::kBF16) { qDataType = DATA_TYPE_BF16; kvDataType = DATA_TYPE_BF16; @@ -3177,7 +3175,7 @@ int AttentionOp::initialize() noexcept } mEnableXQA = (mEnableXQA || mIsSpecDecodingEnabled) - && (mType == tensorrt_llm::DataType::kHALF || mType == tensorrt_llm::DataType::kBF16) && mUseKVCache; + && (mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16) && mUseKVCache; if (mEnableXQA) { @@ -3187,12 +3185,12 @@ int AttentionOp::initialize() noexcept fixedParams.isMLA = mIsGenerationMLA; // TODO: support more combinations. // Update Q and O dtype. - if (mType == tensorrt_llm::DataType::kHALF) + if (mType == nvinfer1::DataType::kHALF) { fixedParams.inputDataType = DATA_TYPE_FP16; fixedParams.outputDataType = DATA_TYPE_FP16; } - else if (mType == tensorrt_llm::DataType::kBF16) + else if (mType == nvinfer1::DataType::kBF16) { fixedParams.inputDataType = DATA_TYPE_BF16; fixedParams.outputDataType = DATA_TYPE_BF16; @@ -3260,6 +3258,10 @@ int AttentionOp::initialize() noexcept reserveSemaphoreArray(mNbMultiBlockSemaphores); } + if (isBuilding()) + { + return 0; + } #if ENABLE_MULTI_DEVICE if (mCpSize > 1 && COMM_SESSION.getSize() > 1) { diff --git a/cpp/tensorrt_llm/common/attentionOp.h b/cpp/tensorrt_llm/common/attentionOp.h index 438489348577..f7337c9c9cb2 100644 --- a/cpp/tensorrt_llm/common/attentionOp.h +++ b/cpp/tensorrt_llm/common/attentionOp.h @@ -20,7 +20,6 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/opUtils.h" #include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fused_multihead_attention_common.h" #include "tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_gemm.h" #include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderXQARunner.h" @@ -57,11 +56,10 @@ class AttentionOp [[nodiscard]] size_t getFmhaMultiCtasKvScratchSize() const noexcept; [[nodiscard]] int getHeadSize(bool checkInit = true) const; [[nodiscard]] int getMaxNumSeqLenTile(int batch_beam_size = 1) const; - [[nodiscard]] size_t getWorkspaceSizeForContext(tensorrt_llm::DataType type, int32_t nbReq, - int32_t max_input_length, int32_t cross_kv_length = 0, int32_t max_num_tokens = 0, - int32_t total_kv_len = 0) const noexcept; + [[nodiscard]] size_t getWorkspaceSizeForContext(nvinfer1::DataType type, int32_t nbReq, int32_t max_input_length, + int32_t cross_kv_length = 0, int32_t max_num_tokens = 0, int32_t total_kv_len = 0) const noexcept; // total_num_seq is the sum of beam_width for multiple requests - [[nodiscard]] size_t getWorkspaceSizeForGeneration(tensorrt_llm::DataType type, int32_t total_num_seq, + [[nodiscard]] size_t getWorkspaceSizeForGeneration(nvinfer1::DataType type, int32_t total_num_seq, int32_t max_attention_window_size, int32_t max_num_tokens, int32_t max_blocks_per_sequence) const noexcept; template <typename T> @@ -183,14 +181,14 @@ class AttentionOp if (this->context_lengths && batch_size > 0) { ss << "context_lengths: " - << *(runtime::ITensor::wrap((void*) this->context_lengths, tensorrt_llm::DataType::kINT32, + << *(runtime::ITensor::wrap((void*) this->context_lengths, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))) << std::endl; } if (this->sequence_lengths && batch_size > 0) { ss << "sequence_lengths: " - << *(runtime::ITensor::wrap((void*) this->sequence_lengths, tensorrt_llm::DataType::kINT32, + << *(runtime::ITensor::wrap((void*) this->sequence_lengths, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))) << std::endl; } @@ -480,7 +478,7 @@ class AttentionOp int mTpSize = 1; int mTpRank = 0; bool mUnfuseQkvGemm = false; - tensorrt_llm::DataType mType; + nvinfer1::DataType mType; int32_t mMaxContextLength = 0; int32_t mMaxSeqLen = 0; int32_t mMaxNumRequests = 0; @@ -503,7 +501,6 @@ class AttentionOp int32_t mSpecDecodingMaxGenerationLength = 1; // Static spec-dec tree length used by FMHA autotuning. int32_t mSpecDecodingTargetMaxGenLen = 0; - bool mForcePrepareSpecDecTreeMask = false; bool mIsMLAEnabled = false; bool mIsGenerationMLA = false; bool mUseGenFlashMLA = false; @@ -574,11 +571,11 @@ class AttentionOp mCrossAttention, mMaxDistance, mPosShiftEnabled, mPagedContextFMHA, mFP8ContextFMHA, mFP8AttenOutput, mFP8ContextMLA, mFP8GenerationMLA, mChunkPrefillBufferBatchSize, mDenseContextFMHA, mHasFullAttentionMask, mIsSpecDecodingEnabled, mUseSpecDecoding, mIsSpecDecTree, mSpecDecodingIsGenerationLengthVariable, - mSpecDecodingMaxGenerationLength, mSpecDecodingTargetMaxGenLen, mForcePrepareSpecDecTreeMask, mIsMLAEnabled, - mIsGenerationMLA, mUseGenFlashMLA, mUseSparseAttention, mUseTllmGenSparseAttentionPaged, - mUseTllmGenSparseAttention, mMLAParams.data(), mCpSize, mCpRank, mCpGroup, mNumAttnHeads, mNumAttnKVHeads, - mNumKVHeadsOrigin, mAttnTpSize, mAttnTpRank, mAttnCpSize, mAttnCpRank, mUlyssesMQABroadcast, - mEnableContextFMHA, mFMHAForceFP32Acc, mMultiBlockMode, mEnableXQA, mUseKVCache, mSkipAttn, mFuseFp4Quant, + mSpecDecodingMaxGenerationLength, mSpecDecodingTargetMaxGenLen, mIsMLAEnabled, mIsGenerationMLA, + mUseGenFlashMLA, mUseSparseAttention, mUseTllmGenSparseAttentionPaged, mUseTllmGenSparseAttention, + mMLAParams.data(), mCpSize, mCpRank, mCpGroup, mNumAttnHeads, mNumAttnKVHeads, mNumKVHeadsOrigin, + mAttnTpSize, mAttnTpRank, mAttnCpSize, mAttnCpRank, mUlyssesMQABroadcast, mEnableContextFMHA, + mFMHAForceFP32Acc, mMultiBlockMode, mEnableXQA, mUseKVCache, mSkipAttn, mFuseFp4Quant, mFusesDsv4InvRopeFp8Quant, mNbMultiBlockSemaphores, mAttentionChunkSize.value_or(-1), mSkipSoftmaxThresholdScaleFactorPrefill, mSkipSoftmaxThresholdScaleFactorDecode, mSageAttnNumEltsPerBlkQ, mSageAttnNumEltsPerBlkK, mSageAttnNumEltsPerBlkV, mSageAttnQkInt8); diff --git a/cpp/tensorrt_llm/common/cudaFp8Utils.cu b/cpp/tensorrt_llm/common/cudaFp8Utils.cu index 03aae114c94f..9cd0227739d8 100644 --- a/cpp/tensorrt_llm/common/cudaFp8Utils.cu +++ b/cpp/tensorrt_llm/common/cudaFp8Utils.cu @@ -421,45 +421,6 @@ __global__ void computeFP8QuantizeScale(T_S* quant_ptr, const T_W* weights, cons } } -// Vectorized PER_TENSOR amax→scale for bf16 input: 128-bit loads (8 bf16/thread), -// block reduce, one atomicMax per block. Paired with a numel-sized grid -// (scaleMatrixVecGridSize) it launches only as many blocks as the input needs, so -// a skinny decode activation ([64, 2048]) contends ~16 blocks on the scale address -// instead of the scalar path's fixed grid(1024) — whose ~1024-way single-address -// atomicMax plus ~900 idle blocks give a data-independent ~6us floor. Mirrors the -// existing vectorized PER_TENSOR *apply* kernel (scaleMatrixPerTensorVec). -template <typename T_S> -__global__ void computeFP8QuantizeScalePerTensorVec(T_S* quant_ptr, __nv_bfloat16 const* weights, int64_t const numel) -{ - constexpr float min_scaling_factor = 1.0f / (FP8_E4M3_MAX * 512.f); - int64_t const vecElements = numel / kVecSize; - int64_t const stride = static_cast<int64_t>(blockDim.x) * gridDim.x; - float max = 0.f; - for (int64_t vi = threadIdx.x + static_cast<int64_t>(blockIdx.x) * blockDim.x; vi < vecElements; vi += stride) - { - float4 raw = *reinterpret_cast<float4 const*>(weights + vi * kVecSize); - __nv_bfloat162 const* pairs = reinterpret_cast<__nv_bfloat162 const*>(&raw); -#pragma unroll - for (int p = 0; p < kPairsPerVec; ++p) - { - float2 const f2 = __bfloat1622float2(pairs[p]); - max = fmaxf(max, fmaxf(fabsf(f2.x), fabsf(f2.y))); - } - } - // scalar tail (numel not divisible by 8) - int64_t const tailStart = vecElements * kVecSize; - for (int64_t i = tailStart + threadIdx.x + static_cast<int64_t>(blockIdx.x) * blockDim.x; i < numel; i += stride) - { - max = fmaxf(max, fabsf(static_cast<float>(weights[i]))); - } - max = blockReduceMax<float>(max); - if (threadIdx.x == 0) - { - auto const scale = (T_S) std::max(max / FP8_E4M3_MAX, min_scaling_factor); - atomicMaxExtd(quant_ptr, scale); - } -} - template <typename T_S, typename T_W> void invokeComputeFP8QuantizeScale(T_S* quant_ptr, const T_W* weights, const int64_t numel, const int64_t lda, QuantizeMode quantize_mode, cudaStream_t stream) @@ -480,34 +441,11 @@ void invokeComputeFP8QuantizeScale(T_S* quant_ptr, const T_W* weights, const int } else if (quantize_mode == QuantizeMode::PER_TENSOR) { + dim3 block(1024); + dim3 grid(1024); cudaMemsetAsync(quant_ptr, 0, sizeof(T_S), stream); sync_check_cuda_error(stream); - // Size the grid to the input (scaleMatrixVecGridSize), NOT a fixed 1024, so - // small activations launch only a handful of blocks — this is the fix for - // the ~6us fixed floor at skinny decode shapes. For 16B-aligned bf16 input - // use the vectorized amax kernel; otherwise the scalar kernel, still with a - // numel-sized grid. - dim3 const block(CTA_SIZE); - dim3 const grid(static_cast<unsigned int>(scaleMatrixVecGridSize(numel))); - bool const aligned = (reinterpret_cast<uintptr_t>(weights) % 16 == 0); - if constexpr (std::is_same_v<T_W, __nv_bfloat16>) - { - if (aligned) - { - computeFP8QuantizeScalePerTensorVec<<<grid, block, 0, stream>>>( - quant_ptr, reinterpret_cast<__nv_bfloat16 const*>(weights), numel); - } - else - { - computeFP8QuantizeScale<QuantizeMode::PER_TENSOR> - <<<grid, block, 0, stream>>>(quant_ptr, weights, numel, lda); - } - } - else - { - computeFP8QuantizeScale<QuantizeMode::PER_TENSOR> - <<<grid, block, 0, stream>>>(quant_ptr, weights, numel, lda); - } + computeFP8QuantizeScale<QuantizeMode::PER_TENSOR><<<grid, block, 0, stream>>>(quant_ptr, weights, numel, lda); } sync_check_cuda_error(stream); } @@ -597,28 +535,11 @@ void invokeComputeScalesAndQuantizeMatrix(T_OUT* output, T_S* quant_ptr, const T } else if (quantize_mode == QuantizeMode::PER_TENSOR) { + dim3 block(1024); + dim3 grid(1024); cudaMemsetAsync(quant_ptr, 0, sizeof(T_S), stream); sync_check_cuda_error(stream); - // Size the amax grid to the input (scaleMatrixVecGridSize), NOT a fixed 1024, - // so skinny decode activations launch only a handful of blocks — this is the - // fix for the ~6us fixed floor. bf16 (16B-aligned) uses the vectorized amax - // kernel; otherwise the scalar kernel, still numel-sized. Then apply as before. - dim3 const block(CTA_SIZE); - dim3 const grid(static_cast<unsigned int>(scaleMatrixVecGridSize(numel))); - bool const aligned = (reinterpret_cast<uintptr_t>(input) % 16 == 0); - if constexpr (std::is_same_v<T_IN, __nv_bfloat16>) - { - if (aligned) - computeFP8QuantizeScalePerTensorVec<<<grid, block, 0, stream>>>( - quant_ptr, reinterpret_cast<__nv_bfloat16 const*>(input), numel); - else - computeFP8QuantizeScale<QuantizeMode::PER_TENSOR> - <<<grid, block, 0, stream>>>(quant_ptr, input, numel, lda); - } - else - { - computeFP8QuantizeScale<QuantizeMode::PER_TENSOR><<<grid, block, 0, stream>>>(quant_ptr, input, numel, lda); - } + computeFP8QuantizeScale<QuantizeMode::PER_TENSOR><<<grid, block, 0, stream>>>(quant_ptr, input, numel, lda); sync_check_cuda_error(stream); invokeQuantizeMatrix(output, quant_ptr, input, numel, lda, quantize_mode, stream); } diff --git a/cpp/tensorrt_llm/common/envUtils.cpp b/cpp/tensorrt_llm/common/envUtils.cpp index 9dcf6c81634e..efef0362f596 100644 --- a/cpp/tensorrt_llm/common/envUtils.cpp +++ b/cpp/tensorrt_llm/common/envUtils.cpp @@ -401,12 +401,6 @@ bool getEnvTryZCopyForKVCacheTransfer() return zcopyForSysmmetricKVCache; } -bool getEnvDisaggEnableInflightCancel() -{ - static bool const enabled = getBoolEnv("TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL"); - return enabled; -} - bool getEnvForceDeterministic() { static bool const forceDeterministic = getBoolEnv("FORCE_DETERMINISTIC"); @@ -517,10 +511,10 @@ uint16_t getEnvNixlPort() return nixlPort; } -bool getEnvNixlDisableCoalesce() +bool getEnvNixlEnableCoalesce() { - static bool const disableCoalesce = getBoolEnv("TRTLLM_NIXL_DISABLE_COALESCE"); - return disableCoalesce; + static bool const enableCoalesce = getBoolEnv("TRTLLM_NIXL_ENABLE_COALESCE"); + return enableCoalesce; } bool getEnvDisaggBenchmarkGenOnly() diff --git a/cpp/tensorrt_llm/common/envUtils.h b/cpp/tensorrt_llm/common/envUtils.h index 13ad0399d574..649813253772 100644 --- a/cpp/tensorrt_llm/common/envUtils.h +++ b/cpp/tensorrt_llm/common/envUtils.h @@ -115,9 +115,6 @@ std::string const& getEnvKVCacheTimeOutputPath(); bool getEnvTryZCopyForKVCacheTransfer(); -// Opt-in for disaggregated KV transfer in-flight cancellation and fail-closed transfer-buffer quarantine. -bool getEnvDisaggEnableInflightCancel(); - // Force deterministic behavior for all kernels. bool getEnvForceDeterministic(); @@ -151,8 +148,7 @@ bool getEnvKVCachePoolUseFabricMemory(); uint16_t getEnvNixlPort(); -// Whether to disable coalescing of contiguous NIXL transfer descriptors (coalescing is on by default). -bool getEnvNixlDisableCoalesce(); +bool getEnvNixlEnableCoalesce(); bool getEnvDisaggBenchmarkGenOnly(); diff --git a/cpp/tensorrt_llm/common/opUtils.cpp b/cpp/tensorrt_llm/common/opUtils.cpp index 560750c2ba93..ff9b57cdd099 100644 --- a/cpp/tensorrt_llm/common/opUtils.cpp +++ b/cpp/tensorrt_llm/common/opUtils.cpp @@ -17,7 +17,6 @@ #include "tensorrt_llm/common/opUtils.h" #include "tensorrt_llm/common/ncclUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/ipcNvlsMemory.h" #include "tensorrt_llm/runtime/utils/mpiTags.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" @@ -34,18 +33,18 @@ TRTLLM_NAMESPACE_BEGIN #if ENABLE_MULTI_DEVICE -std::unordered_map<tensorrt_llm::DataType, ncclDataType_t>* getDtypeMap() +std::unordered_map<nvinfer1::DataType, ncclDataType_t>* getDtypeMap() { - static std::unordered_map<tensorrt_llm::DataType, ncclDataType_t> dtypeMap = { - {tensorrt_llm::DataType::kFLOAT, ncclFloat32}, - {tensorrt_llm::DataType::kHALF, ncclFloat16}, - {tensorrt_llm::DataType::kBF16, ncclBfloat16}, - {tensorrt_llm::DataType::kFP8, ncclInt8}, - {tensorrt_llm::DataType::kBOOL, ncclInt8}, - {tensorrt_llm::DataType::kINT32, ncclInt32}, - {tensorrt_llm::DataType::kINT64, ncclInt64}, - {tensorrt_llm::DataType::kUINT8, ncclUint8}, - {tensorrt_llm::DataType::kINT8, ncclInt8}, + static std::unordered_map<nvinfer1::DataType, ncclDataType_t> dtypeMap = { + {nvinfer1::DataType::kFLOAT, ncclFloat32}, + {nvinfer1::DataType::kHALF, ncclFloat16}, + {nvinfer1::DataType::kBF16, ncclBfloat16}, + {nvinfer1::DataType::kFP8, ncclInt8}, + {nvinfer1::DataType::kBOOL, ncclInt8}, + {nvinfer1::DataType::kINT32, ncclInt32}, + {nvinfer1::DataType::kINT64, ncclInt64}, + {nvinfer1::DataType::kUINT8, ncclUint8}, + {nvinfer1::DataType::kINT8, ncclInt8}, }; return &dtypeMap; } diff --git a/cpp/tensorrt_llm/common/opUtils.h b/cpp/tensorrt_llm/common/opUtils.h index 22169843a9f5..72e5a5ea3e09 100644 --- a/cpp/tensorrt_llm/common/opUtils.h +++ b/cpp/tensorrt_llm/common/opUtils.h @@ -21,7 +21,7 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/workspace.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cublasLt.h> #include <cublas_v2.h> #include <cuda_runtime.h> @@ -62,14 +62,14 @@ void read(char const*& buffer, T& val) buffer += sizeof(T); } -inline cudaDataType_t trtToCublasDtype(tensorrt_llm::DataType type) +inline cudaDataType_t trtToCublasDtype(nvinfer1::DataType type) { switch (type) { - case tensorrt_llm::DataType::kFLOAT: return CUDA_R_32F; - case tensorrt_llm::DataType::kHALF: return CUDA_R_16F; + case nvinfer1::DataType::kFLOAT: return CUDA_R_32F; + case nvinfer1::DataType::kHALF: return CUDA_R_16F; #if defined(NV_TENSORRT_MAJOR) && NV_TENSORRT_MAJOR >= 9 - case tensorrt_llm::DataType::kBF16: return CUDA_R_16BF; + case nvinfer1::DataType::kBF16: return CUDA_R_16BF; #endif default: TLLM_THROW("Not supported data type for cuBLAS"); } @@ -185,6 +185,13 @@ struct hash void const* getCommSessionHandle(); } // namespace common::op +inline bool isBuilding() +{ + auto constexpr key = "IS_BUILDING"; + auto const val = getenv(key); + return val != nullptr && std::string(val) == "1"; +} + #if ENABLE_MULTI_DEVICE #define NCCLCHECK(cmd) \ do \ @@ -207,7 +214,7 @@ void const* getCommSessionHandle(); } \ } while (0) -std::unordered_map<tensorrt_llm::DataType, ncclDataType_t>* getDtypeMap(); +std::unordered_map<nvinfer1::DataType, ncclDataType_t>* getDtypeMap(); std::shared_ptr<ncclComm_t> getComm(std::set<int> const& group); diff --git a/cpp/tensorrt_llm/common/safetensors.cpp b/cpp/tensorrt_llm/common/safetensors.cpp index 8bd91ccfbd51..9171f79e44e5 100644 --- a/cpp/tensorrt_llm/common/safetensors.cpp +++ b/cpp/tensorrt_llm/common/safetensors.cpp @@ -18,7 +18,7 @@ #include "nlohmann/json.hpp" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cstdint> #include <fstream> #include <map> @@ -30,7 +30,7 @@ TRTLLM_NAMESPACE_BEGIN namespace common::safetensors { -using tensorrt_llm::DataType; +using nvinfer1::DataType; static DataType convertDataTypeStrToEnum(std::string const& str) { diff --git a/cpp/tensorrt_llm/common/safetensors.h b/cpp/tensorrt_llm/common/safetensors.h index bdecf95e5909..e31225f1be24 100644 --- a/cpp/tensorrt_llm/common/safetensors.h +++ b/cpp/tensorrt_llm/common/safetensors.h @@ -18,7 +18,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cstdint> #include <map> #include <memory> @@ -34,13 +34,13 @@ class INdArray [[nodiscard]] virtual void const* data() const = 0; [[nodiscard]] virtual int ndim() const = 0; [[nodiscard]] virtual std::vector<int64_t> const& dims() const = 0; - [[nodiscard]] virtual tensorrt_llm::DataType dtype() const = 0; + [[nodiscard]] virtual nvinfer1::DataType dtype() const = 0; - [[nodiscard]] tensorrt_llm::Dims trtDims() const + [[nodiscard]] nvinfer1::Dims trtDims() const { - tensorrt_llm::Dims dims; + nvinfer1::Dims dims; dims.nbDims = ndim(); - TLLM_CHECK(dims.nbDims <= tensorrt_llm::Dims::MAX_DIMS); + TLLM_CHECK(dims.nbDims <= nvinfer1::Dims::MAX_DIMS); memset(dims.d, 0, sizeof(dims.d)); for (int i = 0; i < dims.nbDims; ++i) { diff --git a/cpp/tensorrt_llm/executor/CMakeLists.txt b/cpp/tensorrt_llm/executor/CMakeLists.txt index 358494e9989c..ca1ab298d7f4 100644 --- a/cpp/tensorrt_llm/executor/CMakeLists.txt +++ b/cpp/tensorrt_llm/executor/CMakeLists.txt @@ -27,7 +27,9 @@ set(SRCS contextPhaseParams.cpp debugConfig.cpp decodingConfig.cpp + executor.cpp executorConfig.cpp + executorImpl.cpp executorKVCacheEventManager.cpp extendedRuntimePerfKnobConfig.cpp guidedDecodingConfig.cpp @@ -50,6 +52,7 @@ set(SRCS response.cpp samplingConfig.cpp dynamicBatchConfig.cpp + dynamicBatchTuner.cpp schedulerConfig.cpp serialization.cpp speculativeDecodingConfig.cpp @@ -57,6 +60,7 @@ set(SRCS types.cpp requestUtils.cpp contextPhaseParams.cpp + disaggServerUtil.cpp cacheTransceiverConfig.cpp) if(NOT WIN32) diff --git a/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.cpp b/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.cpp index 2be557ac1f54..a21d470b898a 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.cpp @@ -189,45 +189,8 @@ void AgentConnection::send(DataContext const& ctx, void const* data, size_t size NotificationInfo notificationInfo{syncInfo}; std::stringstream ss; NotificationInfo::serialize(notificationInfo, ss); - bool const inflightCancelEnabled = common::getEnvDisaggEnableInflightCancel(); - TransferState transferState; - if (!inflightCancelEnabled) - { - transferState = status->wait(); - } - else - { - static constexpr int64_t kCancelPollTimeoutMs = 100; - transferState = TransferState::kIN_PROGRESS; - while (transferState == TransferState::kIN_PROGRESS) - { - transferState = status->wait(kCancelPollTimeoutMs); - if (transferState == TransferState::kIN_PROGRESS - && ctx.getTransferTerminate().load(std::memory_order_relaxed)) - { - bool const released = status->release(); - TLLM_LOG_WARNING( - "AgentConnection::send cancelled while transfer was in progress (ctx tag=%d, remote=%s, " - "releaseAccepted=%d)", - ctx.getTag(), mRemoteAgentName.c_str(), released); - TLLM_CHECK_WITH_INFO( - released, "AgentConnection::send cancel could not release the backend transfer handle"); - TLLM_THROW("AgentConnection::send cancelled mid-transfer"); - } - } - } + TransferState transferState = status->wait(); TLLM_CHECK_WITH_INFO(transferState == TransferState::kSUCCESS, "AgentConnection::send failed"); - if (inflightCancelEnabled && ctx.getTransferTerminate().load(std::memory_order_relaxed)) - { - bool const released = status->release(); - TLLM_LOG_WARNING( - "AgentConnection::send cancelled after transfer completed but before notify (ctx tag=%d, remote=%s, " - "releaseAccepted=%d)", - ctx.getTag(), mRemoteAgentName.c_str(), released); - TLLM_CHECK_WITH_INFO( - released, "AgentConnection::send pre-notify cancel could not release the backend transfer handle"); - TLLM_THROW("AgentConnection::send cancelled pre-notify"); - } // TODO: there is a bug in request_with_notify https://github.com/ai-dynamo/nixl/pull/252 mAgentConnectionManager->getAgent()->notifySyncMessage(mRemoteAgentName, ss.str()); } @@ -236,19 +199,11 @@ void AgentConnection::recv(DataContext const& ctx, void* data, size_t size) cons { NotificationSyncInfo syncInfo{mAgentName, ctx}; - bool const received - = mAgentConnectionManager->waitForSyncInfo(mRemoteAgentName, syncInfo, ctx.getTransferTerminate()); - if (common::getEnvDisaggEnableInflightCancel()) - { - TLLM_CHECK_WITH_INFO(received, - "AgentConnection::recv ended before receiving sync notification (ctx tag=%d, remote=%s)", ctx.getTag(), - mRemoteAgentName.c_str()); - } + mAgentConnectionManager->waitForSyncInfo(mRemoteAgentName, syncInfo, ctx.getTransferTerminate()); } void AgentConnection::sendRequestAndBufferInfo(batch_manager::RequestInfo& requestInfo, - std::vector<std::optional<size_t>> const& cacheBufferIds, int connectionIdx, - std::atomic<bool> const* perRequestCancel) + std::vector<std::optional<size_t>> const& cacheBufferIds, int connectionIdx) { TLLM_CHECK(!common::getEnvTryZCopyForKVCacheTransfer()); @@ -300,11 +255,6 @@ void AgentConnection::sendRequestAndBufferInfo(batch_manager::RequestInfo& reque std::stringstream ss; NotificationInfo notificationInfo{requestAndBufferInfo}; NotificationInfo::serialize(notificationInfo, ss); - if (common::getEnvDisaggEnableInflightCancel() && perRequestCancel != nullptr - && perRequestCancel->load(std::memory_order_relaxed)) - { - TLLM_THROW("sendRequestAndBufferInfo cancelled before notify"); - } mAgentConnectionManager->getAgent()->notifySyncMessage(mRemoteAgentName, ss.str()); } @@ -341,17 +291,9 @@ void AgentConnection::sendReadySignal(DataContext const& ctx, bool isReady) cons } bool AgentConnection::recvReadySignal(DataContext const& ctx) const -{ - return recvReadySignalWithStatus(ctx).value_or(false); -} - -std::optional<bool> AgentConnection::recvReadySignalWithStatus(DataContext const& ctx) const { ReadySignalInfo readySignalInfo{mAgentName, ctx, false}; - if (!mAgentConnectionManager->waitForReadySignal(mRemoteAgentName, readySignalInfo, ctx.getTransferTerminate())) - { - return std::nullopt; - } + mAgentConnectionManager->waitForReadySignal(mRemoteAgentName, readySignalInfo, ctx.getTransferTerminate()); return readySignalInfo.mIsReady; } @@ -750,7 +692,7 @@ int AgentConnectionManager::getDeviceId() const } template <typename NotificationType> -bool AgentConnectionManager::waitForNotification( +void AgentConnectionManager::waitForNotification( std::string const& remoteAgentName, NotificationType& expectedInfo, std::atomic<bool> const& terminateFlag) { while (!terminateFlag.load()) @@ -758,7 +700,7 @@ bool AgentConnectionManager::waitForNotification( if (!mIsRunning) { - return false; + return; } updateUnhandledNotifications(); std::scoped_lock lock(mNotificationMutex); @@ -791,7 +733,7 @@ bool AgentConnectionManager::waitForNotification( { it = mUnhandledNotifications.erase(it); } - return true; + return; } } } @@ -811,7 +753,7 @@ bool AgentConnectionManager::waitForNotification( { it = mUnhandledNotifications.erase(it); } - return true; + return; } } } @@ -831,25 +773,24 @@ bool AgentConnectionManager::waitForNotification( } } } - return false; } // Explicit template instantiations -template bool AgentConnectionManager::waitForNotification<NotificationSyncInfo>( +template void AgentConnectionManager::waitForNotification<NotificationSyncInfo>( std::string const& remoteAgentName, NotificationSyncInfo& expectedInfo, std::atomic<bool> const& terminateFlag); -template bool AgentConnectionManager::waitForNotification<ReadySignalInfo>( +template void AgentConnectionManager::waitForNotification<ReadySignalInfo>( std::string const& remoteAgentName, ReadySignalInfo& expectedInfo, std::atomic<bool> const& terminateFlag); -bool AgentConnectionManager::waitForSyncInfo( +void AgentConnectionManager::waitForSyncInfo( std::string const& remoteAgentName, NotificationSyncInfo& syncInfo, std::atomic<bool> const& terminateFlag) { - return waitForNotification(remoteAgentName, syncInfo, terminateFlag); + waitForNotification(remoteAgentName, syncInfo, terminateFlag); } -bool AgentConnectionManager::waitForReadySignal( +void AgentConnectionManager::waitForReadySignal( std::string const& remoteAgentName, ReadySignalInfo& readySignalInfo, std::atomic<bool> const& terminateFlag) { - return waitForNotification(remoteAgentName, readySignalInfo, terminateFlag); + waitForNotification(remoteAgentName, readySignalInfo, terminateFlag); } std::string const& AgentConnectionManager::getAgentName() const diff --git a/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.h b/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.h index 410eff0248c1..25283d1341a1 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.h +++ b/cpp/tensorrt_llm/executor/cache_transmission/agent_utils/connection.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -260,15 +260,13 @@ class AgentConnection : public Connection void send(DataContext const& ctx, void const* data, size_t size) const override; void recv(DataContext const& ctx, void* data, size_t size) const override; void sendRequestAndBufferInfo(batch_manager::RequestInfo& requestInfo, - std::vector<std::optional<size_t>> const& cacheBufferIds, int validConnectionIdx, - std::atomic<bool> const* perRequestCancel = nullptr); - void setSenderState(std::vector<MemoryDesc> cacheReceiverBufferDescs, int validSegmentIdx, + std::vector<std::optional<size_t>> const& cacheBufferIds, int validConnectionIdx); + void setSenderState(std::vector<MemoryDesc> cacheReceiverBufferDescs, int valideSegmentIdx, std::vector<std::pair<size_t, size_t>> offsetRatios, std::vector<uint8_t> bufferKinds); void setHasLoadRemoteAgent(bool hasLoadRemoteAgent); [[nodiscard]] bool hasLoadRemoteAgent() const; void sendReadySignal(DataContext const& ctx, bool isReady) const; bool recvReadySignal(DataContext const& ctx) const; - std::optional<bool> recvReadySignalWithStatus(DataContext const& ctx) const; void activateBuffer(uint8_t kind) const override; [[nodiscard]] std::optional<size_t> getPreAssignedBufferId(uint8_t kind) const override; @@ -322,11 +320,11 @@ class AgentConnectionManager : public ConnectionManager [[nodiscard]] std::string const& getAgentName() const; template <typename NotificationType> - bool waitForNotification( + void waitForNotification( std::string const& remoteAgentName, NotificationType& expectedInfo, std::atomic<bool> const& terminateFlag); - bool waitForSyncInfo( + void waitForSyncInfo( std::string const& remoteAgentName, NotificationSyncInfo& syncInfo, std::atomic<bool> const& terminateFlag); - bool waitForReadySignal( + void waitForReadySignal( std::string const& remoteAgentName, ReadySignalInfo& readySignalInfo, std::atomic<bool> const& terminateFlag); [[nodiscard]] bool isRunning() const override; diff --git a/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.cu b/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.cu index d57ed7530e28..6e8c68d7efa7 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.cu +++ b/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.cu @@ -20,7 +20,6 @@ #include "tensorrt_llm/common/cudaFp8Utils.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/tensor.h" #include "tensorrt_llm/executor/types.h" @@ -28,6 +27,7 @@ #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include <NvInferRuntimeBase.h> #include <cstddef> #include <cstdint> #include <sstream> @@ -467,7 +467,7 @@ void concatKVCache(runtime::ITensor::SharedPtr* inputBlocks, int inputBlockNum, blockInfos[outputBlockNum * inputAllRankNum + oi] = fillBlockInfo(oCacheState, outputBlocks[oi], oRank); } runtime::BufferManager::IBufferPtr blockInfosDeviceBuffer - = bufferManager.gpu(sizeof(BlockInfo<T>) * (blockInfos.size()), tensorrt_llm::DataType::kUINT8); + = bufferManager.gpu(sizeof(BlockInfo<T>) * (blockInfos.size()), nvinfer1::DataType::kUINT8); bufferManager.copy((blockInfos.data()), *blockInfosDeviceBuffer, runtime::MemoryType::kCPU); BlockInfo<T>* iBlockInfoDevice = static_cast<BlockInfo<T>*>(blockInfosDeviceBuffer->data()); @@ -594,7 +594,7 @@ void concatKVCacheDispatch(runtime::ITensor::SharedPtr* inputBlocks, int inputBl } } -tensorrt_llm::Dims makeShapeFromCacheState(kv_cache::CacheState const& cacheState) +nvinfer1::Dims makeShapeFromCacheState(kv_cache::CacheState const& cacheState) { int64_t blockSize = static_cast<int64_t>(cacheState.getModelConfig().mNbKvHeadsPerLayer[0] @@ -1130,8 +1130,8 @@ void splitKVCache(std::map<SizeType32, std::vector<runtime::ITensor::SharedPtr>> std::vector<SizeType32> layersInWindow; size_t cacheBlockSizeSum = 0; size_t inputBlockLayerNumSum = 0; - auto cacheDataType = isIndexerKCache ? tensorrt_llm::DataType::kUINT8 - : kVCacheBlocksPerWindow.begin()->second.front()->getDataType(); + auto cacheDataType + = isIndexerKCache ? nvinfer1::DataType::kUINT8 : kVCacheBlocksPerWindow.begin()->second.front()->getDataType(); for (auto const& [window, blocks] : kVCacheBlocksPerWindow) { @@ -1170,7 +1170,7 @@ void splitKVCache(std::map<SizeType32, std::vector<runtime::ITensor::SharedPtr>> bool const isWindow = windowSizes.size() > 1; runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(T*)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -1182,7 +1182,7 @@ void splitKVCache(std::map<SizeType32, std::vector<runtime::ITensor::SharedPtr>> windowInfoHostBuffer.insert(windowInfoHostBuffer.end(), blockNumInwindow.begin(), blockNumInwindow.end()); windowInfoHostBuffer.insert(windowInfoHostBuffer.end(), layersInWindow.begin(), layersInWindow.end()); - windowInfoDeviceBuffer = bufferManager.gpu(windowInfoHostBuffer.size(), tensorrt_llm::DataType::kINT32); + windowInfoDeviceBuffer = bufferManager.gpu(windowInfoHostBuffer.size(), nvinfer1::DataType::kINT32); bufferManager.copy(windowInfoHostBuffer.data(), *windowInfoDeviceBuffer, runtime::MemoryType::kCPU); for (auto layerNum : layersInWindow) @@ -1404,8 +1404,8 @@ void splitKVCacheDispatch(std::map<SizeType32, std::vector<runtime::ITensor::Sha bool isIndexerKCache) { TLLM_CHECK(!kVCacheBlocksPerWindow.empty()); - auto dataType = isIndexerKCache ? tensorrt_llm::DataType::kUINT8 - : kVCacheBlocksPerWindow.begin()->second.front()->getDataType(); + auto dataType + = isIndexerKCache ? nvinfer1::DataType::kUINT8 : kVCacheBlocksPerWindow.begin()->second.front()->getDataType(); auto dataSize = tensorrt_llm::common::getDTypeSize(dataType); switch (dataSize) @@ -1513,7 +1513,7 @@ void concatKVCache(std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlo } cachePtrs.insert(cachePtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); bool const isWindow = windowSizes.size() > 1; @@ -1525,7 +1525,7 @@ void concatKVCache(std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlo windowInfoHostBuffer.insert(windowInfoHostBuffer.end(), blockNumInwindow.begin(), blockNumInwindow.end()); windowInfoHostBuffer.insert(windowInfoHostBuffer.end(), layersInWindow.begin(), layersInWindow.end()); - windowInfoDeviceBuffer = bufferManager.gpu(windowInfoHostBuffer.size(), tensorrt_llm::DataType::kINT32); + windowInfoDeviceBuffer = bufferManager.gpu(windowInfoHostBuffer.size(), nvinfer1::DataType::kINT32); bufferManager.copy(windowInfoHostBuffer.data(), *windowInfoDeviceBuffer, runtime::MemoryType::kCPU); } constexpr int subWarpSize = 8; diff --git a/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h b/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h index 80816bc5c3a0..c7036b219612 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h +++ b/cpp/tensorrt_llm/executor/cache_transmission/cacheSplitConcat.h @@ -27,7 +27,7 @@ #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntimeBase.h> namespace tensorrt_llm::executor::kv_cache { @@ -78,7 +78,7 @@ void concatKVCacheDispatch(runtime::ITensor::SharedPtr* inputBlocks, int inputBl runtime::ITensor::SharedPtr* outputBlocks, int outputBlockNum, int selfRank, kv_cache::CacheState const& selfCacheState, runtime::BufferManager const& bufferManager); -tensorrt_llm::Dims makeShapeFromCacheState(kv_cache::CacheState const& cacheState); +nvinfer1::Dims makeShapeFromCacheState(kv_cache::CacheState const& cacheState); void splitKVCacheDispatch(std::map<SizeType32, std::vector<runtime::ITensor::SharedPtr>> const& kVCacheBlocksPerWindow, std::vector<runtime::ITensor::SharedPtr>& ouputSplitBlocks, kv_cache::CacheState const& peerCacheState, @@ -147,7 +147,7 @@ void concatRnnSsmStateDispatch(std::vector<runtime::ITensor::SharedPtr> const& i void splitUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr>& outputSplitBlocks, kv_cache::CacheState const& destCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, - size_t ssmBytes, size_t blockSizeBytes, tensorrt_llm::DataType ssmDataType, + size_t ssmBytes, size_t blockSizeBytes, nvinfer1::DataType ssmDataType, runtime::BufferManager const& bufferManager); /** @@ -156,7 +156,7 @@ void splitUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, void splitUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr>& outputSplitBlocks, kv_cache::CacheState const& destCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, - size_t ssmBytes, size_t blockSizeBytes, tensorrt_llm::DataType convDataType, + size_t ssmBytes, size_t blockSizeBytes, nvinfer1::DataType convDataType, runtime::BufferManager const& bufferManager); /** @@ -165,7 +165,7 @@ void splitUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, void concatUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlocks, kv_cache::CacheState const& srcCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, size_t ssmBytes, - size_t blockSizeBytes, tensorrt_llm::DataType ssmDataType, runtime::BufferManager const& bufferManager); + size_t blockSizeBytes, nvinfer1::DataType ssmDataType, runtime::BufferManager const& bufferManager); /** * @brief Concat conv state from per-source buffers into unified pool blocks (section-aware). @@ -173,6 +173,6 @@ void concatUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, void concatUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlocks, kv_cache::CacheState const& srcCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, size_t ssmBytes, - size_t blockSizeBytes, tensorrt_llm::DataType convDataType, runtime::BufferManager const& bufferManager); + size_t blockSizeBytes, nvinfer1::DataType convDataType, runtime::BufferManager const& bufferManager); } // namespace tensorrt_llm::executor::rnn_cache diff --git a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/agentBindings.cpp b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/agentBindings.cpp index 040979e7197d..a987ac108f3b 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/agentBindings.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/agentBindings.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -261,35 +261,28 @@ NB_MODULE(tensorrt_llm_transfer_agent_binding, m) nb::class_<kvc::NixlTransferAgent, kvc::BaseTransferAgent>(m, "NixlTransferAgent") .def(nb::init<kvc::BaseAgentConfig const&>(), nb::arg("config"), nb::call_guard<nb::gil_scoped_release>()) .def("shutdown", &kvc::NixlTransferAgent::shutdown, nb::call_guard<nb::gil_scoped_release>()) - .def("register_memory", &kvc::NixlTransferAgent::registerMemory, nb::arg("descs"), - nb::call_guard<nb::gil_scoped_release>()) - .def("deregister_memory", &kvc::NixlTransferAgent::deregisterMemory, nb::arg("descs"), - nb::call_guard<nb::gil_scoped_release>()) + .def("register_memory", &kvc::NixlTransferAgent::registerMemory, nb::arg("descs")) + .def("deregister_memory", &kvc::NixlTransferAgent::deregisterMemory, nb::arg("descs")) .def("load_remote_agent", nb::overload_cast<std::string const&, kvc::AgentDesc const&>(&kvc::NixlTransferAgent::loadRemoteAgent), - nb::arg("name"), nb::arg("agent_desc"), nb::call_guard<nb::gil_scoped_release>()) + nb::arg("name"), nb::arg("agent_desc")) .def("load_remote_agent_by_connection", nb::overload_cast<std::string const&, kvc::ConnectionInfoType const&>( &kvc::NixlTransferAgent::loadRemoteAgent), - nb::arg("name"), nb::arg("connection_info"), nb::call_guard<nb::gil_scoped_release>()) - .def("get_local_agent_desc", &kvc::NixlTransferAgent::getLocalAgentDesc, - nb::call_guard<nb::gil_scoped_release>()) - .def("get_local_connection_info", &kvc::NixlTransferAgent::getLocalConnectionInfo, - nb::call_guard<nb::gil_scoped_release>()) - .def("invalidate_remote_agent", &kvc::NixlTransferAgent::invalidateRemoteAgent, nb::arg("name"), - nb::call_guard<nb::gil_scoped_release>()) + nb::arg("name"), nb::arg("connection_info")) + .def("get_local_agent_desc", &kvc::NixlTransferAgent::getLocalAgentDesc) + .def("get_local_connection_info", &kvc::NixlTransferAgent::getLocalConnectionInfo) + .def("invalidate_remote_agent", &kvc::NixlTransferAgent::invalidateRemoteAgent, nb::arg("name")) .def( "submit_transfer_requests", [](kvc::NixlTransferAgent& self, kvc::TransferRequest const& request) { return self.submitTransferRequests(request).release(); }, nb::arg("request"), nb::rv_policy::take_ownership, nb::call_guard<nb::gil_scoped_release>(), nb::keep_alive<0, 1>()) - .def("notify_sync_message", &kvc::NixlTransferAgent::notifySyncMessage, nb::arg("name"), - nb::arg("sync_message"), nb::call_guard<nb::gil_scoped_release>()) - .def("get_notified_sync_messages", &kvc::NixlTransferAgent::getNotifiedSyncMessages, - nb::call_guard<nb::gil_scoped_release>()) - .def("check_remote_descs", &kvc::NixlTransferAgent::checkRemoteDescs, nb::arg("name"), nb::arg("memory_descs"), - nb::call_guard<nb::gil_scoped_release>()); + .def( + "notify_sync_message", &kvc::NixlTransferAgent::notifySyncMessage, nb::arg("name"), nb::arg("sync_message")) + .def("get_notified_sync_messages", &kvc::NixlTransferAgent::getNotifiedSyncMessages) + .def("check_remote_descs", &kvc::NixlTransferAgent::checkRemoteDescs, nb::arg("name"), nb::arg("memory_descs")); #endif // NOTE: MooncakeTransferAgent/MooncakeTransferStatus class bindings are intentionally diff --git a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.cpp b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.cpp index 5ab589d7ca75..563b6e6f2a95 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -321,7 +321,6 @@ void NixlHelper::posixFileToGpuFallback(MemoryDescs const& memoryDescs, FileDesc NixlTransferStatus::NixlTransferStatus(std::weak_ptr<nixlAgent> agent, nixlXferReqH* handle) : mWeakAgent{std::move(agent)} , mHandle{handle} - , mSynchronizeHandleAccess{common::getEnvDisaggEnableInflightCancel()} { TLLM_CHECK(!mWeakAgent.expired()); TLLM_CHECK(mHandle); @@ -329,13 +328,19 @@ NixlTransferStatus::NixlTransferStatus(std::weak_ptr<nixlAgent> agent, nixlXferR NixlTransferStatus::~NixlTransferStatus() noexcept { + if (mHandle == nullptr) + { + return; + } + // Skip release if the owning agent was reset; the underlying nixlXferReqH is already gone. + auto agent = mWeakAgent.lock(); + if (!agent) + { + return; + } try { - if (!release()) - { - TLLM_LOG_WARNING( - "NIXL transfer handle release failed during destruction; backend handle may remain active"); - } + agent->releaseXferReq(mHandle); } catch (std::exception const& e) { @@ -347,13 +352,179 @@ NixlTransferStatus::~NixlTransferStatus() noexcept } } +[[nodiscard]] MemoryDescs NixlHelper::coalesceMemoryDescs(MemoryDescs const& descs) +{ + auto const& descVec = descs.getDescs(); + + // If empty or single element, return as-is + if (descVec.size() <= 1) + { + return descs; + } + + size_t const numDescs = descVec.size(); + + // Create index array and sort by address + std::vector<size_t> sortedIndices(numDescs); + std::iota(sortedIndices.begin(), sortedIndices.end(), 0); + + std::sort(sortedIndices.begin(), sortedIndices.end(), + [&descVec](size_t lhs, size_t rhs) + { + // Sort by deviceId first, then by address + if (descVec[lhs].getDeviceId() != descVec[rhs].getDeviceId()) + { + return descVec[lhs].getDeviceId() < descVec[rhs].getDeviceId(); + } + return descVec[lhs].getAddr() < descVec[rhs].getAddr(); + }); + + std::vector<MemoryDesc> coalesced; + coalesced.reserve(numDescs); + + // Start with the first entry + size_t firstIdx = sortedIndices[0]; + uintptr_t currentAddr = descVec[firstIdx].getAddr(); + size_t currentLen = descVec[firstIdx].getLen(); + uint32_t currentDeviceId = descVec[firstIdx].getDeviceId(); + + for (size_t idx = 1; idx < numDescs; ++idx) + { + size_t sortedIdx = sortedIndices[idx]; + auto const& desc = descVec[sortedIdx]; + + // Check if current can be coalesced with previous + bool isContiguous = (currentAddr + currentLen == desc.getAddr()) && (currentDeviceId == desc.getDeviceId()); + + if (isContiguous) + { + // Coalesce: extend the current region + currentLen += desc.getLen(); + } + else + { + // Cannot coalesce: save the current region and start a new one + coalesced.emplace_back(currentAddr, currentLen, currentDeviceId); + + currentAddr = desc.getAddr(); + currentLen = desc.getLen(); + currentDeviceId = desc.getDeviceId(); + } + } + + // Add the last region + coalesced.emplace_back(currentAddr, currentLen, currentDeviceId); + + TLLM_LOG_DEBUG("NixlHelper::coalesceMemoryDescs: coalesced %zu -> %zu entries", descVec.size(), coalesced.size()); + + return MemoryDescs{descs.getType(), std::move(coalesced)}; +} + +[[nodiscard]] std::pair<MemoryDescs, MemoryDescs> NixlHelper::coalesceTransferDescs( + TransferDescs const& srcDescs, TransferDescs const& dstDescs) +{ + auto const& srcVec = srcDescs.getDescs(); + auto const& dstVec = dstDescs.getDescs(); + + // If sizes don't match or empty, return as-is + if (srcVec.size() != dstVec.size() || srcVec.empty()) + { + return {srcDescs, dstDescs}; + } + + size_t const numDescs = srcVec.size(); + + // Create index array and sort by src address + // This allows us to find contiguous regions even if the original order is scattered + std::vector<size_t> sortedIndices(numDescs); + std::iota(sortedIndices.begin(), sortedIndices.end(), 0); + + std::sort(sortedIndices.begin(), sortedIndices.end(), + [&srcVec](size_t lhs, size_t rhs) + { + // Sort by deviceId first, then by address + if (srcVec[lhs].getDeviceId() != srcVec[rhs].getDeviceId()) + { + return srcVec[lhs].getDeviceId() < srcVec[rhs].getDeviceId(); + } + return srcVec[lhs].getAddr() < srcVec[rhs].getAddr(); + }); + + std::vector<MemoryDesc> coalescedSrc; + std::vector<MemoryDesc> coalescedDst; + coalescedSrc.reserve(numDescs); + coalescedDst.reserve(numDescs); + + // Start with the first entry (using sorted order) + size_t firstIdx = sortedIndices[0]; + uintptr_t currentSrcAddr = srcVec[firstIdx].getAddr(); + size_t currentSrcLen = srcVec[firstIdx].getLen(); + uint32_t currentSrcDeviceId = srcVec[firstIdx].getDeviceId(); + + uintptr_t currentDstAddr = dstVec[firstIdx].getAddr(); + size_t currentDstLen = dstVec[firstIdx].getLen(); + uint32_t currentDstDeviceId = dstVec[firstIdx].getDeviceId(); + + for (size_t idx = 1; idx < numDescs; ++idx) + { + size_t sortedIdx = sortedIndices[idx]; + auto const& src = srcVec[sortedIdx]; + auto const& dst = dstVec[sortedIdx]; + + // Check if current src and dst can be coalesced with previous + bool srcContiguous + = (currentSrcAddr + currentSrcLen == src.getAddr()) && (currentSrcDeviceId == src.getDeviceId()); + bool dstContiguous + = (currentDstAddr + currentDstLen == dst.getAddr()) && (currentDstDeviceId == dst.getDeviceId()); + + if (srcContiguous && dstContiguous) + { + // Coalesce: extend the current region + currentSrcLen += src.getLen(); + currentDstLen += dst.getLen(); + } + else + { + // Cannot coalesce: save the current region and start a new one + coalescedSrc.emplace_back(currentSrcAddr, currentSrcLen, currentSrcDeviceId); + coalescedDst.emplace_back(currentDstAddr, currentDstLen, currentDstDeviceId); + + currentSrcAddr = src.getAddr(); + currentSrcLen = src.getLen(); + currentSrcDeviceId = src.getDeviceId(); + + currentDstAddr = dst.getAddr(); + currentDstLen = dst.getLen(); + currentDstDeviceId = dst.getDeviceId(); + } + } + + // Don't forget to add the last region + coalescedSrc.emplace_back(currentSrcAddr, currentSrcLen, currentSrcDeviceId); + coalescedDst.emplace_back(currentDstAddr, currentDstLen, currentDstDeviceId); + + TLLM_LOG_DEBUG( + "NixlHelper::coalesceTransferDescs: coalesced %zu -> %zu transfer entries", srcVec.size(), coalescedSrc.size()); + + return {MemoryDescs{srcDescs.getType(), std::move(coalescedSrc)}, + MemoryDescs{dstDescs.getType(), std::move(coalescedDst)}}; +} + TransferState NixlTransferStatus::wait(int64_t timeout_ms) const { auto startTime = std::chrono::steady_clock::now(); while (true) { - auto const status = queryStatus(); + auto agent = mWeakAgent.lock(); + if (!agent) + { + // Owning agent was reset; report failure so callers don't deref a null status. + mLastStatus.store(static_cast<int>(NIXL_ERR_INVALID_PARAM), std::memory_order_relaxed); + return TransferState::kFAILURE; + } + auto status = agent->getXferStatus(mHandle); + mLastStatus.store(static_cast<int>(status), std::memory_order_relaxed); if (status == NIXL_SUCCESS) { return TransferState::kSUCCESS; @@ -395,64 +566,15 @@ std::string NixlTransferStatus::getLastStatusStr() const [[nodiscard]] bool NixlTransferStatus::isCompleted() const { - return queryStatus() == NIXL_SUCCESS; -} - -nixl_status_t NixlTransferStatus::queryStatus() const -{ - auto const query = [this]() - { - if (mHandle == nullptr) - { - mLastStatus.store(static_cast<int>(NIXL_ERR_INVALID_PARAM), std::memory_order_relaxed); - return NIXL_ERR_INVALID_PARAM; - } - auto agent = mWeakAgent.lock(); - if (!agent) - { - // Owning agent was reset; report failure so callers don't deref a null status. - mLastStatus.store(static_cast<int>(NIXL_ERR_INVALID_PARAM), std::memory_order_relaxed); - return NIXL_ERR_INVALID_PARAM; - } - auto const status = agent->getXferStatus(mHandle); - mLastStatus.store(static_cast<int>(status), std::memory_order_relaxed); - return status; - }; - - if (mSynchronizeHandleAccess) - { - std::lock_guard<std::mutex> lock(mHandleMutex); - return query(); - } - return query(); -} - -[[nodiscard]] bool NixlTransferStatus::release() -{ - std::lock_guard<std::mutex> lock(mHandleMutex); - if (mHandle == nullptr) - { - return true; - } - auto agent = mWeakAgent.lock(); if (!agent) { - mHandle = nullptr; mLastStatus.store(static_cast<int>(NIXL_ERR_INVALID_PARAM), std::memory_order_relaxed); - return true; + return false; } - - auto status = agent->releaseXferReq(mHandle); + auto status = agent->getXferStatus(mHandle); mLastStatus.store(static_cast<int>(status), std::memory_order_relaxed); - if (status == NIXL_SUCCESS) - { - mHandle = nullptr; - return true; - } - - TLLM_LOG_WARNING("NIXL releaseXferReq failed with status: %s", nixlEnumStrings::statusStr(status).c_str()); - return false; + return status == NIXL_SUCCESS; } NixlTransferAgent::NixlTransferAgent(BaseAgentConfig const& config) @@ -535,8 +657,12 @@ void NixlTransferAgent::registerMemory(RegisterDescs const& descs) auto detectedRegionMap = VmmDescSplitter::detectVramRegionMap(descs); mLocalVramRegionInfo.merge(detectedRegionMap); + // Coalesce contiguous memory regions to reduce registration overhead (disabled by default) + // Set TRTLLM_NIXL_ENABLE_COALESCE=1 to enable this optimization + auto coalescedDescs = common::getEnvNixlEnableCoalesce() ? NixlHelper::coalesceMemoryDescs(splitDescs) : splitDescs; + nixl_status_t status; - status = mRawAgent->registerMem(NixlHelper::convertRegDlist(splitDescs), &mExtraParams); + status = mRawAgent->registerMem(NixlHelper::convertRegDlist(coalescedDescs), &mExtraParams); TLLM_CHECK(status == NIXL_SUCCESS); std::string localMD; @@ -551,8 +677,12 @@ void NixlTransferAgent::deregisterMemory(RegisterDescs const& descs) // Split using per-region registry info to match what was registered auto splitDescs = VmmDescSplitter::splitDescsWithRegionMap(descs, mLocalVramRegionInfo); + // Coalesce contiguous memory regions to match what was registered (disabled by default) + // Set TRTLLM_NIXL_ENABLE_COALESCE=1 to enable this optimization + auto coalescedDescs = common::getEnvNixlEnableCoalesce() ? NixlHelper::coalesceMemoryDescs(splitDescs) : splitDescs; + nixl_status_t status; - status = mRawAgent->deregisterMem(NixlHelper::convertRegDlist(splitDescs), &mExtraParams); + status = mRawAgent->deregisterMem(NixlHelper::convertRegDlist(coalescedDescs), &mExtraParams); TLLM_CHECK(status == NIXL_SUCCESS); // Remove entries from registry @@ -577,7 +707,7 @@ void NixlTransferAgent::loadRemoteAgent(std::string const& name, AgentDesc const name == remoteName, "loadRemoteAgent gets error agent name: %s != %s", name.c_str(), remoteName.c_str()); // Store remote VMM region info for chunk boundary calculations in - // VmmDescSplitter::splitAndCoalesceTransferDescs. Per-agent map because different remote agents may have + // VmmDescSplitter::splitTransferDescsWithRegionMaps. Per-agent map because different remote agents may have // overlapping virtual addresses. auto const& regions = agentDesc.getVramRegions(); if (!regions.empty()) @@ -598,13 +728,14 @@ AgentDesc NixlTransferAgent::getLocalAgentDesc() nixl_status_t status = mRawAgent->getLocalMD(nixlBlob); TLLM_CHECK(status == NIXL_SUCCESS); - // Pack ALL local region info (VMM multi-chunk and single-allocation alike) so remote agents can - // compute chunk boundaries and never coalesce transfer descs across separately registered regions. + // Pack local VMM region info so remote agents can compute chunk boundaries. std::vector<VramRegionMeta> regions; - regions.reserve(mLocalVramRegionInfo.size()); for (auto const& [base, info] : mLocalVramRegionInfo) { - regions.push_back({base, info.totalLen, info.chunkSize}); + if (info.chunkSize > 0) + { + regions.push_back({base, info.totalLen, info.chunkSize}); + } } return AgentDesc{nixlBlob, std::move(regions)}; @@ -642,25 +773,32 @@ void NixlTransferAgent::invalidateRemoteAgent(std::string const& name) { reqParams.hasNotif = false; } - // Split transfer descriptors at VMM chunk boundaries to match registered memory, then coalesce - // contiguous pieces. A coalesced descriptor never crosses a chunk boundary or a registered - // region boundary on either side, so every descriptor still falls within a single registered - // memory region on both local and remote sides. Set TRTLLM_NIXL_DISABLE_COALESCE=1 to fall back - // to split-only descriptors. Find remote agent's region map (empty map if not found — e.g. the - // peer's AgentDesc carried no region info; addresses missing from a map are never coalesced, - // so an empty remote map degrades to split-only rather than risking merges across unknown - // remote chunk/registration boundaries). + // Split transfer descriptors at VMM chunk boundaries to match registered memory. + // Both src and dst are split at chunk boundaries to ensure each descriptor + // falls within a single registered memory region on both local and remote sides. + // Find remote agent's VMM region map (empty map if not found). static VramRegionMap const kEmptyMap; auto remoteIt = mRemoteVramRegionInfo.find(request.getRemoteName()); auto const& remoteRegionMap = (remoteIt != mRemoteVramRegionInfo.end()) ? remoteIt->second : kEmptyMap; - auto [xferSrc, xferDst] = VmmDescSplitter::splitAndCoalesceTransferDescs(request.getSrcDescs(), - request.getDstDescs(), mLocalVramRegionInfo, remoteRegionMap, !common::getEnvNixlDisableCoalesce()); + auto [splitSrc, splitDst] = VmmDescSplitter::splitTransferDescsWithRegionMaps( + request.getSrcDescs(), request.getDstDescs(), mLocalVramRegionInfo, remoteRegionMap); + // Coalesce contiguous memory regions to reduce transfer count (disabled by default) + // This matches the coalescing done during registerMemory() + // Set TRTLLM_NIXL_ENABLE_COALESCE=1 to enable this optimization + if (common::getEnvNixlEnableCoalesce()) + { + NVTX3_SCOPED_RANGE(coalesceTransferDescs_CreateXferReq); + auto [coalescedSrc, coalescedDst] = NixlHelper::coalesceTransferDescs(splitSrc, splitDst); + status + = mRawAgent->createXferReq(NixlHelper::convert(request.getOp()), NixlHelper::convertXferDist(coalescedSrc), + NixlHelper::convertXferDist(coalescedDst), request.getRemoteName(), handle, &reqParams); + } + else { - NVTX3_SCOPED_RANGE(createXferReq); - status = mRawAgent->createXferReq(NixlHelper::convert(request.getOp()), NixlHelper::convertXferDist(xferSrc), - NixlHelper::convertXferDist(xferDst), request.getRemoteName(), handle, &reqParams); + status = mRawAgent->createXferReq(NixlHelper::convert(request.getOp()), NixlHelper::convertXferDist(splitSrc), + NixlHelper::convertXferDist(splitDst), request.getRemoteName(), handle, &reqParams); } TLLM_CHECK_WITH_INFO(status == NIXL_SUCCESS, diff --git a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h index 31e62fd6d822..a8f3a0e71005 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h +++ b/cpp/tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h @@ -21,7 +21,6 @@ #include "tensorrt_llm/executor/transferAgent.h" #include <atomic> #include <memory> -#include <mutex> #include <shared_mutex> #include <thread> @@ -39,6 +38,21 @@ struct NixlHelper [[nodiscard]] static nixl_xfer_dlist_t convertXferDist(FileDescs const& descs); static void posixGpuToFileFallback(MemoryDescs const& memoryDesc, FileDescs const& fileDescs); static void posixFileToGpuFallback(MemoryDescs const& memoryDesc, FileDescs const& fileDescs); + + /// @brief Coalesce contiguous memory regions to reduce memory registration overhead. + /// Adjacent memory regions with the same deviceId will be merged into a single region. + /// @param descs Memory descriptors to coalesce + /// @return Coalesced MemoryDescs + [[nodiscard]] static MemoryDescs coalesceMemoryDescs(MemoryDescs const& descs); + + /// @brief Coalesce contiguous memory regions in src and dst to reduce transfer count. + /// If src[i] and src[i+1] are contiguous, and dst[i] and dst[i+1] are also contiguous + /// (with same deviceId), they will be merged into a single transfer. + /// @param srcDescs Source memory descriptors + /// @param dstDescs Destination memory descriptors + /// @return Pair of coalesced (src, dst) MemoryDescs + [[nodiscard]] static std::pair<MemoryDescs, MemoryDescs> coalesceTransferDescs( + TransferDescs const& srcDescs, TransferDescs const& dstDescs); }; class NixlTransferStatus final : public TransferStatus @@ -59,17 +73,11 @@ class NixlTransferStatus final : public TransferStatus [[nodiscard]] int getLastStatus() const noexcept; [[nodiscard]] std::string getLastStatusStr() const; - [[nodiscard]] bool release() override; - private: - [[nodiscard]] nixl_status_t queryStatus() const; - // weak_ptr so the status outliving the owning agent is safe (lock() returns null after reset). std::weak_ptr<nixlAgent> mWeakAgent; nixlXferReqH* mHandle{}; mutable std::atomic<int> mLastStatus{0}; - bool const mSynchronizeHandleAccess; - mutable std::mutex mHandleMutex; }; class NixlTransferAgent final : public BaseTransferAgent diff --git a/cpp/tensorrt_llm/executor/cache_transmission/rnnCacheSplitConcat.cu b/cpp/tensorrt_llm/executor/cache_transmission/rnnCacheSplitConcat.cu index f464c41aadc9..fb6837e8d188 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/rnnCacheSplitConcat.cu +++ b/cpp/tensorrt_llm/executor/cache_transmission/rnnCacheSplitConcat.cu @@ -24,7 +24,6 @@ #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/tensor.h" #include "tensorrt_llm/executor/types.h" @@ -32,6 +31,7 @@ #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include <NvInferRuntimeBase.h> #include <cstddef> #include <cstdint> #include <sstream> @@ -366,7 +366,7 @@ void splitRnnConvState(std::vector<runtime::ITensor::SharedPtr> const& inputConv // Allocate and copy pointer array to device runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -497,7 +497,7 @@ void splitRnnSsmState(std::vector<runtime::ITensor::SharedPtr> const& inputSsmBl cachePtrs.insert(cachePtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -625,7 +625,7 @@ void concatRnnConvState(std::vector<runtime::ITensor::SharedPtr> const& inputSpl cachePtrs.insert(cachePtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -747,7 +747,7 @@ void concatRnnSsmState(std::vector<runtime::ITensor::SharedPtr> const& inputSpli cachePtrs.insert(cachePtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); runtime::BufferManager::IBufferPtr PtrsDeviceBuffer - = bufferManager.gpu(cachePtrs.size(), tensorrt_llm::DataType::kINT64); + = bufferManager.gpu(cachePtrs.size(), nvinfer1::DataType::kINT64); TLLM_CHECK(PtrsDeviceBuffer->getSizeInBytes() == cachePtrs.size() * sizeof(uint64_t)); bufferManager.copy(cachePtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); @@ -1348,7 +1348,7 @@ void splitUnifiedPoolSsm(runtime::ITensor::SharedPtr const& pool, std::vector<Si } allPtrs.insert(allPtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); - auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), tensorrt_llm::DataType::kINT64); + auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), nvinfer1::DataType::kINT64); bufferManager.copy(allPtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); T const** inputPtrsDev = static_cast<T const**>(PtrsDeviceBuffer->data()); @@ -1481,10 +1481,10 @@ void splitUnifiedPoolConv(runtime::ITensor::SharedPtr const& pool, std::vector<S sectionInfo.insert(sectionInfo.end(), sectionDimsDomainTP.begin(), sectionDimsDomainTP.end()); sectionInfo.insert(sectionInfo.end(), sectionOffsetsLocal.begin(), sectionOffsetsLocal.end()); - auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), tensorrt_llm::DataType::kINT64); + auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), nvinfer1::DataType::kINT64); bufferManager.copy(allPtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); - auto sectionInfoBuffer = bufferManager.gpu(sectionInfo.size(), tensorrt_llm::DataType::kINT32); + auto sectionInfoBuffer = bufferManager.gpu(sectionInfo.size(), nvinfer1::DataType::kINT32); bufferManager.copy(sectionInfo.data(), *sectionInfoBuffer, runtime::MemoryType::kCPU); T const** inputPtrsDev = static_cast<T const**>(PtrsDeviceBuffer->data()); @@ -1606,7 +1606,7 @@ void concatUnifiedPoolSsm(runtime::ITensor::SharedPtr const& pool, std::vector<S } allPtrs.insert(allPtrs.end(), prefixLayerNum.begin(), prefixLayerNum.end()); - auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), tensorrt_llm::DataType::kINT64); + auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), nvinfer1::DataType::kINT64); bufferManager.copy(allPtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); T** outputPtrsDev = static_cast<T**>(PtrsDeviceBuffer->data()); @@ -1731,10 +1731,10 @@ void concatUnifiedPoolConv(runtime::ITensor::SharedPtr const& pool, std::vector< sectionInfo.insert(sectionInfo.end(), sectionDimsDomainTP.begin(), sectionDimsDomainTP.end()); sectionInfo.insert(sectionInfo.end(), sectionOffsetsLocal.begin(), sectionOffsetsLocal.end()); - auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), tensorrt_llm::DataType::kINT64); + auto PtrsDeviceBuffer = bufferManager.gpu(allPtrs.size(), nvinfer1::DataType::kINT64); bufferManager.copy(allPtrs.data(), *PtrsDeviceBuffer, runtime::MemoryType::kCPU); - auto sectionInfoBuffer = bufferManager.gpu(sectionInfo.size(), tensorrt_llm::DataType::kINT32); + auto sectionInfoBuffer = bufferManager.gpu(sectionInfo.size(), nvinfer1::DataType::kINT32); bufferManager.copy(sectionInfo.data(), *sectionInfoBuffer, runtime::MemoryType::kCPU); T** outputPtrsDev = static_cast<T**>(PtrsDeviceBuffer->data()); @@ -1810,8 +1810,7 @@ void concatUnifiedPoolConv(runtime::ITensor::SharedPtr const& pool, std::vector< void splitUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr>& outputSplitBlocks, kv_cache::CacheState const& destCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, - size_t ssmBytes, size_t blockSizeBytes, tensorrt_llm::DataType ssmDataType, - runtime::BufferManager const& bufferManager) + size_t ssmBytes, size_t blockSizeBytes, nvinfer1::DataType ssmDataType, runtime::BufferManager const& bufferManager) { auto dataSize = tensorrt_llm::common::getDTypeSize(ssmDataType); switch (dataSize) @@ -1835,7 +1834,7 @@ void splitUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, void splitUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr>& outputSplitBlocks, kv_cache::CacheState const& destCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, - size_t ssmBytes, size_t blockSizeBytes, tensorrt_llm::DataType convDataType, + size_t ssmBytes, size_t blockSizeBytes, nvinfer1::DataType convDataType, runtime::BufferManager const& bufferManager) { auto dataSize = tensorrt_llm::common::getDTypeSize(convDataType); @@ -1860,7 +1859,7 @@ void splitUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, void concatUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlocks, kv_cache::CacheState const& srcCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, size_t ssmBytes, - size_t blockSizeBytes, tensorrt_llm::DataType ssmDataType, runtime::BufferManager const& bufferManager) + size_t blockSizeBytes, nvinfer1::DataType ssmDataType, runtime::BufferManager const& bufferManager) { auto dataSize = tensorrt_llm::common::getDTypeSize(ssmDataType); switch (dataSize) @@ -1884,7 +1883,7 @@ void concatUnifiedPoolSsmDispatch(runtime::ITensor::SharedPtr const& pool, void concatUnifiedPoolConvDispatch(runtime::ITensor::SharedPtr const& pool, std::vector<SizeType32> const& realBlockIndices, std::vector<runtime::ITensor::SharedPtr> const& inputSplitBlocks, kv_cache::CacheState const& srcCacheState, kv_cache::CacheState const& selfCacheState, int selfIdx, size_t ssmBytes, - size_t blockSizeBytes, tensorrt_llm::DataType convDataType, runtime::BufferManager const& bufferManager) + size_t blockSizeBytes, nvinfer1::DataType convDataType, runtime::BufferManager const& bufferManager) { auto dataSize = tensorrt_llm::common::getDTypeSize(convDataType); switch (dataSize) diff --git a/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp b/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp index 147c99b37329..fd60651dc6a0 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/transferAgent.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,12 +19,9 @@ #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/executor/serializeUtils.h" -#include <algorithm> #include <cuda.h> #include <dlfcn.h> -#include <numeric> #include <sstream> -#include <tuple> namespace tensorrt_llm::executor::kv_cache { @@ -155,9 +152,8 @@ MemoryDescs VmmDescSplitter::splitDescsWithRegionMap(MemoryDescs const& descs, V return MemoryDescs{descs.getType(), std::move(result)}; } -std::pair<MemoryDescs, MemoryDescs> VmmDescSplitter::splitAndCoalesceTransferDescs(MemoryDescs const& srcDescs, - MemoryDescs const& dstDescs, VramRegionMap const& localRegionMap, VramRegionMap const& remoteRegionMap, - bool enableCoalesce) +std::pair<MemoryDescs, MemoryDescs> VmmDescSplitter::splitTransferDescsWithRegionMaps(MemoryDescs const& srcDescs, + MemoryDescs const& dstDescs, VramRegionMap const& localRegionMap, VramRegionMap const& remoteRegionMap) { if (srcDescs.getType() != MemoryType::kVRAM) return {srcDescs, dstDescs}; @@ -165,143 +161,54 @@ std::pair<MemoryDescs, MemoryDescs> VmmDescSplitter::splitAndCoalesceTransferDes auto const& srcVec = srcDescs.getDescs(); auto const& dstVec = dstDescs.getDescs(); TLLM_CHECK(srcVec.size() == dstVec.size()); - if (srcVec.empty()) - return {srcDescs, dstDescs}; - - // Sort pair indices by (src deviceId, src addr) so pairs that are contiguous in memory become - // adjacent, maximizing coalescing regardless of input order. Pairs are independent transfers, - // so reordering is safe. The sort only serves coalescing; with it disabled, keep input order. - std::vector<size_t> order(srcVec.size()); - std::iota(order.begin(), order.end(), 0); - if (enableCoalesce) - { - std::sort(order.begin(), order.end(), - [&srcVec](size_t lhs, size_t rhs) - { - if (srcVec[lhs].getDeviceId() != srcVec[rhs].getDeviceId()) - { - return srcVec[lhs].getDeviceId() < srcVec[rhs].getDeviceId(); - } - return srcVec[lhs].getAddr() < srcVec[rhs].getAddr(); - }); - } - - std::vector<MemoryDesc> outSrc, outDst; - outSrc.reserve(srcVec.size()); - outDst.reserve(dstVec.size()); - // Region info of the last emitted piece. Invariant: every emitted desc lies within a single - // chunk on both sides, so a merge is legal iff the new piece is contiguous, in the same region, - // and does not start on a chunk boundary (starting on a boundary means the merge would cross it). - size_t prevSrcChunkSize = 0, prevDstChunkSize = 0; - uintptr_t prevSrcBase = 0, prevDstBase = 0; + std::vector<MemoryDesc> splitSrc, splitDst; + splitSrc.reserve(srcVec.size()); + splitDst.reserve(dstVec.size()); - auto emitPiece - = [&](uintptr_t srcAddr, uintptr_t dstAddr, size_t len, uint32_t srcDev, uint32_t dstDev, size_t srcChunkSize, - uintptr_t srcBase, size_t dstChunkSize, uintptr_t dstBase, bool regionsKnown) - { - if (enableCoalesce && regionsKnown && !outSrc.empty()) - { - auto const& lastSrc = outSrc.back(); - auto const& lastDst = outDst.back(); - bool contiguous = lastSrc.getAddr() + lastSrc.getLen() == srcAddr && lastSrc.getDeviceId() == srcDev - && lastDst.getAddr() + lastDst.getLen() == dstAddr && lastDst.getDeviceId() == dstDev; - bool sameSrcRegion = srcChunkSize == prevSrcChunkSize && srcBase == prevSrcBase; - bool sameDstRegion = dstChunkSize == prevDstChunkSize && dstBase == prevDstBase; - bool srcWithinChunk = srcChunkSize == 0 || (srcAddr - srcBase) % srcChunkSize != 0; - bool dstWithinChunk = dstChunkSize == 0 || (dstAddr - dstBase) % dstChunkSize != 0; - if (contiguous && sameSrcRegion && sameDstRegion && srcWithinChunk && dstWithinChunk) - { - outSrc.back() = MemoryDesc{lastSrc.getAddr(), lastSrc.getLen() + len, srcDev}; - outDst.back() = MemoryDesc{lastDst.getAddr(), lastDst.getLen() + len, dstDev}; - return; - } - } - outSrc.emplace_back(srcAddr, len, srcDev); - outDst.emplace_back(dstAddr, len, dstDev); - prevSrcChunkSize = srcChunkSize; - prevSrcBase = srcBase; - prevDstChunkSize = dstChunkSize; - prevDstBase = dstBase; - }; - - // One-entry region cache per side: after sorting, consecutive pairs almost always fall in the - // same region (typically one KV pool), so the O(log R) map lookup is skipped on cache hits. - struct RegionCache - { - uintptr_t base = 0; - size_t totalLen = 0; - size_t chunkSize = 0; - bool valid = false; - }; - - // Returns {chunkSize, regionBase, found}. A miss means the address is not covered by any - // region metadata (e.g. the peer did not send its region info). Two misses both look like - // {0, 0} yet may belong to two distinct regions, so pieces with a missed lookup on either - // side are never merged — a merge could silently cross a chunk or registration boundary. - auto cachedLookup = [](uintptr_t addr, VramRegionMap const& regionMap, RegionCache& cache) + for (size_t i = 0; i < srcVec.size(); ++i) { - if (cache.valid && addr >= cache.base && addr - cache.base < cache.totalLen) - { - return std::tuple<size_t, uintptr_t, bool>{cache.chunkSize, cache.base, true}; - } - auto it = regionMap.upper_bound(addr); - if (it != regionMap.begin()) + auto [srcChunkSize, srcBase] = lookupChunkInfo(srcVec[i].getAddr(), localRegionMap); + auto [dstChunkSize, dstBase] = lookupChunkInfo(dstVec[i].getAddr(), remoteRegionMap); + + // If neither side is multi-chunk VMM, no splitting is needed. + if (srcChunkSize == 0 && dstChunkSize == 0) { - --it; - if (addr >= it->first && addr - it->first < it->second.totalLen) - { - cache = {it->first, it->second.totalLen, it->second.chunkSize, true}; - return std::tuple<size_t, uintptr_t, bool>{cache.chunkSize, cache.base, true}; - } + splitSrc.push_back(srcVec[i]); + splitDst.push_back(dstVec[i]); + continue; } - return std::tuple<size_t, uintptr_t, bool>{0, 0, false}; - }; - - RegionCache srcCache, dstCache; - size_t numPieces = 0; - for (size_t idx : order) - { - auto const& src = srcVec[idx]; - auto const& dst = dstVec[idx]; - auto [srcChunkSize, srcBase, srcFound] = cachedLookup(src.getAddr(), localRegionMap, srcCache); - auto [dstChunkSize, dstBase, dstFound] = cachedLookup(dst.getAddr(), remoteRegionMap, dstCache); - uintptr_t srcAddr = src.getAddr(); - uintptr_t dstAddr = dst.getAddr(); - size_t remaining = src.getLen(); + uintptr_t srcAddr = srcVec[i].getAddr(); + uintptr_t dstAddr = dstVec[i].getAddr(); + size_t remaining = srcVec[i].getLen(); while (remaining > 0) { size_t srcPieceSize = remaining; if (srcChunkSize > 0) { - srcPieceSize = srcChunkSize - static_cast<size_t>((srcAddr - srcBase) % srcChunkSize); + size_t srcOffsetInChunk = static_cast<size_t>((srcAddr - srcBase) % srcChunkSize); + srcPieceSize = srcChunkSize - srcOffsetInChunk; } size_t dstPieceSize = remaining; if (dstChunkSize > 0) { - dstPieceSize = dstChunkSize - static_cast<size_t>((dstAddr - dstBase) % dstChunkSize); + size_t dstOffsetInChunk = static_cast<size_t>((dstAddr - dstBase) % dstChunkSize); + dstPieceSize = dstChunkSize - dstOffsetInChunk; } size_t pieceSize = std::min({remaining, srcPieceSize, dstPieceSize}); - emitPiece(srcAddr, dstAddr, pieceSize, src.getDeviceId(), dst.getDeviceId(), srcChunkSize, srcBase, - dstChunkSize, dstBase, srcFound && dstFound); + splitSrc.emplace_back(srcAddr, pieceSize, srcVec[i].getDeviceId()); + splitDst.emplace_back(dstAddr, pieceSize, dstVec[i].getDeviceId()); srcAddr += pieceSize; dstAddr += pieceSize; remaining -= pieceSize; - ++numPieces; } } - if (outSrc.size() != srcVec.size()) - { - TLLM_LOG_DEBUG("VmmDescSplitter::splitAndCoalesceTransferDescs: %zu pairs -> %zu pieces -> %zu transfers", - srcVec.size(), numPieces, outSrc.size()); - } - - return {MemoryDescs{srcDescs.getType(), std::move(outSrc)}, MemoryDescs{dstDescs.getType(), std::move(outDst)}}; + return {MemoryDescs{srcDescs.getType(), std::move(splitSrc)}, MemoryDescs{dstDescs.getType(), std::move(splitDst)}}; } MemoryDescs VmmDescSplitter::splitVmmDescs(MemoryDescs const& descs, size_t& detectedChunkSize) diff --git a/cpp/tensorrt_llm/executor/cache_transmission/ucx_utils/ucxCacheCommunicator.cpp b/cpp/tensorrt_llm/executor/cache_transmission/ucx_utils/ucxCacheCommunicator.cpp index 3cd388625640..4ad1e7bffc86 100644 --- a/cpp/tensorrt_llm/executor/cache_transmission/ucx_utils/ucxCacheCommunicator.cpp +++ b/cpp/tensorrt_llm/executor/cache_transmission/ucx_utils/ucxCacheCommunicator.cpp @@ -611,41 +611,23 @@ UcxConnection::ConnectionIdType UcxConnectionManager::getNewConnectionId(std::sh Connection const* UcxConnectionManager::recvConnect(DataContext const& ctx, void* data, size_t size) { - // Co-owned by the completion callback: the terminate path returns while the - // cancelled request may still write the buffer and fire the callback. - struct RecvState - { - std::vector<char> mBuffer; - std::promise<void> mPromise; - }; + std::vector<char> buffer(size + sizeof(UcxConnection::ConnectionIdType)); + std::promise<void> promise; + std::future<void> future = promise.get_future(); + auto completionCallback = [&](ucs_status_t, ucxx::RequestCallbackUserData) -> void { promise.set_value(); }; - auto state = std::make_shared<RecvState>(); - state->mBuffer.resize(size + sizeof(UcxConnection::ConnectionIdType)); - std::future<void> future = state->mPromise.get_future(); - auto completionCallback - = [state](ucs_status_t, ucxx::RequestCallbackUserData) -> void { state->mPromise.set_value(); }; - - std::shared_ptr<ucxx::Request> req = mWorkersPool.front()->tagRecv(state->mBuffer.data(), state->mBuffer.size(), - ucxx::Tag(ctx.getTag()), ucxx::TagMask(0xFFFFFFFF), false, completionCallback); + std::shared_ptr<ucxx::Request> req = mWorkersPool.front()->tagRecv( + buffer.data(), buffer.size(), ucxx::Tag(ctx.getTag()), ucxx::TagMask(0xFFFFFFFF), false, completionCallback); if (!req->isCompleted()) { - // Poll with timeout to allow checking the terminate flag - auto const& terminate = ctx.getTransferTerminate(); - while (future.wait_for(std::chrono::milliseconds(100)) != std::future_status::ready) - { - if (terminate.load()) - { - req->cancel(); - return nullptr; - } - } + future.get(); } TLLM_CHECK_WITH_INFO(req->isCompleted(), "recv SendConnectionId should be completed"); req->checkError(); - memcpy(data, state->mBuffer.data(), size); + memcpy(data, buffer.data(), size); UcxConnection::ConnectionIdType connectionId - = *reinterpret_cast<UcxConnection::ConnectionIdType*>(state->mBuffer.data() + size); + = *reinterpret_cast<UcxConnection::ConnectionIdType*>(buffer.data() + size); std::scoped_lock lock(mConnectionsMutex, mConnectionFuturesMutex); TLLM_CHECK_WITH_INFO(mConnectionFutures.find(connectionId) != mConnectionFutures.end(), "connectionFuture not found In recvConnect connectionId : %lu , worldRank: %d", connectionId, mRank); @@ -660,7 +642,7 @@ Connection const* UcxConnectionManager::recvConnect(DataContext const& ctx, void TLLM_CHECK(!mConnections[connectionId]->isFromRequester()); TLLM_LOG_DEBUG(mRank, "recvConnect connectionId: %lu , sendIDData:%lu", connectionId, - *reinterpret_cast<uint64_t*>(state->mBuffer.data())); + *reinterpret_cast<uint64_t*>(buffer.data())); return mConnections[connectionId].get(); } diff --git a/cpp/tensorrt_llm/executor/disaggServerUtil.cpp b/cpp/tensorrt_llm/executor/disaggServerUtil.cpp new file mode 100644 index 000000000000..6be2e4fb8ae8 --- /dev/null +++ b/cpp/tensorrt_llm/executor/disaggServerUtil.cpp @@ -0,0 +1,555 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/executor/disaggServerUtil.h" +#include "tensorrt_llm/common/utils.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include <mutex> + +namespace tensorrt_llm::executor::disagg_executor +{ + +class DisaggExecutorOrchestrator::Impl +{ +public: + Impl(std::vector<std::filesystem::path> const& ctxEnginePaths, + std::vector<std::filesystem::path> const& genEnginePaths, + std::vector<texec::ExecutorConfig> const& ctxExecutorConfigs, + std::vector<texec::ExecutorConfig> const& genExecutorConfigs, bool hasContextAwaitThreads, + bool hasGenAwaitThreads) + : mhasContextAwaitThreads(hasContextAwaitThreads) + , mhasGenAwaitThreads(hasGenAwaitThreads) + { + TLLM_CHECK(ctxEnginePaths.size() == ctxExecutorConfigs.size()); + TLLM_CHECK(genEnginePaths.size() == genExecutorConfigs.size()); + TLLM_CHECK(!(ctxEnginePaths.empty() || genEnginePaths.empty())); + int worldRank = tensorrt_llm::mpi::MpiComm::world().getRank(); + mIsOrchestrator = (worldRank == 0); + auto contextNum = ctxEnginePaths.size(); + mContextReqIdToGlobalId = std::vector<std::unordered_map<IdType, IdType>>(contextNum); + mContextMapMutexs = std::vector<std::mutex>(contextNum); + auto genNum = genEnginePaths.size(); + mGenerationReqIdToGlobalId = std::vector<std::unordered_map<IdType, IdType>>(genNum); + mGenerationMapMutexs = std::vector<std::mutex>(genNum); + + for (size_t cN = 0; cN < contextNum; cN++) + { + mContextExecutors.push_back(std::make_unique<texec::Executor>( + ctxEnginePaths[cN], texec::ModelType::kDECODER_ONLY, ctxExecutorConfigs[cN])); + } + + for (size_t gN = 0; gN < genNum; gN++) + { + mGenerationExecutors.push_back(std::make_unique<texec::Executor>( + genEnginePaths[gN], texec::ModelType::kDECODER_ONLY, genExecutorConfigs[gN])); + } + + if (mIsOrchestrator) + { + if (mhasContextAwaitThreads) + { + for (size_t contextIdx = 0; contextIdx < contextNum; contextIdx++) + { + mContextThreads.emplace_back( + [this, contextIdx]() { this->waitResponseAndAppendThreadFun(true, contextIdx); }); + } + } + if (mhasGenAwaitThreads) + { + + for (size_t genIdx = 0; genIdx < genNum; genIdx++) + { + mGenerationThreads.emplace_back( + [this, genIdx]() { this->waitResponseAndAppendThreadFun(false, genIdx); }); + } + } + } + tensorrt_llm::mpi::MpiComm::world().barrier(); + } + + std::vector<IdType> enqueueContext(std::vector<texec::Request> const& requests, + std::optional<int> selectContextId = std::nullopt, bool batch = false) + { + + std::vector<IdType> globalReqIds; + for (auto const& request : requests) + { + globalReqIds.push_back(generatedGlobalId()); + TLLM_CHECK(request.getRequestType() == tensorrt_llm::executor::RequestType::REQUEST_TYPE_CONTEXT_ONLY); + } + + if (batch) + { + size_t contextId = selectContextId.has_value() ? selectContextId.value() : selectContextExecutor(); + auto contextReqIds = mContextExecutors[contextId]->enqueueRequests(requests); + { + std::scoped_lock<std::mutex> lock{mContextMapMutexs[contextId]}; + for (size_t i = 0; i < requests.size(); ++i) + { + mContextReqIdToGlobalId[contextId][contextReqIds[i]] = globalReqIds[i]; + } + } + } + else + { + for (size_t i = 0; i < requests.size(); ++i) + { + size_t contextId = selectContextId.has_value() ? selectContextId.value() : selectContextExecutor(); + + auto contextReqId = mContextExecutors[contextId]->enqueueRequest(requests[i]); + { + std::scoped_lock<std::mutex> lock{mContextMapMutexs[contextId]}; + mContextReqIdToGlobalId[contextId][contextReqId] = globalReqIds[i]; + } + } + } + return globalReqIds; + } + + void enqueueGeneration(std::vector<texec::Request> const& requests, std::vector<IdType> const& globalRequestIds, + std::optional<int> selectGenIdx = std::nullopt, bool batch = false) + { + + TLLM_CHECK(globalRequestIds.size() == requests.size()); + + for (auto const& request : requests) + { + + TLLM_CHECK(request.getRequestType() == tensorrt_llm::executor::RequestType::REQUEST_TYPE_GENERATION_ONLY); + } + if (batch) + { + size_t genIdx = selectGenIdx.has_value() ? selectGenIdx.value() : selectGenerationExecutor(); + auto genReqIds = mGenerationExecutors[genIdx]->enqueueRequests(requests); + { + std::scoped_lock<std::mutex> lock{mGenerationMapMutexs[genIdx]}; + for (size_t i = 0; i < requests.size(); ++i) + { + mGenerationReqIdToGlobalId[genIdx][genReqIds[i]] = globalRequestIds[i]; + } + } + } + else + { + for (size_t i = 0; i < requests.size(); ++i) + { + size_t genIdx = selectGenIdx.has_value() ? selectGenIdx.value() : selectGenerationExecutor(); + + auto genReqId = mGenerationExecutors[genIdx]->enqueueRequest(requests[i]); + { + std::scoped_lock<std::mutex> lock{mGenerationMapMutexs[genIdx]}; + mGenerationReqIdToGlobalId[genIdx][genReqId] = globalRequestIds[i]; + } + } + } + } + + std::vector<ResponseWithId> awaitContextResponses( + std::optional<int> contextIdx, std::optional<std::chrono::milliseconds> const& timeout) + { + + std::vector<ResponseWithId> responses; + + if (mhasContextAwaitThreads) + { + + std::unique_lock<std::mutex> lock(mResponsesContextMtx); + auto pred = [&mShutdown = mShutdown, &resp = this->mContextResponses]() -> bool + { return !resp.empty() || mShutdown; }; + auto storeResponses = [&resp = this->mContextResponses, &responses]() + { + responses = std::move(resp); + resp.clear(); + }; + if (timeout) + { + if (mContextResponsesCV.wait_for(lock, timeout.value(), pred)) + { + storeResponses(); + } + } + else + { + mContextResponsesCV.wait(lock, pred); + storeResponses(); + } + TLLM_CHECK_WITH_INFO( + !contextIdx.has_value(), "contextIdx should not be provided when mhasContextAwaitThreads is true"); + + return responses; + } + + if (contextIdx.has_value()) + { + TLLM_CHECK(!mhasContextAwaitThreads); + auto responseFromExecutor = mContextExecutors[contextIdx.value()]->awaitResponses(timeout); + for (auto&& resp : responseFromExecutor) + { + + auto reqId = resp.getRequestId(); + IdType globalId{0}; + + { + std::scoped_lock<std::mutex> lock{mContextMapMutexs.at(contextIdx.value())}; + globalId = mContextReqIdToGlobalId.at(contextIdx.value()).at(reqId); + } + TLLM_CHECK(globalId != 0); + responses.emplace_back(std::move(resp), globalId); + } + return responses; + } + TLLM_CHECK(timeout.has_value()); + auto timeouP = timeout.value() / mContextExecutors.size(); + for (size_t ci = 0; ci < mContextExecutors.size(); ci++) + { + auto responseFromExecutor = mContextExecutors.at(ci)->awaitResponses(timeouP); + for (auto&& resp : responseFromExecutor) + { + auto reqId = resp.getRequestId(); + IdType globalId{0}; + + { + std::scoped_lock<std::mutex> lock{mContextMapMutexs.at(ci)}; + globalId = mContextReqIdToGlobalId.at(ci).at(reqId); + } + TLLM_CHECK(globalId != 0); + responses.emplace_back(std::move(resp), globalId); + } + } + + return responses; + }; + + std::vector<ResponseWithId> awaitGenerationResponses( + std::optional<int> genIdx, std::optional<std::chrono::milliseconds> const& timeout) + { + + std::vector<ResponseWithId> responses; + + if (mhasGenAwaitThreads) + { + + std::unique_lock<std::mutex> lock(mResponseGenerationMtx); + auto pred = [&mShutdown = mShutdown, &resp = this->mGenerationResponses]() -> bool + { return !resp.empty() || mShutdown; }; + auto storeResponses = [&resp = this->mGenerationResponses, &responses]() + { + responses = std::move(resp); + resp.clear(); + }; + if (timeout) + { + if (mGenerationResponsesCv.wait_for(lock, timeout.value(), pred)) + { + storeResponses(); + } + } + else + { + mGenerationResponsesCv.wait(lock, pred); + storeResponses(); + } + TLLM_CHECK_WITH_INFO(!genIdx.has_value(), "genIdx should not be provided when mhasGenAwaitThreads is true"); + return responses; + } + + if (genIdx.has_value()) + { + TLLM_CHECK(!mhasGenAwaitThreads); + auto responseFromExecutor = mGenerationExecutors[genIdx.value()]->awaitResponses(timeout); + for (auto&& resp : responseFromExecutor) + { + + auto reqId = resp.getRequestId(); + IdType globalId{0}; + + { + std::scoped_lock<std::mutex> lock{mGenerationMapMutexs.at(genIdx.value())}; + globalId = mGenerationReqIdToGlobalId.at(genIdx.value()).at(reqId); + } + TLLM_CHECK(globalId != 0); + responses.emplace_back(std::move(resp), globalId); + } + return responses; + } + TLLM_CHECK(timeout.has_value()); + auto timeouP = timeout.value() / mGenerationExecutors.size(); + + for (size_t gi = 0; gi < mGenerationExecutors.size(); gi++) + { + auto responseFromExecutor = mGenerationExecutors.at(gi)->awaitResponses(timeouP); + for (auto&& resp : responseFromExecutor) + { + + auto reqId = resp.getRequestId(); + IdType globalId{0}; + + { + std::scoped_lock<std::mutex> lock{mGenerationMapMutexs.at(gi)}; + globalId = mGenerationReqIdToGlobalId.at(gi).at(reqId); + } + TLLM_CHECK(globalId != 0); + responses.emplace_back(std::move(resp), globalId); + } + } + + return responses; + }; + + [[nodiscard]] bool canEnqueue() const + { + return mIsOrchestrator; + } + + [[nodiscard]] std::vector<std::unique_ptr<texec::Executor>> const& getContextExecutors() const + { + return mContextExecutors; + } + + [[nodiscard]] std::vector<std::unique_ptr<texec::Executor>> const& getGenExecutors() const + { + return mGenerationExecutors; + } + + ~Impl() + { + + mShutdown = true; + + mContextResponsesCV.notify_all(); + mGenerationResponsesCv.notify_all(); + for (auto&& executor : mContextExecutors) + { + executor->shutdown(); + } + for (auto&& executor : mGenerationExecutors) + { + executor->shutdown(); + } + + if (mIsOrchestrator) + { + if (mhasContextAwaitThreads) + { + for (auto&& contextThread : mContextThreads) + { + if (contextThread.joinable()) + { + contextThread.join(); + } + } + } + if (mhasGenAwaitThreads) + { + for (auto&& genThread : mGenerationThreads) + { + if (genThread.joinable()) + { + genThread.join(); + } + } + } + } + } + +private: + IdType generatedGlobalId() + { + return (++mLastId % UINT64_MAX); + }; + + size_t selectContextExecutor() + { + static size_t selectContextId = 0; + auto contextId = (selectContextId++) % mContextExecutors.size(); + if (selectContextId >= mContextExecutors.size()) + { + selectContextId = 0; + } + return contextId; + } + + size_t selectGenerationExecutor() + { + static size_t selectGenerationId = 0; + auto generationIdx = (selectGenerationId++) % mGenerationExecutors.size(); + if (selectGenerationId >= mGenerationExecutors.size()) + { + selectGenerationId = 0; + } + return generationIdx; + } + + void appendNewContextResponse(std::vector<ResponseWithId>&& newResponses) + { + { + std::scoped_lock<std::mutex> lock(mResponsesContextMtx); + for (auto&& response : newResponses) + { + mContextResponses.emplace_back(std::move(response)); + } + } + mContextResponsesCV.notify_all(); + } + + void appendNewGenerationResponse(std::vector<ResponseWithId>&& newResponses) + { + { + std::scoped_lock<std::mutex> lock(mResponseGenerationMtx); + for (auto&& response : newResponses) + { + mGenerationResponses.emplace_back(std::move(response)); + } + } + mGenerationResponsesCv.notify_all(); + } + + void waitResponseAndAppendThreadFun(bool isContext, int executorIdx) + { + + tensorrt_llm::common::setThreadName("waitResponseAndAppendThreadFun"); + + auto& executor = isContext ? mContextExecutors[executorIdx] : mGenerationExecutors[executorIdx]; + + while (!mShutdown) + { + auto responses = executor->awaitResponses(); + + if (responses.empty()) + { + continue; + } + std::vector<ResponseWithId> responseWithIds; + if (isContext) + { + for (auto&& response : responses) + { + auto reqId = response.getRequestId(); + IdType globalId{0}; + + { + std::scoped_lock<std::mutex> lock{mContextMapMutexs.at(executorIdx)}; + globalId = mContextReqIdToGlobalId.at(executorIdx).at(reqId); + } + TLLM_CHECK(globalId != 0); + responseWithIds.emplace_back(std::move(response), globalId); + } + if (responseWithIds.size() > 0) + { + appendNewContextResponse(std::move(responseWithIds)); + } + } + else + { + + for (auto&& response : responses) + { + auto reqId = response.getRequestId(); + IdType globalId{0}; + + { + std::scoped_lock<std::mutex> lock{mGenerationMapMutexs.at(executorIdx)}; + globalId = mGenerationReqIdToGlobalId.at(executorIdx).at(reqId); + } + TLLM_CHECK(globalId != 0); + responseWithIds.emplace_back(std::move(response), globalId); + } + if (responseWithIds.size() > 0) + { + appendNewGenerationResponse(std::move(responseWithIds)); + } + } + } + }; + + std::vector<std::unique_ptr<texec::Executor>> mContextExecutors; + std::vector<std::unique_ptr<texec::Executor>> mGenerationExecutors; + std::vector<std::thread> mContextThreads; + std::vector<std::thread> mGenerationThreads; + + std::atomic<IdType> mLastId{0}; + std::vector<std::unordered_map<IdType, IdType>> mContextReqIdToGlobalId; + std::vector<std::unordered_map<IdType, IdType>> mGenerationReqIdToGlobalId; + std::vector<std::mutex> mContextMapMutexs; + std::vector<std::mutex> mGenerationMapMutexs; + std::vector<ResponseWithId> mContextResponses; + std::condition_variable mContextResponsesCV; + std::mutex mResponsesContextMtx; + + std::vector<ResponseWithId> mGenerationResponses; + std::condition_variable mGenerationResponsesCv; + std::mutex mResponseGenerationMtx; + std::atomic<bool> mShutdown{false}; + std::atomic<bool> mhasContextAwaitThreads{false}; + std::atomic<bool> mhasGenAwaitThreads{false}; + bool mIsOrchestrator{false}; +}; + +DisaggExecutorOrchestrator::DisaggExecutorOrchestrator(std::vector<std::filesystem::path> const& ctxEnginePaths, + std::vector<std::filesystem::path> const& genEnginePaths, + std::vector<executor::ExecutorConfig> const& ctxExecutorConfigs, + std::vector<executor::ExecutorConfig> const& genExecutorConfigs, bool hasContextAwaitThreads, + bool hasGenAwaitThreads) + : mImpl(std::make_unique<DisaggExecutorOrchestrator::Impl>(ctxEnginePaths, genEnginePaths, ctxExecutorConfigs, + genExecutorConfigs, hasContextAwaitThreads, hasGenAwaitThreads)) +{ +} + +std::vector<IdType> DisaggExecutorOrchestrator::enqueueContext( + std::vector<texec::Request> const& requests, std::optional<int> selectContextId, bool batch) +{ + return mImpl->enqueueContext(requests, selectContextId, batch); +} + +void DisaggExecutorOrchestrator::enqueueGeneration(std::vector<texec::Request> const& requests, + std::vector<IdType> const& globalRequestIds, std::optional<int> selectGenIdx, bool batch) +{ + mImpl->enqueueGeneration(requests, globalRequestIds, selectGenIdx, batch); +} + +std::vector<ResponseWithId> DisaggExecutorOrchestrator::awaitContextResponses( + std::optional<std::chrono::milliseconds> const& timeout, std::optional<int> contextIdx) +{ + return mImpl->awaitContextResponses(contextIdx, timeout); +} + +std::vector<ResponseWithId> DisaggExecutorOrchestrator::awaitGenerationResponses( + std::optional<std::chrono::milliseconds> const& timeout, std::optional<int> genIdx) +{ + return mImpl->awaitGenerationResponses(genIdx, timeout); +} + +bool DisaggExecutorOrchestrator::canEnqueue() const +{ + return mImpl->canEnqueue(); +}; + +std::vector<std::unique_ptr<texec::Executor>> const& DisaggExecutorOrchestrator::getContextExecutors() const +{ + return mImpl->getContextExecutors(); +} + +std::vector<std::unique_ptr<texec::Executor>> const& DisaggExecutorOrchestrator::getGenExecutors() const +{ + return mImpl->getGenExecutors(); +} + +DisaggExecutorOrchestrator::~DisaggExecutorOrchestrator() = default; + +} // namespace tensorrt_llm::executor::disagg_executor diff --git a/cpp/tensorrt_llm/executor/dynamicBatchTuner.cpp b/cpp/tensorrt_llm/executor/dynamicBatchTuner.cpp new file mode 100644 index 000000000000..b7cd49f5430b --- /dev/null +++ b/cpp/tensorrt_llm/executor/dynamicBatchTuner.cpp @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/executor/dynamicBatchTuner.h" +#include "tensorrt_llm/common/logger.h" +#include <cmath> + +namespace +{ +using namespace tensorrt_llm::executor; + +void updateStats(SizeType32 value, std::deque<SizeType32>& stats, int64_t& sum, SizeType32 windowSize) +{ + while (static_cast<SizeType32>(stats.size()) >= windowSize) + { + sum -= stats.front(); + stats.pop_front(); + } + stats.push_back(value); + sum += value; +} +} // namespace + +namespace tensorrt_llm::executor +{ + +DynamicBatchTuner::DynamicBatchTuner(DynamicBatchConfig const& config) + : mEnableBatchSizeTuning(config.getEnableBatchSizeTuning()) + , mEnableMaxNumTokensTuning(config.getEnableMaxNumTokensTuning()) + , mDynamicBatchMovingAverageWindow(config.getDynamicBatchMovingAverageWindow()) + , mBatchSizeTable(config.getBatchSizeTable()) +{ + TLLM_CHECK_WITH_INFO(!mBatchSizeTable.empty(), "Batch size table is empty."); + for (size_t i = 1; i < mBatchSizeTable.size(); ++i) + { + TLLM_CHECK_WITH_INFO(mBatchSizeTable[i - 1].first < mBatchSizeTable[i].first, + "Batch size table is not sorted in ascending order."); + } +} + +void DynamicBatchTuner::updateStats(SizeType32 inputLength, SizeType32 outputLength) +{ + ::updateStats(inputLength, mInputLengthStats, mInputLengthSum, mDynamicBatchMovingAverageWindow); + ::updateStats(outputLength, mOutputLengthStats, mOutputLengthSum, mDynamicBatchMovingAverageWindow); +} + +double DynamicBatchTuner::getAverageInputLength() const +{ + return mInputLengthStats.empty() ? 0 : static_cast<double>(mInputLengthSum) / mInputLengthStats.size(); +} + +double DynamicBatchTuner::getAverageOutputLength() const +{ + return mOutputLengthStats.empty() ? 0 : static_cast<double>(mOutputLengthSum) / mOutputLengthStats.size(); +} + +SizeType32 DynamicBatchTuner::getRuntimeBatchSize(SizeType32 maxCapacityBatchSize) const +{ + for (auto const& [batchSizeLimit, batchSize] : mBatchSizeTable) + { + if (maxCapacityBatchSize < batchSizeLimit) + { + return batchSize; + } + } + SizeType32 threshold = maxCapacityBatchSize / kBatchSizeFallbackGranularity * kBatchSizeFallbackGranularity; + if (maxCapacityBatchSize < (threshold + kBatchSizeFallbackThreshold)) + { + return threshold; + } + return maxCapacityBatchSize; +} + +SizeType32 DynamicBatchTuner::getRuntimeMaxNumTokens(SizeType32 maxRuntimeBatchSize) const +{ + // calculate max num token in fully overlapped case + SizeType32 adjustedNumTokens + = 1.0 * (maxRuntimeBatchSize * getAverageInputLength() / getAverageOutputLength() + maxRuntimeBatchSize); + SizeType32 tokenThreshold; + // context heavy (avg ISL/OSL > kMaxNumTokensRatioContextHeavy) + if (getAverageInputLength() / getAverageOutputLength() > kMaxNumTokensRatioContextHeavy) + { + tokenThreshold = kMaxNumTokensThresholdContextHeavy; + } + // balanced case (kMaxNumTokensRatioBalanced < avg ISL/OSL < kMaxNumTokensRatioContextHeavy) + else if (getAverageInputLength() / getAverageOutputLength() > kMaxNumTokensRatioBalanced) + { + tokenThreshold = kMaxNumTokensThresholdBalanced; + } + // gen heavy (avg ISL/OSL < kMaxNumTokensRatioBalanced) + else + { + tokenThreshold = kMaxNumTokensThresholdGenHeavy; + } + // pad it to pow of 2 and max of this value and threshold. + return (std::max(1 << int(ceil(log2(adjustedNumTokens))), tokenThreshold)); +} + +} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/dynamicBatchTuner.h b/cpp/tensorrt_llm/executor/dynamicBatchTuner.h new file mode 100644 index 000000000000..df38cd157fde --- /dev/null +++ b/cpp/tensorrt_llm/executor/dynamicBatchTuner.h @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" + +#include <deque> + +namespace tensorrt_llm::executor +{ + +/// @brief A class that maintains runtime input and output length statistics and computes runtime dynamic batch size. +class DynamicBatchTuner +{ +public: + explicit DynamicBatchTuner(DynamicBatchConfig const& config); + + /// @brief Check if dynamic batch size tuning is enabled. + [[nodiscard]] bool isBatchSizeTuningEnabled() const + { + return mEnableBatchSizeTuning; + } + + /// @brief Check if max num tokens tuning is enabled. + [[nodiscard]] bool isMaxNumTokensTuningEnabled() const + { + return mEnableMaxNumTokensTuning; + } + + /// @brief Update current stats given the input and output length from a single request. + void updateStats(SizeType32 inputLen, SizeType32 outputLen); + + /// @brief Get average input length. + [[nodiscard]] double getAverageInputLength() const; + + /// @brief Get average output length. + [[nodiscard]] double getAverageOutputLength() const; + + /// @brief Get the dynamic batch size based on the current statistics. + [[nodiscard]] SizeType32 getRuntimeBatchSize(SizeType32 maxCapacityBatchSize) const; + + /// @brief Get the dynamic max num tokens based on the current statistics. + [[nodiscard]] SizeType32 getRuntimeMaxNumTokens(SizeType32 runtimeBatchSize) const; + +private: + bool mEnableBatchSizeTuning = false; + + bool mEnableMaxNumTokensTuning = false; + + SizeType32 mDynamicBatchMovingAverageWindow = 0; + + std::vector<std::pair<SizeType32, SizeType32>> mBatchSizeTable; + + int64_t mInputLengthSum = 0; + std::deque<SizeType32> mInputLengthStats; + + int64_t mOutputLengthSum = 0; + std::deque<SizeType32> mOutputLengthStats; + + static SizeType32 const kBatchSizeFallbackGranularity = 512; + static SizeType32 const kBatchSizeFallbackThreshold = 128; + + static double constexpr kMaxNumTokensRatioContextHeavy = 2.0; + static double constexpr kMaxNumTokensRatioBalanced = 0.5; + + static SizeType32 const kMaxNumTokensThresholdContextHeavy = 8192; + static SizeType32 const kMaxNumTokensThresholdBalanced = 4096; + static SizeType32 const kMaxNumTokensThresholdGenHeavy = 2048; +}; + +} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/executor.cpp b/cpp/tensorrt_llm/executor/executor.cpp new file mode 100644 index 000000000000..091bb5128230 --- /dev/null +++ b/cpp/tensorrt_llm/executor/executor.cpp @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <tensorrt_llm/executor/executor.h> +#include <tensorrt_llm/executor/executorImpl.h> + +namespace tensorrt_llm::executor +{ + +Executor::Executor(std::filesystem::path const& modelPath, ModelType modelType, ExecutorConfig const& executorConfig) + : mImpl(std::make_unique<Executor::Impl>(modelPath, std::nullopt, modelType, executorConfig)) +{ +} + +Executor::Executor(std::filesystem::path const& encoderModelPath, std::filesystem::path const& decoderModelPath, + ModelType modelType, ExecutorConfig const& executorConfig) + : mImpl(std::make_unique<Executor::Impl>(decoderModelPath, encoderModelPath, modelType, executorConfig)) +{ +} + +Executor::Executor(BufferView const& engineBuffer, std::string const& jsonConfigStr, ModelType modelType, + ExecutorConfig const& executorConfig, std::optional<std::map<std::string, Tensor>> const& managedWeights) + : mImpl(std::make_unique<Executor::Impl>( + engineBuffer, jsonConfigStr, std::nullopt, std::nullopt, modelType, executorConfig, managedWeights)) +{ +} + +Executor::Executor(BufferView const& encoderEngineBuffer, std::string const& encoderJsonConfigStr, + BufferView const& decoderEngineBuffer, std::string const& decoderJsonConfigStr, ModelType modelType, + ExecutorConfig const& executorConfig) + : mImpl(std::make_unique<Executor::Impl>(decoderEngineBuffer, decoderJsonConfigStr, encoderEngineBuffer, + encoderJsonConfigStr, modelType, executorConfig, std::nullopt)) +{ +} + +Executor::Executor(std::shared_ptr<Model> model, ExecutorConfig const& executorConfig) + : mImpl(std::make_unique<Executor::Impl>(std::move(model), std::nullopt, executorConfig)) +{ +} + +Executor::Executor( + std::shared_ptr<Model> encoderModel, std::shared_ptr<Model> decoderModel, ExecutorConfig const& executorConfig) + : mImpl(std::make_unique<Executor::Impl>(std::move(decoderModel), std::move(encoderModel), executorConfig)) +{ +} + +Executor::~Executor() = default; + +IdType Executor::enqueueRequest(Request const& llmRequest) +{ + return mImpl->enqueueRequest(llmRequest); +} + +std::vector<IdType> Executor::enqueueRequests(std::vector<Request> const& llmRequests) +{ + return mImpl->enqueueRequests(llmRequests); +} + +std::vector<Response> Executor::awaitResponses(std::optional<std::chrono::milliseconds> const& timeout) +{ + return mImpl->awaitResponses(timeout); +} + +std::vector<Response> Executor::awaitResponses( + IdType const& requestId, std::optional<std::chrono::milliseconds> const& timeout) +{ + return mImpl->awaitResponses(requestId, timeout); +} + +std::vector<std::vector<Response>> Executor::awaitResponses( + std::vector<IdType> const& requestIds, std::optional<std::chrono::milliseconds> const& timeout) +{ + return mImpl->awaitResponses(requestIds, timeout); +} + +SizeType32 Executor::getNumResponsesReady(std::optional<IdType> const& requestId) const +{ + return mImpl->getNumResponsesReady(requestId); +} + +void Executor::cancelRequest(IdType requestId) +{ + return mImpl->cancelRequest(requestId); +} + +void Executor::shutdown() +{ + return mImpl->shutdown(); +} + +std::deque<IterationStats> Executor::getLatestIterationStats() +{ + return mImpl->getLatestIterationStats(); +} + +std::deque<RequestStatsPerIteration> Executor::getLatestRequestStats() +{ + return mImpl->getLatestRequestStats(); +} + +std::deque<DebugTensorsPerIteration> Executor::getLatestDebugTensors() +{ + return mImpl->getLatestDebugTensors(); +} + +bool Executor::canEnqueueRequests() const +{ + return mImpl->canEnqueueRequests(); +} + +bool Executor::isParticipant() const +{ + return mImpl->isParticipant(); +} + +std::optional<std::shared_ptr<KVCacheEventManager>> Executor::getKVCacheEventManager() const +{ + return mImpl->getKVCacheEventManager(); +} + +KVCacheEvent::KVCacheEvent( + size_t eventId, KVCacheEventData data, SizeType32 windowSize, std::optional<SizeType32> attentionDpRank) + : eventId{eventId} + , data{std::move(data)} + , windowSize{windowSize} + , attentionDpRank{attentionDpRank} +{ +} + +} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/executorImpl.cpp b/cpp/tensorrt_llm/executor/executorImpl.cpp new file mode 100644 index 000000000000..9f7fb654a2d5 --- /dev/null +++ b/cpp/tensorrt_llm/executor/executorImpl.cpp @@ -0,0 +1,2791 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/executor/executorImpl.h" +#include "tensorrt_llm/batch_manager/trtEncoderModel.h" +#include "tensorrt_llm/batch_manager/trtGptModelFactory.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaProfilerUtils.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/timestampUtils.h" +#include "tensorrt_llm/common/utils.h" +#include "tensorrt_llm/executor/dataTransceiverState.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/orchestratorUtils.h" +#include "tensorrt_llm/executor/requestUtils.h" +#include "tensorrt_llm/executor/serialization.h" +#include "tensorrt_llm/executor/serializeUtils.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/executor/version.h" +#include "tensorrt_llm/runtime/loraCache.h" +#include "tensorrt_llm/runtime/memoryCounters.h" +#include "tensorrt_llm/runtime/utils/mpiTags.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <cuda_profiler_api.h> +#include <iterator> +#include <memory> +#include <optional> +#include <utility> + +namespace tensorrt_llm::executor +{ + +namespace +{ + +[[nodiscard]] bool executorConfigIsValid( + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, runtime::ModelConfig const& modelConfig) +{ + // Make sure logic in this function matches fixExecutorConfig + if (executorConfig.getEnableChunkedContext()) + { + if (modelConfig.isRnnBased() || !modelConfig.isKVCacheEnabled() || !modelConfig.getPagedContextFMHA()) + { + return false; + } + } + return true; +} + +[[nodiscard]] ::tensorrt_llm::executor::ExecutorConfig fixExecutorConfig( + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, runtime::ModelConfig const& modelConfig) +{ + // Make sure logic in this function matches executorConfigIsValid + auto fixedExecutorConfig = executorConfig; + // Disable chunked context when not supported + if (executorConfig.getEnableChunkedContext()) + { + if (modelConfig.isRnnBased() || !modelConfig.isKVCacheEnabled() || !modelConfig.getPagedContextFMHA()) + { + fixedExecutorConfig.setEnableChunkedContext(false); + TLLM_LOG_WARNING( + "Chunked context is not supported for this configuration and will be disabled. " + "Related configs: RNNBased: %d, KVCacheEnabled: %d, PagedContextFMHA: %d", + modelConfig.isRnnBased(), modelConfig.isKVCacheEnabled(), modelConfig.getPagedContextFMHA()); + } + } + return fixedExecutorConfig; +} + +[[nodiscard]] bool statsBufferIsEnabled(SizeType32 maxIterations) +{ + return maxIterations != 0; +} + +[[nodiscard]] bool statsBufferIsBounded(SizeType32 maxIterations) +{ + return maxIterations > 0; +} + +SizeType32 getNumChildRequests(Request const& request) +{ + auto samplingConfig = request.getSamplingConfig(); + return samplingConfig.getBeamWidth() > 1 ? 0 : samplingConfig.getNumReturnSequences().value_or(1) - 1; +} + +} // namespace + +/// @brief Version of TRT-LLM as defined in tensorrt_llm/version.py +char const* version() noexcept +{ + return kTensorRtLlmVersion; +} + +class CancelledRequestsAsyncSend +{ +public: + CancelledRequestsAsyncSend(std::shared_ptr<tensorrt_llm::mpi::MpiComm> const& commSession, + std::unordered_set<IdType> const& cancelledReqIds, int peer) + { + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + mNumReq = static_cast<int64_t>(cancelledReqIds.size()); + TLLM_LOG_DEBUG("start send %ld cancelled requests to rank %d", mNumReq, peer); + mRequest1 + = commSession->sendAsync(&mNumReq, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kCancelledRequestsNumReq); + if (mNumReq > 0) + { + mIds.assign(cancelledReqIds.begin(), cancelledReqIds.end()); + mRequest2 = commSession->sendAsync( + mIds.data(), mIds.size(), mpi::MpiType::kUINT64, peer, mpi::MpiTag::kCancelledRequestsIds); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + } + + ~CancelledRequestsAsyncSend() + { + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + mRequest1->wait(); + if (mRequest2) + { + mRequest2->wait(); + } + TLLM_LOG_DEBUG("end send cancelled requests"); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + } + + CancelledRequestsAsyncSend(CancelledRequestsAsyncSend const& executor) = delete; + CancelledRequestsAsyncSend& operator=(CancelledRequestsAsyncSend const& executor) = delete; + CancelledRequestsAsyncSend(CancelledRequestsAsyncSend&&) = delete; + CancelledRequestsAsyncSend& operator=(CancelledRequestsAsyncSend&&) = delete; + + static std::unordered_set<IdType> cancelledRequestsRecv( + std::shared_ptr<tensorrt_llm::mpi::MpiComm> const& commSession, int peer) + { + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + TLLM_LOG_DEBUG("start recv cancelled requests from rank %d", peer); + std::unordered_set<IdType> cancelledReqIds; + int64_t numReq{0}; + commSession->recv(&numReq, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kCancelledRequestsNumReq); + TLLM_LOG_DEBUG("recv %ld cancelled requests", numReq); + if (numReq > 0) + { + std::vector<IdType> buffer(numReq); + commSession->recv( + buffer.data(), buffer.size(), mpi::MpiType::kUINT64, peer, mpi::MpiTag::kCancelledRequestsIds); + cancelledReqIds = std::unordered_set<IdType>(buffer.begin(), buffer.end()); + } + TLLM_LOG_DEBUG("end recv cancelled requests from rank %d", peer); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return cancelledReqIds; + } + +private: + int64_t mNumReq; + std::vector<IdType> mIds; + std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest1; + std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest2; +}; + +class RequestWithIdAsyncSend +{ +public: + RequestWithIdAsyncSend(std::shared_ptr<tensorrt_llm::mpi::MpiComm> const& commSession, + std::vector<RequestWithId> const& reqWithIds, int peer) + { + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + TLLM_LOG_DEBUG("start send requests to rank %d", peer); + mNumReq = static_cast<int64_t>(reqWithIds.size()); + mRequest1 = commSession->sendAsync(&mNumReq, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kRequestWithIdNumReq); + if (mNumReq > 0) + { + mPacked = RequestWithId::serializeReqWithIds(reqWithIds); + mVecSize = static_cast<int64_t>(mPacked.size()); + mRequest2 + = commSession->sendAsync(&mVecSize, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kRequestWithIdVecSize); + mRequest3 = commSession->sendAsync( + mPacked.data(), mPacked.size(), mpi::MpiType::kCHAR, peer, mpi::MpiTag::kRequestWithIdPacked); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + } + + ~RequestWithIdAsyncSend() + { + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + mRequest1->wait(); + if (mRequest2) + { + mRequest2->wait(); + } + if (mRequest3) + { + mRequest3->wait(); + } + TLLM_LOG_DEBUG("end send requests"); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + } + + RequestWithIdAsyncSend(RequestWithIdAsyncSend const& executor) = delete; + RequestWithIdAsyncSend& operator=(RequestWithIdAsyncSend const& executor) = delete; + RequestWithIdAsyncSend(RequestWithIdAsyncSend&&) = delete; + RequestWithIdAsyncSend& operator=(RequestWithIdAsyncSend&&) = delete; + + static std::vector<RequestWithId> requestWithIdRecv( + std::shared_ptr<tensorrt_llm::mpi::MpiComm> const& commSession, int peer) + { + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + TLLM_LOG_DEBUG("start recv requests from rank %d", peer); + std::vector<RequestWithId> reqWithIds; + int64_t numReq{0}; + commSession->recv(&numReq, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kRequestWithIdNumReq); + if (numReq > 0) + { + std::vector<char> buffer; + int64_t vecSize = 0; + commSession->recv(&vecSize, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kRequestWithIdVecSize); + buffer.resize(vecSize); + commSession->recv( + buffer.data(), buffer.size(), mpi::MpiType::kCHAR, peer, mpi::MpiTag::kRequestWithIdPacked); + reqWithIds = RequestWithId::deserializeReqWithIds(buffer); + } + TLLM_LOG_DEBUG("end recv requests from rank %d", peer); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return reqWithIds; + } + +private: + int64_t mNumReq; + int64_t mVecSize; + std::vector<char> mPacked; + std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest1; + std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest2; + std::shared_ptr<tensorrt_llm::mpi::MpiRequest> mRequest3; +}; + +void Executor::Impl::loadModel(std::optional<std::filesystem::path> const& modelPathOpt, + std::optional<BufferView> const& engineBufferOpt, runtime::GptJsonConfig const& jsonConfig, + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, bool isEncoder, + std::optional<std::map<std::string, Tensor>> const& managedWeightsOpt) +{ + auto const gpusPerNode = jsonConfig.getGpusPerNode(); + auto const tp = jsonConfig.getTensorParallelism(); + auto const pp = jsonConfig.getPipelineParallelism(); + auto const cp = jsonConfig.getContextParallelism(); + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + auto worldConfig = runtime::WorldConfig::mpi(gpusPerNode, tp, pp, cp, parallelConfig.getDeviceIds()); + + TLLM_CHECK_WITH_INFO(modelPathOpt.has_value() || engineBufferOpt.has_value(), + "Either engine path or deserialized engine buffer should be given to load the model properly."); + auto rawEngine = engineBufferOpt.has_value() + ? runtime::RawEngine(engineBufferOpt.value().data(), engineBufferOpt.value().size()) + : runtime::RawEngine(modelPathOpt.value() / jsonConfig.engineFilename(worldConfig)); + + if (rawEngine.getType() != tensorrt_llm::runtime::RawEngine::FilePath) + { + if (modelPathOpt.has_value()) + { + rawEngine.setPath(modelPathOpt.value() / jsonConfig.engineFilename(worldConfig)); + if (managedWeightsOpt.has_value()) + { + TLLM_LOG_WARNING( + "Executor::Impl::loadModel: managedWeightsOpt argument is ignored when loading engine from file."); + } + } + else if (managedWeightsOpt.has_value()) + { + rawEngine.setManagedWeightsMap(managedWeightsOpt.value()); + } + } + + auto const& modelConfig = jsonConfig.getModelConfig(); + + if (isEncoder) + { + mEncoderModel = createEncoderModel(rawEngine, modelConfig, worldConfig, executorConfig); + } + else + { + mModel = createModel(rawEngine, modelConfig, worldConfig, executorConfig); + } +}; + +Executor::Impl::Impl(std::filesystem::path const& modelPath, + std::optional<std::filesystem::path> const& encoderModelPath, ModelType const modelType, + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig) +{ + auto decoderJsonConfig = runtime::GptJsonConfig::parse(modelPath / "config.json"); + + // for now, assume encoder & decoder models share the same MPI config + auto const tp = decoderJsonConfig.getTensorParallelism(); + auto const pp = decoderJsonConfig.getPipelineParallelism(); + auto const cp = decoderJsonConfig.getContextParallelism(); + initializeCommAndWorkers(tp, pp, cp, executorConfig, modelType, modelPath, std::nullopt, decoderJsonConfig); + + if (mIsWorker) + { + if (modelType == ModelType::kENCODER_DECODER) + { + if (encoderModelPath.has_value()) + { + auto const encoderJsonConfig = runtime::GptJsonConfig::parse(encoderModelPath.value() / "config.json"); + + auto const encoderMaxInputLen = encoderJsonConfig.getModelConfig().getMaxInputLen(); + auto const encoderHiddenSize = encoderJsonConfig.getModelConfig().getHiddenSize() + * encoderJsonConfig.getTensorParallelism(); // recover full hidden size + // add encoder info to decoder for encoder-decoder models + // note: GptJsonConfig can no longer have modelConfig as const member since it must be mutable here + decoderJsonConfig.getModelConfigMutable().setMaxEncoderLen(encoderMaxInputLen); + decoderJsonConfig.getModelConfigMutable().setEncoderHiddenSize(encoderHiddenSize); + + loadModel( + encoderModelPath.value(), std::nullopt, encoderJsonConfig, executorConfig, true, std::nullopt); + } + else + { + TLLM_LOG_WARNING("Encoder model path not provided. Skipping Encoder Run."); + } + } + loadModel(modelPath, std::nullopt, decoderJsonConfig, executorConfig, false, std::nullopt); + } + initialize(executorConfig); +} + +Executor::Impl::Impl(BufferView const& engineBufferView, std::string const& jsonConfigStr, + std::optional<BufferView> const& encoderEngineBufferView, std::optional<std::string> const& encoderJsonConfigStr, + ModelType const modelType, ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, + std::optional<std::map<std::string, Tensor>> const& managedWeightsOpt) +{ + auto decoderJsonConfig = runtime::GptJsonConfig::parse(jsonConfigStr); + + // for now, assume encoder & decoder models share the same MPI config + auto const tp = decoderJsonConfig.getTensorParallelism(); + auto const pp = decoderJsonConfig.getPipelineParallelism(); + auto const cp = decoderJsonConfig.getContextParallelism(); + initializeCommAndWorkers(tp, pp, cp, executorConfig, modelType, std::nullopt, std::nullopt, decoderJsonConfig); + + if (mIsWorker) + { + if (modelType == ModelType::kENCODER_DECODER) + { + TLLM_CHECK(encoderEngineBufferView.has_value() && encoderJsonConfigStr.has_value()); + TLLM_CHECK_WITH_INFO( + !managedWeightsOpt.has_value(), "Managed weights are not supported for enc-dec models"); + + auto const encoderJsonConfig = runtime::GptJsonConfig::parse(encoderJsonConfigStr.value()); + + auto const encoderMaxInputLen = encoderJsonConfig.getModelConfig().getMaxInputLen(); + auto const encoderHiddenSize = encoderJsonConfig.getModelConfig().getHiddenSize() + * encoderJsonConfig.getTensorParallelism(); // recover full hidden size + // add encoder info to decoder for encoder-decoder models + // note: GptJsonConfig can no longer have modelConfig as const member since it must be mutable here + decoderJsonConfig.getModelConfigMutable().setMaxEncoderLen(encoderMaxInputLen); + decoderJsonConfig.getModelConfigMutable().setEncoderHiddenSize(encoderHiddenSize); + + loadModel( + std::nullopt, encoderEngineBufferView.value(), encoderJsonConfig, executorConfig, true, std::nullopt); + } + loadModel(std::nullopt, engineBufferView, decoderJsonConfig, executorConfig, false, managedWeightsOpt); + } + initialize(executorConfig); +} + +Executor::Impl::Impl(std::shared_ptr<Model> model, std::optional<std::shared_ptr<Model>> encoderModel, + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig) +{ + auto const& worldConfig = model->getWorldConfig(); + auto const tp = worldConfig.getTensorParallelism(); + auto const pp = worldConfig.getPipelineParallelism(); + auto const cp = worldConfig.getContextParallelism(); + auto const modelType = encoderModel.has_value() ? ModelType::kENCODER_DECODER : ModelType::kDECODER_ONLY; + initializeCommAndWorkers(tp, pp, cp, executorConfig, modelType, std::nullopt, worldConfig); + if (modelType == ModelType::kENCODER_DECODER) + { + mEncoderModel = encoderModel.value(); + } + mModel = std::move(model); + initialize(executorConfig); +} + +Executor::Impl::~Impl() +{ + shutdown(); +} + +void Executor::Impl::initialize(::tensorrt_llm::executor::ExecutorConfig const& executorConfig) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + mShutdown = false; + mShutdownCalled = false; + mIterStatsMaxIterations = executorConfig.getIterStatsMaxIterations(); + mRequestStatsMaxIterations = executorConfig.getRequestStatsMaxIterations(); + mDebugTensorsMaxIterations + = executorConfig.getDebugConfig() ? executorConfig.getDebugConfig()->getDebugTensorsMaxIterations() : 0; + TLLM_CHECK_WITH_INFO(mDebugTensorsMaxIterations == 0 || mCommMode == CommunicationMode::kLEADER, + "debugTensorsMaxIterations > 0 is only allowed in leader mode."); + mBatchingType = executorConfig.getBatchingType(); + mIsSchedulerMaxUtilization = (executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy() + == CapacitySchedulerPolicy::kMAX_UTILIZATION); + mIsSchedulerGuaranteedNoEvict = (executorConfig.getSchedulerConfig().getCapacitySchedulerPolicy() + == CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT); + mIsChunkedContext = executorConfig.getEnableChunkedContext(); + mPromptTableOffloading = executorConfig.getPromptTableOffloading(); + mMaxQueueSize = executorConfig.getMaxQueueSize(); + + mLastReqId = 1; + + auto const& logitsProcConfig = executorConfig.getLogitsPostProcessorConfig(); + if (logitsProcConfig.has_value()) + { + mLogitsPostProcessorMap = logitsProcConfig.value().getProcessorMap().value_or(LogitsPostProcessorMap{}); + initializeLogitsPostProcessorBatched(logitsProcConfig.value()); + if (!logitsProcConfig.value().getReplicate()) + { + mModel->setReplicateLogitsPostProcessor(false); + } + } + + auto const& commComm = COMM_SESSION; + int32_t const commSize = commComm.getSize(); + if (mIsWorker) + { + if (commSize > 1) + { + auto const& worldConfig = mModel->getWorldConfig(); + auto const& commSession = COMM_SESSION; + auto const& rank = commSession.getRank(); + auto const& tp = worldConfig.getTensorParallelism(); + auto const& cp = worldConfig.getContextParallelism(); + + mCommTensorParallel = std::make_shared<tensorrt_llm::mpi::MpiComm>( + commSession.split(rank / tp, worldConfig.getTensorParallelRank())); + mCommContextParallel = std::make_shared<tensorrt_llm::mpi::MpiComm>( + commSession.split(rank / (tp * cp) * tp + rank % tp, worldConfig.getContextParallelRank())); + mCommPipelineParallel = std::make_shared<tensorrt_llm::mpi::MpiComm>( + commSession.split(rank % (tp * cp), worldConfig.getPipelineParallelRank())); + + if (worldConfig.isPipelineParallel()) + { + mRequestWithIdWaitThread = std::make_unique<tensorrt_llm::mpi::MpiWaitThread>( + "requestWithIdWaitThread", [this]() { mRequestWithIdAsyncSndHdl.reset(nullptr); }); + mCancelledRequestsWaitThread = std::make_unique<tensorrt_llm::mpi::MpiWaitThread>( + "cancelledRequestsWaitThread", [this]() { mCancelledRequestsAsyncSndHdl.reset(nullptr); }); + if (mIsLeader) + { + mRequestWithIdLeaderThread + = std::make_unique<std::thread>(&Executor::Impl::requestWithIdLeaderThread, this); + mCancelledRequestsLeaderThread + = std::make_unique<std::thread>(&Executor::Impl::cancelledRequestsLeaderThread, this); + } + } + } + // Launch the execution thread + mMaxNumActiveRequests = mModel->getMaxNumSequences(); + mExecutionThread = std::thread(&Impl::executionLoop, this); + } + + mEnableBlockReuse = executorConfig.getKvCacheConfig().getEnableBlockReuse(); + + auto const& dynamicBatchConfig = executorConfig.getSchedulerConfig().getDynamicBatchConfig(); + if (dynamicBatchConfig) + { + if (mIsWorker) + { + if (mModel->getModelConfig().isTransformerBased() && mModel->getModelConfig().isKVCacheEnabled()) + { + mDynamicBatchTuner = std::make_shared<DynamicBatchTuner>(dynamicBatchConfig.value()); + } + else + { + TLLM_LOG_WARNING("Dynamic batch tuner can only support transformer models that use KV cache."); + } + } + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +std::shared_ptr<Model> Executor::Impl::createModel(runtime::RawEngine const& rawEngine, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig) +{ + auto const gptModelType = [&executorConfig, &modelConfig]() + { + switch (executorConfig.getBatchingType()) + { + case BatchingType::kSTATIC: + TLLM_THROW( + "Static batching type is deprecated. Please use in-flight batching with " + "CapacitySchedulerPolicy::kSTATIC_BATCH instead."); + case BatchingType::kINFLIGHT: + return modelConfig.isRnnBased() ? batch_manager::TrtGptModelType::InflightBatching + : batch_manager::TrtGptModelType::InflightFusedBatching; + default: TLLM_THROW("Invalid batching strategy"); + } + }(); + + bool const isLeaderInOrchMode = (mCommMode == CommunicationMode::kORCHESTRATOR) && mIsLeader; + auto const& fixedExecutorConfig = executorConfigIsValid(executorConfig, modelConfig) + ? executorConfig + : fixExecutorConfig(executorConfig, modelConfig); + + return batch_manager::TrtGptModelFactory::create( + rawEngine, modelConfig, worldConfig, gptModelType, fixedExecutorConfig, isLeaderInOrchMode); +} + +std::shared_ptr<Model> Executor::Impl::createEncoderModel(runtime::RawEngine const& rawEngine, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig) +{ + auto fixedExecutorConfig = ExecutorConfig{}; + fixedExecutorConfig.setSchedulerConfig(executorConfig.getSchedulerConfig()); + return std::make_shared<batch_manager::TrtEncoderModel>( + modelConfig, worldConfig, rawEngine, std::make_shared<runtime::TllmLogger>(), fixedExecutorConfig); +} + +void Executor::Impl::setOrchLeaderComm( + SizeType32 tp, SizeType32 pp, SizeType32 cp, ParallelConfig const& parallelConfig) +{ +#if ENABLE_MULTI_DEVICE + auto optOrchestratorConfig = parallelConfig.getOrchestratorConfig(); + if (optOrchestratorConfig.value().getIsOrchestrator()) + { + TLLM_CHECK_WITH_INFO(mWorldRank == 0, "Rank 0 must be orchestrator"); + } + + TLLM_CHECK_WITH_INFO(parallelConfig.getParticipantIds(), + "When not spawning processes in orchestrator mode, participant IDs must be provided"); + auto participantIds = parallelConfig.getParticipantIds().value(); + + TLLM_CHECK_WITH_INFO(static_cast<SizeType32>(participantIds.size()) == tp * pp * cp, + "When specifying participantIds, participantIds size must be equal to tp*pp*cp"); + + bool isLeader = (mWorldRank == participantIds.front()); + bool isOrchestrator = (mWorldRank == 0); + + // OrchLeaderComm rank 0 is orchestrator, rank 1 is leader + mOrchRank = 0; + mLeaderRank = 1; + + // Create a leaderOrch comm + std::vector<int32_t> leaderOrchRanks{0, participantIds.front()}; + + MPI_Group worldGroup = nullptr; + MPICHECK(MPI_Comm_group(MPI_COMM_WORLD, &worldGroup)); // NOLINT + int worldGroupRank = 0; + MPI_Group_rank(worldGroup, &worldGroupRank); + + int worldSize = 0; + MPICHECK(MPI_Group_size(worldGroup, &worldSize)); // NOLINT + TLLM_CHECK_WITH_INFO(participantIds.front() < worldSize, "Not enough ranks in world"); + + MPI_Group leaderOrchCommGroup = nullptr; + MPICHECK( + MPI_Group_incl(worldGroup, leaderOrchRanks.size(), leaderOrchRanks.data(), &leaderOrchCommGroup)); // NOLINT + int leaderOrchGroupRank = 0; + int leaderOrchGroupSize = 0; + MPI_Group_rank(leaderOrchCommGroup, &leaderOrchGroupRank); + MPI_Group_size(leaderOrchCommGroup, &leaderOrchGroupSize); + + if (isOrchestrator || isLeader) + { + MPI_Comm leaderOrchComm = nullptr; + MPICHECK(MPI_Comm_create_group( + MPI_COMM_WORLD, leaderOrchCommGroup, participantIds.front(), &leaderOrchComm)); // NOLINT + mOrchLeaderComm = std::make_shared<tensorrt_llm::mpi::MpiComm>(leaderOrchComm, false); + } + else + { + mOrchLeaderComm = nullptr; + } +#endif // ENABLE_MULTI_DEVICE +} + +void Executor::Impl::initializeCommAndWorkers(SizeType32 tp, SizeType32 pp, SizeType32 cp, + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, std::optional<ModelType> modelType, + std::optional<std::filesystem::path> const& modelPath, std::optional<runtime::WorldConfig> const& worldConfig, + std::optional<runtime::GptJsonConfig> const& decoderGptJsonConfig) +{ + if (modelType.has_value() && modelType.value() == ModelType::kENCODER_DECODER) + { + TLLM_CHECK_WITH_INFO(pp == 1, + "Encoder-Decoder C++ runtime doesn't support Pipeline Parallelism currently. Please switch to Python " + "runtime for PP mode, if necessary."); + } + + tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE); + mWorldRank = tensorrt_llm::mpi::MpiComm::world().getRank(); + mUsePipelineParallel = pp > 1; + + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + validateParallelConfig(parallelConfig, modelType, modelPath); + + mCommMode = parallelConfig.getCommunicationMode(); + auto optOrchestratorConfig = parallelConfig.getOrchestratorConfig(); + + mRecvPollPeriodMs = executorConfig.getRecvPollPeriodMs(); + + // Need to create communicator between orchestrator and leader if not spawning processes in orchestrator mode + if (mCommMode == CommunicationMode::kORCHESTRATOR && !optOrchestratorConfig.value().getSpawnProcesses()) + { + setOrchLeaderComm(tp, pp, cp, parallelConfig); + } + + if (mCommMode == CommunicationMode::kORCHESTRATOR && optOrchestratorConfig.value().getIsOrchestrator()) + { + initializeOrchestrator(tp, pp, cp, executorConfig, parallelConfig, modelType.value(), modelPath.value()); + } + else + { + initializeWorkers(tp, pp, cp, parallelConfig, worldConfig, decoderGptJsonConfig); + } +} + +void Executor::Impl::validateParallelConfig(ParallelConfig const& parallelConfig, std::optional<ModelType> modelType, + std::optional<std::filesystem::path> const& modelPath) +{ + TLLM_CHECK_WITH_INFO(parallelConfig.getCommunicationType() == CommunicationType::kMPI, + "Only CommunicationType kMPI is supported for now."); + + auto optOrchestratorConfig = parallelConfig.getOrchestratorConfig(); + + if (parallelConfig.getCommunicationMode() == CommunicationMode::kORCHESTRATOR) + { + TLLM_CHECK_WITH_INFO( + optOrchestratorConfig, "OrchestratorConfig must be set when using ORCHESTRATOR communication mode."); + + TLLM_CHECK_WITH_INFO(modelPath, "OrchestratorMode only supports reading model weight from disk currently."); + + TLLM_CHECK_WITH_INFO(modelType, "OrchestratorMode requires modelType to be specified."); + } +} + +void Executor::Impl::initializeOrchestrator(SizeType32 tp, SizeType32 pp, SizeType32 cp, + ::tensorrt_llm::executor::ExecutorConfig const& executorConfig, ParallelConfig parallelConfig, ModelType modelType, + std::filesystem::path const& modelPath) +{ +#if ENABLE_MULTI_DEVICE + namespace su = tensorrt_llm::executor::serialize_utils; + + auto const& worldComm = tensorrt_llm::mpi::MpiComm::world(); + int32_t const worldSize = worldComm.getSize(); + + auto orchestratorConfig = parallelConfig.getOrchestratorConfig().value(); + + mIsWorker = false; + mIsLeader = false; + mIsPipelineLeader = false; + mIsOrchestrator = true; + + // Verify that worldSize is 1 + if (orchestratorConfig.getSpawnProcesses()) + { + TLLM_CHECK_WITH_INFO(worldSize == 1, + "When using the orchestrator mode and isOrchestrator is true, expect MPI worldSize to be 1."); + + // Spawn the worker threads + auto workerExecPath = orchestratorConfig.getWorkerExecutablePath(); + MPI_Comm intercomm = nullptr; + MPI_Info mpiInfo = nullptr; + MPICHECK(MPI_Info_create(&mpiInfo)); + MPICHECK(MPI_Info_set(mpiInfo, "env", "FORCE_NCCL_ALL_REDUCE_STRATEGY")); + + // Binding policy is not inherited for dynamically spawned jobs, resulting in the worker being bound + // to a single core. Override the setting to avoid perf issue - see https://nvbugs/4574329 + MPICHECK(MPI_Info_set(mpiInfo, "bind_to", "none")); + + MPICHECK(MPI_Comm_spawn(workerExecPath.c_str(), MPI_ARGV_NULL, tp * pp * cp, mpiInfo, 0, MPI_COMM_SELF, + &intercomm, MPI_ERRCODES_IGNORE)); + + mOrchLeaderComm = std::make_shared<tensorrt_llm::mpi::MpiComm>(intercomm, true); + // With intercomm, leader is rank 0 in the local group + mLeaderRank = 0; + mOrchRank = 0; + + // Copy the executor config, but set the orchestrator flag to false + auto newOrchConfig = OrchestratorConfig(false, orchestratorConfig.getWorkerExecutablePath()); + parallelConfig.setOrchestratorConfig(newOrchConfig); + auto execConfig = executorConfig; + execConfig.setParallelConfig(parallelConfig); + + // Serialize and send the executorConfig, the modelType and the modelPath + std::ostringstream oStream; + su::serialize(modelPath.string(), oStream); + su::serialize(modelType, oStream); + su::serialize(execConfig, oStream); + + auto str = oStream.str(); + std::vector<char> buffer(str.begin(), str.end()); + auto bufferSize = static_cast<int64_t>(buffer.size()); + mOrchLeaderComm->bcast(&bufferSize, 1, mpi::MpiType::kINT64, MPI_ROOT); + mOrchLeaderComm->bcast(buffer.data(), buffer.size(), mpi::MpiType::kCHAR, MPI_ROOT); + + // Wait for workers to have created their executor instance + MPICHECK(MPI_Barrier(intercomm)); + } + + // Spawn the thread responsible for sending new requests to the leader of the model + mOrchSendReqThread = std::thread(&Impl::orchSendReqThread, this); + + // Spawn the thread responsible for receiving new responses from the leader of the model + mOrchRecvThread + = std::thread([&]() { this->orchRecvThread(mpi::MpiTag::kOrchestratorId, mpi::MpiTag::kOrchestratorData); }); + +#endif // ENABLE_MULTI_DEVICE +} + +void Executor::Impl::initializeWorkers(SizeType32 tp, SizeType32 pp, SizeType32 cp, ParallelConfig& parallelConfig, + std::optional<runtime::WorldConfig> const& worldConfig, + std::optional<runtime::GptJsonConfig> const& decoderGptJsonConfig) +{ + auto const& worldComm = tensorrt_llm::mpi::MpiComm::world(); + int32_t const worldSize = worldComm.getSize(); + + auto const& orchestratorConfig = parallelConfig.getOrchestratorConfig(); + mIsOrchestrator = mCommMode == CommunicationMode::kORCHESTRATOR && orchestratorConfig.value().getIsOrchestrator(); + + TLLM_CHECK_WITH_INFO(mCommMode != CommunicationMode::kORCHESTRATOR || orchestratorConfig.has_value(), + "When using ORCHESTRATOR mode, orchestrator config must be set"); + + if (mCommMode == CommunicationMode::kORCHESTRATOR && !orchestratorConfig.value().getSpawnProcesses()) + { + TLLM_CHECK_WITH_INFO(parallelConfig.getParticipantIds(), + "When not spawning processes in orchestrator mode, participant IDs must be provided"); + + // Check that rank 0 is reserved for the orchestrator + auto const participantIds = parallelConfig.getParticipantIds().value(); + for (auto const& participantId : participantIds) + { + TLLM_CHECK_WITH_INFO(participantId != 0, "Rank 0 is reserved for the orchestrator"); + } + } + + // Participant ids + std::vector<SizeType32> participantIds; + if (!parallelConfig.getParticipantIds()) + { + TLLM_CHECK_WITH_INFO(worldSize == tp * pp * cp, + "With communicationMode kLEADER, MPI worldSize is expected to be equal to tp*pp*cp when " + "participantIds are not specified"); + + participantIds.resize(tp * pp * cp); + std::iota(participantIds.begin(), participantIds.end(), 0); + } + else + { + if (mCommMode == CommunicationMode::kORCHESTRATOR && orchestratorConfig.value().getSpawnProcesses()) + { + TLLM_THROW( + "Participant ids should not be set when using CommunicationMode::kORCHESTRATOR with " + "spawnProcesses=true"); + } + participantIds = parallelConfig.getParticipantIds().value(); + TLLM_CHECK_WITH_INFO(static_cast<SizeType32>(participantIds.size()) == tp * pp * cp, + tensorrt_llm::common::fmtstr("When specifying participantIds, participantIds size (%lu) must be equal to " + "tp*pp*cp (tp is %u, pp is %u, cp is %u)", + participantIds.size(), tp, pp, cp)); + } + + // If deviceIds are specified, check that they match tp*pp*cp + if (parallelConfig.getDeviceIds()) + { + auto deviceIds = parallelConfig.getDeviceIds().value(); + auto const hasNumNodes = parallelConfig.getNumNodes().has_value(); + if (hasNumNodes || static_cast<SizeType32>(deviceIds.size()) != tp * pp * cp) + { + auto const numNodes = hasNumNodes ? parallelConfig.getNumNodes().value() : tensorrt_llm::mpi::getNumNodes(); + TLLM_CHECK_WITH_INFO(static_cast<SizeType32>(deviceIds.size() * numNodes) == tp * pp * cp, + tensorrt_llm::common::fmtstr("When specifying deviceIds, deviceIds (%lu) * numNodes (%u) must be equal " + "to tp*pp*cp (tp is %u, pp is %u, cp is %u)", + deviceIds.size(), numNodes, tp, pp, cp)); + } + } + + // Bool that indicates if current process is worker for this model or not + auto participantIt = std::find(participantIds.begin(), participantIds.end(), mWorldRank); + mIsWorker = participantIt != participantIds.end(); + // Bool that indicates if current ranks is leader for this model + mIsLeader = (mWorldRank == participantIds.front()); + mIsPipelineLeader = (mWorldRank == participantIds[tp * (pp - 1)]); + +#if ENABLE_MULTI_DEVICE + if (mIsWorker) + { + // Create a session, but only assign to COMM_SESSION for ranks participating in this model + MPI_Group worldGroup = MPI_GROUP_NULL; + MPICHECK(MPI_Comm_group(MPI_COMM_WORLD, &worldGroup)); // NOLINT + MPI_Group sessionGroup = MPI_GROUP_NULL; + if (pp > 1) + { + // reverse participantIds to move leader to last pp rank. retain order in each tp group + std::reverse(participantIds.begin(), participantIds.end()); + if (tp > 1) + { + for (SizeType32 ppRank = 0; ppRank < pp; ppRank++) + { + std::reverse(participantIds.begin() + ppRank * tp, participantIds.begin() + (ppRank + 1) * tp); + } + } + } + MPICHECK(MPI_Group_incl(worldGroup, participantIds.size(), participantIds.data(), &sessionGroup)); // NOLINT + MPI_Comm sessionComm = MPI_COMM_NULL; + MPICHECK( + MPI_Comm_create_group(MPI_COMM_WORLD, sessionGroup, 1000 + participantIds.front(), &sessionComm)); // NOLINT + + tensorrt_llm::mpi::MpiComm::setSession(tensorrt_llm::mpi::MpiComm(sessionComm, false)); + } + + if (mIsLeader && mCommMode == CommunicationMode::kORCHESTRATOR) + { + auto optOrchestratorConfig = parallelConfig.getOrchestratorConfig(); + if (orchestratorConfig.has_value() && orchestratorConfig.value().getSpawnProcesses()) + { + mOrchLeaderComm = optOrchestratorConfig.value().getOrchLeaderComm(); + } + else + { + // mOrchLeaderComm has already been created + } + TLLM_CHECK(mOrchLeaderComm.get() != nullptr); + + TLLM_CHECK(worldConfig.has_value() || decoderGptJsonConfig.has_value()); + if (worldConfig.has_value()) + { + mDeviceId = worldConfig->getDevice(); + } + else + { + auto gpusPerNode = decoderGptJsonConfig->getGpusPerNode(); + auto worldConfig = runtime::WorldConfig::mpi(gpusPerNode, tp, pp, cp, parallelConfig.getDeviceIds()); + mDeviceId = worldConfig.getDevice(); + } + // Spawn the thread responsible for receiving new requests from the orchestrator + mLeaderRecvReqThread = std::thread(&Impl::leaderRecvReqThread, this); + + // Spawn the thread responsible for sending new responses to the orchestrator + mLeaderSendThread = std::thread([&]() + { this->leaderSendThread(mSendQueue, mpi::MpiTag::kOrchestratorId, mpi::MpiTag::kOrchestratorData); }); + } +#endif // ENABLE_MULTI_DEVICE +} + +void Executor::Impl::initializeLogitsPostProcessorBatched(LogitsPostProcessorConfig const& logitsProcConfig) +{ + if (logitsProcConfig.getProcessorBatched().has_value()) + { + mLogitsPostProcessorBatched + = [cb = logitsProcConfig.getProcessorBatched().value()]( + std::vector<batch_manager::LlmRequest::RequestIdType> const& reqIdsVec, + std::vector<batch_manager::LlmRequest::TensorPtr>& logitsVec, + std::vector<std::reference_wrapper<batch_manager::LlmRequest::BeamTokens const>> const& beamTokensVec, + CudaStreamPtr const& cudaStreamPtr, + std::vector<std::optional<batch_manager::LlmRequest::RequestIdType>> const& clientIdsVec) + { + std::vector<Tensor> cbLogitsVec; + cbLogitsVec.reserve(logitsVec.size()); + for (auto& logits : logitsVec) + { + cbLogitsVec.emplace_back(executor::detail::ofITensor(logits)); + } + + cb(reqIdsVec, cbLogitsVec, beamTokensVec, cudaStreamPtr, clientIdsVec); + }; + + mModel->setLogitsPostProcessorBatched(mLogitsPostProcessorBatched); + } +} + +IdType Executor::Impl::enqueueRequest(Request const& request) +{ + return enqueueRequests({&request, 1}).at(0); +} + +std::vector<IdType> Executor::Impl::enqueueRequests(std::vector<Request> const& requests) +{ + return enqueueRequests({requests.data(), requests.size()}); +} + +std::vector<IdType> Executor::Impl::enqueueRequests(common::ArrayView<Request const> const& requests) +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called, cannot enqueue requests"); + checkParallelApiUsage(__func__); + + TLLM_LOG_DEBUG("Enqueuing %lu requests", requests.size()); + std::vector<RequestWithId> requestWithIds; + requestWithIds.reserve(requests.size()); + + // First check valid of request in enqueue thread, so Exceptions can be thrown to user. + for (auto const& req : requests) + { + auto logitsPostProcessorName = req.getLogitsPostProcessorName(); + if (logitsPostProcessorName && logitsPostProcessorName.value() != Request::kBatchedPostProcessorName) + { + getLogitsPostProcessor(*logitsPostProcessorName); + } + } + + std::vector<IdType> ids; + { + auto now = std::chrono::steady_clock::now(); + for (auto const& req : requests) + { + ids.emplace_back(generateReqId(req)); + TLLM_LOG_DEBUG("Enqueue new request with id %d", ids.back()); + + std::vector<IdType> childReqIds; + auto numChildRequests = getNumChildRequests(req); + if (numChildRequests > 0) + { + childReqIds.reserve(numChildRequests); + for (int childId = 0; childId < numChildRequests; childId++) + { + childReqIds.emplace_back(generateLocalReqId()); + TLLM_LOG_DEBUG("Add new child request with id %d", childReqIds.back()); + } + } + requestWithIds.emplace_back(RequestWithId{req, ids.back(), std::move(childReqIds), now}); + } + } + + if (mCommMode == CommunicationMode::kLEADER) + { + { + std::scoped_lock<std::mutex> const lck(mQueuedReqMtx); + if (mMaxQueueSize) + { + auto const maxQueueSize = mMaxQueueSize.value(); + + auto totalRequestSize = 0; + for (auto&& reqWithId : requestWithIds) + { + totalRequestSize += (getNumChildRequests(reqWithId.req) + 1); + } + + if (maxQueueSize > 0 && mQueuedRequests.size() + totalRequestSize > static_cast<size_t>(maxQueueSize)) + { + TLLM_THROW("Maximum queue size of %d has been reached, please try again later", maxQueueSize); + } + } + + for (auto&& req : requestWithIds) + { + insertRequestInOrder(mQueuedRequests, std::move(req)); + } + } + mQueuedReqCv.notify_one(); + } + else if (mCommMode == CommunicationMode::kORCHESTRATOR) + { + MpiMessage message(MpiId::PENDING_REQUEST); + message.data = PendingRequestData{std::move(requestWithIds)}; + mSendQueue.push(std::move(message)); + } + return ids; +} + +std::vector<Response> Executor::Impl::awaitResponses(std::optional<std::chrono::milliseconds> const& timeout) +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); + checkParallelApiUsage(__func__); + std::unique_lock<std::mutex> lck(mResponsesMtx); + auto pred = [this]() -> bool { return !mResponses.empty() || mShutdown; }; + auto storeResponses = [this]() + { + std::vector<Response> responses; + for (auto it = mResponses.begin(); it != mResponses.end();) + { + responses.insert(responses.end(), it->second.begin(), it->second.end()); + addTerminatedReqId(it->second, it->first); + it = mResponses.erase(it); + } + return responses; + }; + + std::vector<Response> responses; + if (timeout) + { + if (mResponsesCv.wait_for(lck, timeout.value(), pred)) + { + responses = storeResponses(); + } + } + else + { + mResponsesCv.wait(lck, pred); + responses = storeResponses(); + } + return responses; +} + +std::vector<Response> Executor::Impl::awaitResponses( + IdType const& reqId, std::optional<std::chrono::milliseconds> const& timeout) +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); + checkParallelApiUsage(__func__); + std::unique_lock<std::mutex> lck(mResponsesMtx); + auto pred = [this, reqId]() -> bool + { return (mResponses.find(reqId) != mResponses.end() && !mResponses.at(reqId).empty()) || mShutdown; }; + auto storeIdResponse = [this, reqId]() + { + std::vector<Response> responses; + responses.swap(mResponses.at(reqId)); + mResponses.erase(reqId); + addTerminatedReqId(responses, reqId); + return responses; + }; + + // We don't process a terminated request again. Terminated request is defined as a response + // with isFinal = true for a given requestId. + if (mTerminatedReqIds.contains(reqId)) + { + if (mResponses.find(reqId) != mResponses.end()) + { + TLLM_THROW("ReqId should already be removed from responses!"); + } + std::string const err = "ReqId " + std::to_string(reqId) + " has already been processed and was terminated."; + TLLM_LOG_ERROR("%s", err.c_str()); + + return {Response(reqId, err)}; + } + + std::vector<Response> responses; + if (timeout) + { + if (mResponsesCv.wait_for(lck, timeout.value(), pred)) + { + responses = storeIdResponse(); + } + } + else + { + mResponsesCv.wait(lck, pred); + responses = storeIdResponse(); + } + return responses; +} + +std::vector<std::vector<Response>> Executor::Impl::awaitResponses( + std::vector<IdType> const& requestIds, std::optional<std::chrono::milliseconds> const& timeout) +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); + checkParallelApiUsage(__func__); + std::vector<std::vector<Response>> responses; + responses.reserve(requestIds.size()); + if (timeout) + { + auto const start_time = std::chrono::high_resolution_clock::now(); + for (auto const requestId : requestIds) + { + auto const elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>( + std::chrono::high_resolution_clock::now() - start_time); + responses.emplace_back(awaitResponses( + requestId, timeout.value() > elapsed_ms ? timeout.value() - elapsed_ms : std::chrono::milliseconds{0})); + } + } + else + { + for (auto const requestId : requestIds) + { + responses.emplace_back(awaitResponses(requestId)); + } + } + return responses; +} + +SizeType32 Executor::Impl::getNumResponsesReady(std::optional<IdType> const& optId) const +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); + checkParallelApiUsage(__func__); + std::scoped_lock<std::mutex> lck(mResponsesMtx); + SizeType32 numResponsesReady = 0; + if (optId) + { + auto const reqId = optId.value(); + auto const respIt = mResponses.find(reqId); + if (respIt != mResponses.end()) + { + numResponsesReady = static_cast<SizeType32>(respIt->second.size()); + } + } + else + { + for (auto const& [id, responses] : mResponses) + { + numResponsesReady += static_cast<SizeType32>(responses.size()); + } + } + return numResponsesReady; +} + +void Executor::Impl::shutdown() +{ + // Cannot call shutdown multiple times + if (mShutdownCalled) + { + return; + } + mShutdownCalled = true; + + if (!mShutdown) + { + if (mCommMode == CommunicationMode::kLEADER && mIsLeader) + { + // Enqueue a request to indicate to other ranks to terminate + enqueueTerminateRequest(); + } + else if (mCommMode == CommunicationMode::kORCHESTRATOR) + { + if (mIsOrchestrator) + { + // Send to the leader the termination signal + mShutdown = true; + mResponsesCv.notify_all(); + + mSendQueue.push(MpiMessage(MpiId::TERMINATION)); + + // Wait for sender thread to exit + if (mOrchSendReqThread.joinable()) + { + mOrchSendReqThread.join(); + } + // Wait for recv response thread to exit + if (mOrchRecvThread.joinable()) + { + mOrchRecvThread.join(); + } + } + else if (mIsLeader) + { + // Wait for sender thread to exit + if (mLeaderRecvReqThread.joinable()) + { + mLeaderRecvReqThread.join(); + } + // Wait for send response thread to exit + if (mLeaderSendThread.joinable()) + { + mLeaderSendThread.join(); + } + } + } + } + + // Wait for execution thread to terminate + if (mExecutionThread.joinable()) + { + mExecutionThread.join(); + } + + // If we overwrote COMM_SESSION with split, free it now. Otherwise, since + // COMM_SESSION is a global static object, it will be destroyed in an + // undefined order and can cause crashes on program exit. + if (mIsWorker) + { + tensorrt_llm::mpi::MpiComm::setSession(tensorrt_llm::mpi::MpiComm(MPI_COMM_WORLD, false)); + } +} + +void Executor::Impl::cancelRequest(IdType requestId) +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); + checkParallelApiUsage(__func__); + + // Check if the request is terminated already. If so, return + { + std::scoped_lock<std::mutex> lckResp(mResponsesMtx); + if (mTerminatedReqIds.contains(requestId)) + { + TLLM_LOG_INFO("Ignoring already terminated request %lu", requestId); + return; + } + } + + if (mCommMode == CommunicationMode::kLEADER) + { + std::scoped_lock<std::mutex> lck(mCancelReqMtx); + auto& selCancelledReqIds = mUsePipelineParallel ? mPipelineCancelledReqIds : mCancelledReqIds; + selCancelledReqIds.insert(requestId); + } + else if (mCommMode == CommunicationMode::kORCHESTRATOR) + { + MpiMessage message(MpiId::CANCEL_REQUEST); + std::vector<IdType> cancelledReqIds{requestId}; + message.data = RequestIdsData{std::move(cancelledReqIds)}; + mSendQueue.push(std::move(message)); + } +} + +std::deque<IterationStats> Executor::Impl::getLatestIterationStats() +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); + checkParallelApiUsage(__func__); + std::scoped_lock<std::mutex> lck(mIterStatsMtx); + return std::exchange(mIterationStats, {}); +} + +std::deque<RequestStatsPerIteration> Executor::Impl::getLatestRequestStats() +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); + checkParallelApiUsage(__func__); + + std::scoped_lock<std::mutex> lck(mRequestStatsMtx); + return std::exchange(mRequestStats, {}); +} + +std::deque<DebugTensorsPerIteration> Executor::Impl::getLatestDebugTensors() +{ + TLLM_CHECK_WITH_INFO(!mShutdownCalled, "Shutdown called"); + if (mCommMode == CommunicationMode::kORCHESTRATOR) + { + TLLM_LOG_WARNING("getLatestDebugTensors is not supported in ORCHESTRATOR mode yet"); + return {}; + } + if (mEncoderModel) + { + TLLM_LOG_WARNING("getLatestDebugTensors is not supported for encoder model yet"); + } + std::scoped_lock<std::mutex> lck(mDebugTensorsMtx); + return std::exchange(mDebugTensors, {}); +} + +bool Executor::Impl::canEnqueueRequests() const +{ + return !mShutdownCalled + && ((mCommMode == CommunicationMode::kLEADER && mIsLeader) + || (mCommMode == CommunicationMode::kORCHESTRATOR && mIsOrchestrator)); +} + +bool Executor::Impl::isParticipant() const +{ + return mIsWorker; +} + +std::optional<std::shared_ptr<KVCacheEventManager>> Executor::Impl::getKVCacheEventManager() const +{ + if (!mModel) + { + return std::nullopt; + } + auto cacheEventManager = mModel->getKVCacheManager(); + return cacheEventManager ? std::optional(std::make_shared<KVCacheEventManager>(cacheEventManager)) : std::nullopt; +} + +void Executor::Impl::requestWithIdLeaderThread() +{ + TLLM_CUDA_CHECK(cudaSetDevice(mModel->getWorldConfig().getDevice())); + auto constexpr peer = 0; + while (true) + { + int64_t numActiveRequests; + mCommPipelineParallel->recv( + &numActiveRequests, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kExecutorNumActiveRequests); + if (numActiveRequests < 0) + { + break; + } + + bool lowestPriorityActiveHasValue; + std::optional<PriorityType> lowestPriorityActive; + mCommPipelineParallel->recv(&lowestPriorityActiveHasValue, 1, mpi::MpiType::kBOOL, peer, + mpi::MpiTag::kExecutorLowestPriorityActiveHasValue); + if (lowestPriorityActiveHasValue) + { + PriorityType lowestPriorityActiveValue; + mCommPipelineParallel->recv( + &lowestPriorityActiveValue, 1, mpi::MpiType::kFLOAT, peer, mpi::MpiTag::kExecutorLowestPriorityActive); + lowestPriorityActive = lowestPriorityActiveValue; + } + + auto reqWithIds = getLeaderNewReqWithIds(numActiveRequests, lowestPriorityActive); + setupDynamicLogitsPostProcessors(reqWithIds); + auto requestWithIdAsyncSndHdl + = std::make_unique<RequestWithIdAsyncSend>(mCommPipelineParallel, reqWithIds, peer); + requestWithIdAsyncSndHdl.reset(nullptr); + } +} + +void Executor::Impl::cancelledRequestsLeaderThread() +{ + TLLM_CUDA_CHECK(cudaSetDevice(mModel->getWorldConfig().getDevice())); + auto constexpr peer = 0; + while (true) + { + bool shouldExit; + mCommPipelineParallel->recv(&shouldExit, 1, mpi::MpiType::kBOOL, peer, mpi::MpiTag::kExecutorShouldExit); + if (shouldExit) + { + break; + } + + std::unique_ptr<CancelledRequestsAsyncSend> cancelledRequestsAsyncSndHdl; + { + std::scoped_lock<std::mutex> lck(mCancelReqMtx); + cancelledRequestsAsyncSndHdl + = std::make_unique<CancelledRequestsAsyncSend>(mCommPipelineParallel, mPipelineCancelledReqIds, peer); + mPipelineCancelledReqIds.clear(); + } + cancelledRequestsAsyncSndHdl.reset(nullptr); + } +} + +std::vector<RequestWithId> Executor::Impl::getLeaderNewReqWithIds( + SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive) +{ + std::unique_lock<std::mutex> lck(mQueuedReqMtx); + mQueuedReqCv.wait(lck, [&]() { return (!mQueuedRequests.empty() || numActiveRequests > 0 || mShutdown); }); + + std::vector<RequestWithId> reqWithIds; + + if (mQueuedRequests.empty() || mShutdown) + { + return reqWithIds; + } + + if (mQueuedRequests.front().id == kTerminateReqId) + { + reqWithIds.emplace_back(std::move(mQueuedRequests.front())); + mQueuedRequests.pop_front(); + return reqWithIds; + } + + auto const& firstRequest = mQueuedRequests.front(); + auto const firstBeamWidth = firstRequest.req.getSamplingConfig().getBeamWidth(); + auto const operatingBeamWidth = numActiveRequests > 0 ? mModel->getOperatingBeamWidth() : firstBeamWidth; + + auto const tryInsertQueuedRequestIntoReqWithIds = [this, &reqWithIds, operatingBeamWidth]() -> bool + { + auto& nextRequest = mQueuedRequests.front(); + auto const beamWidth = nextRequest.req.getSamplingConfig().getBeamWidth(); + if (beamWidth != operatingBeamWidth) + { + TLLM_LOG_INFO( + "Can't dequeue request with ID %ld because beam width %d differs from operating beam width %d.", + nextRequest.id, beamWidth, operatingBeamWidth); + return false; + } + + TLLM_LOG_DEBUG("Dequeue request with ID %ld", nextRequest.id); + reqWithIds.emplace_back(std::move(nextRequest)); + mQueuedRequests.pop_front(); + return true; + }; + + auto const maxNewRequests = static_cast<size_t>(std::max(mMaxNumActiveRequests - numActiveRequests, 0)); + for (size_t req = 0; !mQueuedRequests.empty() && req < maxNewRequests;) + { + req += (getNumChildRequests(mQueuedRequests.front().req) + 1); + if (req > maxNewRequests) + { + break; + } + if (!tryInsertQueuedRequestIntoReqWithIds()) + { + break; + } + } + + if (lowestPriorityActive) + { + while (!mQueuedRequests.empty() && mQueuedRequests.front().req.getPriority() > (*lowestPriorityActive)) + { + if (!tryInsertQueuedRequestIntoReqWithIds()) + { + break; + } + } + } + return reqWithIds; +} + +std::vector<RequestWithId> Executor::Impl::getNewReqWithIds( + SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + auto const& worldConfig = mModel->getWorldConfig(); + + if (worldConfig.isPipelineParallel()) + { + mRequestWithIdWaitThread->waitStop(); + } + + TLLM_CUDA_CHECK(cudaSetDevice(mModel->getWorldConfig().getDevice())); + std::vector<RequestWithId> reqWithIds; + if (mIsPipelineLeader) + { + if (!worldConfig.isPipelineParallel()) + { + reqWithIds = getLeaderNewReqWithIds(numActiveRequests, lowestPriorityActive); + setupDynamicLogitsPostProcessors(reqWithIds); + } + else + { + auto const peer = worldConfig.getPipelineParallelism() - 1; + auto numActiveRequestsValue = static_cast<int64_t>(numActiveRequests); + auto request1 = mCommPipelineParallel->sendAsync( + &numActiveRequestsValue, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kExecutorNumActiveRequests); + bool lowestPriorityActiveHasValue = lowestPriorityActive.has_value(); + auto request2 = mCommPipelineParallel->sendAsync(&lowestPriorityActiveHasValue, 1, mpi::MpiType::kBOOL, + peer, mpi::MpiTag::kExecutorLowestPriorityActiveHasValue); + auto request3 = lowestPriorityActiveHasValue + ? mCommPipelineParallel->sendAsync(&lowestPriorityActive.value(), 1, mpi::MpiType::kFLOAT, peer, + mpi::MpiTag::kExecutorLowestPriorityActive) + : nullptr; + request1->wait(); + request2->wait(); + if (request3) + { + request3->wait(); + } + reqWithIds = RequestWithIdAsyncSend::requestWithIdRecv(mCommPipelineParallel, peer); + } + if (worldConfig.isTensorParallel() || worldConfig.isContextParallel()) + { + auto packed = RequestWithId::serializeReqWithIds(reqWithIds); + if (worldConfig.isTensorParallel()) + { + mCommTensorParallel->bcast(packed, 0); + } + if (worldConfig.isContextParallel()) + { + mCommContextParallel->bcast(packed, 0); + } + } + } + else + { + if (worldConfig.isFirstPipelineParallelRank()) + { + std::vector<char> buffer; + mCommTensorParallel->bcast(buffer, 0); + mCommContextParallel->bcast(buffer, 0); + reqWithIds = RequestWithId::deserializeReqWithIds(buffer); + } + else + { + auto const peer = worldConfig.getPipelineParallelRank() - 1; + reqWithIds = RequestWithIdAsyncSend::requestWithIdRecv(mCommPipelineParallel, peer); + } + } + if (!worldConfig.isLastPipelineParallelRank()) + { + auto const peer = worldConfig.getPipelineParallelRank() + 1; + mRequestWithIdAsyncSndHdl = std::make_unique<RequestWithIdAsyncSend>(mCommPipelineParallel, reqWithIds, peer); + mRequestWithIdWaitThread->notifyStart(); + } + TLLM_CUDA_CHECK(cudaSetDevice(mModel->getWorldConfig().getDevice())); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return reqWithIds; +} + +std::tuple<Executor::Impl::RequestList, double> Executor::Impl::fetchNewRequests( + SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_SCOPED_RANGE(fetchNewRequests); + + // If grab requests from queue, do exchange between ranks + auto reqWithIds = getNewReqWithIds(numActiveRequests, lowestPriorityActive); + RequestList newRequests; + double newActiveRequestsQueueLatencyMS{0.}; + for (auto& reqWithId : reqWithIds) + { + if (reqWithId.id == kTerminateReqId) + { + mShutdown = true; + mResponsesCv.notify_all(); + return {}; + } + + try + { + std::optional<LlmRequestLogitsPostProcessor> llmRequestLogitsPostProcessor; + bool applyLogitsPostProcessorBatched{false}; + if (mModel->getWorldConfig().isLastPipelineParallelRank()) + { + auto logitsPostProcessorName = reqWithId.req.getLogitsPostProcessorName(); + if (logitsPostProcessorName) + { + if (logitsPostProcessorName.value() == Request::kBatchedPostProcessorName) + { + TLLM_CHECK_WITH_INFO( + mLogitsPostProcessorBatched, "Batched logits post processor is not defined."); + applyLogitsPostProcessorBatched = true; + } + else + { + if (logitsPostProcessorName->compare(0, + std::char_traits<char>::length(Request::kDynamicPostProcessorNamePrefix), + Request::kDynamicPostProcessorNamePrefix) + == 0) + { + TLLM_CHECK_WITH_INFO(!mModel->getReplicateLogitsPostProcessor() + || mModel->getWorldConfig().getTensorParallelism() == 1, + "Dynamic logits postprocessor must be used with replicate=false or no tensor " + "parallelism."); + } + if (mModel->getWorldConfig().isFirstTensorParallelRank() + || mModel->getReplicateLogitsPostProcessor()) + { + llmRequestLogitsPostProcessor = getLogitsPostProcessor(logitsPostProcessorName.value()); + } + else + { + llmRequestLogitsPostProcessor + = [](IdType reqId, RtTensorPtr& logits, BeamTokens const& beamTokens, + CudaStreamPtr const& cudaStreamPtr, std::optional<IdType> clientId) {}; + } + } + } + } + auto newLlmReq = std::make_shared<batch_manager::LlmRequest>( + reqWithId.id, reqWithId.req, llmRequestLogitsPostProcessor, applyLogitsPostProcessorBatched); + + auto numReturnSequences = newLlmReq->getNumSubRequests(); + if (numReturnSequences > 1) + { + TLLM_CHECK(reqWithId.childReqIds.size() == static_cast<size_t>(numReturnSequences - 1)); + mChildReqIdsMap[reqWithId.id] = reqWithId.childReqIds; + } + + for (auto seqIdx = 0; seqIdx < numReturnSequences; seqIdx++) + { + auto newReq + = seqIdx == 0 ? newLlmReq : newLlmReq->createChildRequest(reqWithId.childReqIds.at(seqIdx - 1)); + + // If static batching and streaming, disable streaming and exclude input + if (mBatchingType == BatchingType::kSTATIC && newReq->isStreaming()) + { + newReq->setStreaming(false); + newReq->setExcludeInputFromOutput(true); + } + + // Validate the request parameters + newReq->validate(mModel->getMaxInputLen(), mModel->getMaxSequenceLen(), mModel->getMaxDraftLen(), + mModel->getVocabSizePadded(), + mEncoderModel ? std::optional<SizeType32>(mEncoderModel->getMaxInputLen()) : std::nullopt, + mEnableBlockReuse); + + TLLM_CHECK_WITH_INFO(!mEncoderModel || !mIsSchedulerMaxUtilization, + "Encoder or Encoder-Decoder model don't support max utilization scheduler yet. Only max requests " + "or guaranteed no evict."); + + // When streaming is enabled and scheduling policy permits evict/restart, need to guard against the case + // where the sequence is truncated on eviction (to respect maxInputLen limits), resulting in loss of + // some tokens that have been streamed out. In this case, resuming generation may result in different + // completion for locations whose tokens have already been returned. There is no way to protect against + // this, so disallowing. + if (newReq->isStreaming() && !mIsSchedulerGuaranteedNoEvict && !mIsChunkedContext) + { + auto const maxReqSeqLen = newReq->mPromptLen + newReq->mMaxNewTokens; + auto const maxRestartLen = maxReqSeqLen - 1; + TLLM_CHECK_WITH_INFO(maxRestartLen <= mModel->getMaxInputLen(), + "Request sequence length is potentially greater than max input length. This cannot be run " + "unless streaming is disabled, context chunking is enabled or the GUARANTEED_NO_EVICT " + "scheduling policy is used"); + } + + // Create the encoder output tensor + if (mEncoderModel) + { + TLLM_CHECK_WITH_INFO(mModel || (!mModel && newReq->getReturnEncoderOutput()), + "Encoder-Decoder models allow optionally returning encoder output. But if it is Encoder-only " + "models, please make sure returnEncoderOutput is always true."); + + // gpu buffers for passing to the next phase + newReq->allocEncoderOutput(mEncoderModel->getBufferManager(), mEncoderModel->getLogitDataType()); + newReq->allocEncoderHiddenStates( + mEncoderModel->getBufferManager(), mEncoderModel->getLogitDataType()); + // pinned buffers for returning results to host + if (newReq->getReturnEncoderOutput()) + { + newReq->allocEncoderOutputHost( + mEncoderModel->getHiddenSize() * mEncoderModel->getWorldConfig().getTensorParallelism(), + mEncoderModel->getLogitDataType()); + } + } + + if (!mEncoderModel && newReq->getEncoderInputFeatures()) + { + TLLM_LOG_INFO("Allocating buffers for encoder output"); + // gpu buffers for passing to the next phase + newReq->allocEncoderOutput(mModel->getBufferManager(), mModel->getLogitDataType()); + newReq->allocEncoderHiddenStates(mModel->getBufferManager(), mModel->getLogitDataType()); + } + + // Create the context logits tensor + if (newReq->getReturnContextLogits()) + { + TLLM_CHECK_WITH_INFO(mModel->getModelConfig().computeContextLogits(), + "Return context logit need to build engine with gather_context_logits"); + newReq->allocContextLogitsHost(mModel->getVocabSizePadded(), mModel->getLogitDataType()); + } + + // Create the generation logits tensor + if (newReq->getReturnGenerationLogits()) + { + TLLM_CHECK_WITH_INFO(mModel->getGatherGenerationLogits(), + "To return generation logits, gather_generation_logits must be enabled in ExecutorConfig"); + + if (mModel->getModelConfig().getSpeculativeDecodingMode().isDraftTokensExternal() + && newReq->hasDraftTokens()) + { + newReq->allocTargetModelAcceptedTokenLogitsHost( + mModel->getVocabSizePadded(), mModel->getLogitDataType()); + } + else + { + newReq->allocGenerationLogitsHost(mModel->getVocabSizePadded(), mModel->getLogitDataType()); + } + } + + if (mModel->getWorldConfig().isLastPipelineParallelRank() && newReq->getGuidedDecodingParams()) + { + TLLM_CHECK_WITH_INFO(mModel->hasGuidedDecoder(), + "Request is specified with GuidedDecodingParams, but GuidedDecoder is not setup. Please " + "provide a valid GuidedDecodingConfig to setup GuidedDecoder."); + } + + if (mModel->getWorldConfig().isLastPipelineParallelRank() && newReq->hasAdditionalOutputs()) + { + newReq->allocAdditionalOutputs([this](std::string const& name) + { return mModel->getTensorDataType(name); }, + [this](std::string const& name) { return mModel->getTensorShape(name); }); + } + + mModel->updatePeftCache(newReq); + + newRequests.emplace_back(std::move(newReq)); + } + + auto queuedEnd = std::chrono::steady_clock::now(); + auto reqQueueLatencyMS + = std::chrono::duration<double, std::milli>(queuedEnd - reqWithId.queuedStart).count(); + newActiveRequestsQueueLatencyMS += reqQueueLatencyMS; + } + catch (runtime::LoraExpectedException const& e) + { + if (mIsLeader) + { + // In case of an expected LoRA exception (e.g. cache full, cache miss), log a warning and enqueue + // response + TLLM_LOG_WARNING("%s", e.what()); + enqueueNewResponses({{reqWithId.id, e.what(), reqWithId.req.getClientId()}}); + } + } + catch (std::exception const& e) + { + if (mIsLeader) + { + // In case of error, create a response with error for this request + auto err = std::string("Encountered an error when fetching new request: ") + e.what(); + TLLM_LOG_ERROR("%s", err.c_str()); + enqueueNewResponses({{reqWithId.id, err, reqWithId.req.getClientId()}}); + } + } + } + TLLM_LOG_DEBUG("[RANK %d] num new requests fetched from queue: %d", COMM_SESSION.getRank(), newRequests.size()); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return {newRequests, newActiveRequestsQueueLatencyMS}; +} + +void Executor::Impl::terminateActiveRequests(RequestList& activeRequests, std::string const& err) +{ + TLLM_LOG_ERROR("%s", err.c_str()); + + // Create a response for all requests and add to queue + for (auto it = activeRequests.cbegin(); it != activeRequests.cend();) + { + auto llmReq = (*it); + + llmReq->setState(batch_manager::LlmRequestState::kGENERATION_COMPLETE); + mModel->terminateRequest(llmReq); + + if (mIsLeader) + { + enqueueNewResponses({{llmReq->mRequestId, err, llmReq->mClientId}}); + } + + // Remove from the requestList + it = activeRequests.erase(it); + } +} + +void Executor::Impl::forwardSync(RequestList& activeRequests) +{ + TLLM_LOG_TRACE("[RANK %d] %s start", COMM_SESSION.getRank(), __PRETTY_FUNCTION__); + try + { + if (mEncoderModel) + { + mEncoderModel->forwardSync(); + } + mModel->forwardSync(); + } + catch (std::exception const& e) + { + std::string const err = std::string("Encountered an error in forwardSync function: ") + e.what(); + terminateActiveRequests(activeRequests, err); + } + TLLM_LOG_TRACE("[RANK %d] %s stop", COMM_SESSION.getRank(), __PRETTY_FUNCTION__); +} + +// The function is used to change the state of a request to context_init from encoder_init for enc-dec model whose +// encoder is skipped. The encoder output is populated accordingly with input features given through model executor of +// decoder. +void Executor::Impl::prepRequestsForEncoderSkip(RequestList& activeRequests) +{ + + for (auto& req : activeRequests) + { + + if (req->isEncoderInitState() && req->getEncoderInputFeatures()) + { + TLLM_LOG_INFO("Changing state of request and setting encoder output to skip encoder run"); + req->setState(batch_manager::LlmRequestState::kCONTEXT_INIT); + req->setEncoderOutput(req->getEncoderInputFeatures()); + } + } +} + +void Executor::Impl::finishTimedOutRequests(RequestList const& activeRequests) +{ + if (mIsLeader) + { + for (auto const& request : activeRequests) + { + if (request->isTimedOut() && !request->isFinished()) + { + // workaround to cancelRequest since it throws an error if + // mCommMode == CommunicationMode::kORCHESTRATOR && !mIsOrchestrator + { + std::scoped_lock<std::mutex> lck(mCancelReqMtx); + auto& selCancelledReqIds = mUsePipelineParallel ? mPipelineCancelledReqIds : mCancelledReqIds; + selCancelledReqIds.insert(request->mRequestId); + } + } + } + } +} + +void Executor::Impl::forwardAsync(RequestList& activeRequests) +{ + try + { + TLLM_LOG_DEBUG("num active requests in scope: %d", activeRequests.size()); + + if (mDynamicBatchTuner) + { + auto const averageInputLength = static_cast<SizeType32>(mDynamicBatchTuner->getAverageInputLength()); + auto const averageOutputLength = static_cast<SizeType32>(mDynamicBatchTuner->getAverageOutputLength()); + auto const maxCapacityBatchSize = mModel->getMaxCapacityBatchSize(averageInputLength, averageOutputLength); + + if (mDynamicBatchTuner->isBatchSizeTuningEnabled()) + { + auto runtimeBatchSize = mDynamicBatchTuner->getRuntimeBatchSize(maxCapacityBatchSize); + mModel->setRuntimeBatchSize(runtimeBatchSize); + } + + if (mDynamicBatchTuner->isMaxNumTokensTuningEnabled()) + { + auto runtimeBatchSize = mModel->getRuntimeBatchSize(); + auto runtimeMaxNumTokens = mDynamicBatchTuner->getRuntimeMaxNumTokens(runtimeBatchSize); + mModel->setRuntimeMaxNumTokens(runtimeMaxNumTokens); + } + } + + if (mEncoderModel) + { + mEncoderModel->forwardAsync(activeRequests); + auto const& encoderStream = *(mEncoderModel->getRuntimeStreamPtr()); + auto const& decoderStream = *(mModel->getRuntimeStreamPtr()); + runtime::CudaEvent encoderFinished; + encoderStream.record(encoderFinished); + decoderStream.wait(encoderFinished); + } + + if (!mEncoderModel) + { + prepRequestsForEncoderSkip(activeRequests); + } + + mModel->forwardAsync(activeRequests); + } + catch (std::exception const& e) + { + std::string err = std::string("Encountered an error in forwardAsync function: ") + e.what(); + terminateActiveRequests(activeRequests, err); + } +} + +IterationStats Executor::Impl::getCurrentIterationStats(RequestList const& activeRequests, double iterLatencyMS, + SizeType32 numNewActiveRequests, double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests) +{ + IterationStats stats; + // Timestamp + stats.timestamp = tensorrt_llm::common::getCurrentTimestamp(); + stats.numNewActiveRequests = numNewActiveRequests; + stats.iterLatencyMS = iterLatencyMS; + stats.newActiveRequestsQueueLatencyMS = newActiveRequestsQueueLatencyMS; + // Active request count + stats.numActiveRequests = static_cast<SizeType32>(activeRequests.size()); + // Queued request count + { + std::scoped_lock<std::mutex> lck(mQueuedReqMtx); + stats.numQueuedRequests = static_cast<SizeType32>(mQueuedRequests.size()); + } + stats.numCompletedRequests = numCompletedRequests; + // Max number of requests + stats.maxNumActiveRequests = mMaxNumActiveRequests; + // Runtime memory allocation statistics + auto const& memoryCounters = runtime::MemoryCounters::getInstance(); + stats.gpuMemUsage = memoryCounters.getGpu(); + stats.cpuMemUsage = memoryCounters.getCpu(); + stats.pinnedMemUsage = memoryCounters.getPinned(); + + // Model specific stats + mModel->getCurrentIterationStats(stats); + return stats; +} + +RequestStatsPerIteration Executor::Impl::getCurrentRequestStats( + RequestList const& activeRequests, RequestList const& finishedRequests) +{ + std::vector<RequestStats> requestStatsVec; + + auto includeDisServingStats = [](LlmRequestPtr const& request, tensorrt_llm::executor::RequestStats& requestStats) + { + auto requestType = request->getLlmRequestType(); + if (requestType == batch_manager::LlmRequestType::LLMREQUEST_TYPE_CONTEXT_ONLY + || requestType == batch_manager::LlmRequestType::LLMREQUEST_TYPE_GENERATION_ONLY) + { + requestStats.disServingStats + = executor::DisServingRequestStats{request->getKvCacheTransferTimeMS(), request->getKvCacheSize()}; + } + }; + + for (auto const& request : activeRequests) + { + RequestStats requestStats; + requestStats.id = request->mRequestId; + requestStats.stage = request->getRequestStage(); + requestStats.contextPrefillPosition = request->getContextCurrentPosition(); + requestStats.numGeneratedTokens = request->getMaxBeamNumTokens() - request->getOrigPromptLen(); + requestStats.avgNumDecodedTokensPerIter = request->getAvgDecodedTokensPerIter(); + includeDisServingStats(request, requestStats); + requestStats.allocTotalBlocksPerRequest = request->getAllocTotalBlocksPerRequest(); + requestStats.allocNewBlocksPerRequest = request->getAllocNewBlocksPerRequest(); + requestStats.reusedBlocksPerRequest = request->getReusedBlocksPerRequest(); + requestStats.missedBlocksPerRequest = request->getMissedBlocksPerRequest(); + requestStats.kvCacheHitRatePerRequest = request->getKVCacheHitRatePerRequest(); + requestStatsVec.emplace_back(requestStats); + } + + { + std::unique_lock<std::mutex> lck(mQueuedReqMtx); + for (auto const& request : mQueuedRequests) + { + // Still waiting for the first scheduling + RequestStats requestStats; + requestStats.id = static_cast<executor::IdType>(request.id); + requestStats.stage = executor::RequestStage::kQUEUED; + requestStats.contextPrefillPosition = 0; + requestStats.numGeneratedTokens = 0; + requestStats.avgNumDecodedTokensPerIter = 0; + requestStats.allocTotalBlocksPerRequest = 0; + requestStats.allocNewBlocksPerRequest = 0; + requestStats.reusedBlocksPerRequest = 0; + requestStats.missedBlocksPerRequest = 0; + requestStats.kvCacheHitRatePerRequest = 0; + requestStatsVec.emplace_back(requestStats); + } + } + + for (auto const& request : finishedRequests) + { + // Still waiting for the first scheduling + RequestStats requestStats; + requestStats.id = static_cast<executor::IdType>(request->mRequestId); + requestStats.stage = executor::RequestStage::kGENERATION_COMPLETE; + requestStats.contextPrefillPosition = request->getContextCurrentPosition(); + requestStats.numGeneratedTokens = request->getMaxBeamNumTokens() - request->getOrigPromptLen(); + requestStats.avgNumDecodedTokensPerIter = request->getAvgDecodedTokensPerIter(); + includeDisServingStats(request, requestStats); + requestStats.allocTotalBlocksPerRequest = request->getAllocTotalBlocksPerRequest(); + requestStats.allocNewBlocksPerRequest = request->getAllocNewBlocksPerRequest(); + requestStats.reusedBlocksPerRequest = request->getReusedBlocksPerRequest(); + requestStats.missedBlocksPerRequest = request->getMissedBlocksPerRequest(); + requestStats.kvCacheHitRatePerRequest = request->getKVCacheHitRatePerRequest(); + requestStatsVec.emplace_back(requestStats); + } + + RequestStatsPerIteration stats{0, std::move(requestStatsVec)}; + + // Model specific stats + mModel->getCurrentRequestStats(stats); + return stats; +} + +void Executor::Impl::appendCurrentIterStats(IterationStats&& currentIterStats) +{ + std::scoped_lock<std::mutex> lck(mIterStatsMtx); + if (statsBufferIsBounded(mIterStatsMaxIterations)) + { + auto const maxIterStats = static_cast<std::size_t>(mIterStatsMaxIterations); + if (mIterationStats.size() >= maxIterStats) + { + mIterationStats.pop_front(); + } + } + mIterationStats.emplace_back(std::move(currentIterStats)); +} + +void Executor::Impl::appendMultipleIterStats(std::vector<IterationStats>&& currentIterStatsVec) +{ + std::scoped_lock<std::mutex> lck(mIterStatsMtx); + mIterationStats.insert(mIterationStats.end(), std::make_move_iterator(currentIterStatsVec.begin()), + std::make_move_iterator(currentIterStatsVec.end())); + if (statsBufferIsBounded(mIterStatsMaxIterations)) + { + auto const maxIterStats = static_cast<std::size_t>(mIterStatsMaxIterations); + while (mIterationStats.size() > maxIterStats) + { + mIterationStats.pop_front(); + } + } +} + +void Executor::Impl::updateIterationStats(RequestList const& activeRequests, double iterLatencyMS, + SizeType32 numNewActiveRequests, double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests, + bool flushToOrchestrator) +{ + NVTX3_SCOPED_RANGE(updateIterationStats); + if (statsBufferIsEnabled(mIterStatsMaxIterations) && mIsLeader) + { + auto currentIterStats = getCurrentIterationStats( + activeRequests, iterLatencyMS, numNewActiveRequests, newActiveRequestsQueueLatencyMS, numCompletedRequests); + // Send the stats to the orchestrator + if (mCommMode == CommunicationMode::kORCHESTRATOR) + { + bool hasSchedThisIter = (currentIterStats.inflightBatchingStats + && currentIterStats.inflightBatchingStats->numScheduledRequests > 0) + || (currentIterStats.staticBatchingStats + && currentIterStats.staticBatchingStats->numScheduledRequests > 0); + appendCurrentIterStats(std::move(currentIterStats)); + if (hasSchedThisIter || flushToOrchestrator) + { + std::deque<IterationStats> iterStatsQueue; + { + std::scoped_lock<std::mutex> lck(mIterStatsMtx); + iterStatsQueue = std::exchange(mIterationStats, {}); + } + MpiMessage message(MpiId::ITER_STATS); + std::vector<IterationStats> iterStates( + std::make_move_iterator(iterStatsQueue.begin()), std::make_move_iterator(iterStatsQueue.end())); + message.data = IterStatsData{std::move(iterStates)}; + mSendQueue.push(std::move(message)); + } + } + else + { + // Add current iteration stats + appendCurrentIterStats(std::move(currentIterStats)); + } + } +} + +void Executor::Impl::appendCurrentRequestStats(RequestStatsPerIteration&& currentRequestStats) +{ + std::scoped_lock<std::mutex> lck(mRequestStatsMtx); + if (statsBufferIsBounded(mRequestStatsMaxIterations)) + { + auto const maxRequestStats = static_cast<std::size_t>(mRequestStatsMaxIterations); + if (mRequestStats.size() >= maxRequestStats) + { + mRequestStats.pop_front(); + } + } + mRequestStats.emplace_back(std::move(currentRequestStats)); +} + +void Executor::Impl::appendMultipleRequestStats(std::vector<RequestStatsPerIteration>&& currentRequestStatsVec) +{ + std::scoped_lock<std::mutex> lck(mRequestStatsMtx); + mRequestStats.insert(mRequestStats.end(), std::make_move_iterator(currentRequestStatsVec.begin()), + std::make_move_iterator(currentRequestStatsVec.end())); + if (statsBufferIsBounded(mRequestStatsMaxIterations)) + { + auto const maxRequestStats = static_cast<std::size_t>(mRequestStatsMaxIterations); + while (mRequestStats.size() > maxRequestStats) + { + mRequestStats.pop_front(); + } + } +} + +void Executor::Impl::updateRequestStats( + RequestList const& activeRequests, RequestList const& finishedRequests, bool flushToOrchestrator) +{ + NVTX3_SCOPED_RANGE(updateRequestStats); + if (statsBufferIsEnabled(mRequestStatsMaxIterations) && mIsLeader) + { + // Add current iteration request stats + auto currentRequestStats = getCurrentRequestStats(activeRequests, finishedRequests); + // Send the stats to the orchestrator + if (mCommMode == CommunicationMode::kORCHESTRATOR) + { + bool hasScheduledReqs = false; + if (!flushToOrchestrator) + { + size_t activeSize = activeRequests.size(); + TLLM_CHECK_WITH_INFO(currentRequestStats.requestStats.size() >= activeSize, + "currentRequestStats num is %ld should >= activeRequest num:%zu", + currentRequestStats.requestStats.size(), activeSize); + hasScheduledReqs = std::any_of(currentRequestStats.requestStats.begin(), + currentRequestStats.requestStats.begin() + static_cast<int64_t>(activeSize), + [](RequestStats const& requestStat) { return requestStat.scheduled; }); + } + appendCurrentRequestStats(std::move(currentRequestStats)); + if (hasScheduledReqs || flushToOrchestrator) + { + std::deque<RequestStatsPerIteration> requestStatsQueue; + { + std::scoped_lock<std::mutex> lck(mRequestStatsMtx); + requestStatsQueue = std::exchange(mRequestStats, {}); + } + std::vector<RequestStatsPerIteration> requestIterStates( + std::make_move_iterator(requestStatsQueue.begin()), + std::make_move_iterator(requestStatsQueue.end())); + MpiMessage message(MpiId::REQUEST_ITER_STATS); + message.data = RequestStatsPerIterationData{std::move(requestIterStates)}; + mSendQueue.push(std::move(message)); + } + } + else + { + // Add current iteration stats + appendCurrentRequestStats(std::move(currentRequestStats)); + } + } +} + +void Executor::Impl::appendCurrentDebugTensors() +{ + if (mDebugTensorsMaxIterations > 0) + { + std::scoped_lock<std::mutex> lck(mDebugTensorsMtx); + if (mDebugTensors.size() >= mDebugTensorsMaxIterations) + { + mDebugTensors.pop_front(); + } + mDebugTensors.emplace_back(mModel->getCurrentDebugTensors()); + } +} + +void Executor::Impl::terminateCancelledRequests(RequestList& activeRequests) +{ + NVTX3_SCOPED_RANGE(terminateCancelledRequests); + auto const& worldConfig = mModel->getWorldConfig(); + auto const broadcastCancelledRequests = [this, &activeRequests, &worldConfig] + { + auto const& commSession = COMM_SESSION; + + if (worldConfig.isPipelineParallel()) + { + mCancelledRequestsWaitThread->waitStop(); + } + + if (commSession.getSize() > 1 && !activeRequests.empty()) + { + if (mIsPipelineLeader) + { + if (worldConfig.isPipelineParallel()) + { + auto const peer = worldConfig.getPipelineParallelism() - 1; + bool shouldExit = false; + mCommPipelineParallel->send( + &shouldExit, 1, mpi::MpiType::kBOOL, peer, mpi::MpiTag::kExecutorShouldExit); + auto pipelineCancelledReqIds + = CancelledRequestsAsyncSend::cancelledRequestsRecv(mCommPipelineParallel, peer); + mCancelledReqIds.insert(pipelineCancelledReqIds.begin(), pipelineCancelledReqIds.end()); + } + + auto numCancelledRequests = static_cast<int64_t>(mCancelledReqIds.size()); + if (worldConfig.isTensorParallel()) + { + mCommTensorParallel->bcastValue(numCancelledRequests, 0); + if (numCancelledRequests > 0) + { + std::vector<IdType> cancelledReqIdsVec(mCancelledReqIds.begin(), mCancelledReqIds.end()); + mCommTensorParallel->bcast( + cancelledReqIdsVec.data(), cancelledReqIdsVec.size(), mpi::MpiType::kUINT64, 0); + } + } + if (worldConfig.isContextParallel()) + { + mCommContextParallel->bcastValue(numCancelledRequests, 0); + if (numCancelledRequests > 0) + { + std::vector<IdType> cancelledReqIdsVec(mCancelledReqIds.begin(), mCancelledReqIds.end()); + mCommContextParallel->bcast( + cancelledReqIdsVec.data(), cancelledReqIdsVec.size(), mpi::MpiType::kUINT64, 0); + } + } + } + // If not leader + else + { + if (worldConfig.isFirstPipelineParallelRank()) + { + int64_t numCancelledRequests = 0; + mCommTensorParallel->bcastValue(numCancelledRequests, 0); + mCommContextParallel->bcastValue(numCancelledRequests, 0); + if (numCancelledRequests > 0) + { + std::vector<IdType> cancelledReqIdsVec(numCancelledRequests); + mCommTensorParallel->bcast( + cancelledReqIdsVec.data(), cancelledReqIdsVec.size(), mpi::MpiType::kUINT64, 0); + mCommContextParallel->bcast( + cancelledReqIdsVec.data(), cancelledReqIdsVec.size(), mpi::MpiType::kUINT64, 0); + mCancelledReqIds + = std::unordered_set<IdType>(cancelledReqIdsVec.begin(), cancelledReqIdsVec.end()); + } + } + else + { + auto const peer = worldConfig.getPipelineParallelRank() - 1; + mCancelledReqIds = CancelledRequestsAsyncSend::cancelledRequestsRecv(mCommPipelineParallel, peer); + } + } + if (!worldConfig.isLastPipelineParallelRank()) + { + auto const peer = worldConfig.getPipelineParallelRank() + 1; + mCancelledRequestsAsyncSndHdl + = std::make_unique<CancelledRequestsAsyncSend>(mCommPipelineParallel, mCancelledReqIds, peer); + mCancelledRequestsWaitThread->notifyStart(); + } + } + }; + + std::unique_lock<std::mutex> lck{mCancelReqMtx, std::defer_lock}; + if (!worldConfig.isPipelineParallel()) + { + lck.lock(); + } + + broadcastCancelledRequests(); + + if (!mCancelledReqIds.empty()) + { + // Loop over active requests and terminate those that have been cancelled + std::unordered_set<IdType> terminatedReqIds; + for (auto& req : activeRequests) + { + auto reqId = req->isChild() ? req->getParentRequestId() : req->mRequestId; + if (mCancelledReqIds.find(reqId) != mCancelledReqIds.end()) + { + auto finishReason = req->isTimedOut() ? FinishReason::kTIMED_OUT : FinishReason::kCANCELLED; + mModel->terminateRequestSync(req, finishReason); + // Parent and child requests share the same request id. + // Mark it terminated first and remove from the set later. + terminatedReqIds.insert(reqId); + } + } + + for (auto const& reqId : terminatedReqIds) + { + mCancelledReqIds.erase(reqId); + } + } +} + +void Executor::Impl::terminateContextFinishedRequests(InTransList& inTransmissionRequests) +{ + NVTX3_SCOPED_RANGE(terminateContextFinishedRequests); + for (auto it = inTransmissionRequests.begin(); it != inTransmissionRequests.end();) + { + auto& item = *it; + auto req = item.request; + if (req->isDisaggContextCompleteState()) + { + // If pinnedBlockIds were tracked, unpin them. Otherwise, just terminate. + auto kvMgr = mModel->getKVCacheManager(); + if (kvMgr && !item.pinnedBlockIds.empty()) + { + kvMgr->unpinBlocksById(item.pinnedBlockIds); + } + else + { + mModel->terminateRequest(req); + } + it = inTransmissionRequests.erase(it); + } + else + { + ++it; + } + } +} + +void Executor::Impl::appendNewResponses(std::vector<Response>&& newResponses) +{ + { + std::scoped_lock<std::mutex> lck(mResponsesMtx); + for (auto& response : newResponses) + { + mResponses[response.getRequestId()].emplace_back(std::move(response)); + } + } + mResponsesCv.notify_all(); +} + +Executor::Impl::RequestList Executor::Impl::populateNewResponses( + RequestList& activeRequests, InTransList& inTransmissionRequests, std::vector<Response>& newResponses) +{ + NVTX3_SCOPED_RANGE(populateNewResponses); + RequestList finishedRequests; + for (auto it = activeRequests.begin(); it != activeRequests.end();) + { + auto const& llmReq = (*it); + bool const requestDone = llmReq->isFinished(); + // Only leader should store responses + if (mIsLeader) + { + auto response = llmReq->createResponse(mModel->hasSpeculativeDecodingFastLogits(), mWorldRank); + if (response) + { + newResponses.emplace_back(std::move(response.value())); + } + } + // Remove from active requests if last response has been generated + if (requestDone) + { + // move the in transmission requests to another tracker + if (llmReq->isDisaggContextTransmissionState()) + { + std::vector<SizeType32> pinnedBlockIds{}; + auto kvMgr = mModel->getKVCacheManager(); + if (kvMgr && kvMgr->isEnableBlockReuse() && !kvMgr->getBlockManager().isVariableWindow()) + { + pinnedBlockIds = kvMgr->storeBlocksForReuse(llmReq->mRequestId, llmReq, /*pinBlocks=*/true); + mModel->terminateRequest(llmReq); + } + inTransmissionRequests.push_back(InTransmissionItem{*it, pinnedBlockIds}); + } + finishedRequests.push_back(*it); + it = activeRequests.erase(it); + } + else + { + ++it; + } + } + return finishedRequests; +} + +void Executor::Impl::executionLoop() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + tensorrt_llm::common::setThreadName("executionLoop"); + + auto const& worldConfig = mModel->getWorldConfig(); + TLLM_CUDA_CHECK(cudaSetDevice(worldConfig.getDevice())); + + auto const [profileIterIdxs, stopIterIdxs] = tensorrt_llm::common::populateIterationIndexes( + kPROFILE_START_STOP_ENV_VAR_NAME, kLEGACY_PROFILE_START_STOP_ENV_VAR_NAME); + + SizeType32 numNewActiveRequests{0}; + std::chrono::time_point<std::chrono::steady_clock> iterStart; + std::chrono::time_point<std::chrono::steady_clock> iterEnd; + bool firstIteration{true}; + RequestList activeRequests; + InTransList inTransmissionRequests; + std::vector<Response> newResponses; + while (!mShutdown || !activeRequests.empty()) + { + double iterLatencyMS{0.0}; + double newActiveRequestsQueueLatencyMS{0.0}; + bool reportFinishedRequests = true; + RequestList finishedRequests; + if (!activeRequests.empty()) + { + finishTimedOutRequests(activeRequests); + terminateCancelledRequests(activeRequests); + forwardSync(activeRequests); + finishedRequests = populateNewResponses(activeRequests, inTransmissionRequests, newResponses); + cleanupDynamicLogitsPostProcessors(finishedRequests); + auto const iterCounter = mModel->getIterCounter(); + auto const stopIter = !stopIterIdxs.empty() && (stopIterIdxs.count(iterCounter - 1) > 0); + if (stopIter) + { + cudaProfilerStop(); + } + + // When there are no active or inflight requests, we need to update the stats before calling + // fetchNewRequests to make sure that the stats are reported accurately. + if (activeRequests.empty() && (!firstIteration)) + { + mModel->resetIterationStats(); + updateIterationStats(activeRequests, iterLatencyMS, numNewActiveRequests, + newActiveRequestsQueueLatencyMS, static_cast<SizeType32>(finishedRequests.size()), true); + updateRequestStats(activeRequests, finishedRequests, true); + reportFinishedRequests = false; + } + if (!newResponses.empty()) + { + enqueueNewResponses(std::move(newResponses)); + newResponses.clear(); + } + iterEnd = std::chrono::steady_clock::now(); + iterLatencyMS = std::chrono::duration<double, std::milli>(iterEnd - iterStart).count(); + } + + if (!inTransmissionRequests.empty()) + { + terminateContextFinishedRequests(inTransmissionRequests); + } + + if (!mShutdown) + { + auto const iterCounter = mModel->getIterCounter(); + auto const profileIter = !profileIterIdxs.empty() && (profileIterIdxs.count(iterCounter) > 0); + if (profileIter) + { + cudaProfilerStart(); + } + iterStart = std::chrono::steady_clock::now(); + std::optional<PriorityType> lowestPriority = std::nullopt; + if (!activeRequests.empty()) + { + lowestPriority = activeRequests.back()->priority(); + } + + auto [newRequests, newActiveRequestsQueueLatency] + = fetchNewRequests(static_cast<SizeType32>(activeRequests.size()), lowestPriority); + newActiveRequestsQueueLatencyMS = newActiveRequestsQueueLatency; + numNewActiveRequests = newRequests.size(); + + if (firstIteration) + { + firstIteration = false; + } + + for (auto const& newRequest : newRequests) + { + insertRequestInOrder(activeRequests, newRequest); + } + + // Update dynamic tuning stats + if (mDynamicBatchTuner) + { + for (auto const& req : activeRequests) + { + auto const inputLength = req->mPromptLen; + auto const outputLength = req->mMaxNewTokens; + mDynamicBatchTuner->updateStats(inputLength, outputLength); + } + } + } + if (!activeRequests.empty()) + { + forwardAsync(activeRequests); + updateIterationStats(activeRequests, iterLatencyMS, numNewActiveRequests, newActiveRequestsQueueLatencyMS, + static_cast<SizeType32>(finishedRequests.size()), false); + // Finished requests were reported once. Avoid reporting it twice. + if (reportFinishedRequests) + { + updateRequestStats(activeRequests, finishedRequests, false); + } + else + { + updateRequestStats(activeRequests, {}, false); + } + appendCurrentDebugTensors(); + } + } + + if (mCancelledRequestsWaitThread) + { + mCancelledRequestsWaitThread.reset(nullptr); + } + if (mRequestWithIdWaitThread) + { + mRequestWithIdWaitThread.reset(nullptr); + } + if (worldConfig.isPipelineParallel() && mIsPipelineLeader) + { + auto const peer = worldConfig.getPipelineParallelism() - 1; + int64_t numActiveRequests = -1; + mCommPipelineParallel->send( + &numActiveRequests, 1, mpi::MpiType::kINT64, peer, mpi::MpiTag::kExecutorNumActiveRequests); + bool shouldExit = true; + mCommPipelineParallel->send(&shouldExit, 1, mpi::MpiType::kBOOL, peer, mpi::MpiTag::kExecutorShouldExit); + } + if (mRequestWithIdLeaderThread) + { + mRequestWithIdLeaderThread->join(); + mRequestWithIdLeaderThread.reset(nullptr); + } + if (mCancelledRequestsLeaderThread) + { + mCancelledRequestsLeaderThread->join(); + mCancelledRequestsLeaderThread.reset(nullptr); + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void Executor::Impl::enqueueTerminateRequest() +{ + { + std::scoped_lock<std::mutex> lck(mQueuedReqMtx); + Request dummyReq({1}, 1); + RequestWithId reqWithId{std::move(dummyReq), kTerminateReqId}; + mQueuedRequests.emplace_back(reqWithId); + } + mQueuedReqCv.notify_one(); +} + +void Executor::Impl::enqueueNewResponses(std::vector<Response>&& newResponses) +{ + TLLM_CHECK_WITH_INFO(mIsLeader, "Only leader should store responses"); + + if (mCommMode == CommunicationMode::kLEADER) + { + appendNewResponses(std::move(newResponses)); + } + else if (mCommMode == CommunicationMode::kORCHESTRATOR) + { + MpiMessage message(MpiId::RESPONSE); + message.data = ResponseData{std::move(newResponses)}; + mSendQueue.push(std::move(message)); + } +} + +// Orchestrator thread sending new requests to leader of the model +void Executor::Impl::orchSendReqThread() +{ + tensorrt_llm::common::setThreadName("orchSendReq"); + + while (true) + { + auto message = mSendQueue.pop(); + + if (message.id == MpiId::TERMINATION) + { + mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mLeaderRank, mpi::MpiTag::kOrchestratorId); + TLLM_LOG_INFO("Orchestrator sendReq thread exiting"); + break; + } + if (message.id == MpiId::PENDING_REQUEST) + { + auto& reqWithIds = std::get<PendingRequestData>(message.data); + auto packed = RequestWithId::serializeReqWithIds(reqWithIds.requests); + + TLLM_LOG_DEBUG("Orchestrator sendReq thread sending %d pending requests", reqWithIds.requests.size()); + // Temporary WAR to indicate to client that we cannot send the serialized request + // because it exceeds int32_t size limit. + // TODO: Should fix as part of https://jirasw.nvidia.com/browse/TRTLLM-708 + if (packed.size() > std::numeric_limits<int32_t>::max()) + { + for (auto const& reqWithId : reqWithIds.requests) + { + { + std::scoped_lock<std::mutex> lck(mResponsesMtx); + mResponses[reqWithId.id].emplace_back(reqWithId.id, + "Request is too large, or you are enqueuing too many requests at once " + "to be sent via MPI_Send, please try to enqueue the request(s) again. " + "This issue will be resolved in a future version of TRT-LLM."); + } + mResponsesCv.notify_all(); + } + } + else + { + mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mLeaderRank, mpi::MpiTag::kOrchestratorId); + mOrchLeaderComm->send( + packed.data(), packed.size(), mpi::MpiType::kCHAR, mLeaderRank, mpi::MpiTag::kOrchestratorData); + } + } + else if (message.id == MpiId::CANCEL_REQUEST) + { + auto& data = std::get<RequestIdsData>(message.data); + + mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mLeaderRank, mpi::MpiTag::kOrchestratorId); + mOrchLeaderComm->send( + data.ids.data(), data.ids.size(), mpi::MpiType::kUINT64, mLeaderRank, mpi::MpiTag::kOrchestratorData); + } + else + { + TLLM_THROW("Invalid message id"); + } + } +} + +// Leader thread receiving new requests from orchestrator +void Executor::Impl::leaderRecvReqThread() +{ + tensorrt_llm::common::setThreadName("leaderRecvReq"); + TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); +#if ENABLE_MULTI_DEVICE + auto& selCancelledReqIds = mUsePipelineParallel ? mPipelineCancelledReqIds : mCancelledReqIds; + while (true) + { + if (mRecvPollPeriodMs > 0) + { + mOrchLeaderComm->recvPoll(mOrchRank, mpi::MpiTag::kOrchestratorId, mRecvPollPeriodMs); + } + + // Blocking is okay: terminate message is expected to arrive here + MPI_Message msg = nullptr; + MPI_Status status; + mOrchLeaderComm->mprobe(mOrchRank, mpi::MpiTag::kOrchestratorId, &msg, &status); + + int32_t count = 0; + MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); // NOLINT + TLLM_CHECK(count == 1); + + MpiId mpiId{}; + MPICHECK(MPI_Mrecv(&mpiId, count, MPI_UINT64_T, &msg, &status)); // NOLINT + + // EXIT condition from receiving TERMINATE msg + if (mpiId == MpiId::TERMINATION) + { + // Enqueue a request to indicate to other ranks to terminate + enqueueTerminateRequest(); + + // Send message to orchestrator to indicate to terminate orch recv thread + mSendQueue.push(MpiMessage(mpiId)); + TLLM_LOG_INFO("Leader recvReq thread exiting"); + break; + } + if (mpiId == MpiId::PENDING_REQUEST) + { + mOrchLeaderComm->mprobe(mOrchRank, mpi::MpiTag::kOrchestratorData, &msg, &status); + MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); // NOLINT + std::vector<char> buffer(count); + MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); // NOLINT + + auto requestWithIds = RequestWithId::deserializeReqWithIds(buffer); + TLLM_LOG_DEBUG("Leader recvReq thread receiving %d pending requests", requestWithIds.size()); + { + std::scoped_lock<std::mutex> lck(mQueuedReqMtx); + if (mMaxQueueSize) + { + auto const maxQueueSize = mMaxQueueSize.value(); + if (maxQueueSize > 0 && mQueuedRequests.size() >= static_cast<size_t>(maxQueueSize)) + { + auto err = tensorrt_llm::common::fmtstr( + "Maximum queue size of %d has been reached, please try again later", maxQueueSize); + TLLM_LOG_ERROR("%s", err.c_str()); + std::vector<Response> responses; + responses.reserve(requestWithIds.size()); + for (auto const& reqWithId : requestWithIds) + { + responses.emplace_back(reqWithId.id, err); + } + enqueueNewResponses(std::move(responses)); + continue; + } + } + for (auto&& req : requestWithIds) + { + req.queuedStart = std::chrono::steady_clock::now(); + insertRequestInOrder(mQueuedRequests, std::move(req)); + } + } + mQueuedReqCv.notify_one(); + } + else if (mpiId == MpiId::CANCEL_REQUEST) + { + // Prepare receiving data + mOrchLeaderComm->mprobe(mOrchRank, mpi::MpiTag::kOrchestratorData, &msg, &status); + MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); // NOLINT + std::vector<uint64_t> cancelledReqIds(count); + MPICHECK(MPI_Mrecv(cancelledReqIds.data(), count, MPI_UINT64_T, &msg, &status)); // NOLINT + + std::scoped_lock<std::mutex> lck(mCancelReqMtx); + selCancelledReqIds.insert(cancelledReqIds.begin(), cancelledReqIds.end()); + } + else + { + TLLM_THROW("Invalid message id"); + } + } +#endif // ENABLE_MULTI_DEVICE +} + +// Leader thread sending responses to orchestrator +void Executor::Impl::leaderSendThread(MpiMessageQueue& sendQueue, mpi::MpiTag idTag, mpi::MpiTag dataTag) +{ + tensorrt_llm::common::setThreadName("leaderSend"); + TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId)); + +#if ENABLE_MULTI_DEVICE + while (true) + { + auto message = sendQueue.pop(); + + if (message.id == MpiId::TERMINATION) + { + mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mOrchRank, idTag); + TLLM_LOG_INFO("Leader sendThread exiting"); + break; + } + if (message.id == MpiId::RESPONSE || message.id == MpiId::ITER_STATS + || message.id == MpiId ::REQUEST_ITER_STATS) + { + std::vector<char> buffer; + if (message.id == MpiId::RESPONSE) + { + auto& responseData = std::get<ResponseData>(message.data); + TLLM_LOG_DEBUG("Leader sendResp thread sending %d responses", responseData.responses.size()); + buffer = Serialization::serialize(responseData.responses); + } + else if (message.id == MpiId::ITER_STATS) + { + auto& iterStatsData = std::get<IterStatsData>(message.data); + TLLM_LOG_DEBUG("Leader sendResp thread sending iter stats"); + buffer = Serialization::serialize(iterStatsData.iterStatsVec); + } + else if (message.id == MpiId::REQUEST_ITER_STATS) + { + auto& requestIterStatsData = std::get<RequestStatsPerIterationData>(message.data); + TLLM_LOG_DEBUG("Leader sendResp thread sending iter request stats"); + buffer = Serialization::serialize(requestIterStatsData.requestStatsPerIterationVec); + } + mOrchLeaderComm->send(&message.id, 1, mpi::MpiType::kUINT64, mOrchRank, idTag); + mOrchLeaderComm->send(buffer.data(), buffer.size(), mpi::MpiType::kCHAR, mOrchRank, dataTag); + } + else + { + TLLM_THROW("Invalid message id"); + } + } +#endif // ENABLE_MULTI_DEVICE +} + +void Executor::Impl::orchRecvThread(mpi::MpiTag idTag, mpi::MpiTag dataTag) +{ + tensorrt_llm::common::setThreadName("orchRecv"); + +#if ENABLE_MULTI_DEVICE + while (true) + { + if (mRecvPollPeriodMs > 0) + { + mOrchLeaderComm->recvPoll(mOrchRank, mpi::MpiTag::kOrchestratorId, mRecvPollPeriodMs); + } + + MPI_Message msg = nullptr; + MPI_Status status; + mOrchLeaderComm->mprobe(mLeaderRank, idTag, &msg, &status); + + int32_t count = 0; + MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); // NOLINT + TLLM_CHECK(count == 1); + + MpiId mpiId{}; + MPICHECK(MPI_Mrecv(&mpiId, count, MPI_UINT64_T, &msg, &status)); // NOLINT + + if (mpiId == MpiId::TERMINATION) + { + TLLM_LOG_INFO("Orchestrator recv thread exiting"); + break; + } + if (mpiId == MpiId::RESPONSE || mpiId == MpiId::ITER_STATS || mpiId == MpiId::REQUEST_ITER_STATS) + { + mOrchLeaderComm->mprobe(mLeaderRank, dataTag, &msg, &status); + MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); // NOLINT + + std::vector<char> buffer(count); + MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); // NOLINT + + if (mpiId == MpiId::RESPONSE) + { + auto newResponses = Serialization::deserializeResponses(buffer); + TLLM_LOG_DEBUG("Orchestrator recv thread receiving %d responses", newResponses.size()); + appendNewResponses(std::move(newResponses)); + } + else if (mpiId == MpiId::ITER_STATS) + { + appendMultipleIterStats(Serialization::deserializeIterationStatsVec(buffer)); + } + else if (mpiId == MpiId::REQUEST_ITER_STATS) + { + appendMultipleRequestStats(Serialization::deserializeRequestStatsPerIterationVec(buffer)); + } + } + else + { + TLLM_THROW("Invalid message id"); + } + } +#endif // ENABLE_MULTI_DEVICE +} + +Executor::Impl::LlmRequestLogitsPostProcessor Executor::Impl::getLogitsPostProcessor(std::string const& name) +{ + auto const postProcIt = mLogitsPostProcessorMap.find(name); + TLLM_CHECK_WITH_INFO( + postProcIt != mLogitsPostProcessorMap.end(), "LogitsPostProcessor %s not found.", name.c_str()); + auto executorLogitsPostProcessor = postProcIt->second; + return [executorLogitsPostProcessor](IdType reqId, RtTensorPtr& logits, BeamTokens const& beamTokens, + CudaStreamPtr const& cudaStreamPtr, std::optional<IdType> clientId) + { + auto logitsTensor = executor::detail::ofITensor(logits); + executorLogitsPostProcessor(reqId, logitsTensor, beamTokens, cudaStreamPtr, clientId); + }; +} + +void Executor::Impl::setupDynamicLogitsPostProcessors(std::vector<RequestWithId>& newReqWithIds) +{ + for (auto& reqWithId : newReqWithIds) + { + auto logitsPostProcessor = reqWithId.req.getLogitsPostProcessor(); + if (logitsPostProcessor) + { + std::string const name = Request::kDynamicPostProcessorNamePrefix + std::to_string(reqWithId.id); + mLogitsPostProcessorMap[name] = logitsPostProcessor.value(); + reqWithId.req.setLogitsPostProcessor(std::nullopt); + reqWithId.req.setLogitsPostProcessorName(name); + } + } +} + +void Executor::Impl::cleanupDynamicLogitsPostProcessors(RequestList const& finishedRequests) +{ + for (auto& req : finishedRequests) + { + std::string const name = Request::kDynamicPostProcessorNamePrefix + std::to_string(req->mRequestId); + auto const postProcIt = mLogitsPostProcessorMap.find(name); + if (postProcIt != mLogitsPostProcessorMap.end()) + { + mLogitsPostProcessorMap.erase(name); + } + } +} + +void Executor::Impl::addTerminatedReqId(std::vector<Response> const& responses, IdType const& reqId) +{ + for (auto const& response : responses) + { + if (response.hasError() || (!response.hasError() && response.getResult().isFinal)) + { + mTerminatedReqIds.insert(reqId); + if (mChildReqIdsMap.find(reqId) != mChildReqIdsMap.end()) + { + for (auto childReqId : mChildReqIdsMap.at(reqId)) + { + mTerminatedReqIds.insert(childReqId); + } + mChildReqIdsMap.erase(reqId); + } + } + } +} + +void Executor::Impl::checkParallelApiUsage(std::string const& methodName) const +{ + // If leader mode, and not leader, throw error + if (mCommMode == CommunicationMode::kLEADER && !mIsLeader) + { + // Non-leader are not expected to call cancelRequest + TLLM_THROW("With LEADER communication mode, only leader rank is expected to call %s", methodName.c_str()); + } + if (mCommMode == CommunicationMode::kORCHESTRATOR && !mIsOrchestrator) + { + TLLM_THROW( + "With ORCHESTRATOR communication mode, only orchestrator rank is expected to call %s", methodName.c_str()); + } +} + +} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/executorImpl.h b/cpp/tensorrt_llm/executor/executorImpl.h new file mode 100644 index 000000000000..f812b55a3fa0 --- /dev/null +++ b/cpp/tensorrt_llm/executor/executorImpl.h @@ -0,0 +1,385 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/common/arrayView.h" +#include "tensorrt_llm/executor/dynamicBatchTuner.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/intervalSet.h" +#include "tensorrt_llm/executor/model.h" +#include "tensorrt_llm/executor/orchestratorUtils.h" +#include "tensorrt_llm/executor/requestWithId.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/gptJsonConfig.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +#include <atomic> +#include <condition_variable> +#include <list> +#include <mutex> +#include <optional> +#include <queue> +#include <thread> +#include <unordered_map> +#include <unordered_set> + +namespace tensorrt_llm::executor +{ + +class RequestWithIdAsyncSend; +class CancelledRequestsAsyncSend; + +class MpiMessageQueue +{ +public: + void push(MpiMessage&& message) + { + std::lock_guard<std::mutex> const lock(mMutex); + mQueue.push(std::move(message)); + mCv.notify_one(); + } + + MpiMessage pop() + { + std::unique_lock<std::mutex> lock(mMutex); + mCv.wait(lock, [this] { return !mQueue.empty(); }); + MpiMessage message = std::move(mQueue.front()); + mQueue.pop(); + return message; + } + +private: + std::queue<MpiMessage> mQueue; + std::mutex mMutex; + std::condition_variable mCv; +}; + +class Executor::Impl + +{ + using LlmRequestPtr = std::shared_ptr<batch_manager::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + // When block reuse is enabled for context worker for disaggregated serving, + // we need to store the pinned block ids so that we can unpin them when + // the request is finished. + struct InTransmissionItem + { + LlmRequestPtr request; + std::vector<SizeType32> pinnedBlockIds; + }; + + using InTransList = std::list<InTransmissionItem>; + +public: + Impl(std::filesystem::path const& modelPath, std::optional<std::filesystem::path> const& encoderModelPath, + [[maybe_unused]] ModelType modelType, ExecutorConfig const& executorConfig); + + Impl(BufferView const& engineBufferView, std::string const& jsonConfigStr, + std::optional<BufferView> const& encoderEngineBufferView, + std::optional<std::string> const& encoderJsonConfigStr, [[maybe_unused]] ModelType modelType, + ExecutorConfig const& executorConfig, std::optional<std::map<std::string, Tensor>> const& managedWeightsOpt); + + Impl(std::shared_ptr<Model> model, std::optional<std::shared_ptr<Model>> encoderModel, + ExecutorConfig const& executorConfig); + + ~Impl(); + + Impl(Impl const& executor) = delete; + Impl& operator=(Impl const& executor) = delete; + Impl(Impl&&) = delete; + Impl& operator=(Impl&&) = delete; + + IdType enqueueRequest(Request const& request); + + std::vector<IdType> enqueueRequests(std::vector<Request> const& requests); + + std::vector<IdType> enqueueRequests(common::ArrayView<Request const> const& requests); + + std::vector<Response> awaitResponses(std::optional<std::chrono::milliseconds> const& timeout = std::nullopt); + + std::vector<Response> awaitResponses( + IdType const& reqId, std::optional<std::chrono::milliseconds> const& optTimeout = std::nullopt); + + std::vector<std::vector<Response>> awaitResponses( + std::vector<IdType> const& requestIds, std::optional<std::chrono::milliseconds> const& timeout); + + SizeType32 getNumResponsesReady(std::optional<IdType> const& optId = std::nullopt) const; + + void cancelRequest(IdType requestId); + + void shutdown(); + + std::deque<IterationStats> getLatestIterationStats(); + std::deque<RequestStatsPerIteration> getLatestRequestStats(); + std::deque<DebugTensorsPerIteration> getLatestDebugTensors(); + + bool canEnqueueRequests() const; + + bool isParticipant() const; + + std::optional<std::shared_ptr<KVCacheEventManager>> getKVCacheEventManager() const; + +private: + using RtTensorPtr = runtime::ITensor::SharedPtr; + using CudaStreamPtr = runtime::BufferManager::CudaStreamPtr; + using LlmRequestLogitsPostProcessor + = std::function<void(IdType, RtTensorPtr&, BeamTokens const&, CudaStreamPtr, std::optional<IdType>)>; + + void initialize(ExecutorConfig const& executorConfig); + + void loadModel(std::optional<std::filesystem::path> const& modelPath, std::optional<BufferView> const& engineBuffer, + runtime::GptJsonConfig const& jsonConfig, ExecutorConfig const& executorConfig, bool isEncoder, + std::optional<std::map<std::string, Tensor>> const& managedWeightsOpt); + + std::shared_ptr<Model> createModel(runtime::RawEngine const& rawEngine, runtime::ModelConfig const& modelConfig, + runtime::WorldConfig const& worldConfig, ExecutorConfig const& executorConfig); + + std::shared_ptr<Model> createEncoderModel(runtime::RawEngine const& rawEngine, + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, + ExecutorConfig const& executorConfig); + + void setOrchLeaderComm(SizeType32 tp, SizeType32 pp, SizeType32 cp, ParallelConfig const& parallelConfig); + + void initializeCommAndWorkers(SizeType32 tp, SizeType32 pp, SizeType32 cp, ExecutorConfig const& executorConfig, + std::optional<ModelType> modelType = std::nullopt, + std::optional<std::filesystem::path> const& modelPath = std::nullopt, + std::optional<runtime::WorldConfig> const& worldConfig = std::nullopt, + std::optional<runtime::GptJsonConfig> const& decoderGptJsonConfig = std::nullopt); + + static void validateParallelConfig(ParallelConfig const& parallelConfig, std::optional<ModelType> modelType, + std::optional<std::filesystem::path> const& modelPath); + + void initializeOrchestrator(SizeType32 tp, SizeType32 pp, SizeType32 cp, ExecutorConfig const& executorConfig, + ParallelConfig parallelConfig, ModelType modelType, std::filesystem::path const& modelPath); + + void initializeWorkers(SizeType32 tp, SizeType32 pp, SizeType32 cp, ParallelConfig& parallelConfig, + std::optional<runtime::WorldConfig> const& worldConfig = std::nullopt, + std::optional<runtime::GptJsonConfig> const& decoderGptJsonConfig = std::nullopt); + + void initializeLogitsPostProcessorBatched(LogitsPostProcessorConfig const& logitsProcConfig); + + IdType generateReqId(Request const& request) + { + // If the request has a disaggregated request id, prefer it. + if (request.getDisaggRequestId().has_value() && request.getDisaggRequestId().value() > kMaxLocalReqId) + { + return request.getDisaggRequestId().value(); + } + // Otherwise, generate a local request id in range [1, kMaxLocalReqId). + return generateLocalReqId(); + } + + IdType generateLocalReqId() + { + return (mLastReqId++ % kMaxLocalReqId); + } + + std::vector<RequestWithId> getLeaderNewReqWithIds( + SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive); + std::vector<RequestWithId> getNewReqWithIds( + SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive); + + std::tuple<Executor::Impl::RequestList, double> fetchNewRequests( + SizeType32 numActiveRequests, std::optional<PriorityType> lowestPriorityActive); + + void forwardSync(RequestList& activeRequests); + + void forwardAsync(RequestList& activeRequests); + + void prepRequestsForEncoderSkip(RequestList& activeRequests); + + void terminateActiveRequests(RequestList& activeRequests, std::string const& err); + + IterationStats getCurrentIterationStats(RequestList const& activeRequests, double iterLatencyMS, + SizeType32 numNewActiveRequests, double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests); + + void appendCurrentIterStats(IterationStats&& currentIterStats); + void appendMultipleIterStats(std::vector<IterationStats>&& currentIterStatsVec); + void updateIterationStats(RequestList const& activeRequests, double iterLatencyMS, SizeType32 numNewActiveRequests, + double newActiveRequestsQueueLatencyMS, SizeType32 numCompletedRequests, bool flushToOrchestrator); + void appendCurrentRequestStats(RequestStatsPerIteration&& currentRequestStats); + void appendMultipleRequestStats(std::vector<RequestStatsPerIteration>&& currentRequestStatsVec); + RequestStatsPerIteration getCurrentRequestStats( + RequestList const& activeRequests, RequestList const& finishedRequests); + void updateRequestStats( + RequestList const& activeRequests, RequestList const& finishedRequests, bool flushToOrchestrator); + + void appendCurrentDebugTensors(); + + void terminateCancelledRequests(RequestList& activeRequests); + + void terminateContextFinishedRequests(InTransList& inTransmissionRequests); + + void appendNewResponses(std::vector<Response>&& newResponses); + + /// @brief Populates new responses from active requests. + /// Active requests that have completed are erased from activeRequests + /// and returned for bookkeeping. + /// @return A list of requests that have completed. + RequestList populateNewResponses( + RequestList& activeRequests, InTransList& inTransmissionRequests, std::vector<Response>& newResponses); + + void executionLoop(); + + void enqueueTerminateRequest(); + void enqueueNewResponses(std::vector<Response>&& newResponses); + + LlmRequestLogitsPostProcessor getLogitsPostProcessor(std::string const& name); + void setupDynamicLogitsPostProcessors(std::vector<RequestWithId>& newReqWithIds); + void cleanupDynamicLogitsPostProcessors(RequestList const& finishedRequests); + + void orchSendReqThread(); + void orchRecvThread(mpi::MpiTag idTag, mpi::MpiTag dataTag); + void leaderRecvReqThread(); + void leaderSendThread(MpiMessageQueue& sendQueue, mpi::MpiTag idTag, mpi::MpiTag dataTag); + + void addTerminatedReqId(std::vector<Response> const& responses, IdType const& reqId); + + // Check that the current process is the leader or orchestrator + void checkParallelApiUsage(std::string const& methodName) const; + + // These functions wait for MPI async sends on separate threads + void requestWithIdWaitThread(); + void cancelledRequestsWaitThread(); + // These functions send data from leader to pipeline leader on separate threads + void requestWithIdLeaderThread(); + void cancelledRequestsLeaderThread(); + + /// @brief mark requests that have timed out before ever being executed as finished. + /// uses cancellation based on communication mode. + /// + /// @param activeRequests [in] List of active requests to check for timeouts + void finishTimedOutRequests(RequestList const& activeRequests); + + // The model to execute + std::shared_ptr<Model> mModel = nullptr; + std::shared_ptr<Model> mEncoderModel = nullptr; + + // The maximum number of activeRequests + SizeType32 mMaxNumActiveRequests; + + // Thread the executes the main loop + std::thread mExecutionThread; + + // Atomic that indicates threads should shutdown + std::atomic<bool> mShutdown; + + // Atomic that indicates if shutdown method has been called + std::atomic<bool> mShutdownCalled = false; + + // Queued requests + std::mutex mQueuedReqMtx; + std::condition_variable mQueuedReqCv; + std::deque<RequestWithId> mQueuedRequests; + std::optional<SizeType32> mMaxQueueSize; + + // Cancelled requests + std::mutex mCancelReqMtx; + std::unordered_set<IdType> mCancelledReqIds; + std::unordered_set<IdType> mPipelineCancelledReqIds; + + // Ready responses + std::unordered_map<IdType, std::vector<Response>> mResponses; + mutable std::mutex mResponsesMtx; + std::condition_variable mResponsesCv; + + // Since the request IDs are generated sequentially, IntervalSet is preferred over unordered_set for its efficient + // memory usage to stores request ID intervals rather than individual request ID numbers. + IntervalSet<IdType> mTerminatedReqIds; + + std::unordered_map<IdType, std::vector<IdType>> mChildReqIdsMap; + + // Iteration stats + SizeType32 mIterStatsMaxIterations; + std::mutex mIterStatsMtx; + std::deque<IterationStats> mIterationStats; + + // Request stats + SizeType32 mRequestStatsMaxIterations; + std::mutex mRequestStatsMtx; + std::deque<RequestStatsPerIteration> mRequestStats; + + // Debug + IterationType mDebugTensorsMaxIterations; + std::mutex mDebugTensorsMtx; + std::deque<DebugTensorsPerIteration> mDebugTensors; + + IdType mLastReqId = 1; + + static constexpr IdType kTerminateReqId = 0; + // Request id > kMaxLocalReqId is reserved for disaggregated requests. + // This max ID is also in Python side. + static constexpr IdType kMaxLocalReqId = 1ULL << 42U; + + BatchingType mBatchingType; + bool mIsSchedulerMaxUtilization; + bool mIsSchedulerGuaranteedNoEvict; + bool mIsChunkedContext; + bool mPromptTableOffloading; + + CommunicationMode mCommMode; + bool mIsWorker = false; + bool mIsLeader = false; + bool mIsPipelineLeader = false; + bool mUsePipelineParallel = false; + + std::unordered_map<std::string, LogitsPostProcessor> mLogitsPostProcessorMap; + std::optional<Model::LogitsPostProcessorBatched> mLogitsPostProcessorBatched; + + bool mIsOrchestrator = false; + std::shared_ptr<tensorrt_llm::mpi::MpiComm> mOrchLeaderComm; + + std::thread mOrchSendReqThread; + std::thread mOrchRecvThread; + std::thread mLeaderRecvReqThread; + std::thread mLeaderSendThread; + + int32_t mRecvPollPeriodMs = 0; + + int32_t mLeaderRank = -1; + int32_t mOrchRank = 0; + int32_t mWorldRank = -1; + int32_t mDeviceId = 0; + + MpiMessageQueue mSendQueue; + + std::shared_ptr<tensorrt_llm::mpi::MpiComm> mCommTensorParallel; + std::shared_ptr<tensorrt_llm::mpi::MpiComm> mCommPipelineParallel; + std::shared_ptr<tensorrt_llm::mpi::MpiComm> mCommContextParallel; + std::unique_ptr<RequestWithIdAsyncSend> mRequestWithIdAsyncSndHdl; + std::unique_ptr<CancelledRequestsAsyncSend> mCancelledRequestsAsyncSndHdl; + std::unique_ptr<std::thread> mRequestWithIdLeaderThread; + std::unique_ptr<std::thread> mCancelledRequestsLeaderThread; + std::unique_ptr<tensorrt_llm::mpi::MpiWaitThread> mRequestWithIdWaitThread; + std::unique_ptr<tensorrt_llm::mpi::MpiWaitThread> mCancelledRequestsWaitThread; + + // for validating requests + bool mEnableBlockReuse; + + inline static std::string const kPROFILE_START_STOP_ENV_VAR_NAME = "TLLM_PROFILE_START_STOP"; + inline static std::string const kLEGACY_PROFILE_START_STOP_ENV_VAR_NAME = "TLLM_GPTM_PROFILE_START_STOP"; + + std::shared_ptr<DynamicBatchTuner> mDynamicBatchTuner; +}; + +} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/intervalSet.h b/cpp/tensorrt_llm/executor/intervalSet.h new file mode 100644 index 000000000000..01f40a685f46 --- /dev/null +++ b/cpp/tensorrt_llm/executor/intervalSet.h @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/executor/types.h" + +namespace tensorrt_llm::executor +{ + +/// @brief An interval inclusive on both ends. +/// A single number interval is represented as [num, num]. +template <typename T> +struct Interval +{ + T lowerEnd; + T upperEnd; +}; + +template <typename T> +bool operator<(Interval<T> const& a, Interval<T> const& b) +{ + return a.lowerEnd < b.lowerEnd; +} + +/// @brief A container to store unique numbers, represented as a vector of ordered and disjoint intervals. +template <typename NumType> +class IntervalSet +{ +public: + /// @brief Check if the given number is in set. + bool contains(NumType num) const + { + // Binary search + SizeType32 left = 0; + SizeType32 right = static_cast<SizeType32>(mIntervals.size()) - 1; + while (left <= right) + { + SizeType32 mid = left + (right - left) / 2; + if (mIntervals[mid].lowerEnd <= num && num <= mIntervals[mid].upperEnd) + { + return true; + } + else if (num < mIntervals[mid].lowerEnd) + { + right = mid - 1; + } + else + { + left = mid + 1; + } + } + return false; + } + + /// @brief Insert a number into set. Do nothing if the number is already in the set. + void insert(NumType num) + { + auto intervalToAdd = Interval<NumType>{num, num}; + + if (mIntervals.size() == 0) + { + mIntervals.insert(mIntervals.begin(), intervalToAdd); + mNumElements++; + return; + } + + // Iter is the first place in mIntervals such that num <= it.lowerEnd + auto iter = std::lower_bound(mIntervals.begin(), mIntervals.end(), intervalToAdd); + + bool iterAtBegin = iter == mIntervals.begin(); + bool iterAtEnd = iter == mIntervals.end(); + + if ((!iterAtEnd && iter->lowerEnd == num) || (!iterAtBegin && num <= (iter - 1)->upperEnd)) + { + // Number falls within the current interval or previous interval. No need to add again. + return; + } + + if (!iterAtBegin && !iterAtEnd && (iter - 1)->upperEnd + 1 == num && iter->lowerEnd - 1 == num) + { + // Merge two adjacent intervals + (iter - 1)->upperEnd = iter->upperEnd; + mIntervals.erase(iter); + } + else if (!iterAtBegin && (iter - 1)->upperEnd + 1 == num) + { + // Number is adjacent to the upper end of the previous interval. Merge left. + (iter - 1)->upperEnd = num; + } + else if (!iterAtEnd && iter->lowerEnd - 1 == num) + { + // Number is adjacent to the lower end of the current interval. Merge right. + iter->lowerEnd = num; + } + else + { + mIntervals.insert(iter, intervalToAdd); + } + mNumElements++; + } + + /// @brief Clear interval set and reset numElements to 0. + void clear() + { + mIntervals.clear(); + mNumElements = 0; + } + + /// @brief Return the size of the set. + SizeType32 getNumElements() const + { + return mNumElements; + } + + /// @brief Return the underlying mIntervals. + std::vector<Interval<NumType>> const& getIntervals() const + { + return mIntervals; + } + +private: + std::vector<Interval<NumType>> mIntervals; + SizeType32 mNumElements{0}; +}; + +} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/model.h b/cpp/tensorrt_llm/executor/model.h new file mode 100644 index 000000000000..52fedf1d1113 --- /dev/null +++ b/cpp/tensorrt_llm/executor/model.h @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/batch_manager/logitsPostProcessor.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/worldConfig.h" + +#include <nlohmann/json.hpp> + +namespace tensorrt_llm::executor +{ + +class Model +{ + using LlmRequestPtr = std::shared_ptr<batch_manager::LlmRequest>; + +public: + Model() = default; + + virtual ~Model() = default; + + /// @brief Function that marks a request Id as complete and cleans up associated state + virtual void terminateRequest(LlmRequestPtr const& llmRequest, bool pause) = 0; + + void terminateRequest(LlmRequestPtr const& llmRequest) + { + terminateRequest(llmRequest, false); + } + + /// @brief Terminate request in the next forwardSync call that includes the request. + virtual void terminateRequestSync(LlmRequestPtr const& llmRequest, FinishReason finishReason) = 0; + + /// @brief Function that synchronizes the decoder + virtual void forwardSync() = 0; + + /// @brief Function that tries to advance the active requests + /// Depending on resources available, it's possible that not all requests will get advanced + /// @param activeRequests The list of request to try to advance + virtual void forwardAsync(batch_manager::RequestList const& activeRequests) = 0; + + /// @brief Override the runtime batch size for the model + virtual void setRuntimeBatchSize(SizeType32 runtimeBatchSize) + { + // By default, we ignore the runtimeBatchSize unless the model actively supports it + } + + /// @brief Get the runtime batch size for the model + [[nodiscard]] virtual SizeType32 getRuntimeBatchSize() const + { + TLLM_CHECK_WITH_INFO(false, "getRuntimeBatchSize is not implemented"); + } + + /// @brieft Override the runtime max num tokens for the model + virtual void setRuntimeMaxNumTokens(SizeType32 runtimeMaxNumTokens) + { + // By default, we ignore the runtimeMaxNumTokens unless the model actively supports it + } + + virtual void updatePeftCache(LlmRequestPtr const& llmRequest) = 0; + + /// @brief Reset the iteration stats when there are no inflight requests + virtual void resetIterationStats() = 0; + + [[nodiscard]] virtual SizeType32 getMaxNumSequences() const = 0; + [[nodiscard]] virtual SizeType32 getMaxInputLen() const = 0; + [[nodiscard]] virtual SizeType32 getHiddenSize() const = 0; + [[nodiscard]] virtual SizeType32 getMaxSequenceLen() const = 0; + [[nodiscard]] virtual SizeType32 getVocabSizePadded() const = 0; + [[nodiscard]] virtual SizeType32 getMaxDraftLen() const = 0; + [[nodiscard]] virtual SizeType32 getNumMicroBatches() const = 0; + [[nodiscard]] virtual SizeType32 getOperatingBeamWidth() const = 0; + [[nodiscard]] virtual nvinfer1::DataType getLogitDataType() const = 0; + [[nodiscard]] virtual runtime::WorldConfig const& getWorldConfig() const = 0; + [[nodiscard]] virtual runtime::ModelConfig const& getModelConfig() const = 0; + [[nodiscard]] virtual runtime::BufferManager const& getBufferManager() const = 0; + [[nodiscard]] virtual runtime::BufferManager::CudaStreamPtr getRuntimeStreamPtr() const = 0; + [[nodiscard]] virtual IterationType getIterCounter() const noexcept = 0; + [[nodiscard]] virtual bool hasSpeculativeDecodingFastLogits() const noexcept = 0; + [[nodiscard]] virtual bool getGatherGenerationLogits() const = 0; + [[nodiscard]] virtual nvinfer1::DataType getTensorDataType(std::string const& name) const = 0; + [[nodiscard]] virtual nvinfer1::Dims getTensorShape(std::string const& name) const = 0; + + /// @brief Function that provides per iteration stats specific to a certain model + /// @param stats The json object to write stats to + virtual void getCurrentIterationStats(IterationStats& stats) const = 0; + + /// @brief Function that provides per request stats specific to a certain model + /// @param stats The request stats to be updated + virtual void getCurrentRequestStats(RequestStatsPerIteration& stats) const = 0; + + [[nodiscard]] virtual DebugTensorsPerIteration getCurrentDebugTensors() const = 0; + + using LogitsPostProcessorBatched = tensorrt_llm::batch_manager::LogitsPostProcessor::LogitsPostProcessorBatched; + + virtual void setLogitsPostProcessorBatched(std::optional<LogitsPostProcessorBatched> logitsPostProcessorBatched) + = 0; + virtual void setReplicateLogitsPostProcessor(bool replicateLogitsPostProcessor) = 0; + [[nodiscard]] virtual bool getReplicateLogitsPostProcessor() const = 0; + + [[nodiscard]] virtual bool hasGuidedDecoder() const noexcept = 0; + + [[nodiscard]] virtual std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager> + getKVCacheManager() = 0; + [[nodiscard]] virtual std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager const> + getKVCacheManager() const = 0; + + //! \brief Get the batch size that can fill the kv cache to the maximum capacity give the sequence length + //! \param seqLen The sequence length + //! \return The batch size that can fill the kv cache to the maximum capacity. If unsuporrted, return 0. + [[nodiscard]] virtual SizeType32 getMaxCapacityBatchSize(SizeType32 inputLength, SizeType32 outputLength) const = 0; +}; + +} // namespace tensorrt_llm::executor diff --git a/cpp/tensorrt_llm/executor/requestImpl.h b/cpp/tensorrt_llm/executor/requestImpl.h index 145336f77f9a..cd0a03cbb349 100644 --- a/cpp/tensorrt_llm/executor/requestImpl.h +++ b/cpp/tensorrt_llm/executor/requestImpl.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -515,7 +515,12 @@ class Request::Impl private: void validate() { - TLLM_CHECK(!mInputTokenIds.empty()); + if (mInputTokenIds.empty()) + { + TLLM_LOG_WARNING( + "Request created with empty inputTokenIds; expected only on empty Helix CP ranks when num_total_blocks " + "< cp_size."); + } TLLM_CHECK(mMaxNewTokens > 0); // Show warning message unless mNumReturnSequences is the default value. diff --git a/cpp/tensorrt_llm/executor/serialization.cpp b/cpp/tensorrt_llm/executor/serialization.cpp index 02077b6857ad..020306e03e56 100644 --- a/cpp/tensorrt_llm/executor/serialization.cpp +++ b/cpp/tensorrt_llm/executor/serialization.cpp @@ -17,7 +17,6 @@ #include "tensorrt_llm/executor/serialization.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/executor/requestImpl.h" @@ -627,8 +626,8 @@ kv_cache::CacheState Serialization::deserializeCacheState(std::istream& is) auto hasRnnConfig = su::deserialize<bool>(is); std::optional<CacheState::RnnModelConfig> rnnModelConfig; std::vector<SizeType32> rnnLayerNumPerPP; - tensorrt_llm::DataType convStateDataType{tensorrt_llm::DataType::kFLOAT}; - tensorrt_llm::DataType ssmStateDataType{tensorrt_llm::DataType::kFLOAT}; + nvinfer1::DataType convStateDataType{nvinfer1::DataType::kFLOAT}; + nvinfer1::DataType ssmStateDataType{nvinfer1::DataType::kFLOAT}; if (hasRnnConfig) { CacheState::RnnModelConfig rnnCfg; @@ -642,8 +641,8 @@ kv_cache::CacheState Serialization::deserializeCacheState(std::istream& is) rnnCfg.mNumHeads = su::deserialize<decltype(CacheState::RnnModelConfig::mNumHeads)>(is); rnnCfg.mConvSectionLayout = static_cast<CacheState::RnnModelConfig::ConvSectionLayout>(su::deserialize<SizeType32>(is)); - convStateDataType = su::deserialize<tensorrt_llm::DataType>(is); - ssmStateDataType = su::deserialize<tensorrt_llm::DataType>(is); + convStateDataType = su::deserialize<nvinfer1::DataType>(is); + ssmStateDataType = su::deserialize<nvinfer1::DataType>(is); rnnLayerNumPerPP = su::deserialize<std::vector<SizeType32>>(is); rnnModelConfig = std::move(rnnCfg); } @@ -765,8 +764,6 @@ DataTransceiverState Serialization::deserializeDataTransceiverState(std::istream { state.setCacheState(std::move(cacheState).value()); } - auto isArbitraryTransferState = su::deserialize<decltype(DataTransceiverState::mIsArbitraryTransferState)>(is); - state.setIsArbitraryTransferState(isArbitraryTransferState); return state; } @@ -774,7 +771,6 @@ void Serialization::serialize(DataTransceiverState const& state, std::ostream& o { su::serialize(state.mCommState, os); su::serialize(state.mCacheState, os); - su::serialize(state.mIsArbitraryTransferState, os); } std::vector<char> Serialization::serialize(DataTransceiverState const& state) @@ -793,7 +789,6 @@ size_t Serialization::serializedSize(DataTransceiverState const& state) size_t totalSize = 0; totalSize += su::serializedSize(state.mCommState); totalSize += su::serializedSize(state.mCacheState); - totalSize += su::serializedSize(state.mIsArbitraryTransferState); return totalSize; } diff --git a/cpp/tensorrt_llm/executor/tensor.cpp b/cpp/tensorrt_llm/executor/tensor.cpp index 9c508c0ec5c3..c38feb0e34b8 100644 --- a/cpp/tensorrt_llm/executor/tensor.cpp +++ b/cpp/tensorrt_llm/executor/tensor.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/executor/tensor.h" #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -54,17 +53,17 @@ DataType Tensor::getDataType() const } switch (mTensor->getDataType()) { - case tensorrt_llm::DataType::kBOOL: return DataType::kBOOL; - case tensorrt_llm::DataType::kINT8: return DataType::kINT8; - case tensorrt_llm::DataType::kINT32: return DataType::kINT32; - case tensorrt_llm::DataType::kUINT8: return DataType::kUINT8; - case tensorrt_llm::DataType::kFP8: return DataType::kFP8; - case tensorrt_llm::DataType::kHALF: return DataType::kFP16; - case tensorrt_llm::DataType::kFLOAT: return DataType::kFP32; - case tensorrt_llm::DataType::kBF16: return DataType::kBF16; - case tensorrt_llm::DataType::kINT64: return DataType::kINT64; - case tensorrt_llm::DataType::kINT4: [[fallthrough]] /* do nothing */; - case tensorrt_llm::DataType::kFP4: [[fallthrough]] /* do nothing */; + case nvinfer1::DataType::kBOOL: return DataType::kBOOL; + case nvinfer1::DataType::kINT8: return DataType::kINT8; + case nvinfer1::DataType::kINT32: return DataType::kINT32; + case nvinfer1::DataType::kUINT8: return DataType::kUINT8; + case nvinfer1::DataType::kFP8: return DataType::kFP8; + case nvinfer1::DataType::kHALF: return DataType::kFP16; + case nvinfer1::DataType::kFLOAT: return DataType::kFP32; + case nvinfer1::DataType::kBF16: return DataType::kBF16; + case nvinfer1::DataType::kINT64: return DataType::kINT64; + case nvinfer1::DataType::kINT4: [[fallthrough]] /* do nothing */; + case nvinfer1::DataType::kFP4: [[fallthrough]] /* do nothing */; default: TLLM_THROW("Unsupported data type"); } } @@ -136,19 +135,19 @@ tr::ITensor::Shape toDims(Shape const& shape) return dims; } -tensorrt_llm::DataType toDataType(DataType dataType) +nvinfer1::DataType toDataType(DataType dataType) { switch (dataType) { - case DataType::kBOOL: return tensorrt_llm::DataType::kBOOL; - case DataType::kUINT8: return tensorrt_llm::DataType::kUINT8; - case DataType::kINT8: return tensorrt_llm::DataType::kINT8; - case DataType::kINT32: return tensorrt_llm::DataType::kINT32; - case DataType::kINT64: return tensorrt_llm::DataType::kINT64; - case DataType::kBF16: return tensorrt_llm::DataType::kBF16; - case DataType::kFP8: return tensorrt_llm::DataType::kFP8; - case DataType::kFP16: return tensorrt_llm::DataType::kHALF; - case DataType::kFP32: return tensorrt_llm::DataType::kFLOAT; + case DataType::kBOOL: return nvinfer1::DataType::kBOOL; + case DataType::kUINT8: return nvinfer1::DataType::kUINT8; + case DataType::kINT8: return nvinfer1::DataType::kINT8; + case DataType::kINT32: return nvinfer1::DataType::kINT32; + case DataType::kINT64: return nvinfer1::DataType::kINT64; + case DataType::kBF16: return nvinfer1::DataType::kBF16; + case DataType::kFP8: return nvinfer1::DataType::kFP8; + case DataType::kFP16: return nvinfer1::DataType::kHALF; + case DataType::kFP32: return nvinfer1::DataType::kFLOAT; case DataType::kUNKNOWN: TLLM_THROW("Unsupported data type"); } diff --git a/cpp/tensorrt_llm/executor_worker/CMakeLists.txt b/cpp/tensorrt_llm/executor_worker/CMakeLists.txt new file mode 100644 index 000000000000..2feb6dfe5790 --- /dev/null +++ b/cpp/tensorrt_llm/executor_worker/CMakeLists.txt @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +set(SRCS executorWorker.cpp) + +include_directories(${PROJECT_SOURCE_DIR}/include) + +set(EXECUTOR_WORKER_TARGET executorWorker) + +add_executable(${EXECUTOR_WORKER_TARGET} ${SRCS}) + +target_link_libraries(${EXECUTOR_WORKER_TARGET} + PUBLIC ${SHARED_TARGET} nvinfer_plugin_tensorrt_llm) + +target_compile_features(${EXECUTOR_WORKER_TARGET} PRIVATE cxx_std_17) diff --git a/cpp/tensorrt_llm/executor_worker/executorWorker.cpp b/cpp/tensorrt_llm/executor_worker/executorWorker.cpp new file mode 100644 index 000000000000..aa1b06c2cb74 --- /dev/null +++ b/cpp/tensorrt_llm/executor_worker/executorWorker.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/serialization.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include <csignal> + +namespace tle = tensorrt_llm::executor; + +int main(int argc, char* argv[]) +{ +#if ENABLE_MULTI_DEVICE + + if (std::getenv("FORCE_NCCL_ALL_REDUCE_STRATEGY") != nullptr) + { + TLLM_LOG_INFO("FORCE_NCCL_ALL_REDUCE_STRATEGY env variable detected in worker"); + } + + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE, true); + + MPI_Comm parentComm; + MPI_Comm_get_parent(&parentComm); + if (parentComm == MPI_COMM_NULL) + { + TLLM_LOG_ERROR("TRT-LLM worker has no parent!"); + return -1; + } + + int size; + MPI_Comm_remote_size(parentComm, &size); + if (size != 1) + { + TLLM_LOG_ERROR("Parent size is %d, must be 1", size); + return -1; + } + + // Since parentComm is an intercommunicator, input root + // is the rank of the parent process in his group + // (always 0 as the parent size is checked before) + + // Receive from the parent the executor configuration + int64_t bufferSize; + MPICHECK(MPI_Bcast(&bufferSize, 1, MPI_INT64_T, 0, parentComm)); + std::vector<char> buffer(bufferSize); + MPICHECK(MPI_Bcast(buffer.data(), bufferSize, MPI_CHAR, 0, parentComm)); + std::istringstream is(std::string(buffer.begin(), buffer.end())); + auto modelPath = tle::Serialization::deserializeString(is); + auto modelType = tle::Serialization::deserializeModelType(is); + auto executorConfig = tle::Serialization::deserializeExecutorConfig(is); + + // Create the orchestrator config for workers + auto orchLeaderComm = std::make_shared<tensorrt_llm::mpi::MpiComm>(parentComm, true); + auto parallelConfig = executorConfig.getParallelConfig(); + TLLM_CHECK_WITH_INFO(parallelConfig.has_value(), "Parallel config should have a value."); + TLLM_CHECK_WITH_INFO( + parallelConfig.value().getOrchestratorConfig().has_value(), "Orchestrator config should have a value."); + auto orchConfig = parallelConfig.value().getOrchestratorConfig().value(); + TLLM_CHECK_WITH_INFO(parallelConfig.has_value(), "Parallel config should have a value."); + auto newOrchConfig = tle::OrchestratorConfig(false, orchConfig.getWorkerExecutablePath(), orchLeaderComm); + parallelConfig.value().setOrchestratorConfig(newOrchConfig); + executorConfig.setParallelConfig(parallelConfig.value()); + // In orchestrator mode, the spawned threads will wait for termination signal from orchestrator + auto executor = tle::Executor(modelPath, modelType, executorConfig); + + // Wait for all workers to have created their instances + MPI_Barrier(parentComm); + TLLM_LOG_INFO("Executor instance created by worker"); + +#endif // ENABLE_MULTI_DEVICE + + return 0; +} diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.cu b/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.cu index cf0ad7040f59..5be8b1c2ff78 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.cu @@ -16,7 +16,6 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h" #include "tensorrt_llm/kernels/quantization.cuh" #include <cooperative_groups.h> @@ -817,7 +816,7 @@ void dispatch_dtype(MiniMaxReduceRMSParams const& params) bool use_float4 = (params.allreduce_in_k != nullptr) && (params.hidden_dim * params.nranks == 6144) && (params.hidden_dim_k * params.nranks == 1024); - if (params.dtype == tensorrt_llm::DataType::kHALF) + if (params.dtype == nvinfer1::DataType::kHALF) { if (use_float4) { @@ -828,7 +827,7 @@ void dispatch_dtype(MiniMaxReduceRMSParams const& params) minimax_reduce_rms_kernel_launcher<half, NRanks>(params); } } - else if (params.dtype == tensorrt_llm::DataType::kBF16) + else if (params.dtype == nvinfer1::DataType::kBF16) { if (use_float4) { @@ -839,7 +838,7 @@ void dispatch_dtype(MiniMaxReduceRMSParams const& params) minimax_reduce_rms_kernel_launcher<__nv_bfloat16, NRanks>(params); } } - else if (params.dtype == tensorrt_llm::DataType::kFLOAT) + else if (params.dtype == nvinfer1::DataType::kFLOAT) { if (use_float4) { diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h b/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h index bf5775f96de9..b0cfd0ca074c 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h @@ -15,7 +15,7 @@ */ #pragma once #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -59,7 +59,7 @@ struct MiniMaxReduceRMSParams { int nranks{}; int rank{}; - tensorrt_llm::DataType dtype; + nvinfer1::DataType dtype; int size_q{}; // numel of Q (num_token * head_dim_q) int hidden_dim{}; // head_dim_q int size_k{}; // numel of K (num_token * head_dim_k) diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu index d9fbc9da0a0c..5a3edda04a70 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.cu @@ -16,7 +16,6 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" #include "tensorrt_llm/kernels/quantization.cuh" #include <cooperative_groups.h> @@ -796,15 +795,15 @@ void allreduce_fusion_op(AllReduceFusionParams const& params) } #define DISPATCH_DTYPE(NRanks) \ - if (params.dtype == tensorrt_llm::DataType::kHALF) \ + if (params.dtype == nvinfer1::DataType::kHALF) \ { \ DISPATCH_PATTERN(half, NRanks); \ } \ - else if (params.dtype == tensorrt_llm::DataType::kBF16) \ + else if (params.dtype == nvinfer1::DataType::kBF16) \ { \ DISPATCH_PATTERN(__nv_bfloat16, NRanks); \ } \ - else if (params.dtype == tensorrt_llm::DataType::kFLOAT) \ + else if (params.dtype == nvinfer1::DataType::kFLOAT) \ { \ DISPATCH_PATTERN(float, NRanks); \ } \ diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h index 769776273ef6..6d2074a6589e 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h @@ -16,7 +16,7 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -124,7 +124,7 @@ struct AllReduceFusionParams { int nranks; int rank; - tensorrt_llm::DataType dtype; + nvinfer1::DataType dtype; int size; int hidden_dim; void** workspace; diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.cu index 09a742494a9a..f1d5c08bda6b 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.cu @@ -21,7 +21,6 @@ #include "tensorrt_llm/common/customAllReduceUtils.h" #include "tensorrt_llm/common/dataType.h" #include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h" #include <cooperative_groups.h> #include <tuple> @@ -1358,7 +1357,7 @@ std::vector<size_t> splitNumber(size_t number) } LowPrecisionAllReduceParams LowPrecisionAllReduceParams::deserialize( - size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size) + size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size) { // Get appropriate static buffer @@ -1402,7 +1401,7 @@ LowPrecisionAllReduceParams LowPrecisionAllReduceParams::deserialize( } LowPrecisionAllReduceParams LowPrecisionAllReduceParams::deserialize_hier( - size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size) + size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size) { // Get appropriate static buffer @@ -1617,7 +1616,7 @@ int32_t max_workspace_size_lowprecision(int32_t tp_size) } void customLowPrecisionAllReduce( - kernels::LowPrecisionAllReduceParams& params, tensorrt_llm::DataType dataType, cudaStream_t stream) + kernels::LowPrecisionAllReduceParams& params, nvinfer1::DataType dataType, cudaStream_t stream) { TLLM_CHECK_WITH_INFO(lowPrecisionConfigurationSupported(params.ranks_per_node, params.elts_total), "Low Precision Custom all-reduce configuration unsupported"); @@ -1626,10 +1625,10 @@ void customLowPrecisionAllReduce( switch (dataType) { - case tensorrt_llm::DataType::kFLOAT: lowPrecisionAllReduceDispatchType<float>(params, stream); break; - case tensorrt_llm::DataType::kHALF: lowPrecisionAllReduceDispatchType<half>(params, stream); break; + case nvinfer1::DataType::kFLOAT: lowPrecisionAllReduceDispatchType<float>(params, stream); break; + case nvinfer1::DataType::kHALF: lowPrecisionAllReduceDispatchType<half>(params, stream); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: lowPrecisionAllReduceDispatchType<__nv_bfloat16>(params, stream); break; + case nvinfer1::DataType::kBF16: lowPrecisionAllReduceDispatchType<__nv_bfloat16>(params, stream); break; #endif default: TLLM_THROW("Unsupported dataType for customAllReduce"); } diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h index 62d19039cc4e..5fc87ef1a523 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h @@ -19,8 +19,8 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/customAllReduceKernels.h" +#include <NvInferRuntime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> #include <vector> @@ -111,15 +111,15 @@ struct LowPrecisionAllReduceParams uint64_t* ag_notify_peer_inside_numa_flags[LP_ALLREDUCE_MAX_BLOCKS * 4]; // 3*flags , 3 is other rank inside numa static LowPrecisionAllReduceParams deserialize( - size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size); + size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size); static LowPrecisionAllReduceParams deserialize_hier( - size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, int token_num, int hidden_size); + size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size); }; bool lowPrecisionConfigurationSupported(size_t msg_size, size_t n_ranks); void customLowPrecisionAllReduce( - kernels::LowPrecisionAllReduceParams& params, tensorrt_llm::DataType dataType, cudaStream_t stream); + kernels::LowPrecisionAllReduceParams& params, nvinfer1::DataType dataType, cudaStream_t stream); int32_t max_workspace_size_lowprecision(int32_t tp_size); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.cu index a8ff3dd431a8..eb44f1638a19 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.cu @@ -33,7 +33,6 @@ #include "tensorrt_llm/common/lamportUtils.cuh" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/quantization.cuh" TRTLLM_NAMESPACE_BEGIN @@ -857,9 +856,9 @@ void oneshotAllreduceFusionOp(AllReduceFusionParams const& params) }; #undef LAUNCH_ALLREDUCE_KERNEL #undef DISPATCH_ALLREDUCE_PATTERN - bool launched = (params.dType == tensorrt_llm::DataType::kBF16 && dispatchImpl((__nv_bfloat16*) nullptr)) - || (params.dType == tensorrt_llm::DataType::kFLOAT && dispatchImpl((float*) nullptr)) - || (params.dType == tensorrt_llm::DataType::kHALF && dispatchImpl((__nv_half*) nullptr)); + bool launched = (params.dType == nvinfer1::DataType::kBF16 && dispatchImpl((__nv_bfloat16*) nullptr)) + || (params.dType == nvinfer1::DataType::kFLOAT && dispatchImpl((float*) nullptr)) + || (params.dType == nvinfer1::DataType::kHALF && dispatchImpl((__nv_half*) nullptr)); if (!launched) { TLLM_CHECK_WITH_INFO(false, "Failed to dispatch MNNVL AllReduceOneShot kernel."); @@ -1247,9 +1246,9 @@ void twoshotAllreduceFusionOp(AllReduceFusionParams const& params) #undef LAUNCH_ALLREDUCE_KERNEL - bool launched = (params.dType == tensorrt_llm::DataType::kFLOAT && dispatchAR((float*) nullptr)) - || (params.dType == tensorrt_llm::DataType::kBF16 && dispatchAR((__nv_bfloat16*) nullptr)) - || (params.dType == tensorrt_llm::DataType::kHALF && dispatchAR((__nv_half*) nullptr)); + bool launched = (params.dType == nvinfer1::DataType::kFLOAT && dispatchAR((float*) nullptr)) + || (params.dType == nvinfer1::DataType::kBF16 && dispatchAR((__nv_bfloat16*) nullptr)) + || (params.dType == nvinfer1::DataType::kHALF && dispatchAR((__nv_half*) nullptr)); if (!launched) { TLLM_CHECK_WITH_INFO(false, "[MNNVL AllReduceTwoShot] Failed to dispatch twoshotAllreduce kernel."); @@ -1389,9 +1388,9 @@ void twoshotAllreduceFusionOp(AllReduceFusionParams const& params) return true; }; - launched = (params.dType == tensorrt_llm::DataType::kFLOAT && dispatchRN((float*) nullptr)) - || (params.dType == tensorrt_llm::DataType::kBF16 && dispatchRN((__nv_bfloat16*) nullptr)) - || (params.dType == tensorrt_llm::DataType::kHALF && dispatchRN((__nv_half*) nullptr)); + launched = (params.dType == nvinfer1::DataType::kFLOAT && dispatchRN((float*) nullptr)) + || (params.dType == nvinfer1::DataType::kBF16 && dispatchRN((__nv_bfloat16*) nullptr)) + || (params.dType == nvinfer1::DataType::kHALF && dispatchRN((__nv_half*) nullptr)); if (!launched) { TLLM_CHECK_WITH_INFO(false, "[MNNVL AllReduceTwoShot] Failed to dispatch rmsnorm lamport kernel."); diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.h index f2006f52240c..2a228e815b8d 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/mnnvlAllreduceKernels.h @@ -17,8 +17,8 @@ #define TRTLLM_MNNVL_ALLREDUCE_KERNELS_H #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" +#include <NvInferRuntime.h> #include <cstdint> TRTLLM_NAMESPACE_BEGIN @@ -39,16 +39,16 @@ struct AllReduceFusionParams //! \name Environmental and Auxiliary Data //! @{ - int nRanks; //!< Total number of participating ranks in the AllReduce operation - int rank; //!< Current rank ID - tensorrt_llm::DataType dType; //!< Data type of the tensors (e.g., FP16, BF16, FP32) - int numTokens; //!< Number of tokens in the input tensor - int tokenDim; //!< Hidden Dimension - void** bufferPtrsDev; //!< Unicast Device pointers to communication buffers for each rank - void* bufferPtrLocal; //!< Local buffer pointer for temporary storage (i.e., bufferPtrsDev[rank]) - void* multicastPtr; //!< Multicast buffer pointer. - uint32_t* bufferFlags; //!< Synchronization flags for coordinating communication phases - bool rmsNormFusion; //!< Whether to fuse RMS normalization with the AllReduce operation + int nRanks; //!< Total number of participating ranks in the AllReduce operation + int rank; //!< Current rank ID + nvinfer1::DataType dType; //!< Data type of the tensors (e.g., FP16, BF16, FP32) + int numTokens; //!< Number of tokens in the input tensor + int tokenDim; //!< Hidden Dimension + void** bufferPtrsDev; //!< Unicast Device pointers to communication buffers for each rank + void* bufferPtrLocal; //!< Local buffer pointer for temporary storage (i.e., bufferPtrsDev[rank]) + void* multicastPtr; //!< Multicast buffer pointer. + uint32_t* bufferFlags; //!< Synchronization flags for coordinating communication phases + bool rmsNormFusion; //!< Whether to fuse RMS normalization with the AllReduce operation ar_fusion::AllReduceFusionPattern pattern = ar_fusion::AllReduceFusionPattern::kAllReduce; //!< Fused epilogue pattern diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.cu index 5b0da0d0e80c..306d42677e2f 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.cu @@ -16,7 +16,6 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h" #include "tensorrt_llm/kernels/quantization.cuh" #include <cooperative_groups.h> @@ -443,11 +442,11 @@ void moereduction_allreduce_fusion_op(MoeReductionAllReduceFusionParams const& p #define MOE_DISPATCH1(DTYPE, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT) \ return moereduction_allreduce_fusion_kernel_launcher<DTYPE, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT>(params); #define MOE_DISPATCH0(NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT) \ - if (params.nranks == NRANKS && params.dtype == tensorrt_llm::DataType::kHALF) \ + if (params.nranks == NRANKS && params.dtype == nvinfer1::DataType::kHALF) \ { \ MOE_DISPATCH1(half, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT); \ } \ - else if (params.nranks == NRANKS && params.dtype == tensorrt_llm::DataType::kBF16) \ + else if (params.nranks == NRANKS && params.dtype == nvinfer1::DataType::kBF16) \ { \ MOE_DISPATCH1(__nv_bfloat16, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT); \ } @@ -557,20 +556,11 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport(MoeFinalizeA } // * MoE finalize - // Accumulate the top-k weighted expert sum and the shared-expert add in - // fp32 (local `facc`), rounding to DType (bf16/fp16) only once when packing - // into `accumulator` for the 128-bit Lamport all-reduce store below. - // Accumulating directly in DType here rounds after every one of the top_k - // terms; across the many routed MoE layers this rounding bias is large - // enough to visibly degrade the routed output, and with attention-DP - // disabled + MTP speculative decoding it drifts the target hidden states - // enough to lower the acceptance length. The non-deferred in-kernel - // finalize (do_finalize=true) already accumulates in fp32; match it here. - float facc[kElemsPerAccess]; + ACC_TYPE accumulator; #pragma unroll for (int i = 0; i < kElemsPerAccess; ++i) { - facc[i] = 0.f; + accumulator.unpacked[i] = static_cast<DType>(0); } for (int k = 0; k < top_k; k++) @@ -592,15 +582,17 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport(MoeFinalizeA permuted_data.packed = reinterpret_cast<float4 const*>(params.allreduce_in)[thread_offset_across_token / kElemsPerAccess]; - // * acc += scale(data) (fp32 accumulation) + // * acc += scale(data) #pragma unroll for (int i = 0; i < kElemsPerAccess; ++i) { - facc[i] += static_cast<float>(permuted_data.unpacked[i]) * block_scale; + // assume computation is done in ScaleType + accumulator.unpacked[i] + += static_cast<DType>((static_cast<float>(permuted_data.unpacked[i]) * block_scale)); } } - // * Add shared expert output (fp32 accumulation) + // * Add shared expert output if (params.shared_expert_output) { // * Load shared expert output @@ -611,18 +603,10 @@ __global__ void moefinalize_allreduce_fusion_kernel_oneshot_lamport(MoeFinalizeA #pragma unroll for (int i = 0; i < kElemsPerAccess; ++i) { - facc[i] += static_cast<float>(shared_expert_output.unpacked[i]); + accumulator.unpacked[i] += shared_expert_output.unpacked[i]; } } - // Round the fp32 accumulator to DType once, packed for the Lamport AR store. - ACC_TYPE accumulator; -#pragma unroll - for (int i = 0; i < kElemsPerAccess; ++i) - { - accumulator.unpacked[i] = static_cast<DType>(facc[i]); - } - // * AR Store int access_id = token_id * params.hidden_dim / kElemsPerAccess + access_id_in_token; int idx = access_id; @@ -743,13 +727,13 @@ void moefinalize_allreduce_fusion_op(MoeFinalizeAllReduceFusionParams const& par #define MOE_FINALIZE_DISPATCH1(DTYPE, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT) \ return moefinalize_allreduce_fusion_kernel_launcher<DTYPE, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT>(params); #define MOE_FINALIZE_DISPATCH0(NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT) \ - if (params.nranks == NRANKS && params.dtype == tensorrt_llm::DataType::kHALF \ - && params.scale_dtype == tensorrt_llm::DataType::kHALF) \ + if (params.nranks == NRANKS && params.dtype == nvinfer1::DataType::kHALF \ + && params.scale_dtype == nvinfer1::DataType::kHALF) \ { \ MOE_FINALIZE_DISPATCH1(half, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT); \ } \ - else if (params.nranks == NRANKS && params.dtype == tensorrt_llm::DataType::kBF16 \ - && params.scale_dtype == tensorrt_llm::DataType::kBF16) \ + else if (params.nranks == NRANKS && params.dtype == nvinfer1::DataType::kBF16 \ + && params.scale_dtype == nvinfer1::DataType::kBF16) \ { \ MOE_FINALIZE_DISPATCH1(__nv_bfloat16, NRANKS, RESIDUAL_OUT, NORM_OUT, QUANT_OUT); \ } diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h index e526a70268b3..556dd4e5cd24 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h @@ -16,7 +16,7 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -44,7 +44,7 @@ struct AllReduceFusionParams { int nranks; int rank; - tensorrt_llm::DataType dtype; + nvinfer1::DataType dtype; // size = token_num * hidden_dim int size; int hidden_dim; @@ -94,7 +94,7 @@ struct MoeFinalizeAllReduceFusionParams : public AllReduceFusionParams // Refer to kernel implementation on layout of those params // number of active experts on current device int top_k; - tensorrt_llm::DataType scale_dtype; + nvinfer1::DataType scale_dtype; // [num_tokens, top_k] void* expert_scale_factor = nullptr; void* shared_expert_output = nullptr; diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu index 74e40dbb2b81..472a5877a80d 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.cu @@ -17,7 +17,6 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/vec_dtypes.cuh" #include "tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h" #include "tensorrt_llm/kernels/quantization.cuh" @@ -130,25 +129,25 @@ using tensorrt_llm::common::launchWithPdlWhenEnabled; #define SWITCH_DTYPE(dtype, TYPE, ...) \ switch (dtype) \ { \ - case tensorrt_llm::DataType::kHALF: \ + case nvinfer1::DataType::kHALF: \ { \ using TYPE = half; \ __VA_ARGS__; \ break; \ } \ - case tensorrt_llm::DataType::kBF16: \ + case nvinfer1::DataType::kBF16: \ { \ using TYPE = __nv_bfloat16; \ __VA_ARGS__; \ break; \ } \ - case tensorrt_llm::DataType::kFLOAT: \ + case nvinfer1::DataType::kFLOAT: \ { \ using TYPE = float; \ __VA_ARGS__; \ break; \ } \ - case tensorrt_llm::DataType::kFP8: \ + case nvinfer1::DataType::kFP8: \ { \ using TYPE = __nv_fp8_e4m3; \ __VA_ARGS__; \ @@ -1404,7 +1403,7 @@ void moe_a2a_combine_launch(MoeA2ACombineParams const& params) // When use_low_precision is set the recv buffers contain FP8 data regardless of params.dtype, // so dispatch the FP8 accumulation kernel in that case. - auto const effective_dtype = params.use_low_precision ? tensorrt_llm::DataType::kFP8 : params.dtype; + auto const effective_dtype = params.use_low_precision ? nvinfer1::DataType::kFP8 : params.dtype; // Launch appropriate kernel with compact macros SWITCH_BOOL(params.enable_rank_mask, ENABLE_RANK_MASK, { diff --git a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h index 5184878ffc51..177293684874 100644 --- a/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h +++ b/cpp/tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h @@ -16,7 +16,7 @@ #pragma once #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> @@ -174,8 +174,8 @@ struct MoeA2ACombineParams // Output tensor void* output_data; // Output buffer [local_num_tokens, elements_per_token] // Payload information - int elements_per_token; // Number of elements per token - tensorrt_llm::DataType dtype; // Data type of the payload (used for combine kernel dispatch) + int elements_per_token; // Number of elements per token + nvinfer1::DataType dtype; // Data type of the payload (used for combine kernel dispatch) bool use_low_precision; // If true, prepare kernel quantizes payload→FP8; combine kernel accumulates FP8→output dtype diff --git a/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.cu b/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.cu index 8af43b2b4914..81e947977797 100644 --- a/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.cu +++ b/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.cu @@ -23,7 +23,6 @@ #include "cutlass/cutlass.h" #include "cutlass/gemm/device/gemm_grouped.h" #include "cutlass/gemm/kernel/default_gemm_grouped.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/splitk_gemm_grouped.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/default_splitk_gemm_grouped.h" @@ -118,16 +117,16 @@ void cudaGraphGroupedGemmTemplate(cutlass::gemm::GemmCoord* problemSizesPtr, int template <int M1, int N1, int K1, int M2, int N2, int K2, int kAlignmentAB, int kAlignmentC, int kStages> void cudaGraphGroupedGemmType(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, - tensorrt_llm::DataType dataType, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream) + nvinfer1::DataType dataType, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream) { - if (dataType == tensorrt_llm::DataType::kHALF) + if (dataType == nvinfer1::DataType::kHALF) { cudaGraphGroupedGemmTemplate<M1, N1, K1, M2, N2, K2, cutlass::half_t, kAlignmentAB, kAlignmentC, kStages>( problemSizesPtr, problemCount, ptrAGpu, ptrBGpu, ptrCGpu, ptrDGpu, ldaGpu, ldbGpu, ldcGpu, lddGpu, hostMaxProblemSizesPtr, stream); } #ifdef ENABLE_BF16 - else if (dataType == tensorrt_llm::DataType::kBF16) + else if (dataType == nvinfer1::DataType::kBF16) { cudaGraphGroupedGemmTemplate<M1, N1, K1, M2, N2, K2, cutlass::bfloat16_t, kAlignmentAB, kAlignmentC, kStages>( problemSizesPtr, problemCount, ptrAGpu, ptrBGpu, ptrCGpu, ptrDGpu, ldaGpu, ldbGpu, ldcGpu, lddGpu, @@ -142,7 +141,7 @@ void cudaGraphGroupedGemmType(cutlass::gemm::GemmCoord* problemSizesPtr, int pro void cudaGraphGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, bool isLoraIn, - tensorrt_llm::DataType dataType, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream) + nvinfer1::DataType dataType, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream) { if (isLoraIn) { @@ -284,17 +283,17 @@ void cudaGraphSplitKGroupedGemmTemplate(cutlass::gemm::GemmCoord* problemSizesPt template <int M1, int N1, int K1, int M2, int N2, int K2, int kAlignmentAB, int kAlignmentC, int kStages> void cudaGraphSplitKGroupedGemmType(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, - tensorrt_llm::DataType dataType, int splitKSlices, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, + nvinfer1::DataType dataType, int splitKSlices, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, int64_t* splitKOffsetsGpu, cudaStream_t stream) { - if (dataType == tensorrt_llm::DataType::kHALF) + if (dataType == nvinfer1::DataType::kHALF) { cudaGraphSplitKGroupedGemmTemplate<M1, N1, K1, M2, N2, K2, cutlass::half_t, kAlignmentAB, kAlignmentC, kStages>( problemSizesPtr, problemCount, ptrAGpu, ptrBGpu, ptrCGpu, ptrDGpu, ldaGpu, ldbGpu, ldcGpu, lddGpu, splitKSlices, hostMaxProblemSizesPtr, splitKOffsetsGpu, stream); } #ifdef ENABLE_BF16 - else if (dataType == tensorrt_llm::DataType::kBF16) + else if (dataType == nvinfer1::DataType::kBF16) { cudaGraphSplitKGroupedGemmTemplate<M1, N1, K1, M2, N2, K2, cutlass::bfloat16_t, kAlignmentAB, kAlignmentC, kStages>(problemSizesPtr, problemCount, ptrAGpu, ptrBGpu, ptrCGpu, ptrDGpu, ldaGpu, ldbGpu, ldcGpu, lddGpu, @@ -309,7 +308,7 @@ void cudaGraphSplitKGroupedGemmType(cutlass::gemm::GemmCoord* problemSizesPtr, i void cudaGraphSplitKGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, - bool isLoraIn, tensorrt_llm::DataType dataType, int splitKSlices, int minKN, + bool isLoraIn, nvinfer1::DataType dataType, int splitKSlices, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, int64_t* splitKOffsetsGpu, cudaStream_t stream) { if (isLoraIn) diff --git a/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.h b/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.h index b447bba3a785..0eecccb78852 100644 --- a/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.h +++ b/cpp/tensorrt_llm/kernels/cuda_graph_grouped_gemm.h @@ -18,7 +18,7 @@ #include "cutlass/gemm_coord.h" #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_runtime.h> TRTLLM_NAMESPACE_BEGIN @@ -45,7 +45,7 @@ namespace kernels */ void cudaGraphGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, bool isLoraIn, - tensorrt_llm::DataType dataType, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream); + nvinfer1::DataType dataType, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, cudaStream_t stream); /** * @brief CUDA Graph compatible wrapper for split-K grouped GEMM operations. @@ -55,7 +55,7 @@ void cudaGraphGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problem */ void cudaGraphSplitKGroupedGemm(cutlass::gemm::GemmCoord* problemSizesPtr, int problemCount, void** ptrAGpu, void** ptrBGpu, void** ptrCGpu, void** ptrDGpu, int64_t* ldaGpu, int64_t* ldbGpu, int64_t* ldcGpu, int64_t* lddGpu, - bool isLoraIn, tensorrt_llm::DataType dataType, int splitKSlices, int minKN, + bool isLoraIn, nvinfer1::DataType dataType, int splitKSlices, int minKN, cutlass::gemm::GemmCoord* hostMaxProblemSizesPtr, int64_t* splitKOffsetsGpu, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu b/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu index ea217d465be1..9cf2b51eb583 100644 --- a/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu +++ b/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu @@ -22,7 +22,6 @@ #include "tensorrt_llm/common/customAllReduceUtils.h" #include "tensorrt_llm/common/dataType.h" #include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include <cooperative_groups.h> #include <cstdint> #include <tuple> @@ -1188,14 +1187,14 @@ bool is_lamport_supported(int token_num, int hidden_size) return true; } -bool is_lamport_supported(tensorrt_llm::DataType dataType, int token_num, int hidden_size) +bool is_lamport_supported(nvinfer1::DataType dataType, int token_num, int hidden_size) { switch (dataType) { - case tensorrt_llm::DataType::kFLOAT: return is_lamport_supported<float>(token_num, hidden_size); - case tensorrt_llm::DataType::kHALF: return is_lamport_supported<half>(token_num, hidden_size); + case nvinfer1::DataType::kFLOAT: return is_lamport_supported<float>(token_num, hidden_size); + case nvinfer1::DataType::kHALF: return is_lamport_supported<half>(token_num, hidden_size); #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: return is_lamport_supported<__nv_bfloat16>(token_num, hidden_size); + case nvinfer1::DataType::kBF16: return is_lamport_supported<__nv_bfloat16>(token_num, hidden_size); #endif default: return false; } @@ -1659,7 +1658,7 @@ static __global__ void __launch_bounds__(512, 1) twoShotAllReduceKernel(AllReduc update_barrier_flag(params.barrier_flag_ptr, params.barrier_flag_counter_ptr); } -bool configurationSupported(AllReduceStrategyType algo, size_t msg_size, size_t n_ranks, tensorrt_llm::DataType type) +bool configurationSupported(AllReduceStrategyType algo, size_t msg_size, size_t n_ranks, nvinfer1::DataType type) { size_t elts_per_thread = 16 / common::getDTypeSize(type); int const msg_align = (algo == AllReduceStrategyType::TWOSHOT) ? n_ranks * elts_per_thread : elts_per_thread; @@ -1895,8 +1894,8 @@ void AllReduceDispatchType(AllReduceParams& params, AllReduceStrategyType strat, } } -AllReduceParams AllReduceParams::deserialize(int64_t* buffer, size_t tpSize, size_t tpRank, - tensorrt_llm::DataType dataType, int token_num, int hidden_size, AllReduceFusionOp op) +AllReduceParams AllReduceParams::deserialize(int64_t* buffer, size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, + int token_num, int hidden_size, AllReduceFusionOp op) { void* const* buffer_ptrs = reinterpret_cast<void* const*>(buffer); int flag_offset; @@ -1934,7 +1933,7 @@ AllReduceParams AllReduceParams::deserialize(int64_t* buffer, size_t tpSize, siz return params; } -void customAllReduce(kernels::AllReduceParams& params, tensorrt_llm::DataType dataType, AllReduceStrategyType strat, +void customAllReduce(kernels::AllReduceParams& params, nvinfer1::DataType dataType, AllReduceStrategyType strat, AllReduceStrategyConfig config, AllReduceFusionOp fusionOp, cudaStream_t stream) { TLLM_CHECK_WITH_INFO(configurationSupported(strat, params.elts_total, params.ranks_per_node, dataType), @@ -1944,10 +1943,10 @@ void customAllReduce(kernels::AllReduceParams& params, tensorrt_llm::DataType da switch (dataType) { - case tensorrt_llm::DataType::kFLOAT: AllReduceDispatchType<float>(params, strat, config, fusionOp, stream); break; - case tensorrt_llm::DataType::kHALF: AllReduceDispatchType<half>(params, strat, config, fusionOp, stream); break; + case nvinfer1::DataType::kFLOAT: AllReduceDispatchType<float>(params, strat, config, fusionOp, stream); break; + case nvinfer1::DataType::kHALF: AllReduceDispatchType<half>(params, strat, config, fusionOp, stream); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: AllReduceDispatchType<__nv_bfloat16>(params, strat, config, fusionOp, stream); break; #endif @@ -1992,22 +1991,22 @@ void launchResidualRmsNormKernel(kernels::AllReduceParams& params, cudaStream_t } void residualRmsNorm( - kernels::AllReduceParams& params, tensorrt_llm::DataType dataType, cudaStream_t stream, AllReduceFusionOp fusionOp) + kernels::AllReduceParams& params, nvinfer1::DataType dataType, cudaStream_t stream, AllReduceFusionOp fusionOp) { sync_check_cuda_error(stream); switch (dataType) { - case tensorrt_llm::DataType::kFLOAT: launchResidualRmsNormKernel<float>(params, stream, fusionOp); break; - case tensorrt_llm::DataType::kHALF: launchResidualRmsNormKernel<half>(params, stream, fusionOp); break; + case nvinfer1::DataType::kFLOAT: launchResidualRmsNormKernel<float>(params, stream, fusionOp); break; + case nvinfer1::DataType::kHALF: launchResidualRmsNormKernel<half>(params, stream, fusionOp); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: launchResidualRmsNormKernel<__nv_bfloat16>(params, stream, fusionOp); break; + case nvinfer1::DataType::kBF16: launchResidualRmsNormKernel<__nv_bfloat16>(params, stream, fusionOp); break; #endif default: TLLM_THROW("Unsupported dataType for customAllReduce"); } sync_check_cuda_error(stream); } -void lamportInitialize(void* buffer, size_t size, tensorrt_llm::DataType dataType, cudaStream_t stream) +void lamportInitialize(void* buffer, size_t size, nvinfer1::DataType dataType, cudaStream_t stream) { sync_check_cuda_error(stream); if (size == 0) @@ -2016,14 +2015,14 @@ void lamportInitialize(void* buffer, size_t size, tensorrt_llm::DataType dataTyp } switch (dataType) { - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: reduce_fusion::lamport_initialize_kernel_launcher<float>(buffer, size, stream); break; - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: reduce_fusion::lamport_initialize_kernel_launcher<half>(buffer, size, stream); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: reduce_fusion::lamport_initialize_kernel_launcher<__nv_bfloat16>(buffer, size, stream); break; #endif diff --git a/cpp/tensorrt_llm/kernels/customAllReduceKernels.h b/cpp/tensorrt_llm/kernels/customAllReduceKernels.h index 93f67ffdd911..f7151f1cd0ab 100644 --- a/cpp/tensorrt_llm/kernels/customAllReduceKernels.h +++ b/cpp/tensorrt_llm/kernels/customAllReduceKernels.h @@ -17,7 +17,7 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> #include <limits> @@ -178,23 +178,23 @@ struct AllReduceParams AllReduceFusionParams fusion_params; - static AllReduceParams deserialize(int64_t* buffer, size_t tpSize, size_t tpRank, tensorrt_llm::DataType dataType, + static AllReduceParams deserialize(int64_t* buffer, size_t tpSize, size_t tpRank, nvinfer1::DataType dataType, int token_num, int hidden_size, AllReduceFusionOp op); }; -bool configurationSupported(AllReduceStrategyType algo, size_t msg_size, size_t n_ranks, tensorrt_llm::DataType type); +bool configurationSupported(AllReduceStrategyType algo, size_t msg_size, size_t n_ranks, nvinfer1::DataType type); -void customAllReduce(kernels::AllReduceParams& params, tensorrt_llm::DataType dataType, AllReduceStrategyType strat, +void customAllReduce(kernels::AllReduceParams& params, nvinfer1::DataType dataType, AllReduceStrategyType strat, AllReduceStrategyConfig config, AllReduceFusionOp fusionOp, cudaStream_t stream); void residualRmsNorm( - kernels::AllReduceParams& params, tensorrt_llm::DataType dataType, cudaStream_t stream, AllReduceFusionOp fusionOp); + kernels::AllReduceParams& params, nvinfer1::DataType dataType, cudaStream_t stream, AllReduceFusionOp fusionOp); -void lamportInitialize(void* buffer, size_t size, tensorrt_llm::DataType dataType, cudaStream_t stream); +void lamportInitialize(void* buffer, size_t size, nvinfer1::DataType dataType, cudaStream_t stream); namespace reduce_fusion { -bool is_lamport_supported(tensorrt_llm::DataType dataType, int token_num, int hidden_size); +bool is_lamport_supported(nvinfer1::DataType dataType, int token_num, int hidden_size); } } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cpp b/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cpp index bcc0eb1165ac..7bba57a03d5e 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cpp +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,6 @@ #include "tensorrt_llm/kernels/cutlass_kernels/cutlass_heuristic.h" #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaBf16Wrapper.h" -#include "tensorrt_llm/common/cudaUtils.h" #ifdef __GNUC__ // Check if the compiler is GCC or Clang #pragma GCC diagnostic push @@ -32,7 +31,6 @@ #pragma GCC diagnostic pop #endif // __GNUC -#include <algorithm> #include <cuda_runtime_api.h> #include <set> #include <vector> @@ -575,7 +573,7 @@ std::vector<CutlassGemmConfig> get_candidate_configs_sm120(CutlassGemmConfig::Ca std::vector<CutlassGemmConfig> candidate_configs; if (config & CutlassGemmConfig::FP8FP4_MIXED) { - // Mixed FP8 x FP4 only supports the 128x128x128B tile. + // Mixed FP8 x FP4: restrict to 128x128x128B only candidate_configs.push_back(CutlassGemmConfig{CutlassTileConfigSM120::CtaShape128x128x128B, MainloopScheduleType::AUTO, EpilogueScheduleType::AUTO, ClusterShape::ClusterShape_1x1x1}); return candidate_configs; @@ -591,34 +589,9 @@ std::vector<CutlassGemmConfig> get_candidate_configs_sm120(CutlassGemmConfig::Ca MainloopScheduleType::AUTO, EpilogueScheduleType::AUTO, ClusterShape::ClusterShape_1x1x1}); candidate_configs.push_back(CutlassGemmConfig{CutlassTileConfigSM120::CtaShape256x128x64B, MainloopScheduleType::AUTO, EpilogueScheduleType::AUTO, ClusterShape::ClusterShape_1x1x1}); + return candidate_configs; } - else - { - TLLM_THROW("Not Implemented: SM120 group GEMM only supports mxfp8-mxfp4 mixed or nvfp4."); - } - // Filter configs by device shared memory. SM100 (B200) has 228 KiB, but - // consumer Blackwell (SM120 RTX PRO 6000, SM121 GB10 / DGX Spark) has only - // 99 KiB. On these constrained devices, keep only CtaShape128x128x64B which - // fits within 99 KiB including FINALIZE epilogue (~80 KiB total). - // CtaShape128x256x64B/256x128x64B overflow with FINALIZE (~100 KiB). - // CtaShape128x128x128B also exceeds 99 KiB at typical stage counts. - { - constexpr int kMinSmemForFullTileSet = 120 * 1024; - int device = 0; - tensorrt_llm::common::check_cuda_error(cudaGetDevice(&device)); - int maxSmem = 0; - tensorrt_llm::common::check_cuda_error( - cudaDeviceGetAttribute(&maxSmem, cudaDevAttrMaxSharedMemoryPerBlockOptin, device)); - - if (maxSmem < kMinSmemForFullTileSet) - { - auto const it = std::remove_if(candidate_configs.begin(), candidate_configs.end(), - [](CutlassGemmConfig const& config) - { return config.tile_config_sm120 != CutlassTileConfigSM120::CtaShape128x128x64B; }); - candidate_configs.erase(it, candidate_configs.end()); - } - } - return candidate_configs; + TLLM_THROW("Not Implemented: SM120 group GEMM only supports mxfp8-mxfp4 mixed or nvfp4."); } else { diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h index 6632f273cc35..dbbed4e08c97 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h @@ -17,7 +17,7 @@ #pragma once #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include "cutlass/half.h" #include <cuda_fp16.h> @@ -38,34 +38,34 @@ namespace kernels namespace cutlass_kernels { /////////////////////////////////////////////////////////////////////////////////////////////////// -// tensorrt_llm::DataType to Cutlass +// nvinfer1::DataType to Cutlass /////////////////////////////////////////////////////////////////////////////////////////////////// -template <tensorrt_llm::DataType> +template <nvinfer1::DataType> struct CutlassType { using type = void; }; template <> -struct CutlassType<tensorrt_llm::DataType::kHALF> +struct CutlassType<nvinfer1::DataType::kHALF> { using type = cutlass::half_t; }; template <> -struct CutlassType<tensorrt_llm::DataType::kBF16> +struct CutlassType<nvinfer1::DataType::kBF16> { using type = cutlass::bfloat16_t; }; template <> -struct CutlassType<tensorrt_llm::DataType::kFP8> +struct CutlassType<nvinfer1::DataType::kFP8> { using type = cutlass::float_e4m3_t; }; template <> -struct CutlassType<tensorrt_llm::DataType::kFP4> +struct CutlassType<nvinfer1::DataType::kFP4> { using type = cutlass::float_e2m1_t; }; diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.cu b/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.cu index adaf60ccc3a8..a4923fdbd072 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.cu +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.cu @@ -52,7 +52,7 @@ __device__ __forceinline__ float reciprocal_approximate_ftz_local(float a) // (8 lanes × 16 BF16 elems = 128 elems). After per-block amax, lanes // 0/8/16/24 each hold one UE8M0 scale byte; lane 0 packs them into a uint32 // and stores in the deep_gemm-expected MN-major layout. -template <int WarpsPerBlock, bool OutputCuteDslSf> +template <int WarpsPerBlock> __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict__ fp8_output, int32_t* __restrict__ packed_scale_output, __nv_bfloat16 const* __restrict__ input, int const m, int const k, int const scale_leading_dim_uint32) @@ -72,7 +72,6 @@ __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict_ bool const row_in_range = (m_idx < m); uint32_t packed = 0u; - uint32_t scale_byte = 0u; if (row_in_range) { int const k_base = packed_sf_k_idx * 512 + lane_id * 16; @@ -121,7 +120,6 @@ __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict_ float const dequant_scale_raw = amax * reciprocal_approximate_ftz_local(448.0f); __nv_fp8_e8m0 ue8m0_scale; ue8m0_scale.__x = __nv_cvt_float_to_e8m0(dequant_scale_raw, __NV_SATFINITE, cudaRoundPosInf); - scale_byte = static_cast<uint32_t>(ue8m0_scale.__x); // Recover quant_scale = 1 / 2^(exp - 127) for fp8 conversion. constexpr uint32_t FP32_EXPONENT_BIAS = 127u; @@ -164,58 +162,32 @@ __global__ void fp8_quantize_1x128_packed_kernel_impl(__nv_fp8_e4m3* __restrict_ } // ---- 5. Pack 4 UE8M0 scales (lanes 0/8/16/24). ---- - if constexpr (!OutputCuteDslSf) + uint32_t const s0 = __shfl_sync(0xFFFFFFFFu, static_cast<uint32_t>(ue8m0_scale.__x), 0); + uint32_t const s1 = __shfl_sync(0xFFFFFFFFu, static_cast<uint32_t>(ue8m0_scale.__x), 8); + uint32_t const s2 = __shfl_sync(0xFFFFFFFFu, static_cast<uint32_t>(ue8m0_scale.__x), 16); + uint32_t const s3 = __shfl_sync(0xFFFFFFFFu, static_cast<uint32_t>(ue8m0_scale.__x), 24); + if (lane_id == 0) { - uint32_t const s0 = __shfl_sync(0xFFFFFFFFu, scale_byte, 0); - uint32_t const s1 = __shfl_sync(0xFFFFFFFFu, scale_byte, 8); - uint32_t const s2 = __shfl_sync(0xFFFFFFFFu, scale_byte, 16); - uint32_t const s3 = __shfl_sync(0xFFFFFFFFu, scale_byte, 24); - if (lane_id == 0) - { - // Mask off scale bytes whose sf_k is past the actual K. - int const num_sf_k = (k + 127) / 128; - int const sf_k_base = packed_sf_k_idx * 4; - if (sf_k_base + 0 < num_sf_k) - packed |= s0; - if (sf_k_base + 1 < num_sf_k) - packed |= (s1 << 8); - if (sf_k_base + 2 < num_sf_k) - packed |= (s2 << 16); - if (sf_k_base + 3 < num_sf_k) - packed |= (s3 << 24); - } + // Mask off scale bytes whose sf_k is past the actual K. + int const num_sf_k = (k + 127) / 128; + int const sf_k_base = packed_sf_k_idx * 4; + if (sf_k_base + 0 < num_sf_k) + packed |= s0; + if (sf_k_base + 1 < num_sf_k) + packed |= (s1 << 8); + if (sf_k_base + 2 < num_sf_k) + packed |= (s2 << 16); + if (sf_k_base + 3 < num_sf_k) + packed |= (s3 << 24); } } - if constexpr (OutputCuteDslSf) - { - // Native MXF8 MMA consumes one UE8M0 scale per 32 K values. Preserve - // the production 1x128 quantization contract by replicating each scale - // four times directly into CUTLASS/CuTe's 128x4 swizzled layout. - if (lane_id % 8 == 0 && m_idx < scale_leading_dim_uint32) - { - int const sf128_idx = packed_sf_k_idx * 4 + lane_id / 8; - int const num_sf128 = (k + 127) / 128; - if (sf128_idx < num_sf128) - { - int const num_sf32 = (k + 31) / 32; - int const num_k_tiles = (num_sf32 + 3) / 4; - int64_t const dst_offset = static_cast<int64_t>(m_idx / 128) * num_k_tiles * 512 - + static_cast<int64_t>(sf128_idx) * 512 + (m_idx % 32) * 16 + ((m_idx % 128) / 32) * 4; - uint32_t const replicated = row_in_range ? scale_byte * 0x01010101u : 0u; - *reinterpret_cast<uint32_t*>(reinterpret_cast<uint8_t*>(packed_scale_output) + dst_offset) = replicated; - } - } - } - else + // Always write the packed scale — `packed` is 0 for padded rows. The grid + // covers the full [0, scale_leading_dim_uint32) leading dim (rounded up to + // WarpsPerBlock), and the m_idx guard drops the few rows past the buffer end. + if (lane_id == 0 && m_idx < scale_leading_dim_uint32) { - // Always write the packed scale — `packed` is 0 for padded rows. The grid - // covers the full [0, scale_leading_dim_uint32) leading dim (rounded up to - // WarpsPerBlock), and the m_idx guard drops the few rows past the buffer end. - if (lane_id == 0 && m_idx < scale_leading_dim_uint32) - { - packed_scale_output[static_cast<int64_t>(packed_sf_k_idx) * scale_leading_dim_uint32 + m_idx] = packed; - } + packed_scale_output[static_cast<int64_t>(packed_sf_k_idx) * scale_leading_dim_uint32 + m_idx] = packed; } #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) @@ -243,27 +215,8 @@ void launch_fp8_quantize_1x128_packed_bf16_e4m3(__nv_fp8_e4m3* fp8_output, int32 dim3 const block(kWarpsPerBlock * 32, 1, 1); tensorrt_llm::common::launchWithPdlWhenEnabled("fp8_quantize_1x128_packed_kernel_impl", - fp8_quantize_1x128_packed_kernel_impl<kWarpsPerBlock, false>, grid, block, 0, stream, fp8_output, - packed_scale_output, input, m, k, scale_leading_dim_uint32); -} - -void launch_fp8_quantize_1x128_cutedsl_bf16_e4m3(__nv_fp8_e4m3* fp8_output, uint8_t* swizzled_scale_output, - __nv_bfloat16 const* input, int m, int k, int padded_m, cudaStream_t stream) -{ - if (m <= 0 || k <= 0) - { - return; - } - - constexpr int kWarpsPerBlock = 4; - int const num_packed_sf_k = (((k + 127) / 128) + 3) / 4; - int const m_blocks = (padded_m + kWarpsPerBlock - 1) / kWarpsPerBlock; - dim3 const grid(num_packed_sf_k, m_blocks, 1); - dim3 const block(kWarpsPerBlock * 32, 1, 1); - - tensorrt_llm::common::launchWithPdlWhenEnabled("fp8_quantize_1x128_cutedsl_kernel_impl", - fp8_quantize_1x128_packed_kernel_impl<kWarpsPerBlock, true>, grid, block, 0, stream, fp8_output, - reinterpret_cast<int32_t*>(swizzled_scale_output), input, m, k, padded_m); + fp8_quantize_1x128_packed_kernel_impl<kWarpsPerBlock>, grid, block, 0, stream, fp8_output, packed_scale_output, + input, m, k, scale_leading_dim_uint32); } } // namespace kernels::fp8_blockscale_gemm diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.h index 7f8c4f07d33d..a4079cd9b054 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_quant_packed.h @@ -47,12 +47,6 @@ namespace kernels::fp8_blockscale_gemm void launch_fp8_quantize_1x128_packed_bf16_e4m3(__nv_fp8_e4m3* fp8_output, int32_t* packed_scale_output, __nv_bfloat16 const* input, int m, int k, int scale_leading_dim_uint32, cudaStream_t stream); -// Quantizes with the same 1x128 scale as above, but replicates each UE8M0 -// scale over four 32-wide groups and writes the native SM100 CuTe/CUTLASS -// 128x4 swizzled scale layout. -void launch_fp8_quantize_1x128_cutedsl_bf16_e4m3(__nv_fp8_e4m3* fp8_output, uint8_t* swizzled_scale_output, - __nv_bfloat16 const* input, int m, int k, int padded_m, cudaStream_t stream); - } // namespace kernels::fp8_blockscale_gemm TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h index ab7ed876257d..24781bec76e7 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h @@ -27,7 +27,7 @@ #include <cuda_fp4.h> #endif #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <array> #include <cuda_runtime_api.h> #include <map> @@ -1032,8 +1032,8 @@ struct GemmProfilerBackend using Config = cutlass_extensions::CutlassGemmConfig; using GemmToProfile = MoeGemmId; - void init(CutlassMoeFCRunnerInterface& runner, GemmToProfile gemm_to_profile, tensorrt_llm::DataType dtype, - tensorrt_llm::DataType wtype, tensorrt_llm::DataType otype, int num_experts, int k, int64_t hidden_size, + void init(CutlassMoeFCRunnerInterface& runner, GemmToProfile gemm_to_profile, nvinfer1::DataType dtype, + nvinfer1::DataType wtype, nvinfer1::DataType otype, int num_experts, int k, int64_t hidden_size, int64_t unpadded_hidden_size, int64_t inter_size, int64_t group_size, ActivationType activation_type, bool bias, bool use_lora, bool min_latency_mode, bool need_weights, MOEParallelismConfig parallelism_config, bool const enable_alltoall, bool use_mxfp8_weight_scaling = false) @@ -1061,21 +1061,20 @@ struct GemmProfilerBackend mSM = common::getSMVersion(); mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::NONE; - if (dtype == tensorrt_llm::DataType::kFP8 - && (wtype == tensorrt_llm::DataType::kFP4 || wtype == tensorrt_llm::DataType::kINT64)) + if (dtype == nvinfer1::DataType::kFP8 + && (wtype == nvinfer1::DataType::kFP4 || wtype == nvinfer1::DataType::kINT64)) { mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::MXFPX; } - else if (dtype == tensorrt_llm::DataType::kFP8 && wtype == tensorrt_llm::DataType::kFP8 - && use_mxfp8_weight_scaling) + else if (dtype == nvinfer1::DataType::kFP8 && wtype == nvinfer1::DataType::kFP8 && use_mxfp8_weight_scaling) { // MXFP8 W8A8: e4m3 acts × e4m3 weights with UE8M0 1x32 block scales on both sides. // Profiler must produce MXFPX block-scaled inputs (otherwise the per-expert SF // pointer arrays stay uninitialized and the kernel reads garbage SF addresses). mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::MXFPX; } - else if ((dtype == tensorrt_llm::DataType::kFP4 || dtype == tensorrt_llm::DataType::kINT64) - && (wtype == tensorrt_llm::DataType::kFP4 || wtype == tensorrt_llm::DataType::kINT64)) + else if ((dtype == nvinfer1::DataType::kFP4 || dtype == nvinfer1::DataType::kINT64) + && (wtype == nvinfer1::DataType::kFP4 || wtype == nvinfer1::DataType::kINT64)) { mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::NVFP4; } @@ -1107,9 +1106,9 @@ struct GemmProfilerBackend int mSampleIndex = 0; - tensorrt_llm::DataType mDType{}; - tensorrt_llm::DataType mWType{}; - tensorrt_llm::DataType mOType{}; + nvinfer1::DataType mDType{}; + nvinfer1::DataType mWType{}; + nvinfer1::DataType mOType{}; // This will be a unique value for every iteration of warmup and actual bench constexpr static int64_t NUM_ROUTING_SAMPLES = 16; diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_grouped_gemm.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_grouped_gemm.h index 15af49fc1839..55ab4e40a3ae 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_grouped_gemm.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_grouped_gemm.h @@ -18,7 +18,7 @@ #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cstdint> #include <cuda_runtime.h> @@ -56,7 +56,7 @@ struct MoeLoraGroupedGemmModule; // stream: CUDA stream to launch onto. using MoeLoraGroupedGemmRunFn = void (*)(MoeLoraGroupedGemmModule const& mod, int64_t num_permuted_tokens, int64_t in_hidden_size, int64_t max_lora_rank, int64_t dtype_bytes, int64_t splitk_slices, void const* input_base, - void* output_base, tensorrt_llm::DataType data_type, cudaStream_t stream); + void* output_base, nvinfer1::DataType data_type, cudaStream_t stream); // Per-module device-resident scratch for the MoE LoRA capture-safe path. // Pointers refer to device memory unless noted. diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h index f3e8940b0c28..e902e2c9d6d3 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h @@ -25,6 +25,7 @@ #ifdef ENABLE_FP4 #include <cuda_fp4.h> #endif +#include <NvInferRuntime.h> #include <array> #include <cuda_runtime_api.h> #include <map> diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_launcher.inl b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_launcher.inl index 9c5ebbdaa19d..0044528b4dff 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_launcher.inl +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_launcher.inl @@ -15,8 +15,6 @@ */ #pragma once -#include "tensorrt_llm/common/cudaUtils.h" - #include "cutlass/array.h" #include "cutlass/numeric_conversion.h" @@ -679,19 +677,6 @@ using namespace cutlass::epilogue; "Workspace is size %zu but only %zu were allocated", calculated_ws_size, \ tma_ws_input.gemm_workspace_size); \ \ - /* Check if kernel SMEM fits on the active device before launch. */ \ - { \ - using GemmKernel_ = typename GemmGrouped::GemmKernel; \ - int smem_size = static_cast<int>(sizeof(typename GemmKernel_::SharedStorage)); \ - int device_ = 0; \ - tensorrt_llm::common::check_cuda_error(cudaGetDevice(&device_)); \ - int maxSmem_ = 0; \ - tensorrt_llm::common::check_cuda_error( \ - cudaDeviceGetAttribute(&maxSmem_, cudaDevAttrMaxSharedMemoryPerBlockOptin, device_)); \ - TLLM_CHECK_WITH_INFO(smem_size <= maxSmem_, \ - "MoE grouped GEMM requires %d bytes shared memory but device supports %d", smem_size, maxSmem_); \ - } \ - \ auto can_implement = gemm.can_implement(args); \ TLLM_CHECK_WITH_INFO(can_implement == cutlass::Status::kSuccess, \ "Grouped GEMM kernel will fail for params. Error: " \ diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_mixed_input_launcher.inl b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_mixed_input_launcher.inl index cdf5ea8dc3c9..f37920dcf73c 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_mixed_input_launcher.inl +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/launchers/moe_gemm_tma_ws_mixed_input_launcher.inl @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,8 +19,6 @@ #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif // __GNUC__ -#include "tensorrt_llm/common/cudaUtils.h" - #include "cutlass/epilogue/collective/default_epilogue.hpp" #include "cutlass/epilogue/thread/linear_combination.h" #include "cutlass/gemm/collective/collective_builder.hpp" @@ -275,17 +273,6 @@ void sm90_generic_mixed_moe_gemm_kernelLauncher(GroupedGemmInput<T, WeightType, // This is not initialized during workspace size calculation so check after TLLM_CHECK_WITH_INFO(hopper_inputs.swap_ab, "swap_ab must be true for mixed dtype WS grouped GEMM"); - { - int smem_size = static_cast<int>(sizeof(typename GemmKernel::SharedStorage)); - int device = 0; - tensorrt_llm::common::check_cuda_error(cudaGetDevice(&device)); - int maxSmem = 0; - tensorrt_llm::common::check_cuda_error( - cudaDeviceGetAttribute(&maxSmem, cudaDevAttrMaxSharedMemoryPerBlockOptin, device)); - TLLM_CHECK_WITH_INFO(smem_size <= maxSmem, - "Mixed dtype WS grouped GEMM requires %d bytes shared memory but device supports %d", smem_size, maxSmem); - } - auto can_implement = gemm.can_implement(arguments); if (can_implement != cutlass::Status::kSuccess) { diff --git a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_kernels.cu b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_kernels.cu index 8bed9c16b58e..b7a32be2e285 100644 --- a/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_kernels.cu +++ b/cpp/tensorrt_llm/kernels/cutlass_kernels/moe_gemm/moe_kernels.cu @@ -58,7 +58,6 @@ #include "tensorrt_llm/kernels/preQuantScaleKernel.h" #include "tensorrt_llm/kernels/quantization.cuh" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/cutlass_kernels/include/moe_lora_pointer_expand.h" #include "tensorrt_llm/kernels/cutlass_kernels/include/moe_util_kernels.h" // NOTE: the grouped-GEMM dispatch (cudaGraph(SplitK)GroupedGemm, @@ -3687,7 +3686,7 @@ void CutlassMoeFCRunner<T, WeightType, OutputType, InputType, BackBoneType, Enab inline void runMoeLoraGroupedGemmModule(::tensorrt_llm::kernels::cutlass_kernels::MoeLoraGroupedGemmModule const& mod, int64_t num_permuted_tokens, int64_t in_hidden_size, int64_t max_lora_rank, int64_t dtype_bytes, int64_t splitk_slices, void const* input_base, void* output_base, - ::tensorrt_llm::kernels::cutlass_kernels::MoeLoraGroupedGemmRunFn run, tensorrt_llm::DataType data_type, + ::tensorrt_llm::kernels::cutlass_kernels::MoeLoraGroupedGemmRunFn run, nvinfer1::DataType data_type, cudaStream_t stream) { TLLM_CHECK_WITH_INFO(mod.permuted_ranks_dev != nullptr, @@ -3699,25 +3698,25 @@ inline void runMoeLoraGroupedGemmModule(::tensorrt_llm::kernels::cutlass_kernels data_type, stream); } -// Map the activation/back-bone type to the DataType enum the +// Map the activation/back-bone type to the nvinfer1 enum the // cuda_graph_grouped_gemm wrappers expect. Only fp16/bf16/fp32 are handled; // anything else is a compile-time error rather than a silent fall-through. template <class ScaleBiasType> -constexpr tensorrt_llm::DataType moeLoraDataType() +constexpr nvinfer1::DataType moeLoraNvInferType() { if constexpr (std::is_same_v<ScaleBiasType, half>) { - return tensorrt_llm::DataType::kHALF; + return nvinfer1::DataType::kHALF; } #if defined(ENABLE_BF16) else if constexpr (std::is_same_v<ScaleBiasType, __nv_bfloat16>) { - return tensorrt_llm::DataType::kBF16; + return nvinfer1::DataType::kBF16; } #endif else if constexpr (std::is_same_v<ScaleBiasType, float>) { - return tensorrt_llm::DataType::kFLOAT; + return nvinfer1::DataType::kFLOAT; } else { @@ -3926,7 +3925,7 @@ auto CutlassMoeFCRunner<T, WeightType, OutputType, InputType, BackBoneType, Enab if (lora_params.grouped_gemm.enabled) { auto const& grouped_gemm = lora_params.grouped_gemm; - tensorrt_llm::DataType const data_type = moeLoraDataType<ScaleBiasType>(); + nvinfer1::DataType const data_type = moeLoraNvInferType<ScaleBiasType>(); // The grouped-GEMM GEMM skips rank-0 rows, but the bias/reorder paths // read lora_fc1_result_ for every valid row. Zero the buffer first so @@ -4021,7 +4020,7 @@ void CutlassMoeFCRunner<T, WeightType, OutputType, InputType, BackBoneType, Enab if (lora_params.grouped_gemm.enabled) { auto const& grouped_gemm = lora_params.grouped_gemm; - tensorrt_llm::DataType const data_type = moeLoraDataType<ScaleBiasType>(); + nvinfer1::DataType const data_type = moeLoraNvInferType<ScaleBiasType>(); // As in loraFC1, zero the output so rank-0 rows the GEMM skips do not // feed stale data into the downstream add. @@ -4751,18 +4750,18 @@ std::map<std::string, std::pair<size_t, size_t>> GemmProfilerBackend::getProfile size_t k = mK; size_t num_expanded_tokens = mMinLatencyMode ? maxM * mNumExpertsPerNode : maxM * k; - TLLM_CHECK(mDType != tensorrt_llm::DataType::kINT4); + TLLM_CHECK(mDType != nvinfer1::DataType::kINT4); // nvllm still uses int64 because torch doesn't have fp4 yet. - bool is_4bit_act = mDType == tensorrt_llm::DataType::kFP4 || mDType == tensorrt_llm::DataType::kINT64; - bool is_4bit_weight = mWType == tensorrt_llm::DataType::kINT4 || mWType == tensorrt_llm::DataType::kFP4 - || mWType == tensorrt_llm::DataType::kINT64; + bool is_4bit_act = mDType == nvinfer1::DataType::kFP4 || mDType == nvinfer1::DataType::kINT64; + bool is_4bit_weight = mWType == nvinfer1::DataType::kINT4 || mWType == nvinfer1::DataType::kFP4 + || mWType == nvinfer1::DataType::kINT64; TLLM_CHECK_WITH_INFO(!is_4bit_act || is_4bit_weight, "Cannot have 4-bit activation with non-4-bit weight"); float dtype_bytes = is_4bit_act ? 0.5f - : static_cast<float>(mWType == tensorrt_llm::DataType::kINT4 ? getDTypeSize(mOType) : getDTypeSize(mDType)); + : static_cast<float>(mWType == nvinfer1::DataType::kINT4 ? getDTypeSize(mOType) : getDTypeSize(mDType)); float weight_bytes = is_4bit_weight ? 0.5f : static_cast<float>(getDTypeSize(mWType)); size_t output_bytes = getDTypeSize(mOType); - size_t gemm_output_bytes = (mOType == tensorrt_llm::DataType::kFP8) + size_t gemm_output_bytes = (mOType == nvinfer1::DataType::kFP8) ? sizeof(TmaWarpSpecializedGroupedGemmInput::OutputTypeAdaptor_t<__nv_fp8_e4m3>) : output_bytes; @@ -4804,18 +4803,18 @@ std::map<std::string, std::pair<size_t, size_t>> GemmProfilerBackend::getProfile // TODO Make quant 2 & 4 bigger for FP8 if we ever change to scaling per expert bool is_int_w_quant - = (mWType == tensorrt_llm::DataType::kINT8 || mWType == tensorrt_llm::DataType::kINT4) && mGroupSize <= 0; + = (mWType == nvinfer1::DataType::kINT8 || mWType == nvinfer1::DataType::kINT4) && mGroupSize <= 0; bool is_int_groupwise_w_quant - = (mWType == tensorrt_llm::DataType::kINT8 || mWType == tensorrt_llm::DataType::kINT4) && mGroupSize > 0; - bool is_fp8_act_quant = mDType == tensorrt_llm::DataType::kFP8; - bool is_fp8_w_quant = mWType == tensorrt_llm::DataType::kFP8; + = (mWType == nvinfer1::DataType::kINT8 || mWType == nvinfer1::DataType::kINT4) && mGroupSize > 0; + bool is_fp8_act_quant = mDType == nvinfer1::DataType::kFP8; + bool is_fp8_w_quant = mWType == nvinfer1::DataType::kFP8; // nvllm still uses int64 because torch doesn't have fp4 yet. - // bool is_fp4_act_quant = mDType == tensorrt_llm::DataType::kFP4 || mDType == tensorrt_llm::DataType::kINT64; - bool is_fp4_w_quant = mWType == tensorrt_llm::DataType::kFP4 || mWType == tensorrt_llm::DataType::kINT64; + // bool is_fp4_act_quant = mDType == nvinfer1::DataType::kFP4 || mDType == nvinfer1::DataType::kINT64; + bool is_fp4_w_quant = mWType == nvinfer1::DataType::kFP4 || mWType == nvinfer1::DataType::kINT64; bool is_w4afp8_quant = is_int_groupwise_w_quant && is_fp8_act_quant; // bool is_wfp4afp8_quant = is_fp4_w_quant && is_fp8_act_quant; - bool is_wfp4a16_quant = (mDType == tensorrt_llm::DataType::kHALF || mDType == tensorrt_llm::DataType::kBF16) - && mWType == tensorrt_llm::DataType::kUINT8; + bool is_wfp4a16_quant = (mDType == nvinfer1::DataType::kHALF || mDType == nvinfer1::DataType::kBF16) + && mWType == nvinfer1::DataType::kUINT8; // Int sizes size_t quant_1_size = is_int_w_quant ? fc1_out_size * num_experts_per_node * dtype_bytes : 0; @@ -5048,19 +5047,19 @@ void GemmProfilerBackend::prepareQuantParams(int num_tokens, char* workspace_ptr GET_WS_PTR(float const*, w4a8_alpha); #undef GET_WS_PTR - if ((mWType == tensorrt_llm::DataType::kINT8 || mWType == tensorrt_llm::DataType::kINT4 - || mWType == tensorrt_llm::DataType::kUINT8) + if ((mWType == nvinfer1::DataType::kINT8 || mWType == nvinfer1::DataType::kINT4 + || mWType == nvinfer1::DataType::kUINT8) && mGroupSize < 0) { TLLM_CHECK(quant_1 && quant_2); mQuantParams = QuantParams::Int(quant_1, quant_2); } - else if (mWType == tensorrt_llm::DataType::kINT4 || mWType == tensorrt_llm::DataType::kUINT8) + else if (mWType == nvinfer1::DataType::kINT4 || mWType == nvinfer1::DataType::kUINT8) { TLLM_CHECK(quant_1 && quant_2); - if (mDType == tensorrt_llm::DataType::kFP8 - || (mWType == tensorrt_llm::DataType::kUINT8 - && (mDType == tensorrt_llm::DataType::kHALF || mDType == tensorrt_llm::DataType::kBF16))) + if (mDType == nvinfer1::DataType::kFP8 + || (mWType == nvinfer1::DataType::kUINT8 + && (mDType == nvinfer1::DataType::kHALF || mDType == nvinfer1::DataType::kBF16))) { TLLM_CHECK(w4a8_alpha); mQuantParams = QuantParams::GroupWise( @@ -5071,7 +5070,7 @@ void GemmProfilerBackend::prepareQuantParams(int num_tokens, char* workspace_ptr mQuantParams = QuantParams::GroupWise(mGroupSize, quant_1, quant_2, nullptr, nullptr, quant_3, quant_4); } } - else if (mWType == tensorrt_llm::DataType::kFP8) + else if (mWType == nvinfer1::DataType::kFP8) { if (mUseMxfp8WeightScaling) { @@ -5090,8 +5089,8 @@ void GemmProfilerBackend::prepareQuantParams(int num_tokens, char* workspace_ptr static_cast<float const*>(quant_3), static_cast<float const*>(quant_4)); } } - else if (mDType == tensorrt_llm::DataType::kFP8 - && (mWType == tensorrt_llm::DataType::kFP4 || mWType == tensorrt_llm::DataType::kINT64)) + else if (mDType == nvinfer1::DataType::kFP8 + && (mWType == nvinfer1::DataType::kFP4 || mWType == nvinfer1::DataType::kINT64)) { TLLM_CHECK(quant_1 && quant_2 && quant_3 && quant_4 && quant_5 && quant_6); mQuantParams = QuantParams::FP8MXFP4(static_cast<float const*>(quant_1), @@ -5100,8 +5099,8 @@ void GemmProfilerBackend::prepareQuantParams(int num_tokens, char* workspace_ptr static_cast<TmaWarpSpecializedGroupedGemmInput::MXFPXElementSF const*>(quant_5), static_cast<float const*>(quant_6)); } - else if ((mDType == tensorrt_llm::DataType::kFP4 || mDType == tensorrt_llm::DataType::kINT64) - && (mWType == tensorrt_llm::DataType::kFP4 || mWType == tensorrt_llm::DataType::kINT64)) + else if ((mDType == nvinfer1::DataType::kFP4 || mDType == nvinfer1::DataType::kINT64) + && (mWType == nvinfer1::DataType::kFP4 || mWType == nvinfer1::DataType::kINT64)) { // nvllm still uses int64 because torch doesn't have fp4 yet. TLLM_CHECK(quant_1 && quant_2 && quant_3 && quant_4 && quant_5 && quant_6); @@ -5121,9 +5120,9 @@ void GemmProfilerBackend::prepareTmaWsInputs(int num_tokens, char* workspace_ptr return; } - bool use_w4afp8 = (mDType == tensorrt_llm::DataType::kFP8 && mWType == tensorrt_llm::DataType::kINT4); - bool use_wfp4a16 = ((mDType == tensorrt_llm::DataType::kHALF || mDType == tensorrt_llm::DataType::kBF16) - && mWType == tensorrt_llm::DataType::kUINT8); + bool use_w4afp8 = (mDType == nvinfer1::DataType::kFP8 && mWType == nvinfer1::DataType::kINT4); + bool use_wfp4a16 = ((mDType == nvinfer1::DataType::kHALF || mDType == nvinfer1::DataType::kBF16) + && mWType == nvinfer1::DataType::kUINT8); bool const use_finalize_fusion = fusion == TmaWarpSpecializedGroupedGemmInput::EpilogueFusion::FINALIZE; bool const finalize_fusion_not_supported = !mInterface->use_fused_finalize_ || mMinLatencyMode || use_wfp4a16 || mGemmToProfile != GemmToProfile::GEMM_2; diff --git a/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/xqaParams.h b/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/xqaParams.h index dc7794752e47..e421be0a6bd7 100644 --- a/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/xqaParams.h +++ b/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/xqaParams.h @@ -63,7 +63,6 @@ struct XQAParams int64_t* spec_decoding_bl_tree_mask_offset; // for blackwell spec-dec tree mask offset uint32_t* spec_decoding_bl_tree_mask; // for blackwell spec-dec tree mask int32_t* spec_bl_tree_first_sparse_mask_offset_kv; // for blackwell spec-dec tree first sparse mask offset kv - bool force_prepare_spec_dec_tree_mask = false; int32_t const* mrope_position_deltas = nullptr; // Helix parallelism params. int32_t const* helix_position_offsets = nullptr; diff --git a/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu b/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu index 6e26dfce65bf..e06b0f200e4b 100644 --- a/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu +++ b/cpp/tensorrt_llm/kernels/fusedQKNormRopeKernel.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -174,17 +174,6 @@ __global__ void fusedQKNormRopeKernel( // pos_id is selected per rotary half-dim (interleaved mRoPE); for plain RoPE // selectMRopePosId always returns position_ids[tokenIdx]. - // Hoist log2(base) and the loop-invariant constants out of the per-thread - // per-elem loop. powf(base, -2*hd/rd) == exp2f(-2*hd/rd * log2(base)); base - // and rotary_dim are kernel-uniform, so one MUFU.LG2 per warp instead of - // one per (thread, iter). Uses the fast __log2f intrinsic (a few ULPs of - // error, absorbed by bf16 downcast at store time). - float const neg2_log2base_over_rd = -2.0f * __log2f(base) / static_cast<float>(rotary_dim); - // rotary_dim is even by contract; when it's also a power of 2 (always in - // practice — 64/128/256) '% rotary_dim' becomes '& (rotary_dim - 1)'. - // The bool is warp-uniform → predicated select, no branch divergence. - int const rd_mask = rotary_dim - 1; - bool const rd_is_pow2 = ((rotary_dim & rd_mask) == 0); // TODO: cos sin calculation could be halved. if constexpr (interleave) { @@ -202,7 +191,7 @@ __global__ void fusedQKNormRopeKernel( int dim_idx = laneId * numElemsPerThread + i; int half_dim = dim_idx / 2; - float freq = exp2f(static_cast<float>(half_dim) * neg2_log2base_over_rd); + float freq = powf(base, -2.0f * half_dim / static_cast<float>(rotary_dim)); if (factor != 1.0f) { @@ -243,9 +232,9 @@ __global__ void fusedQKNormRopeKernel( } int dim_idx = laneId * numElemsPerThread + i; - dim_idx = rd_is_pow2 ? ((dim_idx * 2) & rd_mask) : ((dim_idx * 2) % rotary_dim); + dim_idx = (dim_idx * 2) % rotary_dim; int half_dim = dim_idx / 2; - float freq = exp2f(static_cast<float>(half_dim) * neg2_log2base_over_rd); + float freq = powf(base, -2.0f * half_dim / static_cast<float>(rotary_dim)); if (factor != 1.0f) { diff --git a/cpp/tensorrt_llm/kernels/gptKernels.h b/cpp/tensorrt_llm/kernels/gptKernels.h index d855aade79c2..e13e9bca4d6a 100644 --- a/cpp/tensorrt_llm/kernels/gptKernels.h +++ b/cpp/tensorrt_llm/kernels/gptKernels.h @@ -16,7 +16,6 @@ #pragma once #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fused_multihead_attention_common.h" #include "tensorrt_llm/runtime/iTensor.h" #include <cstdint> @@ -228,11 +227,11 @@ struct BuildDecoderInfoParams std::string toString() const { std::stringstream ss; - auto printTensor = [&ss](char const* name, void* ptr, tensorrt_llm::Dims shape) + auto printTensor = [&ss](char const* name, void* ptr, nvinfer1::Dims shape) { ss << name << ": "; if (ptr) - ss << *(runtime::ITensor::wrap((void*) ptr, tensorrt_llm::DataType::kINT32, shape)); + ss << *(runtime::ITensor::wrap((void*) ptr, nvinfer1::DataType::kINT32, shape)); else ss << "nullptr"; ss << std::endl; diff --git a/cpp/tensorrt_llm/kernels/groupGemm.cu b/cpp/tensorrt_llm/kernels/groupGemm.cu index b41021ffe6f7..5b8c0d929150 100644 --- a/cpp/tensorrt_llm/kernels/groupGemm.cu +++ b/cpp/tensorrt_llm/kernels/groupGemm.cu @@ -28,7 +28,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" TRTLLM_NAMESPACE_BEGIN @@ -64,7 +63,7 @@ template <int M1, int N1, int K1, int M2, int N2, int K2, typename cutlassType, void groupedGemm_(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - tensorrt_llm::DataType dataType, cudaStream_t stream) + nvinfer1::DataType dataType, cudaStream_t stream) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); using ElementA = cutlassType; @@ -179,20 +178,20 @@ template <int M1, int N1, int K1, int M2, int N2, int K2, int kAlignmentAB, int void groupedGemmType_(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - tensorrt_llm::DataType dataType, cudaStream_t stream) + nvinfer1::DataType dataType, cudaStream_t stream) { - if (dataType == tensorrt_llm::DataType::kHALF) + if (dataType == nvinfer1::DataType::kHALF) { groupedGemm_<M1, N1, K1, M2, N2, K2, cutlass::half_t, kAlignmentAB, kAlignmentC, kStages>(problem_sizes, ptrA, ptrB, ptrC, ptrD, gemmParamsWorkSpace, gemmParamsWorkSpaceSize, gemmWorkSpace, gemmWorkspaceSize, dataType, stream); } - else if (dataType == tensorrt_llm::DataType::kFLOAT) + else if (dataType == nvinfer1::DataType::kFLOAT) { TLLM_CHECK_WITH_INFO(false, "not support float input/output"); } #ifdef ENABLE_BF16 - else if (dataType == tensorrt_llm::DataType::kBF16) + else if (dataType == nvinfer1::DataType::kBF16) { groupedGemm_<M1, N1, K1, M2, N2, K2, cutlass::bfloat16_t, kAlignmentAB, kAlignmentC, kStages>(problem_sizes, ptrA, ptrB, ptrC, ptrD, gemmParamsWorkSpace, gemmParamsWorkSpaceSize, gemmWorkSpace, gemmWorkspaceSize, @@ -204,7 +203,7 @@ void groupedGemmType_(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std:: void groupedGemm(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - bool isLoraIn, tensorrt_llm::DataType dataType, int minKN, cudaStream_t stream) + bool isLoraIn, nvinfer1::DataType dataType, int minKN, cudaStream_t stream) { TLLM_LOG_TRACE("%s start, isLoraIn: %d, minKN = %d", __PRETTY_FUNCTION__, static_cast<int>(isLoraIn), minKN); if (isLoraIn) diff --git a/cpp/tensorrt_llm/kernels/groupGemm.h b/cpp/tensorrt_llm/kernels/groupGemm.h index c526e08e986c..dbc1e498b7b2 100644 --- a/cpp/tensorrt_llm/kernels/groupGemm.h +++ b/cpp/tensorrt_llm/kernels/groupGemm.h @@ -17,7 +17,7 @@ #include "cutlass/gemm_coord.h" #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> TRTLLM_NAMESPACE_BEGIN @@ -29,7 +29,7 @@ int64_t getGroupedGemmParamsWorkSpaceSize(int64_t problem_count); void groupedGemm(std::vector<cutlass::gemm::GemmCoord> problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkspace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - bool isLoraIn, tensorrt_llm::DataType dataType, int minKN, cudaStream_t stream); + bool isLoraIn, nvinfer1::DataType dataType, int minKN, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.cu b/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.cu index 3332c2918a3f..409968bb510d 100644 --- a/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.cu +++ b/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.cu @@ -22,7 +22,6 @@ #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/reduceKernelUtils.cuh" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h" TRTLLM_NAMESPACE_BEGIN @@ -656,9 +655,9 @@ void GroupRMSNormBaseKernelLauncher(GroupRMSParams<n>& params) switch (params.dtype) { - case tensorrt_llm::DataType::kHALF: GROUP_RMS_NORM_DISPATCH(half); break; - case tensorrt_llm::DataType::kBF16: GROUP_RMS_NORM_DISPATCH(__nv_bfloat16); break; - case tensorrt_llm::DataType::kFLOAT: GROUP_RMS_NORM_DISPATCH(float); break; + case nvinfer1::DataType::kHALF: GROUP_RMS_NORM_DISPATCH(half); break; + case nvinfer1::DataType::kBF16: GROUP_RMS_NORM_DISPATCH(__nv_bfloat16); break; + case nvinfer1::DataType::kFLOAT: GROUP_RMS_NORM_DISPATCH(float); break; default: TLLM_CHECK_WITH_INFO(false, "Unsupported data type for GroupRMSNorm"); } @@ -751,9 +750,9 @@ void GroupRMSNormKernelLargeBatchLauncher(GroupRMSParams<n>& params) switch (params.dtype) { - case tensorrt_llm::DataType::kHALF: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(half); break; - case tensorrt_llm::DataType::kBF16: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(__nv_bfloat16); break; - case tensorrt_llm::DataType::kFLOAT: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(float); break; + case nvinfer1::DataType::kHALF: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(half); break; + case nvinfer1::DataType::kBF16: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(__nv_bfloat16); break; + case nvinfer1::DataType::kFLOAT: GROUP_RMS_NORM_LARGE_BATCH_DISPATCH(float); break; default: TLLM_CHECK_WITH_INFO(false, "Unsupported data type for GroupRMSNormV2"); } @@ -814,15 +813,15 @@ void GroupRMSNormKernelLauncherWithHeuristic(GroupRMSParams<n>& params) // Choose the appropriate DType switch (params.dtype) { - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: base_warps = calculateNumWarpsBase<half, n>(params); large_batch_warps = calculateNumWarpsLargeBatch<half, n>(params).num_warps_to_launch; break; - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: base_warps = calculateNumWarpsBase<__nv_bfloat16, n>(params); large_batch_warps = calculateNumWarpsLargeBatch<__nv_bfloat16, n>(params).num_warps_to_launch; break; - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: base_warps = calculateNumWarpsBase<float, n>(params); large_batch_warps = calculateNumWarpsLargeBatch<float, n>(params).num_warps_to_launch; break; diff --git a/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h b/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h index 70425f924217..335adf44ed67 100644 --- a/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h +++ b/cpp/tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h @@ -15,7 +15,7 @@ */ #pragma once #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_bf16.h> #include <cuda_fp16.h> #include <map> @@ -44,7 +44,7 @@ struct GroupRMSParams float eps; float weight_bias; bool enable_weights; - tensorrt_llm::DataType dtype; + nvinfer1::DataType dtype; cudaStream_t stream; }; diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/include/moe_kernels.h b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/include/moe_kernels.h index 0b02686f5e53..132990603db9 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/include/moe_kernels.h +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/include/moe_kernels.h @@ -26,7 +26,7 @@ #ifdef ENABLE_FP4 #include <cuda_fp4.h> #endif -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <array> #include <cuda_runtime_api.h> #include <map> @@ -869,8 +869,8 @@ struct GemmProfilerBackend GEMM_2 }; - void init(CutlassMoeFCRunnerInterface& runner, GemmToProfile gemm_to_profile, tensorrt_llm::DataType dtype, - tensorrt_llm::DataType wtype, tensorrt_llm::DataType otype, int num_experts, int k, int64_t hidden_size, + void init(CutlassMoeFCRunnerInterface& runner, GemmToProfile gemm_to_profile, nvinfer1::DataType dtype, + nvinfer1::DataType wtype, nvinfer1::DataType otype, int num_experts, int k, int64_t hidden_size, int64_t inter_size, int64_t group_size, ActivationType activation_type, bool bias, bool use_lora, bool min_latency_mode, bool need_weights, MOEParallelismConfig parallelism_config) { @@ -895,13 +895,13 @@ struct GemmProfilerBackend mSorter.updateNumExperts(mNumExpertsPerNode); mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::NONE; - if (dtype == tensorrt_llm::DataType::kFP8 - && (wtype == tensorrt_llm::DataType::kFP4 || wtype == tensorrt_llm::DataType::kINT64)) + if (dtype == nvinfer1::DataType::kFP8 + && (wtype == nvinfer1::DataType::kFP4 || wtype == nvinfer1::DataType::kINT64)) { mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::MXFPX; } - else if ((dtype == tensorrt_llm::DataType::kFP4 || dtype == tensorrt_llm::DataType::kINT64) - && (wtype == tensorrt_llm::DataType::kFP4 || wtype == tensorrt_llm::DataType::kINT64)) + else if ((dtype == nvinfer1::DataType::kFP4 || dtype == nvinfer1::DataType::kINT64) + && (wtype == nvinfer1::DataType::kFP4 || wtype == nvinfer1::DataType::kINT64)) { mScalingType = TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType::NVFP4; } @@ -932,9 +932,9 @@ struct GemmProfilerBackend int mSampleIndex = 0; - tensorrt_llm::DataType mDType{}; - tensorrt_llm::DataType mWType{}; - tensorrt_llm::DataType mOType{}; + nvinfer1::DataType mDType{}; + nvinfer1::DataType mWType{}; + nvinfer1::DataType mOType{}; // This will be a unique value for every iteration of warmup and actual bench constexpr static int64_t NUM_ROUTING_SAMPLES = 16; diff --git a/cpp/tensorrt_llm/kernels/kvCachePartialCopy.cu b/cpp/tensorrt_llm/kernels/kvCachePartialCopy.cu index 04105721dfca..3b91cf3f1776 100644 --- a/cpp/tensorrt_llm/kernels/kvCachePartialCopy.cu +++ b/cpp/tensorrt_llm/kernels/kvCachePartialCopy.cu @@ -15,7 +15,6 @@ */ #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/kvCachePartialCopy.h" #include <cstdint> #include <cuda_runtime_api.h> @@ -90,42 +89,42 @@ void kvCacheBlockPartialCopy(IBuffer& dst, IBuffer const& src, unsigned int numL TLLM_CHECK_WITH_INFO(dataType == dst.getDataType(), "src and dst dataType does not match"); switch (dataType) { - case tensorrt_llm::DataType::kINT64: + case nvinfer1::DataType::kINT64: hostKVCacheBlockPartialCopy<SizeType64>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case tensorrt_llm::DataType::kINT32: + case nvinfer1::DataType::kINT32: hostKVCacheBlockPartialCopy<std::int32_t>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: hostKVCacheBlockPartialCopy<float>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: hostKVCacheBlockPartialCopy<__nv_bfloat16>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; #endif - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: hostKVCacheBlockPartialCopy<half>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case tensorrt_llm::DataType::kBOOL: + case nvinfer1::DataType::kBOOL: hostKVCacheBlockPartialCopy<bool>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case tensorrt_llm::DataType::kUINT8: + case nvinfer1::DataType::kUINT8: hostKVCacheBlockPartialCopy<std::uint8_t>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; - case tensorrt_llm::DataType::kINT8: + case nvinfer1::DataType::kINT8: hostKVCacheBlockPartialCopy<std::int8_t>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; #ifdef ENABLE_FP8 - case tensorrt_llm::DataType::kFP8: + case nvinfer1::DataType::kFP8: hostKVCacheBlockPartialCopy<__nv_fp8_e4m3>( dst, src, numLayers, numHeads, tokensPerBlock, numHidden, numTokensToCopy, kvFactor, stream); break; diff --git a/cpp/tensorrt_llm/kernels/lora/dora.cpp b/cpp/tensorrt_llm/kernels/lora/dora.cpp index 883d02df9291..43dbf4fdccb0 100644 --- a/cpp/tensorrt_llm/kernels/lora/dora.cpp +++ b/cpp/tensorrt_llm/kernels/lora/dora.cpp @@ -20,13 +20,13 @@ #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/kernels/doraScaling.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <numeric> #include <vector> using tensorrt_llm::kernels::DoraImpl; -DoraImpl::DoraImpl(std::vector<int> const& outHiddenSizes, tensorrt_llm::DataType type) +DoraImpl::DoraImpl(std::vector<int> const& outHiddenSizes, nvinfer1::DataType type) : mType(type) { mCumModuleSizes.resize(outHiddenSizes.size()); @@ -73,14 +73,14 @@ int DoraImpl::run(int64_t numTokens, void const* input, void const* const* loraW auto const* deviceCumModuleSizes = reinterpret_cast<int64_t const*>(workspace); auto const* deviceScalePtrs = reinterpret_cast<void const* const*>((&deviceCumModuleSizes[numModules])); - if (mType == tensorrt_llm::DataType::kHALF) + if (mType == nvinfer1::DataType::kHALF) { tokenPerChannelScale<half>(numel, numModules, numTokens, deviceCumModuleSizes, reinterpret_cast<half const*>(input), reinterpret_cast<half const* const*>(deviceScalePtrs), reinterpret_cast<half*>(outputs[0]), stream); } #ifdef ENABLE_BF16 - else if (mType == tensorrt_llm::DataType::kBF16) + else if (mType == nvinfer1::DataType::kBF16) { tokenPerChannelScale<nv_bfloat16>(numel, numModules, numTokens, deviceCumModuleSizes, reinterpret_cast<nv_bfloat16 const*>(input), reinterpret_cast<nv_bfloat16 const* const*>(deviceScalePtrs), diff --git a/cpp/tensorrt_llm/kernels/lora/dora.h b/cpp/tensorrt_llm/kernels/lora/dora.h index 02cd68e7c1f0..fc21fe669366 100644 --- a/cpp/tensorrt_llm/kernels/lora/dora.h +++ b/cpp/tensorrt_llm/kernels/lora/dora.h @@ -17,7 +17,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> TRTLLM_NAMESPACE_BEGIN @@ -28,7 +28,7 @@ class DoraImpl public: DoraImpl() = delete; - DoraImpl(std::vector<int> const& outHiddenSizes, tensorrt_llm::DataType type); + DoraImpl(std::vector<int> const& outHiddenSizes, nvinfer1::DataType type); ~DoraImpl() = default; @@ -41,7 +41,7 @@ class DoraImpl private: std::vector<int64_t> mCumModuleSizes; std::vector<int64_t> mHostBuf; - tensorrt_llm::DataType mType; + nvinfer1::DataType mType; }; } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/lora/lora.cpp b/cpp/tensorrt_llm/kernels/lora/lora.cpp index 7a2b7d330afc..61f6af00fedc 100644 --- a/cpp/tensorrt_llm/kernels/lora/lora.cpp +++ b/cpp/tensorrt_llm/kernels/lora/lora.cpp @@ -19,7 +19,6 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/groupGemm.h" #include "tensorrt_llm/kernels/lora/lora.h" #include "tensorrt_llm/kernels/splitkGroupGemm.h" @@ -49,9 +48,8 @@ void _getProblemParams(cublasOperation_t& transa, cublasOperation_t& transb, int // TODO should reuse the function in gemmPlugin void _runGemm(int const M, int const N, int const K, bool const transA, bool const transB, - tensorrt_llm::DataType const type, CublasGemmWrapperPtr const& cublasWrapperPtr, void const* act, - void const* weight, void* output, std::optional<cublasLtMatmulHeuristicResult_t> const& heuristic, void* workspace, - cudaStream_t stream) + nvinfer1::DataType const type, CublasGemmWrapperPtr const& cublasWrapperPtr, void const* act, void const* weight, + void* output, std::optional<cublasLtMatmulHeuristicResult_t> const& heuristic, void* workspace, cudaStream_t stream) { cublasWrapperPtr->setStream(stream); cublasWrapperPtr->setWorkspace(workspace); @@ -67,8 +65,7 @@ void _runGemm(int const M, int const N, int const K, bool const transA, bool con } LoraImpl::LoraImpl(int in_hidden_size, std::vector<int> out_hidden_sizes, bool transA, bool transB, - int num_lora_modules, tensorrt_llm::DataType type, int max_low_rank, - std::shared_ptr<CublasGemmWrapper> cublasWrapper) + int num_lora_modules, nvinfer1::DataType type, int max_low_rank, std::shared_ptr<CublasGemmWrapper> cublasWrapper) : mInHiddenSize(in_hidden_size) , mTransA(transA) , mTransB(transB) @@ -85,16 +82,16 @@ LoraImpl::LoraImpl(int in_hidden_size, std::vector<int> out_hidden_sizes, bool t void LoraImpl::setGemmConfig() { TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); - if (mType == tensorrt_llm::DataType::kHALF) + if (mType == nvinfer1::DataType::kHALF) { mCublasWrapper->setFP16GemmConfig(); } - else if (mType == tensorrt_llm::DataType::kFLOAT) + else if (mType == nvinfer1::DataType::kFLOAT) { mCublasWrapper->setFP32GemmConfig(); } #ifdef ENABLE_BF16 - else if (mType == tensorrt_llm::DataType::kBF16) + else if (mType == nvinfer1::DataType::kBF16) { mCublasWrapper->setBF16GemmConfig(); } @@ -124,7 +121,7 @@ int64_t getGemmWorkSpaceSize(int64_t numTokens, int64_t maxLoraModuleNum, int64_ } size_t LoraImpl::getWorkspaceSize( - int64_t const numTokens, int64_t const numReqs, tensorrt_llm::DataType const type) const noexcept + int64_t const numTokens, int64_t const numReqs, nvinfer1::DataType const type) const noexcept { TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); auto const typeSize = tensorrt_llm::common::getDTypeSize(type); diff --git a/cpp/tensorrt_llm/kernels/lora/lora.h b/cpp/tensorrt_llm/kernels/lora/lora.h index 73a2cbe330be..7215a7af74d4 100644 --- a/cpp/tensorrt_llm/kernels/lora/lora.h +++ b/cpp/tensorrt_llm/kernels/lora/lora.h @@ -19,7 +19,7 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cublasMMWrapper.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cassert> #include <vector> @@ -37,10 +37,9 @@ class LoraImpl { public: LoraImpl(int in_hidden_size, std::vector<int> out_hidden_sizes, bool transA, bool transB, int num_lora_modules, - tensorrt_llm::DataType type, int max_low_rank, std::shared_ptr<CublasGemmWrapper> cublasWrapper); + nvinfer1::DataType type, int max_low_rank, std::shared_ptr<CublasGemmWrapper> cublasWrapper); - [[nodiscard]] size_t getWorkspaceSize( - int64_t numTokens, int64_t numReqs, tensorrt_llm::DataType type) const noexcept; + [[nodiscard]] size_t getWorkspaceSize(int64_t numTokens, int64_t numReqs, nvinfer1::DataType type) const noexcept; void setBestTactic(std::optional<Config> config); int run(int64_t numTokens, int64_t numReqs, void const* input, int32_t const* loraRanks, void const* const* loraWeightsPtr, int weightIndex, void* const* outputs, void* workspace, cudaStream_t stream); @@ -55,7 +54,7 @@ class LoraImpl private: bool mTransA; bool mTransB; - tensorrt_llm::DataType mType; + nvinfer1::DataType mType; int mNumLoraModules; // @fixme: seems this is shared across multiple clones. diff --git a/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.cu b/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.cu index 1165f39b3e5d..c3276ea487bc 100644 --- a/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.cu +++ b/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.cu @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include <cub/block/block_load.cuh> #include <cub/block/block_scan.cuh> @@ -359,7 +358,7 @@ void launchLoraGroupGEMMParamFillRowReorderFusion(int32_t* in_sizes, int32_t* ou int64_t a_base, int64_t d_base, int64_t d_prime_base, int32_t const* slot_counts, int32_t const* slot_ranks, int64_t const* slot_offsets, int32_t const* module_out_sizes, int64_t const* module_out_prefix, int64_t const* b_ptrs, int64_t const* b_prime_ptrs, void const* input, int64_t const* sorted_ids, - int32_t module_count, tensorrt_llm::DataType dtype, cudaStream_t stream) + int32_t module_count, nvinfer1::DataType dtype, cudaStream_t stream) { // Determine block dimensions (1D) // Requirements: 1) >= max_lora_count * module_count 2) >= 256 3) divisible by 32 diff --git a/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h b/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h index 835b9f8bed96..3043054ca4b5 100644 --- a/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h +++ b/cpp/tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h @@ -17,7 +17,7 @@ #pragma once #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cstdint> #include <cuda_runtime.h> @@ -70,7 +70,7 @@ void launchLoraGroupGEMMParamFillRowReorderFusion(int32_t* in_sizes, int32_t* ou int64_t a_base, int64_t d_base, int64_t d_prime_base, int32_t const* slot_counts, int32_t const* slot_ranks, int64_t const* slot_offsets, int32_t const* module_out_sizes, int64_t const* module_out_prefix, int64_t const* b_ptrs, int64_t const* b_prime_ptrs, void const* input, int64_t const* sorted_ids, - int32_t module_count, tensorrt_llm::DataType dtype, cudaStream_t stream); + int32_t module_count, nvinfer1::DataType dtype, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.cu b/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.cu deleted file mode 100644 index 45b1a38266b1..000000000000 --- a/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.cu +++ /dev/null @@ -1,443 +0,0 @@ -/* - * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "rmsNormFp4QuantKernels.h" -#include "tensorrt_llm/common/cudaBf16Fallbacks.cuh" -#include "tensorrt_llm/common/cudaTypeUtils.cuh" -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/kernels/quantization.cuh" -#include <cstdint> - -TRTLLM_NAMESPACE_BEGIN - -namespace kernels -{ - -// Self-contained device helpers for this kernel. These mirror the small -// reduce_fusion utilities in customAllReduceKernels.cu but are copied here (in a -// private namespace) so this translation unit does not depend on the AllReduce -// files at all. Kept byte-identical to the originals. -namespace rms_norm_fp4_quant -{ - -static constexpr int kBytesPerAccess = 16; -static constexpr int kWarpSize = 32; -static constexpr int kMaxCtaSize = 1024; - -// Type converter that packs data format to 128-bit data type. -using PackedFloat = union -{ - int4 packed; - float unpacked[4]; -}; - -using PackedHalf = union -{ - int4 packed; - half2 unpacked[4]; -}; - -template <typename T> -struct PackedOn16Bytes -{ -}; - -template <> -struct PackedOn16Bytes<float> -{ - using Type = PackedFloat; -}; - -template <> -struct PackedOn16Bytes<half> -{ - using Type = PackedHalf; -}; - -#ifdef ENABLE_BF16 -using PackedBFloat16 = union -{ - int4 packed; - __nv_bfloat162 unpacked[4]; -}; - -template <> -struct PackedOn16Bytes<__nv_bfloat16> -{ - using Type = PackedBFloat16; -}; -#endif - -// add two 128b data -template <typename T> -inline __device__ int4 add128b(T& a, T& b) -{ - T c; - c.unpacked[0] = a.unpacked[0] + b.unpacked[0]; - c.unpacked[1] = a.unpacked[1] + b.unpacked[1]; - c.unpacked[2] = a.unpacked[2] + b.unpacked[2]; - c.unpacked[3] = a.unpacked[3] + b.unpacked[3]; - return c.packed; -} - -inline __device__ float warp_reduce_sum(float val) -{ - val += __shfl_xor_sync(~0, val, 16); - val += __shfl_xor_sync(~0, val, 8); - val += __shfl_xor_sync(~0, val, 4); - val += __shfl_xor_sync(~0, val, 2); - val += __shfl_xor_sync(~0, val, 1); - return val; -} - -inline __device__ float block_reduce_sum(float val) -{ - __shared__ float smem[kWarpSize]; - int lane_id = threadIdx.x % kWarpSize, warp_id = threadIdx.x / kWarpSize, warp_num = blockDim.x / kWarpSize; - val = warp_reduce_sum(val); - if (lane_id == 0) - { - smem[warp_id] = val; - } - __syncthreads(); - val = lane_id < warp_num ? smem[lane_id] : 0.f; - val = warp_reduce_sum(val); - return val; -} - -template <typename T, typename PackedStruct> -inline __device__ float accumulate(float acc, PackedStruct& vec) -{ - static constexpr int kLoopNum = sizeof(PackedStruct) / sizeof(T); -#pragma unroll - for (int i = 0; i < kLoopNum; ++i) - { - float v = static_cast<float>(reinterpret_cast<T*>(vec.unpacked)[i]); - acc += v * v; - } - return acc; -} - -template <typename T, bool Affine, typename PackedStruct> -inline __device__ int4 rms_norm(float denom, PackedStruct& vec, PackedStruct& weight) -{ - static constexpr int kLoopNum = sizeof(PackedStruct) / sizeof(T); - PackedStruct ret; -#pragma unroll - for (int i = 0; i < kLoopNum; ++i) - { - float v1 = static_cast<float>(reinterpret_cast<T*>(vec.unpacked)[i]); - if constexpr (Affine) - { - float v2 = static_cast<float>(reinterpret_cast<T*>(weight.unpacked)[i]); - reinterpret_cast<T*>(ret.unpacked)[i] = static_cast<T>(v1 * denom * v2); - } - else - { - reinterpret_cast<T*>(ret.unpacked)[i] = static_cast<T>(v1 * denom); - } - } - return ret.packed; -} - -// Fused (optional residual-add +) RMSNorm + NVFP4 input-quantize. Invoked by -// the standalone thop ops fused_add_rmsnorm_fp4_quantize (Residual=true) and -// fused_rmsnorm_fp4_quantize (Residual=false) on the attention-DP path. -// Performs the residual-add and reduction, then emits a per-block -// (SF_VEC_SIZE=16) NVFP4 representation to (quant_out, scale_out). When -// OutNorm=true, BF16 norm_out is also written (so a downstream consumer can read -// the un-quantized value). -// -// Layout assumptions (match cvt_warp_fp16_to_fp4): -// - Each thread accesses kPackedSize=8 BF16/half elements (one int4 = 16 B). -// - Two adjacent threads cover one SF_VEC_SIZE=16 block; SF is computed -// warp-cooperatively via __shfl_xor_sync inside cvt_warp_fp16_to_fp4. -// - Caller guarantees hidden_size % SF_VEC_SIZE == 0. -template <typename T, bool Bias = false, bool Residual = false, bool Affine = false, bool UseSmem = false, - bool OutNorm = false> -__global__ void rmsNormFp4QuantKernel(RmsNormFp4QuantParams params) -{ - static constexpr int kPackedSize = kBytesPerAccess / sizeof(T); - static constexpr int kSfVecSize = 16; - using PackedStruct = typename PackedOn16Bytes<T>::Type; - - extern __shared__ uint8_t smem_ptr[]; - T* smem = reinterpret_cast<T*>(smem_ptr); - - int const bid = blockIdx.x; - int const tid = threadIdx.x; - - T const* bias_buffer = reinterpret_cast<T const*>(params.bias_buffer); - T const* residual_buffer = reinterpret_cast<T const*>(params.residual_buffer); - T const* weight_buffer = reinterpret_cast<T const*>(params.weight_buffer); - T const* intermediate_buffer = reinterpret_cast<T const*>(params.intermediate_buffer); - T* residual_out_buffer = reinterpret_cast<T*>(params.residual_out_buffer); - T* norm_out = reinterpret_cast<T*>(params.norm_out); - - int const block_offset = bid * params.hidden_size; - // Input rows may be strided (e.g. a column slice of a wider projection, - // such as the leading q_lora_rank columns of kv_a_proj_with_mqa). When - // input_row_stride <= 0 it defaults to hidden_size (packed rows), making - // this byte-identical to all existing callers. Only the INPUT read offset - // uses the stride; every output (residual/norm/quant/scale) stays packed. - int const input_block_offset = bid * (params.input_row_stride > 0 ? params.input_row_stride : params.hidden_size); - int const thread_offset = tid * kPackedSize; - - if constexpr (Residual) - { - residual_buffer += block_offset; - // residual_out is packed [m, hidden_size], so it uses the dense block - // offset (not the possibly-strided input offset). - residual_out_buffer += block_offset; - } - intermediate_buffer += input_block_offset; - if constexpr (OutNorm) - { - norm_out += block_offset; - } - -#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) && (__CUDA_ARCH__ < 1200)) - cudaGridDependencySynchronize(); -#endif - - PackedStruct inter_vec, weight_vec; - float acc = 0.f; - for (int offset = thread_offset; offset < params.hidden_size; offset += blockDim.x * kPackedSize) - { - inter_vec.packed = *reinterpret_cast<int4 const*>(intermediate_buffer + offset); - if constexpr (Bias) - { - PackedStruct bias_vec; - bias_vec.packed = *reinterpret_cast<int4 const*>(bias_buffer + offset); - inter_vec.packed = add128b(inter_vec, bias_vec); - } - if constexpr (Residual) - { - PackedStruct residual_vec; - residual_vec.packed = *reinterpret_cast<int4 const*>(residual_buffer + offset); - inter_vec.packed = add128b(inter_vec, residual_vec); - // Write the residual sum to a distinct output buffer (packed offset), - // leaving the input intermediate_buffer untouched. - *reinterpret_cast<int4*>(residual_out_buffer + offset) = inter_vec.packed; - } - acc = accumulate<T>(acc, inter_vec); - if constexpr (UseSmem) - { - *reinterpret_cast<int4*>(&smem[offset]) = inter_vec.packed; - } - } - acc = block_reduce_sum(acc); - float const denom = rsqrtf(acc / params.hidden_size + params.eps); - - float const sf_scale = params.scale_factor_ptr ? *params.scale_factor_ptr : 1.f; - int const hidden_dim_packed = params.hidden_size / kSfVecSize; - - // cvt_warp_fp16_to_fp4 performs full-mask __shfl_xor_sync exchanges, which - // require every lane of the warp to execute the call. When hidden_size / - // kPackedSize is not a multiple of kWarpSize (e.g. hidden_size = 32, 128, - // 8208), the tail warp would otherwise be only partially active in this - // loop -- undefined behavior. Iterate with a warp-uniform bound (the warp's - // lane-0 offset) so all 32 lanes stay converged through the shuffle, and - // mask the per-lane loads/stores instead. Out-of-range lanes feed zeros to - // the shuffle; this is safe because hidden_size % kSfVecSize == 0 makes the - // active region end on an SF-pair boundary, so a padding lane's xor-1 - // partner is always another padding lane. - int const lane_id = tid % kWarpSize; - for (int offset = thread_offset; offset - lane_id * kPackedSize < params.hidden_size; - offset += blockDim.x * kPackedSize) - { - bool const valid = offset < params.hidden_size; - if (valid) - { - if constexpr (UseSmem) - { - inter_vec.packed = *reinterpret_cast<int4 const*>(&smem[offset]); - } - if constexpr (Affine) - { - weight_vec.packed = *reinterpret_cast<int4 const*>(weight_buffer + offset); - } - inter_vec.packed = rms_norm<T, Affine>(denom, inter_vec, weight_vec); - if constexpr (OutNorm) - { - *reinterpret_cast<int4*>(norm_out + offset) = inter_vec.packed; - } - } - else - { - // Benign values for the warp-cooperative SF exchange below (a - // padding lane's first-loop inter_vec may be uninitialized). - inter_vec.packed = make_int4(0, 0, 0, 0); - } - - // FP4 quantize this 8-element packed vec; warp-cooperate with the - // neighbour thread (offset ^ kPackedSize) to compute a single SF for - // their joint 16-element block. All lanes (incl. padding) must execute - // this call; padding lanes pass a null SF pointer and drop the result. - ::tensorrt_llm::kernels::PackedVec<T> pv - = *reinterpret_cast<::tensorrt_llm::kernels::PackedVec<T>*>(&inter_vec); - uint8_t* sf_out_ptr = nullptr; - if (valid) - { - int const access_id_in_token = offset / kPackedSize; - sf_out_ptr = ::tensorrt_llm::kernels::cvt_quant_get_sf_out_offset<uint32_t, 2>(std::nullopt, bid, - access_id_in_token, std::nullopt, hidden_dim_packed, reinterpret_cast<uint32_t*>(params.scale_out), - params.sf_layout); - } - uint32_t const quant_val = ::tensorrt_llm::kernels::cvt_warp_fp16_to_fp4<T, kSfVecSize, /*UE8M0_SF=*/false>( - pv, sf_scale, sf_out_ptr); - if (valid) - { - int const access_id = bid * (params.hidden_size / kPackedSize) + (offset / kPackedSize); - reinterpret_cast<uint32_t*>(params.quant_out)[access_id] = quant_val; - } - } -#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) && (__CUDA_ARCH__ < 1200)) - cudaTriggerProgrammaticLaunchCompletion(); -#endif -} - -template <typename T, bool OutNorm = false> -void launchRmsNormFp4QuantKernel(RmsNormFp4QuantParams const& params, cudaStream_t stream) -{ - static constexpr int kPackedSize = kBytesPerAccess / sizeof(T); - TLLM_CHECK(params.hidden_size % kPackedSize == 0); - TLLM_CHECK(params.hidden_size % 16 == 0); // SF_VEC_SIZE - int need_threads = params.hidden_size / kPackedSize; - int cta_size = need_threads <= kMaxCtaSize ? (need_threads + kWarpSize - 1) / kWarpSize * kWarpSize : kMaxCtaSize; - int cta_num = params.elts_total / params.hidden_size; - bool const need_smem = (cta_size * kBytesPerAccess / sizeof(T) < params.hidden_size); - int smem_size = need_smem ? params.hidden_size * sizeof(T) : 0; - bool const use_smem = need_smem; - - bool const has_bias = params.bias_buffer != nullptr; - bool const has_residual = params.residual_buffer != nullptr; - bool const has_weight = params.weight_buffer != nullptr; - - // Macro-dispatch over Bias/Residual/Affine/UseSmem and the OutNorm template arg. - // Launch through launchWithPdlWhenEnabled so the kernel's PDL primitives - // (cudaGridDependencySynchronize / cudaTriggerProgrammaticLaunchCompletion) - // are actually enabled when TLLM_ENABLE_PDL is set. -#define DISPATCH_FP4_QUANT(BIAS, RESIDUAL, AFFINE, SMEM) \ - if (use_smem == SMEM) \ - { \ - tensorrt_llm::common::launchWithPdlWhenEnabled("rmsNormFp4Quant", \ - rmsNormFp4QuantKernel<T, BIAS, RESIDUAL, AFFINE, SMEM, OutNorm>, dim3(cta_num), dim3(cta_size), \ - static_cast<size_t>(smem_size), stream, params); \ - } - - auto launch = [&]() - { - if (has_bias && has_residual && has_weight) - { - DISPATCH_FP4_QUANT(true, true, true, true) - else DISPATCH_FP4_QUANT(true, true, true, false) - } - else if (!has_bias && has_residual && has_weight) - { - DISPATCH_FP4_QUANT(false, true, true, true) - else DISPATCH_FP4_QUANT(false, true, true, false) - } - else if (has_bias && !has_residual && has_weight) - { - DISPATCH_FP4_QUANT(true, false, true, true) - else DISPATCH_FP4_QUANT(true, false, true, false) - } - else if (!has_bias && !has_residual && has_weight) - { - DISPATCH_FP4_QUANT(false, false, true, true) - else DISPATCH_FP4_QUANT(false, false, true, false) - } - else if (has_bias && has_residual && !has_weight) - { - DISPATCH_FP4_QUANT(true, true, false, true) - else DISPATCH_FP4_QUANT(true, true, false, false) - } - else if (!has_bias && has_residual && !has_weight) - { - DISPATCH_FP4_QUANT(false, true, false, true) - else DISPATCH_FP4_QUANT(false, true, false, false) - } - else if (has_bias && !has_residual && !has_weight) - { - DISPATCH_FP4_QUANT(true, false, false, true) - else DISPATCH_FP4_QUANT(true, false, false, false) - } - else - { - DISPATCH_FP4_QUANT(false, false, false, true) - else DISPATCH_FP4_QUANT(false, false, false, false) - } - }; - launch(); -#undef DISPATCH_FP4_QUANT -} - -} // namespace rms_norm_fp4_quant - -void residualRmsNormFp4Quant(RmsNormFp4QuantParams const& params, tensorrt_llm::DataType dataType, cudaStream_t stream) -{ - // The NVFP4 epilogue (cvt_warp_fp16_to_fp4) is compiled only for - // __CUDA_ARCH__ >= 1000 and emits zeros otherwise, so this kernel is correct - // only on SM 10.x (Blackwell). Fail fast on unsupported archs rather than - // silently producing wrong FP4 (the Python dispatch in rms_norm.py already - // routes those to the unfused path). - int const sm = tensorrt_llm::common::getSMVersion(); - TLLM_CHECK_WITH_INFO(sm >= 100 && sm < 120, - "residualRmsNormFp4Quant requires SM 10.x (Blackwell); got SM %d. The fused NVFP4 epilogue is unsupported on " - "this arch.", - sm); - TLLM_CHECK_WITH_INFO(params.quant_out != nullptr && params.scale_out != nullptr, - "residualRmsNormFp4Quant requires quant_out and scale_out output buffers."); - sync_check_cuda_error(stream); - bool const out_norm = (params.norm_out != nullptr); - if (out_norm) - { - switch (dataType) - { -#ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: - rms_norm_fp4_quant::launchRmsNormFp4QuantKernel<__nv_bfloat16, /*OutNorm=*/true>(params, stream); - break; -#endif - case tensorrt_llm::DataType::kHALF: - rms_norm_fp4_quant::launchRmsNormFp4QuantKernel<half, /*OutNorm=*/true>(params, stream); - break; - default: TLLM_THROW("Unsupported dataType for residualRmsNormFp4Quant"); - } - } - else - { - switch (dataType) - { -#ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: - rms_norm_fp4_quant::launchRmsNormFp4QuantKernel<__nv_bfloat16, /*OutNorm=*/false>(params, stream); - break; -#endif - case tensorrt_llm::DataType::kHALF: - rms_norm_fp4_quant::launchRmsNormFp4QuantKernel<half, /*OutNorm=*/false>(params, stream); - break; - default: TLLM_THROW("Unsupported dataType for residualRmsNormFp4Quant"); - } - } - sync_check_cuda_error(stream); -} - -} // namespace kernels - -TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.h b/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.h deleted file mode 100644 index 0ecba538d567..000000000000 --- a/cpp/tensorrt_llm/kernels/rmsNormFp4QuantKernels.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" -#include "tensorrt_llm/kernels/quantization.h" -#include <cstdint> - -TRTLLM_NAMESPACE_BEGIN - -namespace kernels -{ - -// All parameters for the fused (optional residual-add +) RMSNorm + NVFP4 -// input-quantize kernel: inputs, outputs, and layout config alike, so the -// launcher takes a single struct. Self-contained (does not borrow -// AllReduceParams) since this kernel performs no allreduce — it backs the -// standalone thop ops fused_add_rmsnorm_fp4_quantize / -// fused_rmsnorm_fp4_quantize on the attention-DP path. -struct RmsNormFp4QuantParams -{ - // --- inputs --- - // Input values [m, hidden_size] (read with input_row_stride). Read-only: the - // residual sum is written to residual_out_buffer, never back into this. - void const* intermediate_buffer{nullptr}; - // residual_in [m, hidden_size]; nullptr disables the residual add. - void const* residual_buffer{nullptr}; - // optional bias add [hidden_size]; nullptr disables it. - void const* bias_buffer{nullptr}; - // RMSNorm gamma [hidden_size]; nullptr selects the non-affine path. - void const* weight_buffer{nullptr}; - // Device pointer to the global per-tensor scale (= 448*6 / amax for - // static-quant Linear); nullptr means 1.0. - float const* scale_factor_ptr{nullptr}; - - // --- outputs --- - // Packed FP4 (E2M1) values, 2 per byte. - void* quant_out{nullptr}; - // E4M3 scaling factors (one per SF_VEC_SIZE=16 block), laid out per sf_layout. - void* scale_out{nullptr}; - // Optional BF16/FP16 post-RMSNorm value (packed rows); nullptr to skip. - void* norm_out{nullptr}; - // residual_out [m, hidden_size] (packed): receives input + residual when - // residual_buffer != nullptr. A distinct buffer from intermediate_buffer so - // the input is never mutated (keeps the thop op functionalizable under - // torch.compile) and no pre-kernel copy is needed. nullptr when no residual. - void* residual_out_buffer{nullptr}; - - // --- config --- - int hidden_size{0}; - float eps{0.f}; - // Total element count (= m * hidden_size); used to derive the row count. - int64_t elts_total{0}; - // Scaling-factor layout (typically SWIZZLED). - ::tensorrt_llm::QuantizationSFLayout sf_layout{::tensorrt_llm::QuantizationSFLayout::SWIZZLED}; - // Element stride between input rows in intermediate_buffer. 0 means - // "== hidden_size" (packed rows). Set >0 to read a strided slice (e.g. a - // column-slice of a wider projection) without a preceding contiguous copy. - // Outputs are always written packed. - int input_row_stride{0}; -}; - -// Fused (optional residual-add +) RMSNorm + NVFP4 input-quantize. Folds RMSNorm -// and the next op's NVFP4 input-quant so the (flashinfer RMSNorm + standalone -// fp4_quantize) pair becomes one launch on the attention-DP path. All inputs, -// outputs, and layout configuration are carried in params (see the struct -// field docs above); dataType selects the fp16/bf16 instantiation. -void residualRmsNormFp4Quant(RmsNormFp4QuantParams const& params, tensorrt_llm::DataType dataType, cudaStream_t stream); - -} // namespace kernels - -TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/speculativeDecoding/dynamicTreeKernels.cu b/cpp/tensorrt_llm/kernels/speculativeDecoding/dynamicTreeKernels.cu index caa090f0e25c..3d167660e521 100644 --- a/cpp/tensorrt_llm/kernels/speculativeDecoding/dynamicTreeKernels.cu +++ b/cpp/tensorrt_llm/kernels/speculativeDecoding/dynamicTreeKernels.cu @@ -29,10 +29,12 @@ #include "tensorrt_llm/common/reduceKernelUtils.cuh" #include "tensorrt_llm/common/vec_dtypes.cuh" #include "tensorrt_llm/kernels/decodingCommon.h" +#include <ATen/cuda/CUDAContext.h> #include <algorithm> #include <cfloat> #include <cstdint> #include <limits> +#include <torch/extension.h> TRTLLM_NAMESPACE_BEGIN using namespace tensorrt_llm::common; @@ -41,6 +43,480 @@ using namespace tensorrt_llm::runtime; namespace kernels::speculative_decoding { +// --------------------------------------------------------------------------- +// Two-stage top-k / top-p masking kernels +// Mirrors the approach in invokeBatchTopKSampling (samplingTopKKernels.cu), +// but outputs a masked logits tensor instead of sampling a token. +// --------------------------------------------------------------------------- + +// Stage 1: Parallel top-k reduction across BLOCKS_PER_BEAM_ blocks per row. +// Each block handles (vocabSize / BLOCKS_PER_BEAM_) elements and finds its +// local top-k, writing (global_index, logit_value) pairs into the tmp buffers. +template <typename T, int32_t BLOCK_SIZE_, int32_t BLOCKS_PER_BEAM_> +__global__ void topKProbStage1(T const* __restrict__ logits, T* tmpLogProbs, int32_t* topKTmpIdBuf, T* topKTmpValBuf, + int32_t maxTopK, int32_t const* topKs, int32_t vocabSize) +{ + typedef cub::BlockReduce<TopK_2<T>, BLOCK_SIZE_> BlockReduce; + __shared__ typename BlockReduce::TempStorage tempStorage; + + auto const tid = static_cast<int32_t>(threadIdx.x); + auto const bid = static_cast<int32_t>(blockIdx.x); + auto const rowId = bid / BLOCKS_PER_BEAM_; + auto const blockLane = bid % BLOCKS_PER_BEAM_; // chunk index within the row + + auto const k = (topKs != nullptr) ? topKs[rowId] : maxTopK; + + bool const IS_FP16 = std::is_same<T, half>::value; + T const MAX_T_VAL = IS_FP16 ? HALF_FLT_MAX : FLT_MAX; + + // Base offset into the flat (nRows * vocabSize) logits array for this row. + auto const rowOffset = rowId * vocabSize; + // Base offset into the tmp buffers for this (row, blockLane). + auto const tmpIdxBase = rowId * BLOCKS_PER_BEAM_ * maxTopK + blockLane * k; + + // Copy this block's chunk of logits into tmpLogProbs scratch space. + for (auto elemId = tid + blockLane * BLOCK_SIZE_; elemId < vocabSize; elemId += BLOCK_SIZE_ * BLOCKS_PER_BEAM_) + { + tmpLogProbs[rowOffset + elemId] = logits[rowOffset + elemId]; + } + __syncthreads(); + + // Iteratively find the top-k values via max-reduction, zeroing each found max. + TopK_2<T> partial; + for (int32_t ite = 0; ite < k; ite++) + { + partial.init(); + for (auto elemId = tid + blockLane * BLOCK_SIZE_; elemId < vocabSize; elemId += BLOCK_SIZE_ * BLOCKS_PER_BEAM_) + { + partial.insert(tmpLogProbs[rowOffset + elemId], rowOffset + elemId); + } + + TopK_2<T> total = BlockReduce(tempStorage).Reduce(partial, reduce_topk_op_2<T>); + + if (tid == 0) + { + topKTmpIdBuf[tmpIdxBase + ite] = total.p; // global index (rowOffset + vocabIdx) + topKTmpValBuf[tmpIdxBase + ite] = total.u; // logit value + if (total.p >= 0) + { + tmpLogProbs[total.p] = -MAX_T_VAL; // zero out so next iteration finds next-best + } + } + __syncthreads(); + } +} + +// Stage 2: Merge BLOCKS_PER_BEAM_ * k candidates per row, apply optional top-p, +// then scatter selected logit values back to an output logits tensor (all other +// positions are set to -inf so that a subsequent softmax produces 0 probability). +template <typename T, int32_t BLOCK_SIZE_, int32_t BLOCKS_PER_BEAM_> +__global__ void topKProbStage2ForLogits(int32_t const* __restrict__ topKTmpIdBuf, T* topKTmpValBuf, float* outputLogits, + int32_t maxTopK, int32_t const* topKs, float const* topPs, int32_t vocabSize) +{ + bool const IS_FP16 = std::is_same<T, half>::value; + T const MAX_T_VAL = IS_FP16 ? HALF_FLT_MAX : FLT_MAX; + + auto const tid = static_cast<int32_t>(threadIdx.x); + auto const rowId = static_cast<int32_t>(blockIdx.x); + + auto const k = (topKs != nullptr) ? topKs[rowId] : maxTopK; + // size: number of valid candidates written by Stage 1 for this row. + // stride: row pitch in the tmp buffers (same as in invokeBatchTopKSampling). + auto const size = k * BLOCKS_PER_BEAM_; + auto const stride = maxTopK * BLOCKS_PER_BEAM_; + + typedef cub::BlockReduce<TopK_2<float>, BLOCK_SIZE_> BlockReduce; + __shared__ typename BlockReduce::TempStorage tempStorage; + extern __shared__ char sharedArray[]; + // Shared layout: sId[maxTopK] | sVal2[maxTopK] + auto* sId = reinterpret_cast<int32_t*>(sharedArray); + auto* sVal2 = reinterpret_cast<float*>(sId + maxTopK); + + // Pointer to this row's candidates in the tmp value buffer (modified in-place during reduction). + T* sVal = topKTmpValBuf + rowId * stride; + + // Step 1: Initialize output row to -inf (all threads cooperate for bandwidth). + float* outRow = outputLogits + rowId * vocabSize; + float const negInf = -std::numeric_limits<float>::infinity(); + for (int32_t i = tid; i < vocabSize; i += BLOCK_SIZE_) + { + outRow[i] = negInf; + } + __syncthreads(); + + // Step 2: k-round block-reduction over the k * BLOCKS_PER_BEAM_ valid candidates. + // (Only the first 'size' entries of the row's tmp buffer were written by Stage 1.) + TopK_2<float> partial; + __shared__ float sMaxLogit; + for (int32_t ite = 0; ite < k; ite++) + { + partial.init(); + for (int32_t i = tid; i < size; i += BLOCK_SIZE_) + { + partial.insert(static_cast<float>(sVal[i]), i); + } + + TopK_2<float> total = BlockReduce(tempStorage).Reduce(partial, reduce_topk_op_2<float>); + + if (tid == 0) + { + if (ite == 0) + { + sMaxLogit = total.u; + } + sId[ite] = total.p; + sVal[total.p] = -MAX_T_VAL; // zero out so next iteration finds next-best + sVal2[ite] = total.u; // store raw logit value (not exponentiated) + } + __syncthreads(); + } + + // Step 3: Determine top-p cutoff (tid=0 only). + // sVal2 contains logit values in descending order; we exponentiate to get unnormalized probs. + if (tid == 0) + { + int32_t cutoff = k; + if (topPs != nullptr) + { + float const topP = topPs[rowId]; + if (topP < 1.0f) + { + // Compute unnormalized probabilities and their sum. + float sSum = 0.0f; + for (int32_t ki = 0; ki < k; ki++) + { + sVal2[ki] = __expf(sVal2[ki] - sMaxLogit); // reuse sVal2 to hold exp probs + sSum += sVal2[ki]; + } + // Walk in descending-probability order; stop as soon as cumulative prob >= topP. + float cumProb = 0.0f; + for (int32_t ki = 0; ki < k; ki++) + { + cumProb += sVal2[ki] / sSum; + if (cumProb >= topP) + { + cutoff = ki + 1; // always keep at least this token + break; + } + } + } + } + + // Step 4: Scatter selected logit values back to output. + // topKTmpIdBuf stores (rowOffset + vocabIdx); recover vocabIdx with % vocabSize. + auto const rowStride = rowId * stride; + for (int32_t ki = 0; ki < cutoff; ki++) + { + auto const candidateIdx = sId[ki]; + auto const globalIdx = topKTmpIdBuf[rowStride + candidateIdx]; + if (globalIdx >= 0) + { + auto const vocabIdx = globalIdx % vocabSize; + // sVal2 was overwritten with exp probs when topP < 1; we need the original logit. + // Re-read from the original tmp buffer — the stored value IS the logit (set in Stage 1). + // However sVal[candidateIdx] was zeroed during Stage 2 reduction; but + // topKTmpValBuf still holds the original value at that index (sVal points there). + // We stored the logit as sVal2[ite] = total.u BEFORE any exp, so if topP was not + // applied we can use sVal2[ki] directly. If topP was applied, sVal2[ki] now holds + // the exp prob — we cannot recover the logit. To handle both cases cleanly we use + // log(sVal2[ki]) + sMaxLogit when topPs was applied, otherwise sVal2[ki] directly. + float logitVal; + if (topPs != nullptr && topPs[rowId] < 1.0f) + { + // sVal2[ki] = exp(logit - sMaxLogit), so logit = log(sVal2[ki]) + sMaxLogit + logitVal = __logf(sVal2[ki]) + sMaxLogit; + } + else + { + logitVal = sVal2[ki]; // still the raw logit + } + outRow[vocabIdx] = logitVal; + } + } + } +} + +#define CASE_K_PROB(K_MAX, BLOCK_SIZE_1_, BLOCK_SIZE_2_, BLOCKS_PER_BEAM_) \ + do \ + { \ + topKProbStage1<T, BLOCK_SIZE_1_, BLOCKS_PER_BEAM_> \ + <<<dim3(nRows* BLOCKS_PER_BEAM_, 1), BLOCK_SIZE_1_, 0, stream>>>( \ + logits, tmpLogProbs, topKTmpIdBuf, topKTmpValBuf, maxTopK, topKs, vocabSize); \ + topKProbStage2ForLogits<T, BLOCK_SIZE_2_, BLOCKS_PER_BEAM_> \ + <<<dim3(nRows, 1), BLOCK_SIZE_2_, K_MAX * (sizeof(int32_t) + sizeof(float)), stream>>>( \ + topKTmpIdBuf, topKTmpValBuf, outputLogits, maxTopK, topKs, topPs, vocabSize); \ + } while (0) + +// Host launcher: allocates workspace tensors internally and dispatches the two-stage kernels. +// logits [nRows, vocabSize] – temperature-scaled input (float or half) +// outputLogits [nRows, vocabSize] – output: -inf everywhere except selected top-k-p positions +// topKs [nRows] – per-row k values (int32, on device) +// topPs [nRows] or nullptr – per-row p values (float, on device) +// maxTopK – maximum k across all rows (CPU scalar, 1–1024) +template <typename T> +void invokeTopKTopPMaskingForProbs(T const* logits, float* outputLogits, int32_t const* topKs, float const* topPs, + int32_t maxTopK, int32_t nRows, int32_t vocabSize, cudaStream_t stream) +{ + constexpr int32_t BLOCKS_PER_BEAM = 8; + + // Workspace buffers (allocated as CUDA device tensors via ATen). + auto opts = at::TensorOptions().dtype(torch::kFloat32).device(at::kCUDA); + auto tmpLogProbsTensor = torch::empty({nRows * vocabSize}, opts); + auto topKTmpIdBufTensor + = torch::empty({nRows * BLOCKS_PER_BEAM * maxTopK}, at::TensorOptions().dtype(torch::kInt32).device(at::kCUDA)); + // topKTmpValBuf uses the same dtype as T; we allocate as float and reinterpret for half if needed. + auto topKTmpValBufTensor = torch::empty({nRows * BLOCKS_PER_BEAM * maxTopK}, opts); + + T* tmpLogProbs = reinterpret_cast<T*>(tmpLogProbsTensor.data_ptr<float>()); + int32_t* topKTmpIdBuf = topKTmpIdBufTensor.data_ptr<int32_t>(); + T* topKTmpValBuf = reinterpret_cast<T*>(topKTmpValBufTensor.data_ptr<float>()); + + int32_t logMaxTopK = 0; + int32_t recursor = maxTopK - 1; + while (recursor >>= 1) + { + ++logMaxTopK; + } + + switch (logMaxTopK) + { + case 0: + case 1: + case 2: + case 3: // 0 < maxTopK <= 16 + CASE_K_PROB(16, 128, 128, 8); + break; + case 4: // 16 < maxTopK <= 32 + CASE_K_PROB(32, 256, 128, 8); + break; + case 5: // 32 < maxTopK <= 64 + CASE_K_PROB(64, 256, 256, 8); + break; + case 6: + case 7: + case 8: + case 9: // 64 < maxTopK <= 1024 + CASE_K_PROB(1024, 256, 256, 8); + break; + default: TLLM_CHECK_WITH_INFO(false, "topKProbMasking supports 1 <= k <= 1024 but got k=%d", maxTopK); + } +} + +#undef CASE_K_PROB + +namespace +{ +constexpr double kGreedyTempThreshold = 1e-4; + +torch::Tensor computeSoftmaxForProbOp(torch::Tensor logits) +{ + TORCH_CHECK(logits.is_cuda(), "logits must be a CUDA tensor"); + TORCH_CHECK(logits.dim() == 2, "logits must be a 2D tensor"); + + auto probs = logits.contiguous().to(torch::kFloat32); + auto stream = at::cuda::getCurrentCUDAStream(probs.device().index()); + + BiasSoftmaxParams<float> biasSoftmaxParams; + biasSoftmaxParams.logits = probs.data_ptr<float>(); + biasSoftmaxParams.probs = probs.data_ptr<float>(); + biasSoftmaxParams.batchSize = static_cast<SizeType32>(probs.size(0)); + biasSoftmaxParams.maxBatchSize = static_cast<SizeType32>(probs.size(0)); + biasSoftmaxParams.maxBeamWidth = 1; + biasSoftmaxParams.vocabSize = static_cast<SizeType32>(probs.size(1)); + biasSoftmaxParams.vocabSizePadded = static_cast<SizeType32>(probs.size(1)); + biasSoftmaxParams.skipSoftMax = false; + biasSoftmaxParams.batchSlotsLogits = false; + biasSoftmaxParams.checkParams(); + + invokeAddBiasSoftMax(biasSoftmaxParams, stream); + return probs; +} + +// Fast path for top-K (and optional top-P) filtering using torch::topk instead of a +// full vocab-size sort. kMax must be provided as a CPU integer (the caller computes it +// via topK.max().item() on the Python side). When kMax == 0 or kMax >= vocabSize the +// function falls back to the original sort-based path. +// +// Key advantages over the full-sort path: +// 1. torch::topk with small kMax is O(V * log kMax) vs O(V * log V) for full sort. +// 2. The topk index tensor is [nRows, kMax] instead of [nRows, V] — much smaller. +// 3. No scatter-back of sorted indices needed; masking is done directly on logits. +// 4. For combined top-K + top-P, softmax/cumsum are computed on kMax values (not V). +torch::Tensor applyTopKTopPForProbOp(torch::Tensor logits, torch::optional<torch::Tensor> const& topK, + torch::optional<torch::Tensor> const& topP, int32_t kMax) +{ + int64_t const vocabSize = logits.size(1); + // Host-only checks: the caller is expected to pass nullopt when filtering is fully + // disabled (see SpecMetadata.skip_top_k / skip_top_p). Probing the tensor contents + // via `.item<bool>()` here would force a host-device sync and break CUDA graph + // capture; the per-row `effectiveTopK` formula below already handles disabled rows. + bool const hasTopK = topK.has_value() && topK->defined(); + bool const hasTopP = topP.has_value() && topP->defined(); + + if (!hasTopK && !hasTopP) + { + return logits; + } + + torch::Tensor effectiveTopK; + if (hasTopK) + { + auto topKLong = topK->to(torch::kLong); + effectiveTopK + = torch::where(topKLong > 0, topKLong, torch::full_like(topKLong, vocabSize)).clamp_max(vocabSize); + } + + // Fast path uses `topk(kMax)` which is unsafe when any row has effective top-k > kMax + // (i.e. disabled rows expand to the full vocab). Detecting this requires a tensor + // reduction + `.item<bool>()`, which is incompatible with CUDA graph capture. Only + // probe when the caller explicitly opted into the fast path via kMax > 0 (today only + // the dynamic-tree caller, which is not graph-captured). + bool hasDisabledTopKRows = false; + if (hasTopK && kMax > 0 && kMax < vocabSize) + { + auto topKLong = topK->to(torch::kLong); + hasDisabledTopKRows = topKLong.le(0).any().item<bool>(); + } + + if (hasTopK && !hasDisabledTopKRows && kMax > 0 && kMax < vocabSize) + { + // Fast topk path ───────────────────────────────────────────────────────────── + // topKValues/topKIdx: [nRows, kMax], values in descending order + auto [topKValues, topKIdx] = logits.topk(kMax, /*dim=*/-1, /*largest=*/true, /*sorted=*/true); + + // validTopK[i, j]: True when position j falls within top-K[i] for row i + auto kArange = torch::arange(kMax, torch::TensorOptions().dtype(torch::kInt64).device(logits.device())) + .unsqueeze(0); // [1, kMax] + auto kVals = effectiveTopK.to(torch::kInt64).unsqueeze(1); // [nRows, 1] + auto validTopK = kArange < kVals; // [nRows, kMax] + + // Start with everything masked; scatter will unmark the kept positions. + auto mask = torch::ones( + {logits.size(0), vocabSize}, torch::TensorOptions().dtype(torch::kBool).device(logits.device())); + + if (hasTopP) + { + // Compute top-P on the kMax descending-sorted values only (much cheaper). + // Positions beyond K[i] are treated as -inf so their probability ≈ 0. + auto validTopKValues = topKValues.masked_fill(~validTopK, -std::numeric_limits<float>::infinity()); + auto sortedProbs = validTopKValues.softmax(/*dim=*/-1); // [nRows, kMax] + auto cumsum = sortedProbs.cumsum(/*dim=*/-1); // [nRows, kMax] + // Mask positions where the cumulative probability *before* this token + // already reaches topP — i.e. we have enough probability mass already. + auto topPMask = (cumsum - sortedProbs) >= topP->unsqueeze(1); // [nRows, kMax] + topPMask.select(/*dim=*/1, /*index=*/0).fill_(false); // always keep the top-1 token + // combinedMask: True → mask this vocab position + // False → keep this vocab position + auto combinedMask = topPMask | (~validTopK); // [nRows, kMax] + mask.scatter_(/*dim=*/1, /*index=*/topKIdx, /*src=*/combinedMask); + } + else + { + // Top-K only: unmark the first K[i] positions (those within validTopK). + // ~validTopK is True for positions j >= K[i] → they should stay masked. + mask.scatter_(/*dim=*/1, /*index=*/topKIdx, /*src=*/(~validTopK)); + } + + return logits.masked_fill(mask, -std::numeric_limits<float>::infinity()); + } + + // Fallback: full-sort path (used for top-P only, or when kMax == 0) ──────────── + auto sortResult = logits.sort(/*dim=*/-1, /*descending=*/false); + auto logitsSort = std::get<0>(sortResult); + auto logitsIdx = std::get<1>(sortResult); + + if (hasTopK) + { + auto topKMask = logitsSort.size(1) - effectiveTopK; + topKMask = topKMask.clamp_min(0); + auto topKThreshold = logitsSort.gather(1, topKMask.unsqueeze(1)); + auto mask = logitsSort < topKThreshold; + logitsSort.masked_fill_(mask, -std::numeric_limits<float>::infinity()); + } + + if (hasTopP) + { + auto probsSort = logitsSort.softmax(/*dim=*/-1); + auto probsSum = probsSort.cumsum(/*dim=*/-1, /*dtype=*/probsSort.scalar_type()); + auto topPMask = probsSum <= (1.0 - topP->unsqueeze(1)); + topPMask.select(/*dim=*/1, /*index=*/logitsSort.size(1) - 1).fill_(false); + logitsSort.masked_fill_(topPMask, -std::numeric_limits<float>::infinity()); + } + + return logitsSort.scatter(/*dim=*/-1, /*index=*/logitsIdx, /*src=*/logitsSort); +} + +} // namespace + +torch::Tensor computeProbsFromLogits(torch::Tensor const& logits, torch::Tensor const& temperatures, + torch::optional<torch::Tensor> const& topK, torch::optional<torch::Tensor> const& topP, bool skipTemperature, + int32_t kMax) +{ + TORCH_CHECK(logits.is_cuda(), "logits must be a CUDA tensor"); + TORCH_CHECK(temperatures.is_cuda(), "temperatures must be a CUDA tensor"); + TORCH_CHECK(logits.dim() == 2, "logits must be a 2D tensor"); + TORCH_CHECK(temperatures.dim() == 1, "temperatures must be a 1D tensor"); + TORCH_CHECK(logits.size(0) == temperatures.size(0), "logits and temperatures size mismatch"); + if (topK.has_value() && topK->defined()) + { + TORCH_CHECK(topK->is_cuda(), "top_k must be a CUDA tensor"); + TORCH_CHECK(topK->dim() == 1, "top_k must be a 1D tensor"); + TORCH_CHECK(topK->size(0) == logits.size(0), "top_k and logits size mismatch"); + } + if (topP.has_value() && topP->defined()) + { + TORCH_CHECK(topP->is_cuda(), "top_p must be a CUDA tensor"); + TORCH_CHECK(topP->dim() == 1, "top_p must be a 1D tensor"); + TORCH_CHECK(topP->size(0) == logits.size(0), "top_p and logits size mismatch"); + } + + auto const isGreedy = temperatures <= kGreedyTempThreshold; + auto const safeTemperatures = torch::where(isGreedy, torch::ones_like(temperatures), temperatures); + auto scaledLogits + = (skipTemperature ? logits : logits.div(safeTemperatures.unsqueeze(1))).contiguous().to(torch::kFloat32); + + int64_t const vocabSize = scaledLogits.size(1); + int64_t const nRows = scaledLogits.size(0); + // Host-only presence checks; see comment in applyTopKTopPForProbOp() for why we + // avoid probing tensor contents (would sync and break CUDA graph capture). + bool const hasTopKPresence = topK.has_value() && topK->defined(); + bool const hasTopPPresence = topP.has_value() && topP->defined(); + + // The kernel path produces -inf for rows whose top_k value is 0, so it is only + // safe when every row has an active top_k filter. Determining that requires a + // host-device sync, so only probe when the caller has opted into the kernel + // path (kMax > 0). The kMax > 0 callers (dynamic-tree) are not graph-captured. + bool useKernelPath = false; + if (hasTopKPresence && kMax > 0 && kMax < vocabSize) + { + useKernelPath = torch::logical_and(topK->gt(0), topK->lt(vocabSize)).any().item<bool>(); + } + + torch::Tensor maskedLogits; + if (useKernelPath) + { + // Two-stage CUDA top-k/top-p masking (mirrors invokeBatchTopKSampling). + maskedLogits = torch::empty_like(scaledLogits); + auto topKForKernel = topK->to(torch::kInt32).contiguous(); + auto topPForKernel = hasTopPPresence ? topP->to(torch::kFloat32).contiguous() : torch::Tensor(); + auto stream = at::cuda::getCurrentCUDAStream(scaledLogits.device().index()); + invokeTopKTopPMaskingForProbs<float>(scaledLogits.data_ptr<float>(), maskedLogits.data_ptr<float>(), + topKForKernel.data_ptr<int32_t>(), hasTopPPresence ? topPForKernel.data_ptr<float>() : nullptr, kMax, + static_cast<int32_t>(nRows), static_cast<int32_t>(vocabSize), stream); + } + else + { + // Fallback: PyTorch-based sort path (top-P only or kMax == 0). + maskedLogits = applyTopKTopPForProbOp(scaledLogits, topK, topP, kMax); + } + + auto probs = computeSoftmaxForProbOp(maskedLogits); + + auto argmaxIds = maskedLogits.argmax(/*dim=*/-1, /*keepdim=*/true); + auto oneHot = torch::zeros_like(probs).scatter_(1, argmaxIds, 1.0); + return torch::where(isGreedy.unsqueeze(1), oneHot, probs); +} + //! \param parentList [in] layer-wise parent indices [bs, topK*(depth-1)+1] //! \param selectedIndex [in] resampled history buffer indices [bs, draftTokenNum-1] //! \param treeMask [out] attention mask (which nodes each node can see) diff --git a/cpp/tensorrt_llm/kernels/splitkGroupGemm.cu b/cpp/tensorrt_llm/kernels/splitkGroupGemm.cu index 1f63189ec657..6397396ea6f6 100644 --- a/cpp/tensorrt_llm/kernels/splitkGroupGemm.cu +++ b/cpp/tensorrt_llm/kernels/splitkGroupGemm.cu @@ -26,7 +26,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/splitk_gemm_grouped.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/default_splitk_gemm_grouped.h" #include "tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/splitk_gemm_grouped.h" @@ -204,20 +203,20 @@ template <int M1, int N1, int K1, int M2, int N2, int K2, int kAlignmentAB, int void splitkGroupedGemmType_(std::vector<cutlass::gemm::GemmCoord> const& problemSizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkSpaceSize, - tensorrt_llm::DataType dataType, int splitKSlices, cudaStream_t stream) + nvinfer1::DataType dataType, int splitKSlices, cudaStream_t stream) { - if (dataType == tensorrt_llm::DataType::kHALF) + if (dataType == nvinfer1::DataType::kHALF) { splitkGroupedGemm_<M1, N1, K1, M2, N2, K2, cutlass::half_t, kAlignmentAB, kAlignmentC, kStages>(problemSizes, ptrA, ptrB, ptrC, ptrD, gemmParamsWorkSpace, gemmParamsWorkSpaceSize, gemmWorkSpace, gemmWorkSpaceSize, splitKSlices, stream); } - else if (dataType == tensorrt_llm::DataType::kFLOAT) + else if (dataType == nvinfer1::DataType::kFLOAT) { TLLM_CHECK_WITH_INFO(false, "not support float input/output"); } #ifdef ENABLE_BF16 - else if (dataType == tensorrt_llm::DataType::kBF16) + else if (dataType == nvinfer1::DataType::kBF16) { splitkGroupedGemm_<M1, N1, K1, M2, N2, K2, cutlass::bfloat16_t, kAlignmentAB, kAlignmentC, kStages>( problemSizes, ptrA, ptrB, ptrC, ptrD, gemmParamsWorkSpace, gemmParamsWorkSpaceSize, gemmWorkSpace, @@ -229,7 +228,7 @@ void splitkGroupedGemmType_(std::vector<cutlass::gemm::GemmCoord> const& problem void splitkGroupedGemm(std::vector<cutlass::gemm::GemmCoord> const& problemSizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkSpace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkSpaceSize, - bool isLoraIn, tensorrt_llm::DataType dataType, int splitKSlices, int minKN, cudaStream_t stream) + bool isLoraIn, nvinfer1::DataType dataType, int splitKSlices, int minKN, cudaStream_t stream) { TLLM_LOG_TRACE("%s start, isLoraIn: %d, minKN = %d", __PRETTY_FUNCTION__, static_cast<int>(isLoraIn), minKN); if (isLoraIn) diff --git a/cpp/tensorrt_llm/kernels/splitkGroupGemm.h b/cpp/tensorrt_llm/kernels/splitkGroupGemm.h index bcde457db7d2..6ada8255292e 100644 --- a/cpp/tensorrt_llm/kernels/splitkGroupGemm.h +++ b/cpp/tensorrt_llm/kernels/splitkGroupGemm.h @@ -17,7 +17,7 @@ #include "cutlass/gemm_coord.h" #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <vector> TRTLLM_NAMESPACE_BEGIN @@ -30,7 +30,7 @@ int64_t getSplitkGroupedGemmParamsWorkSpaceSize(int64_t problem_count); void splitkGroupedGemm(std::vector<cutlass::gemm::GemmCoord> const& problem_sizes, std::vector<void*> const& ptrA, std::vector<void*> const& ptrB, std::vector<void*> const& ptrC, std::vector<void*> const& ptrD, void* gemmParamsWorkspace, int64_t gemmParamsWorkSpaceSize, void* gemmWorkSpace, int64_t gemmWorkspaceSize, - bool isLoraIn, tensorrt_llm::DataType dataType, int splitKSlices, int minKN, cudaStream_t stream); + bool isLoraIn, nvinfer1::DataType dataType, int splitKSlices, int minKN, cudaStream_t stream); } // namespace kernels diff --git a/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h b/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h index 8bcc29e0dbdb..d1681c39a56e 100644 --- a/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h +++ b/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h @@ -496,11 +496,7 @@ class TllmGenFmhaKernel tg::CudaRunner::Grid grid{numCtasX, numCtasY, numCtasZ}; // Prepare custom mask for spec-decoding generation kernels if needed. - bool const prepareSpecDecTreeMask = params.mIsSpecDecTree - && (params.mForcePrepareSpecDecTreeMask || params.mLayerIdx == 0 - || (params.mSpecDecodingTargetMaxGenLen > 0 - && params.mMaxSeqLenQ != params.mSpecDecodingTargetMaxGenLen)); - if (prepareSpecDecTreeMask) + if (params.mLayerIdx == 0 && params.mIsSpecDecTree) { int32_t stepQ = options.mTileSizeQ * options.mNumInstsQ; int32_t stepKv = options.mTileSizeKv * options.mNumInstsKv; diff --git a/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h b/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h index 9252650ac67b..83b816c346f5 100644 --- a/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h +++ b/cpp/tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h @@ -369,7 +369,6 @@ struct TllmGenFmhaRunnerParams // row stride ceilDiv(mPackedMaskMaxSeqLenQ, 32) rather than ceilDiv(seqLenQ, 32). int32_t mPackedMaskMaxSeqLenQ = 0; int32_t mSpecDecodingTargetMaxGenLen = 0; - bool mForcePrepareSpecDecTreeMask = false; // set the attention mask type TllmGenFmhaRunnerParams& setAttentionMaskType(std::int8_t maskType) diff --git a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels.h b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels.h index 04bcdfbbe2ac..302c278f7a2a 100644 --- a/cpp/tensorrt_llm/kernels/unfusedAttentionKernels.h +++ b/cpp/tensorrt_llm/kernels/unfusedAttentionKernels.h @@ -17,7 +17,6 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/kernels/mlaKernels.h" @@ -243,42 +242,42 @@ struct QKVPreprocessingParams { ss << "seq_lens: " << *(runtime::ITensor::wrap( - (void*) seq_lens, tensorrt_llm::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); + (void*) seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); } if (cache_seq_lens && batch_size > 0) { ss << "cache_seq_lens: " - << *(runtime::ITensor::wrap((void*) cache_seq_lens, tensorrt_llm::DataType::kINT32, - runtime::ITensor::makeShape({batch_size}))); + << *(runtime::ITensor::wrap( + (void*) cache_seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); } if (encoder_seq_lens && batch_size > 0) { ss << "encoder_seq_lens: " - << *(runtime::ITensor::wrap((void*) encoder_seq_lens, tensorrt_llm::DataType::kINT32, - runtime::ITensor::makeShape({batch_size}))); + << *(runtime::ITensor::wrap( + (void*) encoder_seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); } if (cu_seq_lens && batch_size > 0) { ss << "cu_seq_lens: " << *(runtime::ITensor::wrap( - (void*) cu_seq_lens, tensorrt_llm::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); + (void*) cu_seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); } if (cu_kv_seq_lens && batch_size > 0) { ss << "cu_kv_seq_lens: " - << *(runtime::ITensor::wrap((void*) cu_kv_seq_lens, tensorrt_llm::DataType::kINT32, - runtime::ITensor::makeShape({batch_size}))); + << *(runtime::ITensor::wrap( + (void*) cu_kv_seq_lens, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size}))); } if (sparse_kv_offsets) { ss << "sparse_kv_offsets: " - << *(runtime::ITensor::wrap((void*) sparse_kv_offsets, tensorrt_llm::DataType::kINT32, + << *(runtime::ITensor::wrap((void*) sparse_kv_offsets, nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({batch_size + 1}))); } if (rotary_embedding_inv_freq && batch_size > 0 && rotary_embedding_dim > 0) { ss << "rotary_embedding_inv_freq: " - << *(runtime::ITensor::wrap((void*) rotary_embedding_inv_freq, tensorrt_llm::DataType::kFLOAT, + << *(runtime::ITensor::wrap((void*) rotary_embedding_inv_freq, nvinfer1::DataType::kFLOAT, runtime::ITensor::makeShape({batch_size, rotary_embedding_dim / 2}))); } ss << "rotary_coef_cache_buffer: " << rotary_coef_cache_buffer << std::endl; diff --git a/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.cpp b/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.cpp index d219572d2707..3e19f9ebe72a 100644 --- a/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.cpp +++ b/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.cpp @@ -16,7 +16,6 @@ #include "ub_interface.h" #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaDriverWrapper.h" -#include "tensorrt_llm/common/tllmDataType.h" #include <cuda_runtime.h> #include <cuda_runtime_api.h> @@ -81,13 +80,13 @@ namespace kernels::ub { void allreduce2_userbuff_inplace_launcher(int const handler, size_t const offset, size_t const elements, - tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) + nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { allreduce2_userbuff_inplace_impl(handler, offset, elements, dataType, comm, stream); } int allgather2_userbuff_residual_launcher(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable) { return allgather2_userbuff_residual_impl( @@ -96,7 +95,7 @@ int allgather2_userbuff_residual_launcher(int const handler, size_t const offset int allreduce2_userbuff_rmsnorm_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { return allreduce2_userbuff_rmsnorm_impl(handler, offset, out_handler, out_offset, elements, hidden_size, beta, gamma, eps, residual_in, residual_out, dataType, comm, stream); @@ -104,7 +103,7 @@ int allreduce2_userbuff_rmsnorm_launcher(int const handler, size_t const offset, int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { return allreduce2_userbuff_inplace_rmsnorm_quant_impl(handler, offset, out_handler, out_offset, elements, @@ -114,7 +113,7 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, - void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { return allreduce2_userbuff_inplace_rmsnorm_quant_fp4_impl(handler, offset, out_handler, out_offset, scale_handler, scale_offset, elements, hidden_size, beta, gamma, eps, scalefactor, residual_in, residual_out, dataType, comm, @@ -166,12 +165,12 @@ TRTLLM_NAMESPACE_BEGIN namespace kernels::ub { void allreduce2_userbuff_inplace_launcher(int const handler, size_t const offset, size_t const elements, - tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) + nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { } int allgather2_userbuff_residual_launcher(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable) { return 0; @@ -179,7 +178,7 @@ int allgather2_userbuff_residual_launcher(int const handler, size_t const offset int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { return 0; @@ -188,7 +187,7 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, - void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { return 0; } diff --git a/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.h b/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.h index dc68154fb462..e8a48e2c680b 100644 --- a/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.h +++ b/cpp/tensorrt_llm/kernels/userbuffers/ub_interface.h @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "ub_allocator.h" namespace tensorrt_llm::runtime::ub @@ -41,24 +40,24 @@ namespace kernels::ub using ::tensorrt_llm::runtime::ub::communicator; void allreduce2_userbuff_inplace_launcher(int const handler, size_t const offset, size_t const elements, - tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream = 0); + nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream = 0); int allgather2_userbuff_residual_launcher(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable = false); int allreduce2_userbuff_rmsnorm_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); + void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); int allreduce2_userbuff_inplace_rmsnorm_quant_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_launcher(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, - void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); + void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); } // namespace kernels::ub TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.cu b/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.cu index a19059e9010c..8cb5814e0398 100644 --- a/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.cu +++ b/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.cu @@ -15,7 +15,6 @@ */ #include "tensorrt_llm/common/config.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/quantization.cuh" #include "userbuffers.h" #include "utils.h" @@ -1775,11 +1774,11 @@ int allgather2_userbuff_residual(int const handler, size_t const offset, size_t } void allreduce2_userbuff_inplace_impl(int const handler, size_t const offset, size_t const elements, - tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) + nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { switch (dataType) { - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1792,7 +1791,7 @@ void allreduce2_userbuff_inplace_impl(int const handler, size_t const offset, si break; } #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1810,17 +1809,17 @@ void allreduce2_userbuff_inplace_impl(int const handler, size_t const offset, si } int allgather2_userbuff_residual_impl(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable) { switch (dataType) { - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: return allgather2_userbuff_residual<half>( handler, offset, elements, hidden_size, residual, comm, stream, force_enable); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: return allgather2_userbuff_residual<__nv_bfloat16>( handler, offset, elements, hidden_size, residual, comm, stream, force_enable); break; @@ -1831,11 +1830,11 @@ int allgather2_userbuff_residual_impl(int const handler, size_t const offset, si int allreduce2_userbuff_rmsnorm_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { switch (dataType) { - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1850,7 +1849,7 @@ int allreduce2_userbuff_rmsnorm_impl(int const handler, size_t const offset, int break; } #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1871,12 +1870,12 @@ int allreduce2_userbuff_rmsnorm_impl(int const handler, size_t const offset, int int allreduce2_userbuff_inplace_rmsnorm_quant_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { switch (dataType) { - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1891,7 +1890,7 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_impl(int const handler, size_t con break; } #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1915,11 +1914,11 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_impl(int const handler, size_t con int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, void* residual_in, - void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream) + void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream) { switch (dataType) { - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: { if (kDISABLE_FP32_ACCUMULATION) { @@ -1936,7 +1935,7 @@ int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_impl(int const handler, size_t break; } #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: { if (kDISABLE_FP32_ACCUMULATION) { diff --git a/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.h b/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.h index 5d3ffe0cc950..96f21b748282 100644 --- a/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.h +++ b/cpp/tensorrt_llm/kernels/userbuffers/userbuffers.h @@ -14,7 +14,6 @@ * limitations under the License. */ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/worldConfig.h" @@ -121,25 +120,25 @@ namespace kernels::ub { using namespace ::tensorrt_llm::runtime::ub; void allreduce2_userbuff_inplace_impl(int const handler, size_t const offset, size_t const elements, - tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream = 0); + nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream = 0); // for TP-parallelism, only single node is implemented int allgather2_userbuff_residual_impl(int const handler, size_t const offset, size_t const elements, - int const hidden_size, void* residual, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream, + int const hidden_size, void* residual, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream, bool force_enable); int allreduce2_userbuff_rmsnorm_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); + void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); int allreduce2_userbuff_inplace_rmsnorm_quant_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, - float* scalefactor, void* residual_in, void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, + float* scalefactor, void* residual_in, void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); int allreduce2_userbuff_inplace_rmsnorm_quant_fp4_impl(int const handler, size_t const offset, int const out_handler, size_t const out_offset, int const scale_handler, size_t const scale_offset, size_t const elements, int const hidden_size, void* beta, void* gamma, float eps, float* scalefactor, void* residual_in, - void* residual_out, tensorrt_llm::DataType dataType, communicator* comm, cudaStream_t stream); + void* residual_out, nvinfer1::DataType dataType, communicator* comm, cudaStream_t stream); } // namespace kernels::ub TRTLLM_NAMESPACE_END diff --git a/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h b/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h index c2bf35175391..eb939b57c2db 100644 --- a/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h +++ b/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h @@ -24,6 +24,8 @@ #include "tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h" #include "tensorrt_llm/runtime/common.h" +#include <NvInferRuntime.h> + #include <cassert> #include <cmath> #include <cstdint> diff --git a/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemmNVFP4.h b/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemmNVFP4.h index 6e901846ed25..616f9d25c2bf 100644 --- a/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemmNVFP4.h +++ b/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemmNVFP4.h @@ -24,6 +24,8 @@ #include "tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h" #include "tensorrt_llm/runtime/common.h" +#include <NvInferRuntime.h> + #include <cassert> #include <cmath> #include <cstdint> diff --git a/cpp/tensorrt_llm/kernels/xqaDispatcher.cpp b/cpp/tensorrt_llm/kernels/xqaDispatcher.cpp index 8a5eeee91c6b..35fd02e7f127 100644 --- a/cpp/tensorrt_llm/kernels/xqaDispatcher.cpp +++ b/cpp/tensorrt_llm/kernels/xqaDispatcher.cpp @@ -534,7 +534,6 @@ void XqaDispatcher::runImpl( tllmRunnerParams.generalPackedCustoMaskPtr = params.spec_decoding_packed_mask; tllmRunnerParams.mPackedMaskMaxSeqLenQ = params.spec_decoding_max_generation_length; tllmRunnerParams.mSpecDecodingTargetMaxGenLen = mFixedParams.specDecodingTargetMaxGenLen; - tllmRunnerParams.mForcePrepareSpecDecTreeMask = params.force_prepare_spec_dec_tree_mask; tllmRunnerParams.customMaskPtr = params.spec_decoding_bl_tree_mask; tllmRunnerParams.customMaskOffsetsPtr = params.spec_decoding_bl_tree_mask_offset; tllmRunnerParams.firstSparseMaskOffsetsKvPtr = params.spec_bl_tree_first_sparse_mask_offset_kv; diff --git a/cpp/tensorrt_llm/layers/decodingParams.h b/cpp/tensorrt_llm/layers/decodingParams.h index 76c5cedd637b..1e77b8919ca1 100644 --- a/cpp/tensorrt_llm/layers/decodingParams.h +++ b/cpp/tensorrt_llm/layers/decodingParams.h @@ -17,7 +17,6 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/kernels/beamSearchKernels.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -193,9 +192,9 @@ class ExplicitDraftTokensSetupParams : public DecodingSetupParams public: OptVec<float> temperature; // [setupBatchSize] // Hack to init some data for the context phase in the setup. - TensorPtr randomDataSample; // [maxBatchSize], on gpu - TensorPtr temperatures; // [maxBatchSize], on gpu - tensorrt_llm::DataType dtype; // [1] + TensorPtr randomDataSample; // [maxBatchSize], on gpu + TensorPtr temperatures; // [maxBatchSize], on gpu + nvinfer1::DataType dtype; // [1] }; class EagleSetupParams : public DecodingSetupParams @@ -203,9 +202,9 @@ class EagleSetupParams : public DecodingSetupParams public: OptVec<float> temperature; // [setupBatchSize] // Hack to init some data for the context phase in the setup. - TensorPtr randomDataSample; // [maxBatchSize], on gpu - TensorPtr temperatures; // [maxBatchSize], on gpu - tensorrt_llm::DataType dtype; // [1] + TensorPtr randomDataSample; // [maxBatchSize], on gpu + TensorPtr temperatures; // [maxBatchSize], on gpu + nvinfer1::DataType dtype; // [1] }; class DynamicDecodeSetupParams : public BaseSetupParams diff --git a/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.cpp b/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.cpp index aedeb731574f..e014ee4535e5 100644 --- a/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.cpp +++ b/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.cpp @@ -16,7 +16,6 @@ #include "explicitDraftTokensLayer.h" #include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/penaltyTypes.h" #include "tensorrt_llm/kernels/speculativeDecoding/common.h" #include "tensorrt_llm/kernels/speculativeDecoding/explicitDraftTokensKernels.h" @@ -94,15 +93,15 @@ void ExplicitDraftTokensLayer<T>::setup(SizeType32 batchSize, SizeType32 beamWid batchSlots, getLimitsPenalty(DecodingPenaltyType::Temperature), "temperature penalty"); // Dispatch context buffer fill - if (mDecoderDtype == tensorrt_llm::DataType::kFLOAT) + if (mDecoderDtype == nvinfer1::DataType::kFLOAT) { fillContextBuffers<float>(batchSize, batchSlots, *setupParams, workspace); } - else if (mDecoderDtype == tensorrt_llm::DataType::kHALF) + else if (mDecoderDtype == nvinfer1::DataType::kHALF) { fillContextBuffers<half>(batchSize, batchSlots, *setupParams, workspace); } - else if (mDecoderDtype == tensorrt_llm::DataType::kBF16) + else if (mDecoderDtype == nvinfer1::DataType::kBF16) { fillContextBuffers<__nv_bfloat16>(batchSize, batchSlots, *setupParams, workspace); } @@ -127,15 +126,15 @@ void ExplicitDraftTokensLayer<T>::forwardAsync(std::shared_ptr<BaseDecodingOutpu convertPackedMask(*outputs, *inputs, workspace); // Slice output ids, pos ids, next draft tokens. - if (mDecoderDtype == tensorrt_llm::DataType::kFLOAT) + if (mDecoderDtype == nvinfer1::DataType::kFLOAT) { splitInputDataToBatchSlots<float>(*outputs, *inputs, workspace); } - else if (mDecoderDtype == tensorrt_llm::DataType::kHALF) + else if (mDecoderDtype == nvinfer1::DataType::kHALF) { splitInputDataToBatchSlots<half>(*outputs, *inputs, workspace); } - else if (mDecoderDtype == tensorrt_llm::DataType::kBF16) + else if (mDecoderDtype == nvinfer1::DataType::kBF16) { splitInputDataToBatchSlots<__nv_bfloat16>(*outputs, *inputs, workspace); } diff --git a/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.h b/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.h index 17fca4513cf1..75883ded6e5a 100644 --- a/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.h +++ b/cpp/tensorrt_llm/layers/explicitDraftTokensLayer.h @@ -16,7 +16,6 @@ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/layers/baseLayer.h" #include "tensorrt_llm/layers/decodingParams.h" #include "tensorrt_llm/runtime/common.h" @@ -84,7 +83,7 @@ class ExplicitDraftTokensLayer : public BaseLayer TensorPtr mTemperature; - std::optional<tensorrt_llm::DataType> mDecoderDtype{std::nullopt}; + std::optional<nvinfer1::DataType> mDecoderDtype{std::nullopt}; }; } // namespace tensorrt_llm::layers diff --git a/cpp/tensorrt_llm/layers/lookaheadAlgorithm.cpp b/cpp/tensorrt_llm/layers/lookaheadAlgorithm.cpp index 76da89dfec0d..09843fd7ce44 100644 --- a/cpp/tensorrt_llm/layers/lookaheadAlgorithm.cpp +++ b/cpp/tensorrt_llm/layers/lookaheadAlgorithm.cpp @@ -17,7 +17,6 @@ #include "tensorrt_llm/layers/lookaheadAlgorithm.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" #include "tensorrt_llm/runtime/common.h" @@ -36,14 +35,14 @@ LookaheadAlgorithm::LookaheadAlgorithm( runtime::SizeType32 maxW, runtime::SizeType32 maxN, runtime::SizeType32 maxG, runtime::SizeType32 id) : mPoolManager(maxG) , mPrefillsMax(runtime::BufferManager::cpu( - runtime::ITensor::makeShape({(maxN <= 1 ? 0 : maxN - 2)}), tensorrt_llm::DataType::kINT32)) + runtime::ITensor::makeShape({(maxN <= 1 ? 0 : maxN - 2)}), nvinfer1::DataType::kINT32)) , mPastTokensMax( - runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxW * (maxN - 1)}), tensorrt_llm::DataType::kINT32)) - , mKeyTokensMax(runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxW}), tensorrt_llm::DataType::kINT32)) + runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxW * (maxN - 1)}), nvinfer1::DataType::kINT32)) + , mKeyTokensMax(runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxW}), nvinfer1::DataType::kINT32)) , mGoldenTokensMax( - runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxN * 2 - 1}), tensorrt_llm::DataType::kINT32)) + runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxN * 2 - 1}), nvinfer1::DataType::kINT32)) , mGuessTokensMax( - runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxG * (maxN - 1)}), tensorrt_llm::DataType::kINT32)) + runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxG * (maxN - 1)}), nvinfer1::DataType::kINT32)) , mMaxW(maxW) , mMaxN(maxN) , mMaxG(maxG) @@ -53,13 +52,12 @@ LookaheadAlgorithm::LookaheadAlgorithm( std::tie(maxGeneratedLen, std::ignore, maxDraftLen, std::ignore) = executor::LookaheadDecodingConfig(maxW, maxN, maxG).calculateSpeculativeResource(); mAttentionMask = runtime::BufferManager::cpu( - runtime::ITensor::makeShape({maxDraftLen, maxDraftLen}), tensorrt_llm::DataType::kBOOL); + runtime::ITensor::makeShape({maxDraftLen, maxDraftLen}), nvinfer1::DataType::kBOOL); mDraftTokensMax - = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxDraftLen}), tensorrt_llm::DataType::kINT32); + = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxDraftLen}), nvinfer1::DataType::kINT32); mSampledTokensMax - = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxGeneratedLen}), tensorrt_llm::DataType::kINT32); - mEncodeMapMax - = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxDraftLen}), tensorrt_llm::DataType::kINT32); + = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxGeneratedLen}), nvinfer1::DataType::kINT32); + mEncodeMapMax = runtime::BufferManager::cpu(runtime::ITensor::makeShape({maxDraftLen}), nvinfer1::DataType::kINT32); } void LookaheadAlgorithm::setup(TensorConstPtr const& prompt, SizeType32 w, SizeType32 n, SizeType32 g, uint64_t seed) diff --git a/cpp/tensorrt_llm/layers/lookaheadDecodingLayer.cpp b/cpp/tensorrt_llm/layers/lookaheadDecodingLayer.cpp index 986f0e0b978e..bf6e15080f3c 100644 --- a/cpp/tensorrt_llm/layers/lookaheadDecodingLayer.cpp +++ b/cpp/tensorrt_llm/layers/lookaheadDecodingLayer.cpp @@ -19,7 +19,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/kernels/samplingTopKKernels.h" #include "tensorrt_llm/layers/decodingParams.h" @@ -65,42 +64,38 @@ LookaheadDecodingLayer<T>::CpuAlgorithmResources::CpuAlgorithmResources(DecoderD mPrompts.reserve(maxBatchSize); for (auto bi = 0; bi < maxBatchSize; bi++) { - mPrompts.emplace_back(BufferManager::cpu(ITensor::makeShape({0}), tensorrt_llm::DataType::kINT32)); + mPrompts.emplace_back(BufferManager::cpu(ITensor::makeShape({0}), nvinfer1::DataType::kINT32)); } auto const maxBatchShape1D = ITensor::makeShape({maxBatchSize}); - mBatchSlots = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); mTargetTokens - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT32); - mTokensPerStep = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mEndIds = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), nvinfer1::DataType::kINT32); + mTokensPerStep = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); + mEndIds = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); - mOutputIds - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxNumNewTokens}), tensorrt_llm::DataType::kINT32); + mOutputIds = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxNumNewTokens}), nvinfer1::DataType::kINT32); mNewTokens = BufferManager::cpu( - ITensor::makeShape({maxTokensPerStep, maxBatchSize, beamWidth}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxTokensPerStep, maxBatchSize, beamWidth}), nvinfer1::DataType::kINT32); mPathsOffsets - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), tensorrt_llm::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), nvinfer1::DataType::kINT32); mPathsOffsetsBatch - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), tensorrt_llm::DataType::kINT32); - mNumNewTokens = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mNumNewTokensCumSum = BufferManager::cpu(ITensor::makeShape({maxBatchSize + 1}), tensorrt_llm::DataType::kINT32); - mNextDraftTokens - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxDraftLen}), tensorrt_llm::DataType::kINT32); - mNextDraftPosIds - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxDraftLen}), tensorrt_llm::DataType::kINT32); - mGenerationLengths = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), nvinfer1::DataType::kINT32); + mNumNewTokens = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); + mNumNewTokensCumSum = BufferManager::cpu(ITensor::makeShape({maxBatchSize + 1}), nvinfer1::DataType::kINT32); + mNextDraftTokens = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxDraftLen}), nvinfer1::DataType::kINT32); + mNextDraftPosIds = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxDraftLen}), nvinfer1::DataType::kINT32); + mGenerationLengths = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); mPositionOffsets - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT32); - mPositionIds - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), nvinfer1::DataType::kINT32); + mPositionIds = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), nvinfer1::DataType::kINT32); mAttentionMask - = BufferManager::cpu(ITensor::makeShape({maxTokensPerStep, maxTokensPerStep}), tensorrt_llm::DataType::kBOOL); + = BufferManager::cpu(ITensor::makeShape({maxTokensPerStep, maxTokensPerStep}), nvinfer1::DataType::kBOOL); mPackedMask = BufferManager::cpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep, static_cast<ITensor::DimType64>(divUp(maxTokensPerStep, 32))}), - tensorrt_llm::DataType::kINT32); - mNextDraftLengths = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mSequenceLengths = BufferManager::cpu(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); + mNextDraftLengths = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); + mSequenceLengths = BufferManager::cpu(maxBatchShape1D, nvinfer1::DataType::kINT32); } template <typename T> @@ -122,12 +117,12 @@ LookaheadDecodingLayer<T>::LookaheadDecodingLayer( auto const maxBatchShape2D = ITensor::makeShape({maxBatchSize, maxTokensPerStep}); mWorkspaceSize = getTopKWorkspaceSize<T>(maxBatchSize, maxTokensPerStep, maxTopK, vocabSizePadded); - mTargetTokensDevice = mBufferManager->gpu(maxBatchShape2D, tensorrt_llm::DataType::kINT32); + mTargetTokensDevice = mBufferManager->gpu(maxBatchShape2D, nvinfer1::DataType::kINT32); mCurandStatesDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); mSetupWorkspaceSize = DecodingLayerWorkspace::calculateRequiredWorkspaceSize( - std::make_pair(maxBatchShape1D, tensorrt_llm::DataType::kINT64)); + std::make_pair(maxBatchShape1D, nvinfer1::DataType::kINT64)); TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); } diff --git a/cpp/tensorrt_llm/layers/lookaheadDecodingUtils.h b/cpp/tensorrt_llm/layers/lookaheadDecodingUtils.h index 8e3e8f6c590d..739cf65001ab 100644 --- a/cpp/tensorrt_llm/layers/lookaheadDecodingUtils.h +++ b/cpp/tensorrt_llm/layers/lookaheadDecodingUtils.h @@ -16,7 +16,6 @@ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -319,12 +318,12 @@ class DebugTensor { switch (mTensor.getDataType()) { - case tensorrt_llm::DataType::kBOOL: return values<bool>(); - case tensorrt_llm::DataType::kFLOAT: return values<float>(); - case tensorrt_llm::DataType::kINT8: return values<std::int8_t>(); - case tensorrt_llm::DataType::kINT32: return values<std::int32_t>(); - case tensorrt_llm::DataType::kINT64: return values<std::int64_t>(); - case tensorrt_llm::DataType::kUINT8: return values<std::uint8_t>(); + case nvinfer1::DataType::kBOOL: return values<bool>(); + case nvinfer1::DataType::kFLOAT: return values<float>(); + case nvinfer1::DataType::kINT8: return values<std::int8_t>(); + case nvinfer1::DataType::kINT32: return values<std::int32_t>(); + case nvinfer1::DataType::kINT64: return values<std::int64_t>(); + case nvinfer1::DataType::kUINT8: return values<std::uint8_t>(); default: return std::string(mName + ": Unsupported data type"); } } @@ -377,12 +376,12 @@ class DebugTensor { switch (mTensor.getDataType()) { - case tensorrt_llm::DataType::kBOOL: return randomize<bool>(3); - case tensorrt_llm::DataType::kFLOAT: return randomize<float>(3); - case tensorrt_llm::DataType::kINT8: return randomize<std::int8_t>(3); - case tensorrt_llm::DataType::kINT32: return randomize<std::int32_t>(3); - case tensorrt_llm::DataType::kINT64: return randomize<std::int64_t>(3); - case tensorrt_llm::DataType::kUINT8: return randomize<std::uint8_t>(3); + case nvinfer1::DataType::kBOOL: return randomize<bool>(3); + case nvinfer1::DataType::kFLOAT: return randomize<float>(3); + case nvinfer1::DataType::kINT8: return randomize<std::int8_t>(3); + case nvinfer1::DataType::kINT32: return randomize<std::int32_t>(3); + case nvinfer1::DataType::kINT64: return randomize<std::int64_t>(3); + case nvinfer1::DataType::kUINT8: return randomize<std::uint8_t>(3); default: return; } } @@ -392,12 +391,12 @@ class DebugTensor { switch (mTensor.getDataType()) { - case tensorrt_llm::DataType::kBOOL: return randomize<bool>(0); - case tensorrt_llm::DataType::kFLOAT: return randomize<float>(0); - case tensorrt_llm::DataType::kINT8: return randomize<std::int8_t>(0); - case tensorrt_llm::DataType::kINT32: return randomize<std::int32_t>(0); - case tensorrt_llm::DataType::kINT64: return randomize<std::int64_t>(0); - case tensorrt_llm::DataType::kUINT8: return randomize<std::uint8_t>(0); + case nvinfer1::DataType::kBOOL: return randomize<bool>(0); + case nvinfer1::DataType::kFLOAT: return randomize<float>(0); + case nvinfer1::DataType::kINT8: return randomize<std::int8_t>(0); + case nvinfer1::DataType::kINT32: return randomize<std::int32_t>(0); + case nvinfer1::DataType::kINT64: return randomize<std::int64_t>(0); + case nvinfer1::DataType::kUINT8: return randomize<std::uint8_t>(0); default: return; } } @@ -406,12 +405,12 @@ class DebugTensor { switch (mTensor.getDataType()) { - case tensorrt_llm::DataType::kBOOL: return randomize<bool>(1); - case tensorrt_llm::DataType::kFLOAT: return randomize<float>(1); - case tensorrt_llm::DataType::kINT8: return randomize<std::int8_t>(1); - case tensorrt_llm::DataType::kINT32: return randomize<std::int32_t>(1); - case tensorrt_llm::DataType::kINT64: return randomize<std::int64_t>(1); - case tensorrt_llm::DataType::kUINT8: return randomize<std::uint8_t>(1); + case nvinfer1::DataType::kBOOL: return randomize<bool>(1); + case nvinfer1::DataType::kFLOAT: return randomize<float>(1); + case nvinfer1::DataType::kINT8: return randomize<std::int8_t>(1); + case nvinfer1::DataType::kINT32: return randomize<std::int32_t>(1); + case nvinfer1::DataType::kINT64: return randomize<std::int64_t>(1); + case nvinfer1::DataType::kUINT8: return randomize<std::uint8_t>(1); default: return; } } diff --git a/cpp/tensorrt_llm/layers/lookaheadPoolManager.cpp b/cpp/tensorrt_llm/layers/lookaheadPoolManager.cpp index 397b4262226a..5954bc520ad0 100644 --- a/cpp/tensorrt_llm/layers/lookaheadPoolManager.cpp +++ b/cpp/tensorrt_llm/layers/lookaheadPoolManager.cpp @@ -16,7 +16,6 @@ #include "tensorrt_llm/layers/lookaheadPoolManager.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" #include <cstddef> @@ -68,7 +67,7 @@ void LookaheadPoolManager::accept(TensorConstPtr const& prompt, SizeType32 level for (SizeType32 ti = 0; ti + level - 1 < length; ti++) { auto key = promptRange[ti]; - TensorPtr ngram = BufferManager::cpu(ITensor::makeShape({level - 1}), tensorrt_llm::DataType::kINT32); + TensorPtr ngram = BufferManager::cpu(ITensor::makeShape({level - 1}), nvinfer1::DataType::kINT32); BufferRange<TokenIdType const> sourceRange(*ITensor::slice(prompt, ti + 1, level - 1)); BufferRange<TokenIdType> ngramRange(*ngram); std::copy(sourceRange.begin(), sourceRange.end(), ngramRange.begin()); @@ -108,7 +107,7 @@ void LookaheadPoolManager::update(TensorConstPtr const& keyTokens, TensorConstPt for (SizeType32 wi = 0; wi < window; wi++) { TensorConstPtr source = ITensor::at(ngramTokens, {wi}); - TensorPtr ngram = BufferManager::cpu(source->getShape(), tensorrt_llm::DataType::kINT32); + TensorPtr ngram = BufferManager::cpu(source->getShape(), nvinfer1::DataType::kINT32); BufferRange<TokenIdType const> sourceRange(*source); BufferRange<TokenIdType> ngramRange(*ngram); std::copy(sourceRange.begin(), sourceRange.end(), ngramRange.begin()); diff --git a/cpp/tensorrt_llm/layers/medusaDecodingLayer.cpp b/cpp/tensorrt_llm/layers/medusaDecodingLayer.cpp index 9e4098b34ebf..40eff62c17d6 100644 --- a/cpp/tensorrt_llm/layers/medusaDecodingLayer.cpp +++ b/cpp/tensorrt_llm/layers/medusaDecodingLayer.cpp @@ -16,7 +16,6 @@ #include "medusaDecodingLayer.h" #include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/kernels/samplingTopKKernels.h" #include "tensorrt_llm/kernels/speculativeDecoding/medusaDecodingKernels.h" @@ -89,10 +88,10 @@ void MedusaDecodingLayer<T>::allocateBuffer() mTiledBatchSlotsSetup = BufferManager::pinnedPool( ITensor::makeShape({static_cast<SizeType32>(mDecoderDomain.getBatchSize() * maxDraftPathLen)}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mTiledBatchSlotsForward = BufferManager::pinnedPool( ITensor::makeShape({static_cast<SizeType32>(mDecoderDomain.getBatchSize() * maxDraftPathLen)}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mMedusaInputLogitsPtrs = BufferManager::pinnedPool( ITensor::makeShape({static_cast<SizeType32>(mDecoderDomain.getBatchSize() * maxDraftPathLen)}), TRTDataType<T*>::value); diff --git a/cpp/tensorrt_llm/layers/penaltyLayer.cpp b/cpp/tensorrt_llm/layers/penaltyLayer.cpp index c72b8e463bc6..c6c57ca5034d 100644 --- a/cpp/tensorrt_llm/layers/penaltyLayer.cpp +++ b/cpp/tensorrt_llm/layers/penaltyLayer.cpp @@ -18,7 +18,6 @@ #include "penaltyLayer.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/nvtxUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/penaltyKernels.h" #include "tensorrt_llm/kernels/penaltyTypes.h" #include "tensorrt_llm/layers/defaultDecodingParams.h" @@ -85,11 +84,11 @@ void PenaltyLayer<T>::allocateWorkspace() auto const workspaceSize = mDecoderDomain.getBatchSize() * mDecoderDomain.getMaxDecodingTokens() * mConfiguredBeamWidth * mDecoderDomain.getVocabSize() * 2; - mPenaltyWorkspaceDevice = mBufferManager->gpu(workspaceSize, tensorrt_llm::DataType::kINT32); + mPenaltyWorkspaceDevice = mBufferManager->gpu(workspaceSize, nvinfer1::DataType::kINT32); if (mDecodingMode.isBeamSearch()) { - mPenaltyWorkspacePrevDevice = mBufferManager->gpu(workspaceSize, tensorrt_llm::DataType::kINT32); + mPenaltyWorkspacePrevDevice = mBufferManager->gpu(workspaceSize, nvinfer1::DataType::kINT32); } } @@ -112,27 +111,27 @@ void PenaltyLayer<T>::allocateBuffer() if (mDecodingMode.isUseTemperature()) { - mTemperatureDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kFLOAT); + mTemperatureDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kFLOAT); } if (mDecodingMode.isUseRepetitionPenalty()) { - mRepetitionPenaltyDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kFLOAT); + mRepetitionPenaltyDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kFLOAT); } if (mDecodingMode.isUsePresencePenalty()) { - mPresencePenaltyDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kFLOAT); + mPresencePenaltyDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kFLOAT); } if (mDecodingMode.isUseFrequencyPenalty()) { - mFrequencyPenaltyDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kFLOAT); + mFrequencyPenaltyDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kFLOAT); } if (mDecodingMode.isUseMinLength()) { - mMinLengthDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kINT32); + mMinLengthDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kINT32); } if (mDecodingMode.isUseOccurrencePenalty()) { - mPromptIgnoreLengthDevice = mBufferManager->gpu(batchSizeShape, tensorrt_llm::DataType::kINT32); + mPromptIgnoreLengthDevice = mBufferManager->gpu(batchSizeShape, nvinfer1::DataType::kINT32); } auto const logitsPtrDeviceDesc = std::make_pair(batchSizeShape, TRTDataType<T*>::value); diff --git a/cpp/tensorrt_llm/nanobind/CMakeLists.txt b/cpp/tensorrt_llm/nanobind/CMakeLists.txt index 4d6fbf9c2607..b523ae193871 100755 --- a/cpp/tensorrt_llm/nanobind/CMakeLists.txt +++ b/cpp/tensorrt_llm/nanobind/CMakeLists.txt @@ -14,6 +14,7 @@ set(SRCS batch_manager/llmRequest.cpp common/tllmExceptions.cpp executor/bindings.cpp + executor/executor.cpp executor/executorConfig.cpp executor/request.cpp process_group/bindings.cpp @@ -22,6 +23,7 @@ set(SRCS runtime/moeBindings.cpp suffixAutomaton/bindings.cpp testing/kvCacheManagerTestUtilBinding.cpp + testing/modelSpecBinding.cpp userbuffers/bindings.cpp thop/bindings.cpp ../runtime/ipcNvlsMemory.cu diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/algorithms.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/algorithms.cpp index c13466565342..4070811b2d72 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/algorithms.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/algorithms.cpp @@ -23,11 +23,11 @@ #include "tensorrt_llm/batch_manager/createNewDecoderRequests.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/logitsPostProcessor.h" #include "tensorrt_llm/batch_manager/medusaBuffers.h" #include "tensorrt_llm/batch_manager/microBatchScheduler.h" #include "tensorrt_llm/batch_manager/pauseRequests.h" #include "tensorrt_llm/batch_manager/peftCacheManager.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/nanobind/common/customCasters.h" #include "tensorrt_llm/runtime/decoderState.h" #include "tensorrt_llm/runtime/torch.h" @@ -129,6 +129,13 @@ void tensorrt_llm::nanobind::batch_manager::algorithms::initBindings(nb::module_ nb::call_guard<nb::gil_scoped_release>()) .def("name", [](AllocateKvCache const&) { return AllocateKvCache::name; }); + nb::class_<LogitsPostProcessor>(m, LogitsPostProcessor::name) + .def(nb::init<>()) + .def("__call__", &LogitsPostProcessor::operator(), nb::arg("decoder_input_buffers"), + nb::arg("replicate_logits_post_processor"), nb::arg("world_config"), nb::arg("stream"), + nb::arg("logits_post_processor_batched") = std::nullopt) + .def("name", [](LogitsPostProcessor const&) { return LogitsPostProcessor::name; }); + nb::class_<CreateNewDecoderRequests>(m, CreateNewDecoderRequests::name) .def(nb::init<bool, bool, bool>(), nb::arg("speculative_decoding_fast_logits"), nb::arg("is_leader_in_orch_mode"), nb::arg("is_normalize_log_probs")) @@ -136,7 +143,7 @@ void tensorrt_llm::nanobind::batch_manager::algorithms::initBindings(nb::module_ "__call__", [](CreateNewDecoderRequests& self, tr::ModelConfig const& modelConfig, tr::WorldConfig const& worldConfig, executor::DecodingConfig const& decodingConfig, RequestVector const& contextRequests, - tensorrt_llm::DataType logitsType, DecoderInputBuffers& inputBuffers, + nvinfer1::DataType logitsType, DecoderInputBuffers& inputBuffers, runtime::decoder::DecoderState& decoderState, tensorrt_llm::runtime::CudaStream const& runtimeStream, tensorrt_llm::runtime::CudaStream const& decoderStream, SizeType32 maxSequenceLength, SizeType32 beamWidth) diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp index 3c1823e2a94a..0846663dafad 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp @@ -24,7 +24,6 @@ #include "tensorrt_llm/batch_manager/peftCacheManager.h" #include "tensorrt_llm/batch_manager/rnnStateManager.h" #include "tensorrt_llm/batch_manager/sequenceSlotManager.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/nanobind/common/bindTypes.h" #include "tensorrt_llm/runtime/gptDecoderBatched.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -174,8 +173,6 @@ void initBindings(nb::module_& m) nb::arg("kv_tokens_per_block")) .def_prop_rw( "estimated_reusable_tokens", &GenLlmReq::getEstimatedReusableTokens, &GenLlmReq::setEstimatedReusableTokens) - .def_prop_rw( - "expect_snapshot_points", &GenLlmReq::getExpectedSnapshotPoints, &GenLlmReq::setExpectedSnapshotPoints) .def_prop_rw("guided_decoding_params", &GenLlmReq::getGuidedDecodingParams, &GenLlmReq::setGuidedDecodingParams) .def_prop_rw("context_phase_params", &GenLlmReq::getContextPhaseParams, &GenLlmReq::setContextPhaseParams) .def_prop_ro("is_context_only_request", &GenLlmReq::isContextOnlyRequest) @@ -196,8 +193,6 @@ void initBindings(nb::module_& m) .def_prop_ro("kv_cache_transfer_time_ms", &GenLlmReq::getKvCacheTransferTimeMS) .def_prop_ro("kv_cache_transfer_start", &GenLlmReq::getKvCacheTransferStart) .def_prop_ro("kv_cache_transfer_end", &GenLlmReq::getKvCacheTransferEnd) - .def("get_kv_cache_transfer_start", &GenLlmReq::getKvCacheTransferStart) - .def("get_kv_cache_transfer_end", &GenLlmReq::getKvCacheTransferEnd) .def_prop_ro("kv_cache_size", &GenLlmReq::getKvCacheSize) .def("set_kv_cache_transfer_start", &GenLlmReq::setKvCacheTransferStart, nb::arg("time")) .def("set_kv_cache_transfer_end", &GenLlmReq::setKvCacheTransferEnd, nb::arg("time")) @@ -481,11 +476,7 @@ void initBindings(nb::module_& m) .def("set_first_scheduled_time", &tb::LlmRequest::setFirstScheduledTime) .def("update_perf_metrics", &tb::LlmRequest::updatePerfMetrics, nb::arg("iter_counter")) .def("remove_lora_tensors", &tb::LlmRequest::removeLoraTensors) - // Bind to the single storage owned by libtensorrt_llm.so (reached through - // globalSteadyClockOffset()) instead of an inline-static member, so the - // offset is shared with the native library rather than living in this - // module's separate copy. - .def_rw_static("global_steady_clock_offset", &tb::globalSteadyClockOffset()); + .def_rw_static("global_steady_clock_offset", &tb::LlmRequest::sGlobalSteadyClockOffset); nb::class_<tb::SequenceSlotManager>(m, "SequenceSlotManager") .def(nb::init<tb::SequenceSlotManager::SlotIdType, uint64_t>(), nb::arg("max_num_slots"), @@ -500,7 +491,7 @@ void initBindings(nb::module_& m) nb::arg("max_num_sequences"), nb::arg("model_config"), nb::arg("world_config"), nb::arg("buffer_manager"), nb::call_guard<nb::gil_scoped_release>()) .def(nb::init<tr::SizeType32, tr::SizeType32, tr::SizeType32, tr::SizeType32, tr::SizeType32, tr::SizeType32, - tr::WorldConfig const&, int64_t, tensorrt_llm::DataType, tensorrt_llm::DataType, + tr::WorldConfig const&, int64_t, nvinfer1::DataType, nvinfer1::DataType, std::vector<tr::SizeType32> const&, tr::SizeType32>(), nb::arg("d_state"), nb::arg("d_conv"), nb::arg("num_heads"), nb::arg("n_groups"), nb::arg("head_dim"), nb::arg("max_batch_size"), nb::arg("world_config"), nb::arg("stream"), nb::arg("dtype"), diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/cacheTransceiver.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/cacheTransceiver.cpp index 2268e6f8ed03..8a7120022b09 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/cacheTransceiver.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/cacheTransceiver.cpp @@ -20,7 +20,6 @@ #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/rnnStateManager.h" #include "tensorrt_llm/common/bindingUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/nanobind/common/customCasters.h" #include <ATen/ATen.h> @@ -92,12 +91,6 @@ void tb::CacheTransceiverBindings::initBindings(nb::module_& m) .def("request_and_receive_sync", &BaseCacheTransceiver::requestAndReceiveSync, nb::call_guard<nb::gil_scoped_release>()) .def("request_and_receive_async", &BaseCacheTransceiver::requestAndReceiveAsync) - .def("get_serialized_data_transceiver_state", - [](tb::BaseCacheTransceiver& self) - { - auto serialized = self.getSerializedDataTransceiverState(); - return nb::bytes(serialized.data(), serialized.size()); - }) .def( "check_context_transfer_status", [](tb::BaseCacheTransceiver& self, std::optional<int> const& atLeastRequestNum, bool markComplete = false) @@ -118,8 +111,7 @@ void tb::CacheTransceiverBindings::initBindings(nb::module_& m) .def("check_gen_transfer_status", &BaseCacheTransceiver::checkGenTransferStatus, nb::call_guard<nb::gil_scoped_release>()) .def("check_gen_transfer_complete", &BaseCacheTransceiver::checkGenTransferComplete) - .def("cancel_request", &BaseCacheTransceiver::cancelRequest) - .def("has_poisoned_transfer_buffer", &BaseCacheTransceiver::hasPoisonedTransferBuffer); + .def("cancel_request", &BaseCacheTransceiver::cancelRequest); nb::enum_<executor::kv_cache::CacheState::AttentionType>(m, "AttentionType") .value("DEFAULT", executor::kv_cache::CacheState::AttentionType::kDEFAULT) @@ -127,7 +119,7 @@ void tb::CacheTransceiverBindings::initBindings(nb::module_& m) nb::class_<tb::CacheTransceiver, tb::BaseCacheTransceiver>(m, "CacheTransceiver") .def(nb::init<tb::kv_cache_manager::BaseKVCacheManager*, std::vector<SizeType32>, SizeType32, SizeType32, - runtime::WorldConfig, std::vector<SizeType32>, tensorrt_llm::DataType, + runtime::WorldConfig, std::vector<SizeType32>, nvinfer1::DataType, executor::kv_cache::CacheState::AttentionType, std::optional<executor::CacheTransceiverConfig>, std::vector<SizeType32>>(), nb::arg("cache_manager"), nb::arg("num_kv_heads_per_layer"), nb::arg("size_per_head"), diff --git a/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp index b1c4391c0e26..1496ebbb883d 100644 --- a/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp +++ b/cpp/tensorrt_llm/nanobind/batch_manager/kvCacheManager.cpp @@ -18,7 +18,6 @@ #include "kvCacheManager.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/peftCacheManager.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/nanobind/common/bindTypes.h" #include "tensorrt_llm/nanobind/common/customCasters.h" #include "tensorrt_llm/runtime/torch.h" @@ -349,8 +348,8 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) nb::class_<tbk::PoolConfiguration>(m, "PoolConfiguration") .def(nb::init<>()) - .def(nb::init<SizeType32, SizeType32, tensorrt_llm::DataType>(), nb::arg("window_size"), - nb::arg("size_per_head"), nb::arg("dtype")) + .def(nb::init<SizeType32, SizeType32, nvinfer1::DataType>(), nb::arg("window_size"), nb::arg("size_per_head"), + nb::arg("dtype")) .def_rw("window_size", &tbk::PoolConfiguration::windowSize) .def_rw("size_per_head", &tbk::PoolConfiguration::sizePerHead) .def_rw("dtype", &tbk::PoolConfiguration::dtype); @@ -662,8 +661,8 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) nb::class_<tbk::KVCacheManager, tbk::BaseKVCacheManager>(m, "KVCacheManager") .def(nb::init<std::vector<SizeType32> const&, SizeType32, SizeType32, std::map<SizeType32, std::tuple<SizeType32, SizeType32>> const&, SizeType32, SizeType32, - std::vector<SizeType32> const&, tensorrt_llm::DataType, SizeType32, int64_t, SizeType32, SizeType32, - bool, tbk::CacheType, std::optional<tensorrt_llm::executor::RetentionPriority>, + std::vector<SizeType32> const&, nvinfer1::DataType, SizeType32, int64_t, SizeType32, SizeType32, bool, + tbk::CacheType, std::optional<tensorrt_llm::executor::RetentionPriority>, std::shared_ptr<tbk::KVCacheEventManager>, bool, bool, std::shared_ptr<tbc::KvCacheConnectorManager>, bool, SizeType32, SizeType32, bool, std::optional<tbk::LinearAttentionMetadata>, std::vector<tbk::PoolConfiguration> const&>(), @@ -694,9 +693,7 @@ void tb::kv_cache_manager::KVCacheManagerBindings::initBindings(nb::module_& m) .def("copy_linear_attention_block", &tbk::KVCacheManager::copyLinearAttentionBlock, nb::arg("llm_request"), nb::call_guard<nb::gil_scoped_release>()) .def("copy_linear_attention_block_batch", &tbk::KVCacheManager::copyLinearAttentionBlockBatch, - nb::arg("llm_requests"), nb::call_guard<nb::gil_scoped_release>()) - .def("get_memory_pool_block_indices", &tbk::KVCacheManager::getMemoryPoolBlockIndicesByBlockIds, - nb::arg("block_ids"), nb::arg("window_size"), nb::call_guard<nb::gil_scoped_release>()); + nb::arg("llm_requests"), nb::call_guard<nb::gil_scoped_release>()); } void tb::BasePeftCacheManagerBindings::initBindings(nb::module_& m) diff --git a/cpp/tensorrt_llm/nanobind/bindings.cpp b/cpp/tensorrt_llm/nanobind/bindings.cpp index 2c724adc279c..db263fa639f9 100644 --- a/cpp/tensorrt_llm/nanobind/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/bindings.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,7 +32,6 @@ #include "tensorrt_llm/batch_manager/peftCacheManagerConfig.h" #include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/nanobind/batch_manager/algorithms.h" #include "tensorrt_llm/nanobind/batch_manager/bindings.h" #include "tensorrt_llm/nanobind/batch_manager/buffers.h" @@ -47,6 +46,7 @@ #include "tensorrt_llm/nanobind/runtime/bindings.h" #include "tensorrt_llm/nanobind/suffixAutomaton/bindings.h" #include "tensorrt_llm/nanobind/testing/kvCacheManagerTestUtilBinding.h" +#include "tensorrt_llm/nanobind/testing/modelSpecBinding.h" #include "tensorrt_llm/nanobind/thop/bindings.h" #include "tensorrt_llm/nanobind/userbuffers/bindings.h" #include "tensorrt_llm/runtime/common.h" @@ -168,17 +168,17 @@ NB_MODULE(TRTLLM_NB_MODULE, m) .def_rw("host_cache_size", &tb::PeftCacheManagerConfig::hostCacheSize) .def_rw("lora_prefetch_dir", &tb::PeftCacheManagerConfig::loraPrefetchDir); - nb::enum_<tensorrt_llm::DataType>(m, "DataType") - .value("FLOAT", tensorrt_llm::DataType::kFLOAT) - .value("HALF", tensorrt_llm::DataType::kHALF) - .value("INT8", tensorrt_llm::DataType::kINT8) - .value("INT32", tensorrt_llm::DataType::kINT32) - .value("BOOL", tensorrt_llm::DataType::kBOOL) - .value("UINT8", tensorrt_llm::DataType::kUINT8) - .value("FP8", tensorrt_llm::DataType::kFP8) - .value("BF16", tensorrt_llm::DataType::kBF16) - .value("INT64", tensorrt_llm::DataType::kINT64) - .value("NVFP4", tensorrt_llm::DataType::kFP4) + nb::enum_<nvinfer1::DataType>(m, "DataType") + .value("FLOAT", nvinfer1::DataType::kFLOAT) + .value("HALF", nvinfer1::DataType::kHALF) + .value("INT8", nvinfer1::DataType::kINT8) + .value("INT32", nvinfer1::DataType::kINT32) + .value("BOOL", nvinfer1::DataType::kBOOL) + .value("UINT8", nvinfer1::DataType::kUINT8) + .value("FP8", nvinfer1::DataType::kFP8) + .value("BF16", nvinfer1::DataType::kBF16) + .value("INT64", nvinfer1::DataType::kINT64) + .value("NVFP4", nvinfer1::DataType::kFP4) .export_values(); nb::enum_<tr::ModelConfig::ModelVariant>(m, "GptModelVariant") @@ -295,7 +295,7 @@ NB_MODULE(TRTLLM_NB_MODULE, m) .def(nb::self != nb::self); nb::class_<tr::ModelConfig>(m, "ModelConfig") - .def(nb::init<SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, tensorrt_llm::DataType>(), + .def(nb::init<SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, nvinfer1::DataType>(), nb::arg("vocab_size"), nb::arg("num_layers"), nb::arg("num_attention_layers"), nb::arg("num_rnn_layers"), nb::arg("num_heads"), nb::arg("hidden_size"), nb::arg("data_type")) .def_prop_ro("vocab_size", &tr::ModelConfig::getVocabSize) @@ -512,6 +512,7 @@ NB_MODULE(TRTLLM_NB_MODULE, m) tensorrt_llm::nanobind::process_group::initBindings(mInternalProcessGroup); tpb::Buffers::initBindings(mInternalBatchManager); tensorrt_llm::nanobind::runtime::initBindings(mInternalRuntime); + tensorrt_llm::nanobind::testing::initBindings(mInternalTesting); tensorrt_llm::nanobind::testing::initKvCacheTestUtilBindings(mInternalTesting); tpb::initBindings(mInternalBatchManager); @@ -544,8 +545,4 @@ NB_MODULE(TRTLLM_NB_MODULE, m) m.def("ipc_nvls_supported", &tr::ipcNvlsSupported); m.def("steady_clock_now", []() { return std::chrono::steady_clock::now(); }); - // Global (offset-normalized) steady clock, matching what - // LlmRequest::setKvCacheTransferStart/End expect. Reads the process-global - // steady clock offset, set by PyExecutor at startup. - m.def("global_steady_clock_now", []() { return tb::LlmRequest::getSteadyClockNow(); }); } diff --git a/cpp/tensorrt_llm/nanobind/executor/bindings.cpp b/cpp/tensorrt_llm/nanobind/executor/bindings.cpp index a8d2301fa43d..b0ad31b7347e 100644 --- a/cpp/tensorrt_llm/nanobind/executor/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/executor/bindings.cpp @@ -16,6 +16,7 @@ */ #include "bindings.h" +#include "executor.h" #include "executorConfig.h" #include "request.h" #include "tensorrt_llm/executor/executor.h" @@ -286,6 +287,7 @@ void initBindings(nb::module_& m) tensorrt_llm::nanobind::executor::initRequestBindings(m); tensorrt_llm::nanobind::executor::initConfigBindings(m); + tensorrt_llm::nanobind::executor::Executor::initBindings(m); } } // namespace tensorrt_llm::nanobind::executor diff --git a/cpp/tensorrt_llm/nanobind/executor/executor.cpp b/cpp/tensorrt_llm/nanobind/executor/executor.cpp new file mode 100644 index 000000000000..34cc8182d1bb --- /dev/null +++ b/cpp/tensorrt_llm/nanobind/executor/executor.cpp @@ -0,0 +1,225 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "executor.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/tensor.h" +#include "tensorrt_llm/nanobind/common/customCasters.h" + +#include <nanobind/nanobind.h> +#include <nanobind/ndarray.h> +#include <nanobind/stl/chrono.h> +#include <nanobind/stl/filesystem.h> +#include <nanobind/stl/map.h> +#include <nanobind/stl/optional.h> +#include <nanobind/stl/shared_ptr.h> +#include <nanobind/stl/string.h> +#include <nanobind/stl/vector.h> +#include <torch/extension.h> + +namespace nb = nanobind; +namespace tle = tensorrt_llm::executor; + +namespace nanobind::detail +{ + +template <> +struct dtype_traits<half> +{ + static constexpr dlpack::dtype value{ + (uint8_t) dlpack::dtype_code::Float, // type code + 16, // size in bits + 1 // lanes (simd), usually set to 1 + }; + static constexpr auto name = const_name("float16"); +}; +} // namespace nanobind::detail + +namespace +{ +tle::Tensor numpyToTensor(nb::object const& object) +{ + std::string dtype_name = nb::cast<std::string>(object.attr("dtype").attr("name")); + nb::object metadata = object.attr("dtype").attr("metadata"); + + tle::DataType dtype; + if (dtype_name == "float16") + { + dtype = tle::DataType::kFP16; + } + else if (dtype_name == "float32") + { + dtype = tle::DataType::kFP32; + } + else if (dtype_name == "int8") + { + dtype = tle::DataType::kINT8; + } + else if (dtype_name == "int32") + { + dtype = tle::DataType::kINT32; + } + else if (dtype_name == "int64") + { + dtype = tle::DataType::kINT64; + } + else if (dtype_name == "void8" && !metadata.is_none() && nb::cast<std::string>(metadata["dtype"]) == "float8") + { + dtype = tle::DataType::kFP8; + } + else if (dtype_name == "void16" && !metadata.is_none() && nb::cast<std::string>(metadata["dtype"]) == "bfloat16") + { + dtype = tle::DataType::kBF16; + } + else + { + TLLM_THROW("Unsupported numpy dtype."); + } + + nb::object array_interface = object.attr("__array_interface__"); + nb::object shape_obj = array_interface["shape"]; + std::vector<int64_t> dims; + dims.reserve(nb::len(shape_obj)); + + for (size_t i = 0; i < nb::len(shape_obj); ++i) + { + dims.push_back(nb::cast<int64_t>(shape_obj[i])); + } + + nb::object data_obj = array_interface["data"]; + uintptr_t addr = nb::cast<uintptr_t>(data_obj[0]); + void* data_ptr = reinterpret_cast<void*>(addr); + tle::Shape shape(dims.data(), dims.size()); + return tle::Tensor::of(dtype, data_ptr, shape); +} + +} // namespace + +namespace tensorrt_llm::nanobind::executor +{ + +Executor::Executor( + std::filesystem::path const& modelPath, tle::ModelType modelType, tle::ExecutorConfig const& executorConfig) +{ + mExecutor = std::make_unique<tle::Executor>(modelPath, modelType, executorConfig); +} + +Executor::Executor(std::filesystem::path const& encoderModelPath, std::filesystem::path const& decoderModelPath, + tle::ModelType modelType, tle::ExecutorConfig const& executorConfig) +{ + mExecutor = std::make_unique<tle::Executor>(encoderModelPath, decoderModelPath, modelType, executorConfig); +} + +Executor::Executor(nb::bytes const& engineBuffer, std::string const& jsonConfigStr, tle::ModelType modelType, + tle::ExecutorConfig const& executorConfig, std::optional<nb::dict> managedWeights) +{ + uint8_t const* data = static_cast<uint8_t const*>(engineBuffer.data()); + size_t size = engineBuffer.size(); + std::optional<std::map<std::string, tle::Tensor>> managedWeightsMap = std::nullopt; + if (managedWeights.has_value() && !managedWeights.value().empty()) + { + managedWeightsMap = std::map<std::string, tle::Tensor>(); + for (auto const& [rawName, rawArray] : managedWeights.value()) + { + std::string name = nb::cast<std::string>(rawName); + nb::object array_obj = nb::cast<nb::object>(rawArray); + managedWeightsMap->emplace(name, numpyToTensor(array_obj)); + } + } + mExecutor = std::make_unique<tle::Executor>( + tle::BufferView(data, size), jsonConfigStr, modelType, executorConfig, managedWeightsMap); +} + +Executor::Executor(std::string const& encoderEngineBuffer, std::string const& encoderJsonConfigStr, + std::string const& decoderEngineBuffer, std::string const& decoderJsonConfigStr, tle::ModelType modelType, + tle::ExecutorConfig const& executorConfig) +{ + uint8_t const* encoderData = reinterpret_cast<uint8_t const*>(encoderEngineBuffer.data()); + size_t encoderSize = encoderEngineBuffer.size(); + uint8_t const* decoderData = reinterpret_cast<uint8_t const*>(decoderEngineBuffer.data()); + size_t decoderSize = decoderEngineBuffer.size(); + mExecutor = std::make_unique<tle::Executor>(tle::BufferView(encoderData, encoderSize), encoderJsonConfigStr, + tle::BufferView(decoderData, decoderSize), decoderJsonConfigStr, modelType, executorConfig); +} + +nb::object Executor::enter() +{ + TLLM_CHECK(static_cast<bool>(mExecutor)); + return nb::cast(this); +} + +void Executor::exit( + [[maybe_unused]] nb::handle type, [[maybe_unused]] nb::handle value, [[maybe_unused]] nb::handle traceback) +{ + shutdown(); + mExecutor = nullptr; +} + +void Executor::shutdown() +{ + // NOTE: we must release the GIL here. Executor has spawned a thread for the execution loop. That thread must be + // able to do forward progress for the shutdown process to succeed. It takes the GIL during its callbacks, so + // we release it now. Note that we shouldn't do anything related to python objects after that. + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + nb::gil_scoped_release release; + mExecutor->shutdown(); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void Executor::initBindings(nb::module_& m) +{ + nb::class_<Executor>(m, "Executor") + .def(nb::init<std::filesystem::path const&, tle::ModelType, tle::ExecutorConfig const&>(), + nb::arg("model_path"), nb::arg("model_type"), nb::arg("executor_config")) + .def(nb::init<std::filesystem::path const&, std::filesystem::path const&, tle::ModelType, + tle::ExecutorConfig const&>(), + nb::arg("encoder_model_path"), nb::arg("decoder_model_path"), nb::arg("model_type"), + nb::arg("executor_config")) + .def(nb::init<nb::bytes, std::string const&, tle::ModelType, tle::ExecutorConfig const&, nb::dict>(), + nb::arg("engine_buffer"), nb::arg("json_config_str"), nb::arg("model_type"), nb::arg("executor_config"), + nb::arg("managed_weights") = nb::dict()) + .def(nb::init<std::string const&, std::string const&, std::string const&, std::string const&, tle::ModelType, + tle::ExecutorConfig const&>(), + nb::arg("encoder_engine_buffer"), nb::arg("encoder_json_config_str"), nb::arg("decoder_engine_buffer"), + nb::arg("decoder_json_config_str"), nb::arg("model_type"), nb::arg("executor_config")) + .def("shutdown", &Executor::shutdown) + .def("__enter__", &Executor::enter) + .def("__exit__", &Executor::exit, nb::arg("type").none(), nb::arg("value").none(), nb::arg("traceback").none()) + .def("enqueue_request", &Executor::enqueueRequest, nb::arg("request")) + .def("enqueue_requests", &Executor::enqueueRequests, nb::arg("requests")) + .def("await_responses", + nb::overload_cast<std::optional<std::chrono::milliseconds> const&>(&Executor::awaitResponses), + nb::arg("timeout") = nb::none()) + .def("await_responses", + nb::overload_cast<tle::IdType const&, std::optional<std::chrono::milliseconds> const&>( + &Executor::awaitResponses), + nb::arg("id"), nb::arg("timeout") = nb::none()) + .def("await_responses", + nb::overload_cast<std::vector<tle::IdType> const&, std::optional<std::chrono::milliseconds> const&>( + &Executor::awaitResponses), + nb::arg("ids"), nb::arg("timeout") = nb::none()) + .def("get_num_responses_ready", &Executor::getNumResponsesReady, nb::arg("id") = nb::none()) + .def("cancel_request", &Executor::cancelRequest, nb::arg("id") = nb::none()) + .def("get_latest_iteration_stats", &Executor::getLatestIterationStats) + .def("get_latest_request_stats", &Executor::getLatestRequestStats) + .def("get_latest_debug_tensors", &Executor::getLatestDebugTensors) + .def("can_enqueue_requests", &Executor::canEnqueueRequests) + .def("get_kv_cache_event_manager", &Executor::getKVCacheEventManager); +} + +} // namespace tensorrt_llm::nanobind::executor diff --git a/cpp/tensorrt_llm/nanobind/executor/executor.h b/cpp/tensorrt_llm/nanobind/executor/executor.h new file mode 100644 index 000000000000..22c24abb4bfd --- /dev/null +++ b/cpp/tensorrt_llm/nanobind/executor/executor.h @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include <nanobind/nanobind.h> + +namespace nb = nanobind; +namespace tle = tensorrt_llm::executor; + +namespace tensorrt_llm::nanobind::executor +{ + +class Executor +{ +public: + Executor( + std::filesystem::path const& modelPath, tle::ModelType modelType, tle::ExecutorConfig const& executorConfig); + + Executor(std::filesystem::path const& encoderModelPath, std::filesystem::path const& decoderModelPath, + tle::ModelType modelType, tle::ExecutorConfig const& executorConfig); + + Executor(nb::bytes const& engineBuffer, std::string const& jsonConfigStr, tle::ModelType modelType, + tle::ExecutorConfig const& executorConfig, std::optional<nb::dict> managedWeights); + + Executor(std::string const& encoderEngineBuffer, std::string const& encoderJsonConfigStr, + std::string const& decoderEngineBuffer, std::string const& decoderJsonConfigStr, tle::ModelType modelType, + tle::ExecutorConfig const& executorConfig); + + nb::object enter(); + void exit( + [[maybe_unused]] nb::handle type, [[maybe_unused]] nb::handle value, [[maybe_unused]] nb::handle traceback); + void shutdown(); + + [[nodiscard]] tle::IdType enqueueRequest(tle::Request const& request) + { + return mExecutor->enqueueRequest(request); + } + + [[nodiscard]] std::vector<tle::IdType> enqueueRequests(std::vector<tle::Request> const& requests) + { + return mExecutor->enqueueRequests(requests); + } + + [[nodiscard]] std::vector<tle::Response> awaitResponses( + std::optional<std::chrono::milliseconds> const& timeout = std::nullopt) + { + // Await responses blocks until a response is received. Release GIL so that it can be ran in a background + // thread. + nb::gil_scoped_release release; + return mExecutor->awaitResponses(timeout); + } + + [[nodiscard]] std::vector<tle::Response> awaitResponses( + tle::IdType const& requestId, std::optional<std::chrono::milliseconds> const& timeout = std::nullopt) + { + // Await responses blocks until a response is received. Release GIL so that it can be ran in a background + // thread. + nb::gil_scoped_release release; + return mExecutor->awaitResponses(requestId, timeout); + } + + [[nodiscard]] std::vector<std::vector<tle::Response>> awaitResponses(std::vector<tle::IdType> const& requestIds, + std::optional<std::chrono::milliseconds> const& timeout = std::nullopt) + { + // Await responses blocks until a response is received. Release GIL so that it can be ran in a background + // thread. + nb::gil_scoped_release release; + return mExecutor->awaitResponses(requestIds, timeout); + } + + [[nodiscard]] tle::SizeType32 getNumResponsesReady(std::optional<tle::IdType> const& requestId = std::nullopt) const + { + return mExecutor->getNumResponsesReady(requestId); + } + + void cancelRequest(tle::IdType requestId) + { + mExecutor->cancelRequest(requestId); + } + + std::deque<tle::IterationStats> getLatestIterationStats() + { + return mExecutor->getLatestIterationStats(); + } + + std::deque<tle::RequestStatsPerIteration> getLatestRequestStats() + { + return mExecutor->getLatestRequestStats(); + } + + std::deque<tle::DebugTensorsPerIteration> getLatestDebugTensors() + { + return mExecutor->getLatestDebugTensors(); + } + + [[nodiscard]] bool canEnqueueRequests() const + { + return mExecutor->canEnqueueRequests(); + } + + [[nodiscard]] std::optional<std::shared_ptr<tle::KVCacheEventManager>> getKVCacheEventManager() const + { + return mExecutor->getKVCacheEventManager(); + } + + static void initBindings(nb::module_& m); + +private: + std::unique_ptr<tle::Executor> mExecutor; +}; + +} // namespace tensorrt_llm::nanobind::executor diff --git a/cpp/tensorrt_llm/nanobind/executor/executorConfig.cpp b/cpp/tensorrt_llm/nanobind/executor/executorConfig.cpp index acd33c0df769..830f30ab9c67 100644 --- a/cpp/tensorrt_llm/nanobind/executor/executorConfig.cpp +++ b/cpp/tensorrt_llm/nanobind/executor/executorConfig.cpp @@ -171,9 +171,6 @@ void initConfigBindings(nb::module_& m) .def("__getstate__", kvCacheConfigGetstate) .def("__setstate__", kvCacheConfigSetstate); - // Deprecated: orchestrator mode is non-functional (its executorWorker binary was - // removed with the TensorRT backend); binding kept for compatibility, removal is a - // follow-up pending API-stability review. nb::class_<tle::OrchestratorConfig>(m, "OrchestratorConfig") .def(nb::init<bool, std::string, std::shared_ptr<mpi::MpiComm>, bool>(), nb::arg("is_orchestrator") = true, nb::arg("worker_executable_path") = "", nb::arg("orch_leader_comm").none() = nullptr, diff --git a/cpp/tensorrt_llm/nanobind/runtime/bindings.cpp b/cpp/tensorrt_llm/nanobind/runtime/bindings.cpp index eec3cd79bac1..6d5d70aafb6b 100644 --- a/cpp/tensorrt_llm/nanobind/runtime/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/runtime/bindings.cpp @@ -18,7 +18,6 @@ #include "bindings.h" #include "hostfunc.h" #include "moeBindings.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceWorkspace.h" #include "tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h" #include "tensorrt_llm/kernels/customAllReduceKernels.h" @@ -40,6 +39,7 @@ #include "tensorrt_llm/runtime/loraCache.h" #include "tensorrt_llm/runtime/mcastGPUBuffer.h" #include "tensorrt_llm/runtime/speculativeDecodingMode.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" #include "tensorrt_llm/runtime/torchView.h" #include "tensorrt_llm/runtime/virtualMemory.h" @@ -68,7 +68,7 @@ class PyIGptDecoder : public tr::IGptDecoder void setup(tr::SamplingConfig const& samplingConfig, size_t batchSize, tr::DecodingInput::TensorConstPtr const& batchSlots, std::optional<tr::DecodingOutput> const& output = std::nullopt, - std::optional<tensorrt_llm::DataType> explicitDraftTokensDType = std::nullopt, + std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt, std::optional<std::vector<tr::ITensor::SharedConstPtr>> const& lookaheadPrompt = std::nullopt, std::optional<std::vector<te::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs = std::nullopt) override { @@ -125,6 +125,47 @@ void initBindings(nb::module_& m) .def("materialize_with_tag", &tr::CudaVirtualMemoryManager::materializeWithTag, nb::arg("tag"), nb::call_guard<nb::gil_scoped_release>()); + nb::class_<tr::TllmRuntime>(m, "TllmRuntime") + .def( + "__init__", + [](tr::TllmRuntime* self, std::filesystem::path engine_path, float gpu_weights_percent = 1.0f, + bool use_shape_inference = true) + { + // Using default logger by passing nullptr + new (self) + tr::TllmRuntime(tr::RawEngine(engine_path), nullptr, gpu_weights_percent, use_shape_inference); + }, + nb::arg("engine_path"), nb::arg("gpu_weights_percent") = 1.0f, nb::arg("use_shape_inference") = true) + .def( + "__init__", + [](tr::TllmRuntime* self, nb::ndarray<nb::numpy, uint8_t> engine_buffer, float gpu_weights_percent = 1.0f, + bool use_shape_inference = true) + { + if (engine_buffer.ndim() != 1) + throw std::runtime_error("Expected 1-D array for engine buffer"); + new (self) tr::TllmRuntime(tr::RawEngine(engine_buffer.data(), engine_buffer.size()), nullptr, + gpu_weights_percent, use_shape_inference); + }, + nb::arg("engine_buffer"), nb::arg("gpu_weights_percent") = 1.0f, nb::arg("use_shape_inference") = true) + .def_prop_ro("num_contexts", &tr::TllmRuntime::getNbContexts) + .def_prop_ro("num_profiles", &tr::TllmRuntime::getNbProfiles) + .def("get_opt_profile_id", &tr::TllmRuntime::getOptProfileId, nb::arg("num_tokens"), nb::arg("split_points"), + nb::call_guard<nb::gil_scoped_release>()) + .def("clear_contexts", &tr::TllmRuntime::clearContexts, nb::call_guard<nb::gil_scoped_release>()) + .def("execute_context", &tr::TllmRuntime::executeContext, nb::arg("context_id"), + nb::call_guard<nb::gil_scoped_release>()) + .def_prop_ro("stream_ptr", &tr::TllmRuntime::getStreamPtr) + .def_prop_ro("buffer_manager", + static_cast<tr::BufferManager& (tr::TllmRuntime::*) ()>(&tr::TllmRuntime::getBufferManager)) + .def("set_layer_profiler", &tr::TllmRuntime::setLayerProfiler, nb::call_guard<nb::gil_scoped_release>()) + .def("has_layer_profiler", &tr::TllmRuntime::hasLayerProfiler, nb::arg("context_id"), + nb::call_guard<nb::gil_scoped_release>()) + .def_prop_ro("layer_profiler_info", &tr::TllmRuntime::getLayerProfileInfo) + .def("report_to_profiler", &tr::TllmRuntime::reportToProfiler, nb::arg("context_id"), + nb::call_guard<nb::gil_scoped_release>()) + .def_prop_ro("logits_dtype_from_engine", + [](tr::TllmRuntime& self) { return self.getEngine().getTensorDataType("logits"); }); + nb::class_<tr::LookaheadDecodingBuffers>(m, "LookaheadDecodingBuffers") .def(nb::init<tr::SizeType32, tr::SizeType32, tr::BufferManager const&>(), nb::arg("max_num_sequences"), nb::arg("max_tokens_per_step"), nb::arg("buffer_manager"), nb::call_guard<nb::gil_scoped_release>()) @@ -163,7 +204,7 @@ void initBindings(nb::module_& m) "setup", [](tr::IGptDecoder& self, tr::SamplingConfig const& samplingConfig, size_t batchSize, at::Tensor const& batchSlots, std::optional<tr::DecodingOutput> const& output = std::nullopt, - std::optional<tensorrt_llm::DataType> explicitDraftTokensDType = std::nullopt, + std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt, std::optional<std::vector<tr::ITensor::SharedConstPtr>> const& lookaheadPrompt = std::nullopt, std::optional<std::vector<te::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs = std::nullopt) { diff --git a/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.cpp b/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.cpp new file mode 100644 index 000000000000..caef94c5defd --- /dev/null +++ b/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.cpp @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "modelSpecBinding.h" +#include "tensorrt_llm/nanobind/common/customCasters.h" +#include "tensorrt_llm/testing/modelSpec.h" + +#include <nanobind/nanobind.h> + +namespace nb = nanobind; +using tensorrt_llm::testing::ModelSpec; +using tensorrt_llm::testing::KVCacheType; +using tensorrt_llm::testing::QuantMethod; +using tensorrt_llm::testing::OutputContentType; + +namespace tensorrt_llm::nanobind::testing +{ + +void initBindings(nb::module_& m) +{ + nb::enum_<QuantMethod>(m, "QuantMethod", nb::is_arithmetic(), "Quantization Method") + .value("NONE", QuantMethod::kNONE, "No Quantization") + .value("SMOOTH_QUANT", QuantMethod::kSMOOTH_QUANT, "Smooth Quantization"); + + nb::enum_<OutputContentType>(m, "OutputContentType", nb::is_arithmetic(), "Output Content Type") + .value("NONE", OutputContentType::kNONE, "No Output Content") + .value("CONTEXT_LOGITS", OutputContentType::kCONTEXT_LOGITS, "Context Logits") + .value("GENERATION_LOGITS", OutputContentType::kGENERATION_LOGITS, "Generation Logits") + .value("LOG_PROBS", OutputContentType::kLOG_PROBS, "Log Probs") + .value("CUM_LOG_PROBS", OutputContentType::kCUM_LOG_PROBS, "Cumulative Log"); + + nb::class_<ModelSpec>(m, "ModelSpec") + .def(nb::init<std::string const&, nvinfer1::DataType>()) + .def("use_gpt_plugin", &ModelSpec::useGptAttentionPlugin, nb::rv_policy::reference_internal) + .def("use_packed_input", &ModelSpec::usePackedInput, nb::rv_policy::reference_internal) + .def("set_kv_cache_type", &ModelSpec::setKVCacheType, nb::rv_policy::reference_internal) + .def("use_decoder_per_request", &ModelSpec::useDecoderPerRequest, nb::rv_policy::reference_internal) + .def("use_tensor_parallelism", &ModelSpec::useTensorParallelism, nb::rv_policy::reference_internal) + .def("use_pipeline_parallelism", &ModelSpec::usePipelineParallelism, nb::rv_policy::reference_internal) + .def("use_context_parallelism", &ModelSpec::useContextParallelism, nb::rv_policy::reference_internal) + .def("set_draft_tokens", &ModelSpec::setDraftTokens, nb::rv_policy::reference_internal) + .def("use_accept_by_logits", &ModelSpec::useAcceptByLogits, nb::rv_policy::reference_internal) + .def("use_mamba_plugin", &ModelSpec::useMambaPlugin, nb::rv_policy::reference_internal) + .def("gather_logits", &ModelSpec::gatherLogits, nb::rv_policy::reference_internal) + .def("replace_logits", &ModelSpec::replaceLogits, nb::rv_policy::reference_internal) + .def("return_log_probs", &ModelSpec::returnLogProbs, nb::rv_policy::reference_internal) + .def("smoke_test", &ModelSpec::smokeTest, nb::rv_policy::reference_internal) + .def("use_medusa", &ModelSpec::useMedusa, nb::rv_policy::reference_internal) + .def("use_eagle", &ModelSpec::useEagle, nb::rv_policy::reference_internal) + .def("use_lookahead_decoding", &ModelSpec::useLookaheadDecoding, nb::rv_policy::reference_internal) + .def("use_explicit_draft_tokens_decoding", &ModelSpec::useExplicitDraftTokensDecoding, + nb::rv_policy::reference_internal) + .def("use_draft_tokens_external_decoding", &ModelSpec::useDraftTokensExternalDecoding, + nb::rv_policy::reference_internal) + .def("use_logits", &ModelSpec::useLogits) + .def("use_multiple_profiles", &ModelSpec::useMultipleProfiles, nb::rv_policy::reference_internal) + .def("set_max_input_length", &ModelSpec::setMaxInputLength, nb::rv_policy::reference_internal) + .def("set_max_output_length", &ModelSpec::setMaxOutputLength, nb::rv_policy::reference_internal) + .def("set_quant_method", &ModelSpec::setQuantMethod, nb::rv_policy::reference_internal) + .def("use_lora_plugin", &ModelSpec::useLoraPlugin, nb::rv_policy::reference_internal) + .def("get_input_file", &ModelSpec::getInputFile) + .def("get_model_path", &ModelSpec::getModelPath) + .def("get_results_file", &ModelSpec::getResultsFile) + .def("get_generation_logits_file", &ModelSpec::getGenerationLogitsFile) + .def("get_context_logits_file", &ModelSpec::getContextLogitsFile) + .def("get_cum_log_probs_file", &ModelSpec::getCumLogProbsFile) + .def("get_log_probs_file", &ModelSpec::getLogProbsFile) + .def("enable_context_fmha_fp32_acc", &ModelSpec::enableContextFMHAFp32Acc, nb::rv_policy::reference_internal) + .def("get_enable_context_fmha_fp32_acc", &ModelSpec::getEnableContextFMHAFp32Acc) + .def("__copy__", [](ModelSpec const& self) { return ModelSpec(self); }); +} + +} // namespace tensorrt_llm::nanobind::testing diff --git a/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.h b/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.h new file mode 100644 index 000000000000..1aababc6ff89 --- /dev/null +++ b/cpp/tensorrt_llm/nanobind/testing/modelSpecBinding.h @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <nanobind/nanobind.h> + +namespace nb = nanobind; + +namespace tensorrt_llm::nanobind::testing +{ + +void initBindings(nb::module_& m); + +} // namespace tensorrt_llm::nanobind::testing diff --git a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp index dd1ff0db5410..23bd36d67b82 100644 --- a/cpp/tensorrt_llm/nanobind/thop/bindings.cpp +++ b/cpp/tensorrt_llm/nanobind/thop/bindings.cpp @@ -179,8 +179,7 @@ void initBindings(nb::module_& m) nb::arg("relative_attention_bias") = std::nullopt, nb::arg("relative_attention_max_distance") = 0, nb::arg("spec_decoding_target_max_draft_tokens") = std::nullopt, nb::arg("quant_scale_qkv") = std::nullopt, nb::arg("dsv4_inv_rope_cos_sin_cache") = std::nullopt, nb::arg("enable_dsv4_epilogue_fusion") = false, - nb::arg("force_prepare_spec_dec_tree_mask") = false, "Multi-head attention operation", - nb::call_guard<nb::gil_scoped_release>()); + "Multi-head attention operation", nb::call_guard<nb::gil_scoped_release>()); m.def( "get_helix_workspace_size_per_rank", diff --git a/cpp/tensorrt_llm/plugins/CMakeLists.txt b/cpp/tensorrt_llm/plugins/CMakeLists.txt new file mode 100755 index 000000000000..8b89cccdc813 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/CMakeLists.txt @@ -0,0 +1,183 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +set(PLUGIN_TARGET_NAME nvinfer_plugin_tensorrt_llm) +set(PLUGIN_SHARED_TARGET ${PLUGIN_TARGET_NAME}) + +set(TARGET_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(PLUGIN_EXPORT_MAP ${TARGET_DIR}/exports.map) # Linux +set(PLUGIN_EXPORT_DEF ${TARGET_DIR}/exports.def) # Windows + +if(${CMAKE_BUILD_TYPE} MATCHES "Debug") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") +endif() + +set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --Wno-deprecated-declarations") +set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --diag-suppress 997") + +if(NOT WIN32) + # additional warnings + # + # Ignore overloaded-virtual warning. We intentionally change parameters of + # some methods in derived class. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-overloaded-virtual") + if(WARNING_IS_ERROR) + message(STATUS "Treating warnings as errors in GCC compilation") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") + endif() +else() # Windows + # warning level 4 + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") +endif() + +set(PLUGIN_SOURCES) +set(PLUGIN_CU_SOURCES) + +set(PLUGIN_LISTS + bertAttentionPlugin + cpSplitPlugin + fusedLayernormPlugin + gptAttentionCommon + gptAttentionPlugin + identityPlugin + gemmPlugin + gemmSwigluPlugin + fp8RowwiseGemmPlugin + smoothQuantGemmPlugin + fp4GemmPlugin + quantizePerTokenPlugin + quantizeTensorPlugin + quantizeToFP4Plugin + layernormQuantizationPlugin + rmsnormQuantizationPlugin + weightOnlyGroupwiseQuantMatmulPlugin + weightOnlyQuantMatmulPlugin + lookupPlugin + loraPlugin + doraPlugin + mixtureOfExperts + selectiveScanPlugin + mambaConv1dPlugin + lruPlugin + cumsumLastDimPlugin + topkLastDimPlugin + lowLatencyGemmPlugin + eaglePlugin + lowLatencyGemmSwigluPlugin + qserveGemmPlugin + cudaStreamPlugin + gemmAllReducePlugin) + +foreach(PLUGIN_ITER ${PLUGIN_LISTS}) + include_directories(${PLUGIN_ITER}) + add_subdirectory(${PLUGIN_ITER}) +endforeach(PLUGIN_ITER) + +if(ENABLE_MULTI_DEVICE) + include_directories(ncclPlugin) + add_subdirectory(ncclPlugin) +endif() +include_directories(common) +add_subdirectory(common) + +# Set gencodes +list(APPEND PLUGIN_SOURCES "${PLUGIN_CU_SOURCES}") + +list(APPEND PLUGIN_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/api/tllmPlugin.cpp") + +# ################################# SHARED LIBRARY +# ############################################################################## + +if(WIN32) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS 1) +endif() + +add_library(${PLUGIN_SHARED_TARGET} SHARED ${PLUGIN_SOURCES}) +add_cuda_architectures(${PLUGIN_SHARED_TARGET} 89) + +target_include_directories( + ${PLUGIN_SHARED_TARGET} + PUBLIC ${CUDA_INSTALL_DIR}/include + PUBLIC + $<TARGET_PROPERTY:${INTERNAL_CUTLASS_KERNELS_TARGET},INTERFACE_INCLUDE_DIRECTORIES> + PRIVATE ${TARGET_DIR}) + +if(USING_OSS_CUTLASS_FP4_GEMM) + target_compile_definitions(${PLUGIN_SHARED_TARGET} + PUBLIC USING_OSS_CUTLASS_FP4_GEMM) +endif() + +if(USING_OSS_CUTLASS_ALLREDUCE_GEMM) + target_compile_definitions(${PLUGIN_SHARED_TARGET} + PUBLIC USING_OSS_CUTLASS_ALLREDUCE_GEMM) +endif() + +if(USING_OSS_CUTLASS_MOE_GEMM) + target_compile_definitions(${PLUGIN_SHARED_TARGET} + PUBLIC USING_OSS_CUTLASS_MOE_GEMM) +endif() + +if(ENABLE_MULTI_DEVICE) + target_include_directories(${PLUGIN_SHARED_TARGET} + PUBLIC ${MPI_C_INCLUDE_DIRS}) +endif() + +if(CUDA_VERSION VERSION_LESS 11.0) + target_include_directories(${PLUGIN_SHARED_TARGET} PUBLIC ${CUB_ROOT_DIR}) +endif() + +set_target_properties( + ${PLUGIN_SHARED_TARGET} + PROPERTIES CXX_STANDARD "17" + CXX_STANDARD_REQUIRED "YES" + CXX_EXTENSIONS "NO" + ARCHIVE_OUTPUT_DIRECTORY "${TRT_OUT_DIR}" + LIBRARY_OUTPUT_DIRECTORY "${TRT_OUT_DIR}" + RUNTIME_OUTPUT_DIRECTORY "${TRT_OUT_DIR}") + +if(WIN32) + set_target_properties( + ${PLUGIN_SHARED_TARGET} + PROPERTIES LINK_FLAGS "/DEF:${PLUGIN_EXPORT_DEF} ${UNDEFINED_FLAG}") +else() + set_target_properties( + ${PLUGIN_SHARED_TARGET} + PROPERTIES + LINK_FLAGS + "-Wl,--exclude-libs,ALL -Wl,--version-script=${PLUGIN_EXPORT_MAP} -Wl,-rpath,'$ORIGIN' ${AS_NEEDED_FLAG} ${UNDEFINED_FLAG}" + ) +endif() + +set_property(TARGET ${PLUGIN_SHARED_TARGET} PROPERTY CUDA_STANDARD 17) + +target_link_libraries( + ${PLUGIN_SHARED_TARGET} + ${CUBLAS_LIB} + ${CUBLASLT_LIB} + ${TRT_LIB} + ${CUDA_DRV_LIB} + ${CUDA_RT_LIB} + ${CMAKE_DL_LIBS} + ${SHARED_TARGET}) + +if(WIN32) + target_link_libraries(${PLUGIN_SHARED_TARGET} context_attention_src) +endif() + +if(ENABLE_MULTI_DEVICE) + target_link_libraries(${PLUGIN_SHARED_TARGET} ${MPI_C_LIBRARIES} ${NCCL_LIB}) +endif() diff --git a/cpp/tensorrt_llm/plugins/api/tllmPlugin.cpp b/cpp/tensorrt_llm/plugins/api/tllmPlugin.cpp new file mode 100644 index 000000000000..f0dceb2f4a99 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/api/tllmPlugin.cpp @@ -0,0 +1,313 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "tensorrt_llm/plugins/api/tllmPlugin.h" + +#include "tensorrt_llm/common/stringUtils.h" +#include "tensorrt_llm/runtime/tllmLogger.h" + +#include "tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.h" +#include "tensorrt_llm/plugins/doraPlugin/doraPlugin.h" +#include "tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.h" +#include "tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.h" +#include "tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h" +#include "tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.h" +#include "tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h" +#include "tensorrt_llm/plugins/identityPlugin/identityPlugin.h" +#include "tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.h" +#include "tensorrt_llm/plugins/lookupPlugin/lookupPlugin.h" +#include "tensorrt_llm/plugins/loraPlugin/loraPlugin.h" +#include "tensorrt_llm/plugins/lruPlugin/lruPlugin.h" +#include "tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.h" +#include "tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h" +#include "tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.h" +#if ENABLE_MULTI_DEVICE +#include "tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h" +#include "tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h" +#include "tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.h" +#include "tensorrt_llm/plugins/ncclPlugin/allreducePlugin.h" +#include "tensorrt_llm/plugins/ncclPlugin/recvPlugin.h" +#include "tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.h" +#include "tensorrt_llm/plugins/ncclPlugin/sendPlugin.h" +#endif // ENABLE_MULTI_DEVICE +#include "tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h" +#include "tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.h" +#include "tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.h" +#include "tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.h" +#include "tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.h" +#include "tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.h" +#include "tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h" +#include "tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h" +#include "tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.h" +#include "tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.h" +#include "tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.h" +#include "tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.h" +#include "tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.h" +#include "tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.h" +#include "tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.h" +#include "tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.h" +#include "tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h" + +#include <array> +#include <cstdlib> + +#include <NvInferRuntime.h> + +namespace tc = tensorrt_llm::common; + +namespace +{ + +nvinfer1::IPluginCreator* creatorPtr(nvinfer1::IPluginCreator& creator) +{ + return &creator; +} + +nvinfer1::IPluginCreatorInterface* creatorInterfacePtr(nvinfer1::IPluginCreatorInterface& creator) +{ + return &creator; +} + +auto tllmLogger = tensorrt_llm::runtime::TllmLogger(); + +nvinfer1::ILogger* gLogger{&tllmLogger}; + +class GlobalLoggerFinder : public nvinfer1::ILoggerFinder +{ +public: + nvinfer1::ILogger* findLogger() override + { + return gLogger; + } +}; + +GlobalLoggerFinder gGlobalLoggerFinder{}; + +#if !defined(_MSC_VER) +[[maybe_unused]] __attribute__((constructor)) +#endif +void initOnLoad() +{ + auto constexpr kLoadPlugins = "TRT_LLM_LOAD_PLUGINS"; + auto const loadPlugins = std::getenv(kLoadPlugins); + if (loadPlugins && loadPlugins[0] == '1') + { + initTrtLlmPlugins(gLogger); + } +} + +bool pluginsInitialized = false; + +} // namespace + +namespace tensorrt_llm::plugins::api +{ + +LoggerManager& tensorrt_llm::plugins::api::LoggerManager::getInstance() noexcept +{ + static LoggerManager instance; + return instance; +} + +void LoggerManager::setLoggerFinder(nvinfer1::ILoggerFinder* finder) +{ + std::lock_guard<std::mutex> lk(mMutex); + if (mLoggerFinder == nullptr && finder != nullptr) + { + mLoggerFinder = finder; + } +} + +[[maybe_unused]] nvinfer1::ILogger* LoggerManager::logger() +{ + std::lock_guard<std::mutex> lk(mMutex); + if (mLoggerFinder != nullptr) + { + return mLoggerFinder->findLogger(); + } + return nullptr; +} + +nvinfer1::ILogger* LoggerManager::defaultLogger() noexcept +{ + return gLogger; +} +} // namespace tensorrt_llm::plugins::api + +// New Plugin APIs + +extern "C" +{ + bool initTrtLlmPlugins(void* logger, char const* libNamespace) + { + if (pluginsInitialized) + { + return true; + } + + if (logger) + { + gLogger = static_cast<nvinfer1::ILogger*>(logger); + } + setLoggerFinder(&gGlobalLoggerFinder); + + auto registry = getPluginRegistry(); + + { + std::int32_t nbCreators; + auto creators = getPluginCreators(nbCreators); + + for (std::int32_t i = 0; i < nbCreators; ++i) + { + auto const creator = creators[i]; + creator->setPluginNamespace(libNamespace); + registry->registerCreator(*creator, libNamespace); + if (gLogger) + { + auto const msg = tc::fmtstr("Registered plugin creator %s version %s in namespace %s", + creator->getPluginName(), creator->getPluginVersion(), libNamespace); + gLogger->log(nvinfer1::ILogger::Severity::kVERBOSE, msg.c_str()); + } + } + } + + { + std::int32_t nbCreators; + auto creators = getCreators(nbCreators); + + for (std::int32_t i = 0; i < nbCreators; ++i) + { + auto const creator = creators[i]; + registry->registerCreator(*creator, libNamespace); + } + } + + pluginsInitialized = true; + return true; + } + + [[maybe_unused]] void setLoggerFinder([[maybe_unused]] nvinfer1::ILoggerFinder* finder) + { + tensorrt_llm::plugins::api::LoggerManager::getInstance().setLoggerFinder(finder); + } + + [[maybe_unused]] nvinfer1::IPluginCreator* const* getPluginCreators(std::int32_t& nbCreators) + { + static tensorrt_llm::plugins::IdentityPluginCreator identityPluginCreator; + static tensorrt_llm::plugins::BertAttentionPluginCreator bertAttentionPluginCreator; + static tensorrt_llm::plugins::FusedLayernormPluginCreator fusedLayernormPluginCreator; + static tensorrt_llm::plugins::GPTAttentionPluginCreator gptAttentionPluginCreator; + static tensorrt_llm::plugins::GemmPluginCreator gemmPluginCreator; + static tensorrt_llm::plugins::GemmSwigluPluginCreator gemmSwigluPluginCreator; + static tensorrt_llm::plugins::Fp8RowwiseGemmPluginCreator fp8RowwiseGemmPluginCreator; + static tensorrt_llm::plugins::MixtureOfExpertsPluginCreator moePluginCreator; +#if ENABLE_MULTI_DEVICE + static tensorrt_llm::plugins::SendPluginCreator sendPluginCreator; + static tensorrt_llm::plugins::RecvPluginCreator recvPluginCreator; + static tensorrt_llm::plugins::AllreducePluginCreator allreducePluginCreator; + static tensorrt_llm::plugins::AllgatherPluginCreator allgatherPluginCreator; + static tensorrt_llm::plugins::ReduceScatterPluginCreator reduceScatterPluginCreator; + static tensorrt_llm::plugins::GemmAllReducePluginCreator gemmAllReducePluginCreator; +#endif // ENABLE_MULTI_DEVICE + static tensorrt_llm::plugins::SmoothQuantGemmPluginCreator smoothQuantGemmPluginCreator; + static tensorrt_llm::plugins::QServeGemmPluginCreator qserveGemmPluginCreator; + static tensorrt_llm::plugins::LayernormQuantizationPluginCreator layernormQuantizationPluginCreator; + static tensorrt_llm::plugins::QuantizeToFP4PluginCreator quantizeToFP4PluginCreator; + static tensorrt_llm::plugins::QuantizePerTokenPluginCreator quantizePerTokenPluginCreator; + static tensorrt_llm::plugins::QuantizeTensorPluginCreator quantizeTensorPluginCreator; + static tensorrt_llm::plugins::RmsnormQuantizationPluginCreator rmsnormQuantizationPluginCreator; + static tensorrt_llm::plugins::WeightOnlyGroupwiseQuantMatmulPluginCreator + weightOnlyGroupwiseQuantMatmulPluginCreator; + static tensorrt_llm::plugins::WeightOnlyQuantMatmulPluginCreator weightOnlyQuantMatmulPluginCreator; + static tensorrt_llm::plugins::LookupPluginCreator lookupPluginCreator; + static tensorrt_llm::plugins::LoraPluginCreator loraPluginCreator; + static tensorrt_llm::plugins::SelectiveScanPluginCreator selectiveScanPluginCreator; + static tensorrt_llm::plugins::Fp4GemmPluginCreator fp4GemmPluginCreator; + static tensorrt_llm::plugins::MambaConv1dPluginCreator mambaConv1DPluginCreator; + static tensorrt_llm::plugins::lruPluginCreator lruPluginCreator; + static tensorrt_llm::plugins::CumsumLastDimPluginCreator cumsumLastDimPluginCreator; + static tensorrt_llm::plugins::TopkLastDimPluginCreator topkLastDimPluginCreator; + static tensorrt_llm::plugins::LowLatencyGemmPluginCreator lowLatencyGemmPluginCreator; + static tensorrt_llm::plugins::LowLatencyGemmSwigluPluginCreator lowLatencyGemmSwigluPluginCreator; + static tensorrt_llm::plugins::EagleDecodeDraftTokensPluginCreator eagleDecodeDraftTokensPluginCreator; + static tensorrt_llm::plugins::EagleSampleAndAcceptDraftTokensPluginCreator + eagleSampleAndAcceptDraftTokensPluginCreator; + static tensorrt_llm::plugins::CudaStreamPluginCreator cudaStreamPluginCreator; + + static std::array pluginCreators + = { creatorPtr(identityPluginCreator), + creatorPtr(bertAttentionPluginCreator), + creatorPtr(gptAttentionPluginCreator), + creatorPtr(gemmPluginCreator), + creatorPtr(gemmSwigluPluginCreator), + creatorPtr(fp8RowwiseGemmPluginCreator), + creatorPtr(moePluginCreator), +#if ENABLE_MULTI_DEVICE + creatorPtr(sendPluginCreator), + creatorPtr(recvPluginCreator), + creatorPtr(allreducePluginCreator), + creatorPtr(allgatherPluginCreator), + creatorPtr(reduceScatterPluginCreator), + creatorPtr(gemmAllReducePluginCreator), +#endif // ENABLE_MULTI_DEVICE + creatorPtr(fusedLayernormPluginCreator), + creatorPtr(smoothQuantGemmPluginCreator), + creatorPtr(qserveGemmPluginCreator), + creatorPtr(layernormQuantizationPluginCreator), + creatorPtr(quantizeToFP4PluginCreator), + creatorPtr(quantizePerTokenPluginCreator), + creatorPtr(quantizeTensorPluginCreator), + creatorPtr(rmsnormQuantizationPluginCreator), + creatorPtr(weightOnlyGroupwiseQuantMatmulPluginCreator), + creatorPtr(weightOnlyQuantMatmulPluginCreator), + creatorPtr(lookupPluginCreator), + creatorPtr(loraPluginCreator), + creatorPtr(selectiveScanPluginCreator), + creatorPtr(fp4GemmPluginCreator), + creatorPtr(mambaConv1DPluginCreator), + creatorPtr(lruPluginCreator), + creatorPtr(cumsumLastDimPluginCreator), + creatorPtr(topkLastDimPluginCreator), + creatorPtr(lowLatencyGemmPluginCreator), + creatorPtr(eagleDecodeDraftTokensPluginCreator), + creatorPtr(eagleSampleAndAcceptDraftTokensPluginCreator), + creatorPtr(lowLatencyGemmSwigluPluginCreator), + creatorPtr(cudaStreamPluginCreator), + }; + nbCreators = pluginCreators.size(); + return pluginCreators.data(); + } + + [[maybe_unused]] nvinfer1::IPluginCreatorInterface* const* getCreators(std::int32_t& nbCreators) + { + static tensorrt_llm::plugins::EaglePrepareDrafterInputsPluginCreator eaglePrepareDrafterInputsPluginCreator; +#if ENABLE_MULTI_DEVICE + static tensorrt_llm::plugins::CpSplitPluginCreator cpSplitPluginCreator; +#endif // ENABLE_MULTI_DEVICE + + static tensorrt_llm::plugins::DoraPluginCreator doraPluginCreator; + + static std::array creators + = { creatorInterfacePtr(eaglePrepareDrafterInputsPluginCreator), +#if ENABLE_MULTI_DEVICE + creatorInterfacePtr(cpSplitPluginCreator), +#endif // ENABLE_MULTI_DEVICE + creatorInterfacePtr(doraPluginCreator) }; + + nbCreators = creators.size(); + return creators.data(); + } +} // extern "C" diff --git a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.cpp b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.cpp new file mode 100644 index 000000000000..6acf0b3a9d25 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.cpp @@ -0,0 +1,1206 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "bertAttentionPlugin.h" +#include "tensorrt_llm/kernels/gptKernels.h" +#include "tensorrt_llm/kernels/recoverFromRingAtten.h" +#include "tensorrt_llm/kernels/sageAttentionKernels.h" +#include "tensorrt_llm/kernels/unfusedAttentionKernels.h" +#include "tensorrt_llm/runtime/iBuffer.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +namespace tc = tensorrt_llm::common; + +using tensorrt_llm::plugins::BertAttentionPluginCreator; +using tensorrt_llm::plugins::BertAttentionPlugin; + +static char const* BERT_ATTENTION_PLUGIN_VERSION{"1"}; +static char const* BERT_ATTENTION_PLUGIN_NAME{"BertAttention"}; +PluginFieldCollection BertAttentionPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> BertAttentionPluginCreator::mPluginAttributes; + +BertAttentionPlugin::BertAttentionPlugin(int num_heads, int head_size, float q_scaling, + ContextFMHAType context_fmha_type, nvinfer1::DataType type, bool do_relative_attention, int max_distance, + bool remove_padding, bool sage_attn, int sage_attn_q_block_size, int sage_attn_k_block_size, + int sage_attn_v_block_size, int cp_size, int cp_rank, std::set<int> cp_group) + : mNumHeads(num_heads) + , mHeadSize(head_size) + , mQScaling(q_scaling) + , mType(type) + , mRelativeAttention(do_relative_attention) + , mMaxDistance(max_distance) + , mRemovePadding(remove_padding) + , mEnableContextFMHA(context_fmha_type != ContextFMHAType::DISABLED) + , mFMHAForceFP32Acc(context_fmha_type == ContextFMHAType::ENABLED_WITH_FP32_ACC) + , mSageAttn(sage_attn) + , mCpSize(cp_size) + , mCpRank(cp_rank) + , mCpGroup(std::move(cp_group)) +{ + // pre-check whether FMHA is supported in order to save memory allocation + if (mEnableContextFMHA) + { + mEnableContextFMHA = false; + if (!(mType == DataType::kHALF || mType == DataType::kBF16)) + { + TLLM_LOG_WARNING("Fall back to unfused MHA because of unsupported data type."); + } + else if (mRelativeAttention) + { + TLLM_LOG_WARNING("Fall back to unfused MHA because of relative position embedding."); + } + else + { + mEnableContextFMHA = true; + } + } + + if (mSageAttn) + { + mSageAttnQBlockSize = sage_attn_q_block_size; + mSageAttnKBlockSize = sage_attn_k_block_size; + mSageAttnVBlockSize = sage_attn_v_block_size; + std::vector<int> blockSizeCombination + = {sage_attn_q_block_size, sage_attn_k_block_size, sage_attn_v_block_size}; + if (mSageAttnSupportedBlockSizes.find(blockSizeCombination) == mSageAttnSupportedBlockSizes.end() + || (head_size != 128 && head_size != 72 && head_size != 80)) + { + TLLM_LOG_WARNING(" Q, k ,v quant block size not support. disable sage attention"); + mSageAttn = false; + } + else + { + TLLM_LOG_INFO("SageAttnQBlockSize: %d, SageAttnKBlockSize: %d, SageAttnVBlockSize: %d", mSageAttnQBlockSize, + mSageAttnKBlockSize, mSageAttnVBlockSize); + } + } + + if (cp_group.size() > 1 && !mEnableContextFMHA) + { + TLLM_LOG_ERROR("Unfused MHA do not support context parallel now."); + } +} + +// Parameterized constructor +BertAttentionPlugin::BertAttentionPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mNumHeads); + read(d, mHeadSize); + read(d, mQScaling); + read(d, mQKHalfAccum); + read(d, mEnableContextFMHA); + read(d, mFMHAForceFP32Acc); + read(d, mType); + read(d, mRelativeAttention); + read(d, mMaxDistance); + read(d, mRemovePadding); + read(d, mSageAttn); + read(d, mSageAttnQBlockSize); + read(d, mSageAttnKBlockSize); + read(d, mSageAttnVBlockSize); + read(d, mCpSize); + read(d, mCpRank); + mCpGroup.clear(); + int groupItem = 0; + while (d != a + length) + { + read(d, groupItem); + mCpGroup.insert(groupItem); + } + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* BertAttentionPlugin::clone() const noexcept +{ + auto* plugin = new BertAttentionPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + plugin->initialize(); + return plugin; +} + +nvinfer1::DimsExprs BertAttentionPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + TLLM_CHECK(outputIndex == 0); + auto ret = inputs[0]; + ret.d[mRemovePadding ? 1 : 2] = exprBuilder.constant(ret.d[mRemovePadding ? 1 : 2]->getConstantValue() / 3); + return ret; +} + +bool BertAttentionPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + // inputs: [0] qkv, [1] input_lengths, [2] max_input_length (optional), [3] relative_attention_bias (optional) + // outputs: [X] hidden_states + if (nbInputs == 2) + { // BERT + if (pos == 1) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + if (nbInputs > 2) + { // Encoder in encoder-decoder + if (pos == 1 || pos == 2) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + + return false; +} + +void BertAttentionPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t BertAttentionPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + // if remove padding, inputs[0] "qkv_hidden_states" dim is [num_tokens, 3*hidden_dim] which doesn't have shape + // info should get max_batch_size and max_input_length from inputs[1] "input_lengths" and input[2] + // "max_input_length" + int const batch_size = mRemovePadding ? inputs[1].dims.d[0] : inputs[0].dims.d[0]; + int const input_seq_len = mRemovePadding ? inputs[2].dims.d[0] : inputs[0].dims.d[1]; + int const local_hidden_units_ = inputs[0].dims.d[mRemovePadding ? 1 : 2] / 3; + + auto const size = tensorrt_llm::runtime::BufferDataType(inputs[0].type).getSize(); + + size_t const attention_mask_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * input_seq_len; + size_t const cu_seqlens_size = sizeof(int) * (batch_size + 1); + size_t const q_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * local_hidden_units_; + size_t const k_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * local_hidden_units_; + size_t const v_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * local_hidden_units_; + size_t const qk_buf_size = mEnableContextFMHA ? 0 : size * batch_size * mNumHeads * input_seq_len * input_seq_len; + size_t const qkv_buf_2_size = mEnableContextFMHA ? 0 : size * batch_size * input_seq_len * local_hidden_units_; + size_t const qk_buf_float_size + = mEnableContextFMHA ? 0 : sizeof(float) * batch_size * mNumHeads * input_seq_len * input_seq_len; + size_t const padding_offset_size = mEnableContextFMHA ? 0 : sizeof(int) * batch_size * input_seq_len; + size_t const fmha_scheduler_counter = mEnableContextFMHA ? sizeof(uint32_t) : 0; + int const paddedHeadSize = mSageAttn ? ((mHeadSize + 15) / 16) * 16 : mHeadSize; + const size_t quanted_qkv_size + = mSageAttn ? sizeof(__nv_fp8_e4m3) * batch_size * input_seq_len * mNumHeads * paddedHeadSize * 3 : 0; + const size_t q_scale_size = mSageAttn + ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnQBlockSize - 1) / mSageAttnQBlockSize) * mNumHeads + : 0; + const size_t k_scale_size = mSageAttn + ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnKBlockSize - 1) / mSageAttnKBlockSize) * mNumHeads + : 0; + const size_t v_scale_size = mSageAttn + ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnVBlockSize - 1) / mSageAttnVBlockSize) * mNumHeads + : 0; + const size_t scale_bmm1_device_size = mSageAttn ? sizeof(float) * 2 : 0; + const size_t scale_bmm2_device_size = mSageAttn ? sizeof(float) : 0; + size_t sage_quant_space_size = mSageAttn ? sizeof(float) * batch_size * mNumHeads * mHeadSize : 0; + + if (paddedHeadSize != mHeadSize) + sage_quant_space_size + = sage_quant_space_size < (batch_size * input_seq_len * mNumHeads * paddedHeadSize * sizeof(__nv_bfloat16)) + ? (batch_size * input_seq_len * mNumHeads * paddedHeadSize * sizeof(__nv_bfloat16)) + : sage_quant_space_size; + + // workspace for RingAttention ping-pong buffer + bool const enableRingAttn = (mCpGroup.size() > 1); + const size_t ring_q_buf_size = enableRingAttn ? size * batch_size * input_seq_len * local_hidden_units_ : 0; + const size_t ring_kv_buf_size = enableRingAttn + ? 2 * size * batch_size * input_seq_len * local_hidden_units_ + sizeof(int) * (batch_size + 1) + : 0; + const size_t ring_softmax_stats_buf_size + = enableRingAttn ? 2 * sizeof(float) * batch_size * input_seq_len * mNumHeads : 0; + const size_t ring_softmax_stats_accu_buf_size + = enableRingAttn ? 2 * sizeof(float) * batch_size * input_seq_len * mNumHeads : 0; + const size_t ring_block_output_size = enableRingAttn ? size * batch_size * input_seq_len * local_hidden_units_ : 0; + + int const NUM_BUFFERS = 24; + + size_t workspaces[NUM_BUFFERS]; + workspaces[0] = CUBLAS_WORKSPACE_SIZE; + workspaces[1] = attention_mask_size; + workspaces[2] = cu_seqlens_size; + workspaces[3] = q_buf_2_size; + workspaces[4] = k_buf_2_size; + workspaces[5] = v_buf_2_size; + workspaces[6] = qk_buf_size; + workspaces[7] = qkv_buf_2_size; + workspaces[8] = qk_buf_float_size; + workspaces[9] = padding_offset_size; + workspaces[10] = fmha_scheduler_counter; + workspaces[11] = quanted_qkv_size; + workspaces[12] = q_scale_size; + workspaces[13] = v_scale_size; + workspaces[14] = k_scale_size; + workspaces[15] = scale_bmm1_device_size; + workspaces[16] = scale_bmm2_device_size; + workspaces[17] = sage_quant_space_size; + workspaces[18] = ring_q_buf_size; + workspaces[19] = ring_kv_buf_size; // kv1 + workspaces[20] = ring_kv_buf_size; // kv2 + workspaces[21] = ring_softmax_stats_buf_size; + workspaces[22] = ring_softmax_stats_accu_buf_size; + workspaces[23] = ring_block_output_size; + + return tc::calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); +} + +template <typename T> +int BertAttentionPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) +{ + + // inputs + // input_tensor [batch_size, seq_len, local_hidden_size*3] or [num_tokens, local_hidden_size*3] + // input_lengths [batch_size] + // max_input_length [max_input_length] -- use shape dim to represent max value. If remove padding, this records + // the max input length among sequences; otherwise same as input_tensor's padded dim[1] relative_attention_bias + // [num_heads, num_buckets] (optional) + // outputs + // output_tensor [batch_size, seq_len, local_hidden_size] or [num_tokens, local_hidden_size] + + // if remove padding, inputs[0] dim is [num_tokens] which doesn't have workspace info + // should get max_batch_size from inputs[1] and max_input_length from plugin attribute + int const batch_size = mRemovePadding ? inputDesc[1].dims.d[0] : inputDesc[0].dims.d[0]; + int const input_seq_len = mRemovePadding ? inputDesc[2].dims.d[0] : inputDesc[0].dims.d[1]; + int const num_tokens = mRemovePadding ? inputDesc[0].dims.d[0] : batch_size * input_seq_len; + int const request_batch_size = batch_size; + int const request_seq_len = input_seq_len; + int const local_hidden_units_ = inputDesc[0].dims.d[mRemovePadding ? 1 : 2] / 3; + float const q_scaling = mQScaling; + + T const* attention_input = reinterpret_cast<T const*>(inputs[0]); + int const* input_lengths = reinterpret_cast<int const*>(inputs[1]); + T const* relative_attn_table = mRelativeAttention ? reinterpret_cast<T const*>(inputs[3]) : nullptr; + T* context_buf_ = (T*) (outputs[0]); + + auto cublasHandle = mCublasWrapper->getCublasHandle(); + TLLM_CUDA_CHECK(cublasSetStream(cublasHandle, stream)); + mCublasWrapper->setStream(stream); + mCublasWrapper->setWorkspace(workspace); + if (inputDesc[0].type == DataType::kHALF) + { + mCublasWrapper->setFP16GemmConfig(); + } + else if (inputDesc[0].type == DataType::kFLOAT) + { + mCublasWrapper->setFP32GemmConfig(); + } +#ifdef ENABLE_BF16 + else if constexpr (std::is_same_v<T, __nv_bfloat16>) + { + mCublasWrapper->setBF16GemmConfig(); + } +#endif + + size_t const attention_mask_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * input_seq_len; + size_t const cu_seqlens_size = sizeof(int) * (batch_size + 1); + size_t const q_buf_2_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * local_hidden_units_; + size_t const k_buf_2_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * local_hidden_units_; + size_t const v_buf_2_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * local_hidden_units_; + size_t const qk_buf_size + = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * mNumHeads * input_seq_len * input_seq_len; + size_t const qkv_buf_2_size = mEnableContextFMHA ? 0 : sizeof(T) * batch_size * input_seq_len * local_hidden_units_; + size_t const qk_buf_float_size + = mEnableContextFMHA ? 0 : sizeof(float) * batch_size * mNumHeads * input_seq_len * input_seq_len; + size_t const padding_offset_size = mEnableContextFMHA ? 0 : sizeof(int) * batch_size * input_seq_len; + size_t const fmha_scheduler_counter = mEnableContextFMHA ? sizeof(uint32_t) : 0; + + int const paddedHeadSize = mSageAttn ? ((mHeadSize + 15) / 16) * 16 : mHeadSize; + const size_t quanted_qkv_size + = mSageAttn ? sizeof(__nv_fp8_e4m3) * batch_size * input_seq_len * mNumHeads * paddedHeadSize * 3 : 0; + const size_t q_scale_size = mSageAttn + ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnQBlockSize - 1) / mSageAttnQBlockSize) * mNumHeads + : 0; + const size_t k_scale_size = mSageAttn + ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnKBlockSize - 1) / mSageAttnKBlockSize) * mNumHeads + : 0; + const size_t v_scale_size = mSageAttn + ? sizeof(float) * batch_size * ((input_seq_len + mSageAttnVBlockSize - 1) / mSageAttnVBlockSize) * mNumHeads + : 0; + const size_t scale_bmm1_device_size = mSageAttn ? sizeof(float) * 2 : 0; + const size_t scale_bmm2_device_size = mSageAttn ? sizeof(float) : 0; + size_t sage_quant_space_size = mSageAttn ? sizeof(float) * batch_size * mNumHeads * mHeadSize : 0; + + if (paddedHeadSize != mHeadSize) + sage_quant_space_size + = sage_quant_space_size < (batch_size * input_seq_len * mNumHeads * paddedHeadSize * sizeof(__nv_bfloat16)) + ? (batch_size * input_seq_len * mNumHeads * paddedHeadSize * sizeof(__nv_bfloat16)) + : sage_quant_space_size; + + bool const enableRingAttn = (mCpGroup.size() > 1); + const size_t ring_q_buf_size = enableRingAttn ? sizeof(T) * batch_size * input_seq_len * local_hidden_units_ : 0; + const size_t ring_kv_buf_size + = enableRingAttn ? 2 * sizeof(T) * batch_size * input_seq_len * local_hidden_units_ : 0; + const size_t ring_softmax_stats_buf_size + = enableRingAttn ? 2 * sizeof(float) * batch_size * input_seq_len * mNumHeads : 0; + const size_t ring_block_output_size + = enableRingAttn ? sizeof(T) * batch_size * input_seq_len * local_hidden_units_ : 0; + + // Workspace pointer shift + int8_t* workspace_byte_ptr = reinterpret_cast<int8_t*>(workspace); + size_t offset = CUBLAS_WORKSPACE_SIZE; + + T* attention_mask = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, attention_mask_size)); + int* cu_seqlens = reinterpret_cast<int*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, cu_seqlens_size)); + T* q_buf_2_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, q_buf_2_size)); + T* k_buf_2_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, k_buf_2_size)); + T* v_buf_2_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, v_buf_2_size)); + T* qk_buf_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, qk_buf_size)); + T* qkv_buf_2_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, qkv_buf_2_size)); + float* qk_buf_float_ + = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, qk_buf_float_size)); + int* padding_offset = reinterpret_cast<int*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, padding_offset_size)); + uint32_t* fmha_tile_counter_ptr + = reinterpret_cast<uint32_t*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, fmha_scheduler_counter)); + + __nv_fp8_e4m3* quanted_qkv_ptr + = reinterpret_cast<__nv_fp8_e4m3*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, quanted_qkv_size)); + float* q_scale_ptr = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, q_scale_size)); + float* k_scale_ptr = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, k_scale_size)); + float* v_scale_ptr = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, v_scale_size)); + float* scale_bmm1_ptr + = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, scale_bmm1_device_size)); + float* scale_bmm2_ptr + = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, scale_bmm2_device_size)); + void* sage_quant_space_ptr + = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, sage_quant_space_size)); + + T* ring_q_buf_ = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_q_buf_size)); + T* ring_kv_buf_1_ = reinterpret_cast<T*>( + tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_kv_buf_size + sizeof(int) * (batch_size + 1))); + T* ring_kv_buf_2_ = reinterpret_cast<T*>( + tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_kv_buf_size + sizeof(int) * (batch_size + 1))); + float* ring_softmax_stats_buf_ + = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_softmax_stats_buf_size)); + float* ring_softmax_accu_stats_buf_ + = reinterpret_cast<float*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_softmax_stats_buf_size)); + T* ring_block_output_ + = reinterpret_cast<T*>(tc::nextWorkspacePtr(workspace_byte_ptr, offset, ring_block_output_size)); + + // build attention_mask, cu_seqlens, and padding_offset tensors + BuildDecoderInfoParams<T> params{}; + params.seqQOffsets = cu_seqlens; + params.paddingOffsets = padding_offset; + params.attentionMask = attention_mask; + params.seqQLengths = input_lengths; + params.batchSize = batch_size; + params.maxQSeqLength = input_seq_len; + params.numTokens = num_tokens; + params.attentionMaskType = AttentionMaskType::PADDING; + params.fmhaTileCounter = fmha_tile_counter_ptr; + if (mSageAttn) + { + params.fmhaHostBmm1Scale = 1.0f / (sqrtf(mHeadSize * 1.0f) * q_scaling); + params.fmhaBmm1Scale = scale_bmm1_ptr; + params.fmhaBmm2Scale = scale_bmm2_ptr; + } + invokeBuildDecoderInfo(params, stream); + sync_check_cuda_error(stream); + + auto const gemm_data_type = tc::CudaDataType<T>::value; + int const attention_seq_len_1 = request_seq_len; // q length + int const attention_seq_len_2 = request_seq_len; // kv length + + // If the model has relative attentiona bias, q scaling should be applied in QK gemm stage and use 1 in + // softamax stage (because to get softmax[scale(Q*K) + rel pos bias] here, q_scaling can't be applied during + // softmax phase by qk_scale); otherwise, use 1 in gemm stage and apply scaling in softmax stage + float const qk_scale + = 1.0f / (sqrtf(mHeadSize * 1.0f) * q_scaling); // q_scaling in denominator. by default q_scaling =1.0f + float const qk_scale_gemm = mRelativeAttention ? qk_scale : 1.0f; + T const qk_scale_softmax = static_cast<T>(mRelativeAttention ? 1.0f : qk_scale); + + T* linear_bias_slopes = nullptr; + + // FMHA doesn't apply to MHA with relative attention bias, i.e. softmax(QK + bias) * V + // We update mEnableContextFMHA in constructor to check this condition + if (mEnableContextFMHA) + { + if (enableRingAttn) + { + // make sure the padding part of key/value buffer is 0 + cudaMemsetAsync(ring_kv_buf_1_, 0, + reinterpret_cast<int8_t*>(ring_kv_buf_2_) - reinterpret_cast<int8_t*>(ring_kv_buf_1_), stream); + + cudaMemcpyAsync(ring_q_buf_, attention_input, ring_q_buf_size, cudaMemcpyDeviceToDevice, stream); + cudaMemcpyAsync(ring_kv_buf_1_, + const_cast<char*>(reinterpret_cast<char const*>(attention_input)) + ring_q_buf_size, ring_kv_buf_size, + cudaMemcpyDeviceToDevice, stream); + cudaMemcpyAsync(reinterpret_cast<char*>(ring_kv_buf_1_) + ring_kv_buf_size, cu_seqlens, + sizeof(int) * (batch_size + 1), cudaMemcpyDeviceToDevice, stream); + // init softmax_stats + cudaMemsetAsync(ring_softmax_accu_stats_buf_, 0, ring_softmax_stats_buf_size, stream); + +#if ENABLE_MULTI_DEVICE + // relative position of prev/next rank in cp group + int prev_rank = mCpRank > 0 ? mCpRank - 1 : mCpGroup.size() - 1; + int next_rank = (mCpRank == static_cast<int>(mCpGroup.size() - 1)) ? 0 : mCpRank + 1; +#endif // ENABLE_MULTI_DEVICE + + common::check_cuda_error(cudaStreamCreate(&mNcclStream)); + common::check_cuda_error(cudaStreamSynchronize(stream)); + + uint32_t* fmha_scheduler_counter_h = (uint32_t*) malloc(sizeof(uint32_t)); + cudaMemcpyAsync( + fmha_scheduler_counter_h, fmha_tile_counter_ptr, sizeof(uint32_t), cudaMemcpyDeviceToHost, stream); + for (size_t iter = 0; iter < mCpGroup.size(); ++iter) + { + // KV buffer used by fmha + T* ring_fmha_kv_buf_ = (iter % 2 == 0) ? ring_kv_buf_1_ : ring_kv_buf_2_; +#if ENABLE_MULTI_DEVICE + T* ring_send_kv_buf_ = (iter % 2 == 0) ? ring_kv_buf_1_ : ring_kv_buf_2_; + T* ring_recv_kv_buf_ = (iter % 2 == 0) ? ring_kv_buf_2_ : ring_kv_buf_1_; + if (iter < mCpGroup.size() - 1) + { + NCCLCHECK(ncclGroupStart()); + TLLM_CHECK_WITH_INFO(mNcclComm.get() != nullptr, "mNcclComm should be initialized before used"); + NCCLCHECK(ncclSend(ring_send_kv_buf_, + ring_kv_buf_size / sizeof(T) + sizeof(int) / sizeof(T) * (batch_size + 1), + (*getDtypeMap())[inputDesc[0].type], next_rank, *mNcclComm, mNcclStream)); + NCCLCHECK(ncclRecv(ring_recv_kv_buf_, + ring_kv_buf_size / sizeof(T) + sizeof(int) / sizeof(T) * (batch_size + 1), + (*getDtypeMap())[inputDesc[0].type], prev_rank, *mNcclComm, mNcclStream)); + NCCLCHECK(ncclGroupEnd()); + } +#else + TLLM_LOG_ERROR("Please set ENABLE_MULTI_DEVICE to enable RingAttention"); + return 1; +#endif // ENABLE_MULTI_DEVICE + // Construct the fmha params for running kernels. + MHARunnerParams fmhaParams{}; + fmhaParams.b = request_batch_size; + fmhaParams.qSeqLen = request_seq_len; + fmhaParams.kvSeqLen = request_seq_len; + fmhaParams.totalQSeqLen = request_batch_size * request_seq_len; + // Device buffer pointers. + fmhaParams.qPtr = ring_q_buf_; + fmhaParams.kvPtr = ring_fmha_kv_buf_; + if (iter == 0) + { + fmhaParams.outputPtr = context_buf_; + fmhaParams.softmaxStatsPtr = ring_softmax_accu_stats_buf_; + } + else + { + cudaMemsetAsync(ring_softmax_stats_buf_, 0, ring_softmax_stats_buf_size, stream); + fmhaParams.outputPtr = ring_block_output_; + fmhaParams.softmaxStatsPtr = ring_softmax_stats_buf_; + } + fmhaParams.cuQSeqLenPtr = cu_seqlens; + fmhaParams.cuKvSeqLenPtr + = reinterpret_cast<int*>(reinterpret_cast<char*>(ring_fmha_kv_buf_) + ring_kv_buf_size); + + fmhaParams.tileCounterPtr = fmha_tile_counter_ptr; + fmhaParams.stream = stream; + // Run the fmha kernel. + cudaMemsetAsync(fmhaParams.outputPtr, 0, ring_block_output_size, stream); + cudaMemcpyAsync(fmhaParams.tileCounterPtr, fmha_scheduler_counter_h, sizeof(uint32_t), + cudaMemcpyHostToDevice, stream); + mFmhaDispatcher->run(fmhaParams); + if (iter != 0) + { + invokeRecoverFromRA<T>((T*) context_buf_, (float*) ring_softmax_accu_stats_buf_, + (T*) ring_block_output_, (float*) ring_softmax_stats_buf_, fmhaParams.b, fmhaParams.qSeqLen, + mNumHeads, mHeadSize, cu_seqlens, stream); + } + cudaStreamSynchronize(stream); + cudaStreamSynchronize(mNcclStream); + } + common::check_cuda_error(cudaStreamDestroy(mNcclStream)); + free(fmha_scheduler_counter_h); + } + + else + { + if (mSageAttn && mHeadSize == 72 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 64 + && mSageAttnVBlockSize == 256) + { + sage_quant<72, 80, 64, 64, 256, __nv_bfloat16, __nv_fp8_e4m3, float>( + // host var + batch_size, mNumHeads, input_seq_len, true, true, + // device var + // q k v + attention_input, attention_input + mNumHeads * mHeadSize, + attention_input + 2 * mNumHeads * mHeadSize, + // stride + 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, + cu_seqlens, sage_quant_space_ptr, + // quant q k v + quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, + quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, + // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, + 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, + // scales + q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); + + sync_check_cuda_error(stream); + } + if (mSageAttn && mHeadSize == 80 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 64 + && mSageAttnVBlockSize == 256) + { + sage_quant<80, 80, 64, 64, 256, __nv_bfloat16, __nv_fp8_e4m3, float>( + // host var + batch_size, mNumHeads, input_seq_len, true, true, + // device var + // q k v + attention_input, attention_input + mNumHeads * mHeadSize, + attention_input + 2 * mNumHeads * mHeadSize, + // stride + 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, + cu_seqlens, sage_quant_space_ptr, + // quant q k v + quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, + quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, + // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, + 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, + // scales + q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); + + sync_check_cuda_error(stream); + } + if (mSageAttn && mHeadSize == 128 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 64 + && mSageAttnVBlockSize == 256) + { + sage_quant<128, 128, 64, 64, 256, __nv_bfloat16, __nv_fp8_e4m3, float>( + // host var + batch_size, mNumHeads, input_seq_len, true, true, + // device var + // q k v + attention_input, attention_input + mNumHeads * mHeadSize, + attention_input + 2 * mNumHeads * mHeadSize, + // stride + 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, + cu_seqlens, sage_quant_space_ptr, + // quant q k v + quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, + quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, + // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, + 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, + // scales + q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); + + sync_check_cuda_error(stream); + } + if (mSageAttn && mHeadSize == 128 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 32 + && mSageAttnVBlockSize == 32) + { + sage_quant<128, 128, 64, 32, 32, __nv_bfloat16, __nv_fp8_e4m3, float>( + // host var + batch_size, mNumHeads, input_seq_len, true, true, + // device var + // q k v + attention_input, attention_input + mNumHeads * mHeadSize, + attention_input + 2 * mNumHeads * mHeadSize, + // stride + 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, + cu_seqlens, sage_quant_space_ptr, + // quant q k v + quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, + quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, + // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, + 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, + // scales + q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); + + sync_check_cuda_error(stream); + } + if (mSageAttn && mHeadSize == 80 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 32 + && mSageAttnVBlockSize == 32) + { + sage_quant<80, 80, 64, 32, 32, __nv_bfloat16, __nv_fp8_e4m3, float>( + // host var + batch_size, mNumHeads, input_seq_len, true, true, + // device var + // q k v + attention_input, attention_input + mNumHeads * mHeadSize, + attention_input + 2 * mNumHeads * mHeadSize, + // stride + 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, + cu_seqlens, sage_quant_space_ptr, + // quant q k v + quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, + quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, + // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, + 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, + // scales + q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); + + sync_check_cuda_error(stream); + } + if (mSageAttn && mHeadSize == 72 && mSageAttnQBlockSize == 64 && mSageAttnKBlockSize == 32 + && mSageAttnVBlockSize == 32) + { + sage_quant<72, 80, 64, 32, 32, __nv_bfloat16, __nv_fp8_e4m3, float>( + // host var + batch_size, mNumHeads, input_seq_len, true, true, + // device var + // q k v + attention_input, attention_input + mNumHeads * mHeadSize, + attention_input + 2 * mNumHeads * mHeadSize, + // stride + 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, 3 * mNumHeads * mHeadSize, cu_seqlens, + cu_seqlens, sage_quant_space_ptr, + // quant q k v + quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * paddedHeadSize, + quanted_qkv_ptr + 2 * mNumHeads * paddedHeadSize, + // quanted_qkv_ptr, quanted_qkv_ptr + mNumHeads * mHeadSize, context, + 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, 3 * mNumHeads * paddedHeadSize, + // scales + q_scale_ptr, k_scale_ptr, v_scale_ptr, stream); + + sync_check_cuda_error(stream); + } + + // Construct the fmha params for running kernels. + MHARunnerParams fmhaParams{}; + fmhaParams.b = request_batch_size; + fmhaParams.qSeqLen = request_seq_len; + fmhaParams.kvSeqLen = request_seq_len; + fmhaParams.totalQSeqLen = request_batch_size * request_seq_len; + // Device buffer pointers. + fmhaParams.qkvPtr = attention_input; + fmhaParams.outputPtr = context_buf_; + fmhaParams.cuQSeqLenPtr = cu_seqlens; + fmhaParams.cuKvSeqLenPtr = cu_seqlens; + fmhaParams.tileCounterPtr = fmha_tile_counter_ptr; + fmhaParams.stream = stream; + if (mSageAttn) + { + if (paddedHeadSize != mHeadSize) + fmhaParams.outputPtr = sage_quant_space_ptr; + fmhaParams.qkvPtr = quanted_qkv_ptr; + fmhaParams.scaleBmm1Ptr = scale_bmm1_ptr; + fmhaParams.scaleBmm2Ptr = scale_bmm2_ptr; + fmhaParams.qScalePtr = q_scale_ptr; + fmhaParams.kScalePtr = k_scale_ptr; + fmhaParams.vScalePtr = v_scale_ptr; + fmhaParams.qMaxNBlock = (input_seq_len + mSageAttnQBlockSize - 1) / mSageAttnQBlockSize; + fmhaParams.kMaxNBlock = (input_seq_len + mSageAttnKBlockSize - 1) / mSageAttnKBlockSize; + fmhaParams.vMaxNBlock = (input_seq_len + mSageAttnVBlockSize - 1) / mSageAttnVBlockSize; + } + + // Run the fmha kernel. + + // TODO: set it correctly for contiguous kv buffer (cross-attention). + fmhaParams.totalKvSeqLen = num_tokens; + + fmhaParams.cuKvSeqLenPtr = cu_seqlens; + fmhaParams.cuMaskRowsPtr = cu_seqlens; + fmhaParams.tileCounterPtr = fmha_tile_counter_ptr; + + fmhaParams.scaleBmm1Ptr = scale_bmm1_ptr; + fmhaParams.scaleBmm2Ptr = scale_bmm2_ptr; + fmhaParams.forceFp32Acc = mFMHAForceFP32Acc; + mFmhaDispatcher->run(fmhaParams); + sync_check_cuda_error(stream); + if (mSageAttn) + { + if (paddedHeadSize != mHeadSize && mHeadSize == 72) + { + unpadding<80, 72, __nv_bfloat16>(batch_size, mNumHeads, input_seq_len, sage_quant_space_ptr, + mNumHeads * 72, mNumHeads * 80, cu_seqlens, context_buf_, stream); + } + } + } + } + else + { + // FIXME: a temporary solution to make sure the padding part of key/value buffer is 0 + // NOTE: pointer subtraction is used below since there could be some extra gap due to alignment. + // Otherwise, we could do cudaMemsetAsync(k_buf_2_, 0, k_buf_2_size + v_buf_2_size, stream); + // cudaMemsetAsync(k_buf_2_, 0, reinterpret_cast<int8_t*>(qk_buf_) - reinterpret_cast<int8_t*>(k_buf_2_), + // stream); + // FIXME: the final solution is to change the add_fusedQKV_bias_transpose_kernel to map CTAs corresponding to + // the output shape, and set the padding part to 0. Without zero-initialize guarantee, these workspace buffers + // may contain random NaN values when IFB workload is high. + cudaMemsetAsync(k_buf_2_, 0, + reinterpret_cast<int8_t*>(v_buf_2_) - reinterpret_cast<int8_t*>(k_buf_2_) + v_buf_2_size, stream); + + // only non-FMHA path needs to split Q,K,V from QKV + invokeAddFusedQKVBiasTranspose(q_buf_2_, k_buf_2_, v_buf_2_, const_cast<T*>(attention_input), input_lengths, + mRemovePadding ? padding_offset : nullptr, batch_size, input_seq_len, num_tokens, mNumHeads, mNumHeads, + mHeadSize, 0, 0.0f, RotaryScalingType::kNONE, 0.0f, 0, PositionEmbeddingType::kLEARNED_ABSOLUTE, + (float*) nullptr, 0, stream); + + if (!mQKHalfAccum && gemm_data_type != CUDA_R_32F) + { + mCublasWrapper->stridedBatchedGemm(CUBLAS_OP_T, CUBLAS_OP_N, + attention_seq_len_2, // n + attention_seq_len_1, // m + mHeadSize, // k + qk_scale_gemm, k_buf_2_, gemm_data_type, + mHeadSize, // k + attention_seq_len_2 * mHeadSize, // n * k + q_buf_2_, gemm_data_type, + mHeadSize, // k + attention_seq_len_1 * mHeadSize, // m * k + 0.0f, qk_buf_float_, CUDA_R_32F, + attention_seq_len_2, // n + attention_seq_len_2 * attention_seq_len_1, + request_batch_size * mNumHeads, // global batch size + CUDA_R_32F); + + // add relative position bias + if (mRelativeAttention) + { + // add rel pos bias + // QK is (batch_size, local_head_num, q_length, k_length), rel pos bias is (1, local_head_num, + // max_output_len + 1, max_output_len + 1). broadcast along 1st dim. max_seq_len is already + // max_output_len + 1. In implicit mode, relative_attention_bias is rel attn table + // [num_heads, num_buckets], with necessary params (max_distance, num_buckets) passed at the end + invokeAddRelativeAttentionBiasUnaligned(qk_buf_float_, relative_attn_table, request_batch_size, + mNumHeads, attention_seq_len_1, attention_seq_len_2, stream, mMaxDistance > 0, + inputDesc[3].dims.d[1], mMaxDistance, true /* bidirectional */); + } + + MaskedSoftmaxParam<T, float> param; + param.attention_score = qk_buf_; // (batch_size, head_num, q_length, k_length) + param.qk = qk_buf_float_; // (batch_size, head_num, q_length, k_length) + param.attention_mask = attention_mask; // (batch_size, q_length, k_length) + param.batch_size = request_batch_size; + param.q_length = attention_seq_len_1; + param.k_length = attention_seq_len_2; + param.num_heads = mNumHeads; + param.qk_scale = qk_scale_softmax; + param.linear_bias_slopes = const_cast<T*>(linear_bias_slopes); // (head_num,), optional + invokeMaskedSoftmax(param, stream); + } + else + { + mCublasWrapper->stridedBatchedGemm(CUBLAS_OP_T, CUBLAS_OP_N, attention_seq_len_2, attention_seq_len_1, + mHeadSize, k_buf_2_, mHeadSize, attention_seq_len_2 * mHeadSize, q_buf_2_, mHeadSize, + attention_seq_len_1 * mHeadSize, qk_buf_, attention_seq_len_2, + attention_seq_len_2 * attention_seq_len_1, request_batch_size * mNumHeads, qk_scale_gemm, + 0.0f); // alpha, beta + + // add relative position bias + if (mRelativeAttention) + { + // add rel pos bias + // QK is (batch_size, local_head_num, q_length, k_length), rel pos bias is (1, local_head_num, + // max_output_len + 1, max_output_len + 1). broadcast along 1st dim. max_seq_len is already + // max_output_len + 1. In implicit mode, relative_attention_bias is rel attn table + // [num_heads, num_buckets], with necessary params (max_distance, num_buckets) passed at the end + invokeAddRelativeAttentionBiasUnaligned(qk_buf_, relative_attn_table, request_batch_size, mNumHeads, + attention_seq_len_1, attention_seq_len_2, stream, mMaxDistance > 0, inputDesc[3].dims.d[1], + mMaxDistance, true /* bidirectional */); + } + + MaskedSoftmaxParam<T, T> param; + param.attention_score = qk_buf_; // (batch_size, head_num, q_length, k_length) + param.qk = qk_buf_; // (batch_size, head_num, q_length, k_length) + param.attention_mask = attention_mask; // (batch_size, q_length, k_length) + param.batch_size = request_batch_size; + param.q_length = attention_seq_len_1; + param.k_length = attention_seq_len_2; + param.num_heads = mNumHeads; + param.qk_scale = qk_scale_softmax; + param.linear_bias_slopes = const_cast<T*>(linear_bias_slopes); // (head_num,), optional + invokeMaskedSoftmax(param, stream); + } + + mCublasWrapper->stridedBatchedGemm(CUBLAS_OP_N, CUBLAS_OP_N, mHeadSize, attention_seq_len_1, + attention_seq_len_2, v_buf_2_, mHeadSize, attention_seq_len_2 * mHeadSize, qk_buf_, attention_seq_len_2, + attention_seq_len_1 * attention_seq_len_2, qkv_buf_2_, mHeadSize, attention_seq_len_1 * mHeadSize, + request_batch_size * mNumHeads); + + if (!mRemovePadding) + { + invokeTransposeQKV(context_buf_, qkv_buf_2_, request_batch_size, attention_seq_len_1, mNumHeads, mHeadSize, + (float*) nullptr, 0, stream); + } + else + { + invokeTransposeAttentionOutRemovePadding(qkv_buf_2_, context_buf_, num_tokens, request_batch_size, + request_seq_len, mNumHeads, mHeadSize, padding_offset, (float*) nullptr, 0, stream); + } + } + sync_check_cuda_error(stream); + return 0; +} + +template int BertAttentionPlugin::enqueueImpl<half>(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream); + +template int BertAttentionPlugin::enqueueImpl<float>(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream); + +#ifdef ENABLE_BF16 +template int BertAttentionPlugin::enqueueImpl<__nv_bfloat16>(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream); +#endif + +int BertAttentionPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + if (mType == DataType::kHALF) + { + return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kFLOAT) + { + return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16) + { + return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType BertAttentionPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return inputTypes[0]; +} + +// IPluginV2 Methods + +char const* BertAttentionPlugin::getPluginType() const noexcept +{ + return BERT_ATTENTION_PLUGIN_NAME; +} + +char const* BertAttentionPlugin::getPluginVersion() const noexcept +{ + return BERT_ATTENTION_PLUGIN_VERSION; +} + +int BertAttentionPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int BertAttentionPlugin::initialize() noexcept +{ + auto cublasHandle = getCublasHandle(); + auto cublasLtHandle = getCublasLtHandle(); + mCublasWrapper.reset(new tc::CublasMMWrapper(cublasHandle, cublasLtHandle, nullptr, nullptr)); + if (mEnableContextFMHA) + { + // Pre-checked during constructing. + Data_type data_type; + if (mType == DataType::kHALF) + { + data_type = DATA_TYPE_FP16; + } + else if (mType == DataType::kBF16) + { + data_type = DATA_TYPE_BF16; + } + else + { + TLLM_CHECK_WITH_INFO(false, "GPTAttentionPlugin received wrong data type."); + } + + // Construct the fmha runner. + MHARunnerFixedParams fmhaParams{}; + if (mSageAttn) + { + fmhaParams.dataType = DATA_TYPE_E4M3; + } + else + { + fmhaParams.dataType = data_type; + } + fmhaParams.dataTypeOut = data_type; + fmhaParams.forceFp32Acc = mFMHAForceFP32Acc; + fmhaParams.attentionMaskType = ContextAttentionMaskType::PADDING; + fmhaParams.isSPadded = !mRemovePadding; + fmhaParams.numQHeads = mNumHeads; + fmhaParams.numKvHeads = mNumHeads; + fmhaParams.headSize = mHeadSize; + fmhaParams.qScaling = mQScaling; + fmhaParams.sageBlockSizeQ = mSageAttnQBlockSize; + fmhaParams.sageBlockSizeK = mSageAttnKBlockSize; + fmhaParams.sageBlockSizeV = mSageAttnVBlockSize; + if (mSageAttn) + { + int const paddedHeadSize = ((mHeadSize + 15) / 16) * 16; + fmhaParams.headSize = paddedHeadSize; + } + + if (mCpGroup.size() > 1) + { + fmhaParams.attentionInputLayout = AttentionInputLayout::Q_CONTIGUOUS_KV; + fmhaParams.saveSoftmax = true; + } + + // Load kernels from the pre-compiled cubins. + // The KV input data type. The default is same as dataType. + fmhaParams.dataTypeKv = data_type; + fmhaParams.headSizeV = mHeadSize; + + // Load kernels from the pre-compiled cubins. + mFmhaDispatcher.reset(new FmhaDispatcher(fmhaParams)); + // Fall back to unfused MHA kernels if not supported. + mEnableContextFMHA = mFmhaDispatcher->isSupported(); + } + +#if ENABLE_MULTI_DEVICE + if (mCpGroup.size() > 1 && COMM_SESSION.getSize() > 1) + { + TLLM_LOG_TRACE("%s start for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); + mNcclComm = getComm(mCpGroup); + TLLM_LOG_TRACE("%s stop for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); + } +#endif // ENABLE_MULTI_DEVICE + + return 0; +} + +void BertAttentionPlugin::destroy() noexcept +{ + delete this; +} + +size_t BertAttentionPlugin::getSerializationSize() const noexcept +{ + return sizeof(mNumHeads) + sizeof(mHeadSize) + sizeof(mQScaling) + sizeof(mQKHalfAccum) + sizeof(mEnableContextFMHA) + + sizeof(mFMHAForceFP32Acc) + sizeof(mType) + sizeof(mRelativeAttention) + sizeof(mMaxDistance) + + sizeof(mRemovePadding) + sizeof(mSageAttn) + sizeof(mSageAttnQBlockSize) + sizeof(mSageAttnKBlockSize) + + sizeof(mSageAttnVBlockSize) + sizeof(mCpSize) + sizeof(mCpRank) + sizeof(int32_t) * mCpGroup.size(); +} + +void BertAttentionPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mNumHeads); + write(d, mHeadSize); + write(d, mQScaling); + write(d, mQKHalfAccum); + write(d, mEnableContextFMHA); + write(d, mFMHAForceFP32Acc); + write(d, mType); + write(d, mRelativeAttention); + write(d, mMaxDistance); + write(d, mRemovePadding); + write(d, mSageAttn); + write(d, mSageAttnQBlockSize); + write(d, mSageAttnKBlockSize); + write(d, mSageAttnVBlockSize); + write(d, mCpSize); + write(d, mCpRank); + for (auto it = mCpGroup.begin(); it != mCpGroup.end(); ++it) + { + write(d, *it); + } + TLLM_CHECK(d == a + getSerializationSize()); +} + +void BertAttentionPlugin::terminate() noexcept {} + +/////////////// + +BertAttentionPluginCreator::BertAttentionPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + + mPluginAttributes.emplace_back(PluginField("num_heads", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("head_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("q_scaling", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("context_fmha_type", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("do_relative_attention", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("max_distance", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("remove_padding", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("sage_attn", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("sage_attn_q_block_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("sage_attn_k_block_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("sage_attn_v_block_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("cp_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("cp_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("cp_group", nullptr, PluginFieldType::kINT32)); + + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* BertAttentionPluginCreator::getPluginName() const noexcept +{ + return BERT_ATTENTION_PLUGIN_NAME; +} + +char const* BertAttentionPluginCreator::getPluginVersion() const noexcept +{ + return BERT_ATTENTION_PLUGIN_VERSION; +} + +PluginFieldCollection const* BertAttentionPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* BertAttentionPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int num_heads{}; + int head_size{}; + ContextFMHAType context_fmha_type{}; + float q_scaling{}; + nvinfer1::DataType type{}; + bool do_relative_attention{}; + int max_distance{}; + bool remove_padding{}; + bool sage_attn{}; + int sage_attn_q_block_size{}; + int sage_attn_k_block_size{}; + int sage_attn_v_block_size{}; + int cp_size{}; + int cp_rank{}; + std::set<int> cp_group{}; + + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "num_heads")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + num_heads = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "head_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + head_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "q_scaling")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + q_scaling = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + else if (!strcmp(attrName, "context_fmha_type")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + context_fmha_type = static_cast<ContextFMHAType>(*(static_cast<int8_t const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "do_relative_attention")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + do_relative_attention = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); + } + else if (!strcmp(attrName, "max_distance")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + max_distance = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "remove_padding")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + remove_padding = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); + } + else if (!strcmp(attrName, "sage_attn")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + sage_attn = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); + if (sage_attn) + { + std::cout << "sage attn true!" << std::endl; + } + } + else if (!strcmp(attrName, "sage_attn_q_block_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + sage_attn_q_block_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "sage_attn_k_block_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + sage_attn_k_block_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "sage_attn_v_block_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + sage_attn_v_block_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "cp_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + cp_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "cp_rank")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + cp_rank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "cp_group")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + auto const* r = static_cast<int const*>(fields[i].data); + for (int j = 0; j < fields[i].length; ++j) + { + cp_group.insert(*r); + ++r; + } + } + } + try + { + auto* obj = new BertAttentionPlugin(num_heads, head_size, q_scaling, context_fmha_type, type, + do_relative_attention, max_distance, remove_padding, sage_attn, sage_attn_q_block_size, + sage_attn_k_block_size, sage_attn_v_block_size, cp_size, cp_rank, cp_group); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* BertAttentionPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call BertAttentionPlugin::destroy() + try + { + auto* obj = new BertAttentionPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.h b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.h new file mode 100644 index 000000000000..2eb39086a005 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/bertAttentionPlugin/bertAttentionPlugin.h @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/cublasMMWrapper.h" +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/kernels/fmhaDispatcher.h" +#include "tensorrt_llm/kernels/gptKernels.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include <cassert> +#include <cuda_runtime.h> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class BertAttentionPlugin : public BasePlugin +{ +public: + BertAttentionPlugin() = delete; + + BertAttentionPlugin(int num_heads, int head_size, float q_scaling, + tensorrt_llm::kernels::ContextFMHAType context_fmha_type, nvinfer1::DataType type, + bool do_relative_attention = false, int max_distance = 0, bool remove_padding = false, bool sage_attn = false, + int sage_attn_q_block_size = 0, int sage_attn_k_block_size = 0, int sage_attn_v_block_size = 0, int cp_size = 1, + int cp_rank = 0, std::set<int> cp_group = {}); + + BertAttentionPlugin(void const* data, size_t length); + + ~BertAttentionPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + template <typename T> + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + const std::string mLayerName; + + int mNumHeads; + int mHeadSize; + float mQScaling; + nvinfer1::DataType mType; + bool mRelativeAttention = false; + int mMaxDistance = 0; + bool mRemovePadding = false; + + // unfused mha + bool mQKHalfAccum = false; + + // fmha runner (disable by default) + bool mEnableContextFMHA = false; + bool mFMHAForceFP32Acc = false; + + // sage attention + bool mSageAttn = false; + int mSageAttnQBlockSize = 0; + int mSageAttnKBlockSize = 0; + int mSageAttnVBlockSize = 0; + std::set<std::vector<int>> mSageAttnSupportedBlockSizes{{64, 64, 256}, {64, 32, 32}}; + + int mSM = tensorrt_llm::common::getSMVersion(); + + // comm group for RingAttention + int mCpSize = 1; + int mCpRank = 0; + std::set<int> mCpGroup = {}; +#if ENABLE_MULTI_DEVICE + std::shared_ptr<ncclComm_t> mNcclComm; +#endif // ENABLE_MULTI_DEVICE + cudaStream_t mNcclStream; + + // The default copy constructor will leave them as nullptr. clone() shall initialize it. + UniqPtrWNullCopy<tensorrt_llm::kernels::FmhaDispatcher> mFmhaDispatcher; + UniqPtrWNullCopy<tensorrt_llm::common::CublasMMWrapper> mCublasWrapper; +}; + +class BertAttentionPluginCreator : public BaseCreator +{ +public: + BertAttentionPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/CMakeLists.txt b/cpp/tensorrt_llm/plugins/common/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/common/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.cpp b/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.cpp new file mode 100644 index 000000000000..2aab6b3675d8 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.cpp @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "checkMacrosPlugin.h" + +#include "tensorrt_llm/common/logger.h" + +namespace tensorrt_llm::plugins +{ + +void caughtError(std::exception const& e) +{ + TLLM_LOG_EXCEPTION(e); +} + +void logError(char const* msg, char const* file, char const* fn, int line) +{ + TLLM_LOG_ERROR("Parameter check failed at: %s::%s::%d, condition: %s", file, fn, line, msg); +} + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.h b/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.h new file mode 100644 index 000000000000..d8d8af1ef220 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/common/checkMacrosPlugin.h @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaUtils.h" + +namespace tensorrt_llm::plugins +{ + +void logError(char const* msg, char const* file, char const* fn, int line); + +void caughtError(std::exception const& e); + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.cpp b/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.cpp new file mode 100644 index 000000000000..e5d6650648ab --- /dev/null +++ b/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.cpp @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/common/cublasMMWrapper.h" +#include "tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm.h" +#include "tensorrt_llm/kernels/cutlass_kernels/fpA_intB_gemm/fpA_intB_gemm.h" +#include "tensorrt_llm/kernels/cutlass_kernels/fused_gated_gemm/fused_gated_gemm.h" +#include "tensorrt_llm/kernels/cutlass_kernels/int8_gemm/int8_gemm.h" +#include "tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h" +#include "tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h" +#include "tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h" +#include "tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h" +#if defined(USING_OSS_CUTLASS_FP4_GEMM) +#include "tensorrt_llm/kernels/cutlass_kernels/include/fp4_gemm.h" +#else +#include "fp4_gemm.h" +#endif +#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) +#include "tensorrt_llm/kernels/cutlass_kernels/include/allreduce_gemm_runner.h" +using GemmAllReduceImplInterface = tensorrt_llm::kernels::opened_cutlass_kernels::GemmAllReduceImplInterface; +#else +#include "allreduce_gemm_runner.h" +using GemmAllReduceImplInterface = tensorrt_llm::kernels::cutlass_kernels::GemmAllReduceImplInterface; +#endif + +#include <cstddef> + +namespace tensorrt_llm::plugins +{ + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::GemmPluginProfiler() +{ + mMNKProfileMap = std::make_shared<MNKProfileMap>(); + + // set SKIP_GEMM_PLUGIN_PROFILINGS=1 to avoid tactics profilings + auto const skipEnv = std::getenv("SKIP_GEMM_PLUGIN_PROFILINGS"); + mSkip = (skipEnv != NULL && std::stoi(skipEnv)); + if (mSkip) + { + TLLM_LOG_DEBUG( + "SKIP_GEMM_PLUGIN_PROFILINGS is set. Skipping GEMM plugin profilings. It could result in runtime error " + "if default tactic is not defined."); + } +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::serialize( + char*& buffer, GemmIdType const& gemmId) const +{ + auto mProfileMap = mMNKProfileMap->getMProfileMap(gemmId); + + // Save number of profiles for given GEMM ID + write(buffer, static_cast<int>(mProfileMap->size())); + for (auto const& pair : *mProfileMap) + { + // Save pair of M to the best GEMM config + write(buffer, pair); + } +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::deserialize( + char const*& data, GemmDims& dims, GemmIdType const& gemmId) +{ + // NOTE: this mutex is not needed since each thread owns its private map, but will put here for + // consistency + writer_lock lock(mMNKProfileMap->mutex); + + mDims = dims; + + // GemmId gemmId(dims.n, dims.k); + if (!mMNKProfileMap->existsMProfileMap(gemmId)) + { + // Create GEMM with GEMM ID if it does not exist + mMNKProfileMap->createMProfileMap(gemmId); + } + // Populate map with profiles of GEMM ID + auto profileMap = mMNKProfileMap->getMProfileMap(gemmId); + int selectedMapSize; + read(data, selectedMapSize); + for (int ii = 0; ii < selectedMapSize; ++ii) + { + std::pair<int, std::optional<Config>> config; + read(data, config); + profileMap->insert(config); + } +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +size_t GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::getSerializationSize( + GemmIdType const& gemmId) const +{ + reader_lock lock(mMNKProfileMap->mutex); + return sizeof(int) + // size of the tactics map + mMNKProfileMap->getMProfileMap(gemmId)->size() + * sizeof(std::pair<int, std::optional<Config>>); // size of the tactics map +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +int GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::getMaxProfileM() const +{ + return 8192; +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::initTmpData( + int m, int n, int k, char* workspace, size_t size, cudaStream_t stream) +{ + /* Do nothing */ +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::profileTactics(RunnerPtr const& runner, + nvinfer1::DataType const& type, GemmDims const& dims, GemmIdType const& gemmId, bool hasWeightOnlyCudaKernel) +{ + writer_lock lock(mMNKProfileMap->mutex); + + if (!dims.isInitialized()) + { + return; + } + + mRunner = runner; + mType = type; + + int const maxM = std::min(nextPowerOfTwo(dims.maxM), getMaxProfileM()); + computeTmpSize(maxM, dims.n, dims.k); + + if (!mMNKProfileMap->existsMProfileMap(gemmId)) + { + // Create map for GEMM ID + mMNKProfileMap->createMProfileMap(gemmId); + } + + if (mSkip) + { + return; + } + + auto mProfileMap = mMNKProfileMap->getMProfileMap(gemmId); + bool isAllocated{false}; + + auto profileTactics = [&mProfileMap, &isAllocated, this](int m, int n, int k) + { + if (mProfileMap->count(m) == 0) + { + if (!isAllocated) + { + // Allocate tmp data to run GEMMs + allocateTmpData(); + isAllocated = true; + } + initTmpData(m, n, k, mWorkspaceTmp, mTmpWorkspaceSizeInBytes, mStream); + auto tactics = this->getTactics(m, n, k); + + // Profile different tactics for particular m and insert best config to the map + mProfileMap->insert({m, this->profileTacticsForProblem(m, n, k, tactics)}); + } + }; + + common::check_cuda_error(cudaStreamCreate(&mStream)); + + int const startMinMRounded = nextPowerOfTwo(dims.minM); + + if (hasWeightOnlyCudaKernel) + { + // Profile tactics for finer granularity of M, + // if CUDA kernel is enabled for weight-only plugins + int minM = dims.minM; + for (int m = std::max(1, minM); m < std::min(16, maxM); m += 1) + { + profileTactics(m, dims.n, dims.k); + } + + for (int m = 16; m < maxM; m *= 2) + { + profileTactics(m, dims.n, dims.k); + } + } + else + { + // Profile tactics for CUTLASS kernel only + for (int m = std::max(1, startMinMRounded); m < maxM; m *= 2) + { + profileTactics(m, dims.n, dims.k); + } + } + + profileTactics(maxM, dims.n, dims.k); + + if (isAllocated) + { + // Free tmp data + freeTmpData(); + } + common::check_cuda_error(cudaStreamDestroy(mStream)); +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +std::optional<Config> GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::getBestConfig( + int m, GemmIdType const& gemmId) const +{ + reader_lock lock(mMNKProfileMap->mutex); + + if (mSkip) + { + TLLM_LOG_TRACE("Skip is set, no best config is set for this instance"); + return std::nullopt; + } + + int const mRounded = std::min(std::max(1, nextPowerOfTwo(m)), getMaxProfileM()); + fflush(stdout); + + if (mMNKProfileMap->getMProfileMap(gemmId)->count(m) > 0) + { + return mMNKProfileMap->getMProfileMap(gemmId)->at(m); + } + else if (mMNKProfileMap->getMProfileMap(gemmId)->count(mRounded) > 0) + { + return mMNKProfileMap->getMProfileMap(gemmId)->at(mRounded); + } + else + { + std::ostringstream msg; + msg << "Cannot find best tactic for m=" << m << " and GEMM ID " << gemmId; + TLLM_LOG_WARNING(msg.str()); + return std::nullopt; + } +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::allocateTmpData() +{ + TLLM_CHECK_WITH_INFO(mTmpWorkspaceSizeInBytes > 0, "tmpWorkspaceSizeInBytes must be larger than 0"); + auto const status = cudaMalloc(&mWorkspaceTmp, mTmpWorkspaceSizeInBytes); + TLLM_CHECK_WITH_INFO(status == cudaSuccess, "Can't allocate tmp workspace for GEMM tactics profiling."); +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +void GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::freeTmpData() +{ + auto const status = cudaFree(mWorkspaceTmp); + TLLM_CHECK_WITH_INFO(status == cudaSuccess, "Can't free tmp workspace for GEMM tactics profiling."); +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +std::optional<Config> GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::profileTacticsForProblem( + int m, int n, int k, std::vector<Config> const& tactics) +{ + TLLM_LOG_DEBUG(__PRETTY_FUNCTION__); + + float bestTime = std::numeric_limits<float>::max(); + Config bestConfig; + bool foundOne = false; + + // Iterate over all tactics for given M, N and K + for (size_t ii = 0; ii < tactics.size(); ++ii) + { + Config const& candidateConfig = tactics[ii]; + float time = std::numeric_limits<float>::max(); + try + { + if (!checkTactic(m, n, k, candidateConfig)) + { + continue; + } + // Profile particular tactic for given M, N and K + time = profileTacticForProblem(m, n, k, candidateConfig); + foundOne = true; + } + catch (std::exception const& e) + { + std::ostringstream msg; + msg << "Cannot profile configuration " << ii; + if constexpr (std::is_same_v<Config, tensorrt_llm::cutlass_extensions::CutlassGemmConfig>) + { + msg << ": " << candidateConfig.toString(); + } + msg << "\n (for" + << " m=" << m << ", n=" << n << ", k=" << k << ")" + << ", reason: \"" << e.what() << "\". Skipped"; + TLLM_LOG_TRACE(msg.str()); + cudaGetLastError(); // Reset the last cudaError to cudaSuccess. + continue; + } + + // Choose the fastest tactic + if (time < bestTime) + { + bestConfig = candidateConfig; + bestTime = time; + } + } + + if (!foundOne) + { + std::ostringstream msg; + msg << "Have not found any valid GEMM config for shape (" + << "m=" << m << ", n=" << n << ", k=" << k << "). Will try to use default or fail at runtime"; + TLLM_LOG_WARNING(msg.str()); + return std::nullopt; + } + + return {bestConfig}; +} + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +float GemmPluginProfiler<Config, RunnerPtr, GemmIdType, GemmIdHashType>::profileTacticForProblem( + int m, int n, int k, Config const& tactic) +{ + constexpr int warmup = 5; + constexpr int runs = 10; + + cudaStream_t stream = mStream; + + // Warmup the execution + for (int i = 0; i < warmup; ++i) + { + runTactic(m, n, k, tactic, mWorkspaceTmp, stream); + } + + cudaEvent_t start; + cudaEvent_t stop; + common::check_cuda_error(cudaEventCreate(&start)); + common::check_cuda_error(cudaEventCreate(&stop)); + common::check_cuda_error(cudaStreamSynchronize(stream)); + common::check_cuda_error(cudaEventRecord(start, stream)); + + // Profile GEMM + for (int i = 0; i < runs; ++i) + { + runTactic(m, n, k, tactic, mWorkspaceTmp, stream); + } + + common::check_cuda_error(cudaEventRecord(stop, stream)); + + common::check_cuda_error(cudaEventSynchronize(stop)); + + float elapsed; + common::check_cuda_error(cudaEventElapsedTime(&elapsed, start, stop)); + + common::check_cuda_error(cudaEventDestroy(start)); + common::check_cuda_error(cudaEventDestroy(stop)); + + return elapsed / runs; +} + +template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassInt8GemmRunnerInterface>, GemmIdCore, + GemmIdCoreHash>; + +template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunnerInterface>, GemmIdCore, + GemmIdCoreHash>; + +template class GemmPluginProfiler<cublasLtMatmulHeuristicResult_t, + std::shared_ptr<tensorrt_llm::common::CublasMMWrapper>, GemmIdCublas, GemmIdCublasHash>; + +// TODO I dont like the dependency on the MOE plugin here, but MOE needs the full context to run profiles +template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, MixtureOfExpertsPlugin*, + GemmIDMoe, GemmIDMoeHash>; + +template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFusedGatedGemmRunnerInterface>, GemmIdCore, + GemmIdCoreHash>; + +template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFp8RowwiseGemmRunnerInterface>, GemmIdCore, + GemmIdCoreHash>; + +#if defined(USING_OSS_CUTLASS_FP4_GEMM) +template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFp4GemmRunnerInterface>, GemmIdCore, GemmIdCoreHash>; +#else +template class GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + std::shared_ptr<tensorrt_llm::kernels::internal_cutlass_kernels::CutlassFp4GemmRunnerInterface>, GemmIdCore, + GemmIdCoreHash>; +#endif + +template class GemmPluginProfiler<LowLatencyGemmPluginProfiler::Config, LowLatencyGemmRunnerPtr, GemmIdCore, + GemmIdCoreHash>; + +template class GemmPluginProfiler<LowLatencyGemmSwigluPluginProfiler::Config, LowLatencyGemmSwigluRunnerPtr, GemmIdCore, + GemmIdCoreHash>; + +template class GemmPluginProfiler<GemmAllReduceImplInterface::LaunchConfig, std::shared_ptr<GemmAllReduceImplInterface>, + GemmIdCore, GemmIdCoreHash>; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.h b/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.h new file mode 100644 index 000000000000..fe85b3b7e456 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/common/gemmPluginProfiler.h @@ -0,0 +1,332 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "pluginUtils.h" + +#include <cuda_runtime.h> + +#include <cstdlib> +#include <iostream> +#include <memory> +#include <mutex> +#include <optional> +#include <shared_mutex> +#include <sstream> +#include <unordered_map> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +struct GemmDims +{ + using DimType64 = utils::DimType64; + + DimType64 minM; + DimType64 maxM; + DimType64 n; + DimType64 k; + + GemmDims() + : minM(-1) + , maxM(-1) + , n(-1) + , k(-1) + { + } + + GemmDims(DimType64 minM_, DimType64 maxM_, DimType64 n_, DimType64 k_) + : minM(minM_) + , maxM(maxM_) + , n(n_) + , k(k_) + { + } + + [[nodiscard]] bool isInitialized() const + { + return minM >= 0 && maxM >= 0 && n >= 0 && k >= 0; + } +}; + +// Unique ID of GEMM +// In our case GEMM is uniqly identified by N and K +class GemmIdCore +{ +public: + int n; + int k; + nvinfer1::DataType dtype; + + GemmIdCore(int n_, int k_, nvinfer1::DataType const& dtype_) + : n(n_) + , k(k_) + , dtype(dtype_) + { + } + + GemmIdCore() + : n(-1) + , k(-1) + , dtype(nvinfer1::DataType::kFLOAT) // dtype does not matter here + { + } + + bool operator==(GemmIdCore const& id) const + { + return isEqual(id); + } + + friend std::ostream& operator<<(std::ostream& out, GemmIdCore const& id) + { + out << "(N;K)=(" << id.n << ";" << id.k << "),"; + out << " type=" << static_cast<int>(id.dtype); + return out; + } + +protected: + bool isEqual(GemmIdCore const& id) const + { + return n == id.n && k == id.k && dtype == id.dtype; + } +}; + +// Hash of GemmId +struct GemmIdCoreHash +{ + std::size_t operator()(GemmIdCore const& id) const + { + auto h1 = std::hash<int>{}(id.n); + auto h2 = std::hash<int>{}(id.k); + auto h3 = std::hash<int>{}(static_cast<int>(id.dtype)); + return h1 ^ h2 ^ h3; + } +}; + +class GemmIdCublas : public GemmIdCore +{ +public: + bool transA{}; + bool transB{}; + nvinfer1::DataType outputDtype; + + GemmIdCublas(int n_, int k_, nvinfer1::DataType const& dtype_, bool transA_, bool transB_, + nvinfer1::DataType const& output_dtype_) + : GemmIdCore(n_, k_, dtype_) + , transA(transA_) + , transB(transB_) + , outputDtype(output_dtype_) + { + } + + GemmIdCublas() {} + + bool operator==(GemmIdCublas const& id) const + { + return isEqual(id) && transA == id.transA && transB == id.transB && outputDtype == id.outputDtype; + } + + friend std::ostream& operator<<(std::ostream& out, GemmIdCublas const& id) + { + out << "(N;K)=(" << id.n << ";" << id.k << "),"; + out << " type=" << static_cast<int>(id.dtype); + out << " transA=" << id.transA; + out << " transB=" << id.transB; + out << " outputDtype=" << static_cast<int>(id.outputDtype); + return out; + } +}; + +// Hash of GemmIdCublas +struct GemmIdCublasHash +{ + std::size_t operator()(GemmIdCublas const& id) const + { + auto h1 = std::hash<int>{}(id.n); + auto h2 = std::hash<int>{}(id.k); + auto h3 = std::hash<int>{}(static_cast<int>(id.dtype)); + auto h4 = std::hash<bool>{}(id.transA); + auto h5 = std::hash<bool>{}(id.transB); + auto h6 = std::hash<bool>{}(static_cast<int>(id.outputDtype)); + return h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6; + } +}; + +template <typename Config, typename RunnerPtr, typename GemmIdType, typename GemmIdHashType> +class GemmPluginProfiler +{ +public: + // Map for single GEMM for different Ms (GEMM dimension) to the best config for particular M + using MProfileMap = std::unordered_map<int, std::optional<Config>>; + using MProfileMapPtr = std::shared_ptr<MProfileMap>; + + // requires exclusive ownership to write to *this + using reader_lock = std::unique_lock<std::shared_timed_mutex>; + // requires shared ownership to read from other + using writer_lock = std::shared_lock<std::shared_timed_mutex>; + + // Struct of continuing map if GEMMs to the best profiles for different Ms + struct MNKProfileMap + { + // Mutex guarding map + std::shared_timed_mutex mutex; + // Map from GEMM Id to profile for particular GEMM + std::unordered_map<GemmIdType, MProfileMapPtr, GemmIdHashType> profileMap; + + bool existsMProfileMap(GemmIdType const& id) + { + auto const iter = profileMap.find(id); + return iter != profileMap.end(); + } + + void createMProfileMap(GemmIdType const& id) + { + profileMap[id] = std::make_shared<MProfileMap>(); + } + + MProfileMapPtr getMProfileMap(GemmIdType const& id) + { + auto const iter = profileMap.find(id); + if (iter == profileMap.end()) + { + std::ostringstream msg; + msg << "Cannot find ID (" << id << ") in the profile map. Abort."; + TLLM_THROW(msg.str()); + } + return iter->second; + } + }; + + using MNKProfileMapPtr = std::shared_ptr<MNKProfileMap>; + + GemmPluginProfiler(); + + virtual ~GemmPluginProfiler() = default; + + void serialize(char*& buffer, GemmIdType const& gemmId) const; + + void deserialize(char const*& data, GemmDims& dims, GemmIdType const& gemmId); + size_t getSerializationSize(GemmIdType const& gemmId) const; + + void profileTactics(RunnerPtr const& runner, nvinfer1::DataType const& type, GemmDims const& dims, + GemmIdType const& gemmId, bool hasWeightOnlyCudaKernel = false); + + void setSelectionTactics(MNKProfileMapPtr const& map) + { + mMNKProfileMap = map; + } + + void setTmpWorkspaceSizeInBytes(size_t bytes) + { + mTmpWorkspaceSizeInBytes = bytes; + } + + void setSkip(bool skip) + { + mSkip = mSkip || skip; + } + + std::optional<Config> getBestConfig(int m, GemmIdType const& gemmId) const; + + virtual int getMaxProfileM() const; + +protected: + virtual void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) = 0; + + virtual void computeTmpSize(size_t maxM, size_t n, size_t k) = 0; + + virtual bool checkTactic(int m, int n, int k, Config const& tactic) const + { + return true; + } + + virtual std::vector<Config> getTactics(int m, int n, int k) const = 0; + + virtual void initTmpData(int m, int n, int k, char* workspace, size_t size, cudaStream_t stream); + +private: + void allocateTmpData(); + + void freeTmpData(); + + std::optional<Config> profileTacticsForProblem(int m, int n, int k, std::vector<Config> const& tactics); + + float profileTacticForProblem(int m, int n, int k, Config const& tactic); + + int nextPowerOfTwo(int v) const + { + --v; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return ++v; + } + +protected: + RunnerPtr mRunner{nullptr}; + + nvinfer1::DataType mType{}; + +private: + MNKProfileMapPtr mMNKProfileMap{}; + + size_t mTmpWorkspaceSizeInBytes{0}; + + char* mWorkspaceTmp{nullptr}; + + cudaStream_t mStream; + + GemmDims mDims{}; + + bool mSkip{false}; +}; + +template <typename GemmPluginProfilerType> +class GemmPluginProfilerManager +{ +public: + using MNKProfileMap = typename GemmPluginProfilerType::MNKProfileMap; + using MNKProfileMapPtr = typename GemmPluginProfilerType::MNKProfileMapPtr; + using GemmPluginProfilerPtr = std::shared_ptr<GemmPluginProfilerType>; + + GemmPluginProfilerManager() + { + mMNKProfileMap = std::make_shared<MNKProfileMap>(); + } + + GemmPluginProfilerPtr createGemmPluginProfiler(bool inference, bool skip = false) + { + auto profiler = std::make_shared<GemmPluginProfilerType>(); + profiler->setSkip(skip); + // If the profiler is created during the engine build, + // mMNKProfileMap is shared between different profilers to minimize the time spent on the profiling + // and do not repeat profiling for the GEMMs of the same shape. + if (!inference) + { + profiler->setSelectionTactics(mMNKProfileMap); + } + return profiler; + } + +private: + MNKProfileMapPtr mMNKProfileMap{}; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/common/plugin.cpp b/cpp/tensorrt_llm/plugins/common/plugin.cpp new file mode 100644 index 000000000000..82c8bf93b13c --- /dev/null +++ b/cpp/tensorrt_llm/plugins/common/plugin.cpp @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include "checkMacrosPlugin.h" +#include <cstdint> +#include <functional> + +#ifdef _MSC_VER +#define FN_NAME __FUNCTION__ +#else +#define FN_NAME __func__ +#endif + +PluginFieldParser::PluginFieldParser(int32_t nbFields, nvinfer1::PluginField const* fields) + : mFields{fields} +{ + for (int32_t i = 0; i < nbFields; i++) + { + mMap.emplace(fields[i].name, PluginFieldParser::Record{i}); + } +} + +PluginFieldParser::~PluginFieldParser() +{ + for (auto const& [name, record] : mMap) + { + if (!record.retrieved) + { + std::stringstream ss; + ss << "unused plugin field with name: " << name; + tensorrt_llm::plugins::logError(ss.str().c_str(), __FILE__, FN_NAME, __LINE__); + } + } +} + +template <typename T> +nvinfer1::PluginFieldType toFieldType(); +#define SPECIALIZE_TO_FIELD_TYPE(T, type) \ + template <> \ + nvinfer1::PluginFieldType toFieldType<T>() \ + { \ + return nvinfer1::PluginFieldType::type; \ + } +SPECIALIZE_TO_FIELD_TYPE(half, kFLOAT16) +SPECIALIZE_TO_FIELD_TYPE(float, kFLOAT32) +SPECIALIZE_TO_FIELD_TYPE(double, kFLOAT64) +SPECIALIZE_TO_FIELD_TYPE(int8_t, kINT8) +SPECIALIZE_TO_FIELD_TYPE(int16_t, kINT16) +SPECIALIZE_TO_FIELD_TYPE(int32_t, kINT32) +SPECIALIZE_TO_FIELD_TYPE(char, kCHAR) +SPECIALIZE_TO_FIELD_TYPE(nvinfer1::Dims, kDIMS) +SPECIALIZE_TO_FIELD_TYPE(void, kUNKNOWN) +#undef SPECIALIZE_TO_FIELD_TYPE + +template <typename T> +std::optional<T> PluginFieldParser::getScalar(std::string_view const& name) +{ + auto const iter = mMap.find(name); + if (iter == mMap.end()) + { + return std::nullopt; + } + auto& record = mMap.at(name); + auto const& f = mFields[record.index]; + TLLM_CHECK(toFieldType<T>() == f.type && f.length == 1); + record.retrieved = true; + return std::optional{*static_cast<T const*>(f.data)}; +} + +#define INSTANTIATE_PluginFieldParser_getScalar(T) \ + template std::optional<T> PluginFieldParser::getScalar(std::string_view const&) +INSTANTIATE_PluginFieldParser_getScalar(half); +INSTANTIATE_PluginFieldParser_getScalar(float); +INSTANTIATE_PluginFieldParser_getScalar(double); +INSTANTIATE_PluginFieldParser_getScalar(int8_t); +INSTANTIATE_PluginFieldParser_getScalar(int16_t); +INSTANTIATE_PluginFieldParser_getScalar(int32_t); +INSTANTIATE_PluginFieldParser_getScalar(char); +INSTANTIATE_PluginFieldParser_getScalar(nvinfer1::Dims); +#undef INSTANTIATE_PluginFieldParser_getScalar + +template <typename T> +std::optional<std::set<T>> PluginFieldParser::getSet(std::string_view const& name) +{ + auto const iter = mMap.find(name); + if (iter == mMap.end()) + { + return std::nullopt; + } + auto& record = mMap.at(name); + auto const& f = mFields[record.index]; + TLLM_CHECK(toFieldType<T>() == f.type); + std::set<T> group; + auto const* r = static_cast<T const*>(f.data); + for (int j = 0; j < f.length; ++j) + { + group.insert(*r); + ++r; + } + + record.retrieved = true; + return std::optional{group}; +} + +#define INSTANTIATE_PluginFieldParser_getVector(T) \ + template std::optional<std::set<T>> PluginFieldParser::getSet(std::string_view const&) +INSTANTIATE_PluginFieldParser_getVector(int32_t); +#undef INSTANTIATE_PluginFieldParser_getVector diff --git a/cpp/tensorrt_llm/plugins/common/plugin.h b/cpp/tensorrt_llm/plugins/common/plugin.h new file mode 100644 index 000000000000..a7febe4cc13d --- /dev/null +++ b/cpp/tensorrt_llm/plugins/common/plugin.h @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/common/opUtils.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/plugins/common/checkMacrosPlugin.h" + +#include <NvInferRuntime.h> + +#include <cstring> +#include <map> +#include <memory> +#include <optional> +#include <set> +#include <string> +#include <unordered_map> + +namespace tensorrt_llm::plugins +{ + +using namespace tensorrt_llm::common::op; + +class BasePlugin : public nvinfer1::IPluginV2DynamicExt +{ +public: + void setPluginNamespace(char const* libNamespace) noexcept override + { + mNamespace = libNamespace; + } + + [[nodiscard]] char const* getPluginNamespace() const noexcept override + { + return mNamespace.c_str(); + } + +protected: + std::string mNamespace{api::kDefaultNamespace}; +}; + +class BasePluginV3 : public nvinfer1::IPluginV3, + public nvinfer1::IPluginV3OneCore, + public nvinfer1::IPluginV3OneBuild, + public nvinfer1::IPluginV3OneRuntime +{ +public: + void setPluginNamespace(char const* libNamespace) noexcept + { + mNamespace = libNamespace; + } + + [[nodiscard]] char const* getPluginNamespace() const noexcept override + { + return mNamespace.c_str(); + } + +protected: + std::string mNamespace{api::kDefaultNamespace}; +}; + +class BaseCreator : public nvinfer1::IPluginCreator +{ +public: + void setPluginNamespace(char const* libNamespace) noexcept override + { + mNamespace = libNamespace; + } + + [[nodiscard]] char const* getPluginNamespace() const noexcept override + { + return mNamespace.c_str(); + } + +protected: + std::string mNamespace{api::kDefaultNamespace}; +}; + +class BaseCreatorV3 : public nvinfer1::IPluginCreatorV3One +{ +public: + void setPluginNamespace(char const* libNamespace) noexcept + { + mNamespace = libNamespace; + } + + [[nodiscard]] char const* getPluginNamespace() const noexcept override + { + return mNamespace.c_str(); + } + +protected: + std::string mNamespace{api::kDefaultNamespace}; +}; + +} // namespace tensorrt_llm::plugins + +// Init with O(n) and retrieve with O(1) +class PluginFieldParser +{ +public: + // field array must remain valid when calling getScalar() later. + PluginFieldParser(int32_t nbFields, nvinfer1::PluginField const* fields); + // delete to remind accidental mis-use (copy) which may result in false-alarm warnings about unused fields. + PluginFieldParser(PluginFieldParser const&) = delete; + PluginFieldParser& operator=(PluginFieldParser const&) = delete; + // check if all fields are retrieved and emit warning if some of them are not. + ~PluginFieldParser(); + template <typename T> + std::optional<T> getScalar(std::string_view const& name); + template <typename T> + std::optional<std::set<T>> getSet(std::string_view const& name); + +private: + nvinfer1::PluginField const* mFields; + + struct Record + { + Record(int32_t idx) + : index{idx} + { + } + + int32_t const index; + bool retrieved{false}; + }; + + std::unordered_map<std::string_view, Record> mMap; +}; diff --git a/cpp/tensorrt_llm/plugins/common/pluginUtils.h b/cpp/tensorrt_llm/plugins/common/pluginUtils.h new file mode 100644 index 000000000000..ee3e59d57c6d --- /dev/null +++ b/cpp/tensorrt_llm/plugins/common/pluginUtils.h @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <NvInferRuntime.h> + +#include "tensorrt_llm/common/logger.h" + +namespace tensorrt_llm::plugins::utils +{ +using DimType64 = int64_t; + +inline DimType64 computeMDimension(bool transA, nvinfer1::Dims const& dims) +{ + DimType64 M{1}; + if (transA) + { + for (int i = dims.nbDims - 1; i > 0; --i) + { + M *= dims.d[i]; + } + } + else + { + for (int i = 0; i < dims.nbDims - 1; ++i) + { + M *= dims.d[i]; + } + } + return M; +} + +inline DimType64 computeNDimension(bool transB, nvinfer1::Dims const& dims) +{ + DimType64 N{1}; + if (transB) + { + for (int32_t i = 0; i < dims.nbDims - 1; ++i) + { + N *= dims.d[i]; + } + } + else + { + for (int32_t i = dims.nbDims - 1; i > 0; --i) + { + N *= dims.d[i]; + } + } + return N; +} + +inline std::int32_t logErrorReturn0(char const* variable) +{ + TLLM_LOG_ERROR("Value of %s is out of range for int32_t", variable); + return 0; +} + +#define TLLM_INT32_CAST(value) \ + ((value > 0x7FFFFFFFLL || value < -0x80000000LL) ? tensorrt_llm::plugins::utils::logErrorReturn0(#value) \ + : static_cast<int32_t>(value)) + +} // namespace tensorrt_llm::plugins::utils diff --git a/cpp/tensorrt_llm/plugins/cpSplitPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/cpSplitPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cpSplitPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.cpp b/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.cpp new file mode 100644 index 000000000000..221d5dac2da1 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.cpp @@ -0,0 +1,356 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdio> + +#include "cpSplitPlugin.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::CpSplitPluginCreator; +using tensorrt_llm::plugins::CpSplitPlugin; + +static char const* CPSPLIT_PLUGIN_VERSION{"1"}; +static char const* CPSPLIT_PLUGIN_NAME{"CpSplit"}; +PluginFieldCollection CpSplitPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> CpSplitPluginCreator::mPluginAttributes; + +CpSplitPlugin::CpSplitPlugin() +{ + initFieldsToSerialize(); +} + +CpSplitPlugin::CpSplitPlugin(int cpSize, int cpRank) + : mCpSize(cpSize) + , mCpRank(cpRank) +{ + initFieldsToSerialize(); +} + +void CpSplitPlugin::initFieldsToSerialize() +{ + mDataToSerialize.clear(); + mDataToSerialize.emplace_back(PluginField("cp_size", &mCpSize, PluginFieldType::kINT32, 1)); + mDataToSerialize.emplace_back(PluginField("cp_rank", &mCpRank, PluginFieldType::kINT32, 1)); + mFCToSerialize.nbFields = mDataToSerialize.size(); + mFCToSerialize.fields = mDataToSerialize.data(); +} + +// IPluginV3 methods +nvinfer1::IPluginCapability* CpSplitPlugin::getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept +{ + switch (type) + { + case PluginCapabilityType::kBUILD: return static_cast<IPluginV3OneBuild*>(this); + case PluginCapabilityType::kRUNTIME: return static_cast<IPluginV3OneRuntime*>(this); + case PluginCapabilityType::kCORE: return static_cast<IPluginV3OneCore*>(this); + } + return nullptr; +} + +nvinfer1::IPluginV3* CpSplitPlugin::clone() noexcept +{ + std::unique_ptr<CpSplitPlugin> plugin{std::make_unique<CpSplitPlugin>(*this)}; + plugin->setPluginNamespace(mNamespace.c_str()); + plugin->initFieldsToSerialize(); + return plugin.release(); +} + +// IPluginV3OneCore methods +char const* CpSplitPlugin::getPluginName() const noexcept +{ + return CPSPLIT_PLUGIN_NAME; +} + +char const* CpSplitPlugin::getPluginVersion() const noexcept +{ + return CPSPLIT_PLUGIN_VERSION; +} + +// IPluginV3OneBuild methods +int32_t CpSplitPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept +{ + return 0; +} + +int32_t CpSplitPlugin::getOutputDataTypes( + DataType* outputTypes, int32_t nbOutputs, DataType const* inputTypes, int32_t nbInputs) const noexcept +{ + outputTypes[0] = inputTypes[0]; + outputTypes[1] = DataType::kINT32; + outputTypes[2] = DataType::kINT32; + return 0; +} + +int32_t CpSplitPlugin::getOutputShapes(DimsExprs const* inputs, int32_t nbInputs, DimsExprs const* shapeInputs, + int32_t nbShapeInputs, DimsExprs* outputs, int32_t nbOutputs, IExprBuilder& exprBuilder) noexcept +{ + outputs[0].nbDims = 1; + + auto cpSize = exprBuilder.constant(mCpSize); + auto upper = inputs[0].d[0]; + auto opt = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *upper, *cpSize); + outputs[0].d[0] = exprBuilder.declareSizeTensor(1, *opt, *upper); + + // We must have such an output size tensor (with dim == 0) to notify the shape of output tensor above + outputs[1].nbDims = 0; + outputs[2].nbDims = 1; + outputs[2].d[0] = upper; + return 0; +} + +bool CpSplitPlugin::supportsFormatCombination( + int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept +{ + if (pos == IdxEntry::INPUT_IDS) + { + return ((inOut[pos].desc.type == DataType::kINT32) && (inOut[pos].desc.format == TensorFormat::kLINEAR)); + } + else if (pos == IdxEntry::REQUEST_TYPES || pos == IdxEntry::HOST_CONTEXT_LENGTH) + { + return inOut[pos].desc.type == DataType::kINT32; + } + else + { + return ((inOut[pos].desc.type == DataType::kINT32) && (inOut[pos].desc.format == TensorFormat::kLINEAR)); + } + return false; +} + +int32_t CpSplitPlugin::getNbOutputs() const noexcept +{ + return 3; +} + +size_t CpSplitPlugin::getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept +{ + return 0; +} + +int32_t CpSplitPlugin::getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept +{ + return 0; +} + +int32_t CpSplitPlugin::getNbTactics() noexcept +{ + return 0; +} + +char const* CpSplitPlugin::getTimingCacheID() noexcept +{ + return nullptr; +} + +int32_t CpSplitPlugin::getFormatCombinationLimit() noexcept +{ + return 1; +} + +char const* CpSplitPlugin::getMetadataString() noexcept +{ + return nullptr; +} + +// IPluginV3OneRuntime methods +int32_t CpSplitPlugin::setTactic(int32_t tactic) noexcept +{ + return 0; +} + +int32_t CpSplitPlugin::onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::PluginTensorDesc const* out, int32_t nbOutputs) noexcept +{ + return 0; +} + +int32_t CpSplitPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // @param inputIds [tokenNum] + // @param host_request_types [batchSize]: Tensor = None (On CPU) + // The tensor on the host that indicates if a request is in context or + // generation phase. Its shape is [batch_size]. See Inflight Batching + // in docs/gpt_attention.md, + // @param host_context_lengths [batchSize]: Tensor = None (On CPU) + // A host tensor that contains the lengths of the different inputs + // outputs + // @param outputIds [tokenNum spiltted by cp] + // @param outputLength scalar + // @param joinIdx [tokenNum] + + int64_t tokenNum = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) + { + tokenNum *= inputDesc[0].dims.d[i]; + } + + RequestType const* reqTypes = static_cast<RequestType const*>(inputs[IdxEntry::REQUEST_TYPES]); + int32_t const* hContextLengths = static_cast<int32_t const*>(inputs[IdxEntry::HOST_CONTEXT_LENGTH]); + int const* inputIds = reinterpret_cast<int const*>(inputs[IdxEntry::INPUT_IDS]); + int* outputIds = reinterpret_cast<int*>(outputs[0]); + int32_t* outputLength = reinterpret_cast<int32_t*>(outputs[1]); + int32_t* outputJoinIdx = reinterpret_cast<int32_t*>(outputs[2]); + + int32_t const nbSeq = inputDesc[IdxEntry::HOST_CONTEXT_LENGTH].dims.d[0]; + + int32_t* hInputs = new int[inputDesc[IdxEntry::INPUT_IDS].dims.d[0]]; + int32_t* hOutputs = new int[inputDesc[IdxEntry::INPUT_IDS].dims.d[0]]; + int32_t* hOutputJoinIdx = new int[inputDesc[IdxEntry::INPUT_IDS].dims.d[0]]; + cudaMemcpyAsync( + hInputs, inputIds, sizeof(int32_t) * inputDesc[IdxEntry::INPUT_IDS].dims.d[0], cudaMemcpyDeviceToHost, stream); + sync_check_cuda_error(stream); + + int32_t inputIdx = 0; + int32_t outputIdx = 0; + for (int32_t seqIdx = 0; seqIdx < nbSeq; ++seqIdx) + { + if (reqTypes[seqIdx] == RequestType::kCONTEXT) + { + auto const& ctxLength = hContextLengths[seqIdx]; + int32_t partialAverageLength = (ctxLength + mCpSize - 1) / mCpSize; + int32_t partialLength + = mCpRank == mCpSize - 1 ? ctxLength - partialAverageLength * (mCpSize - 1) : partialAverageLength; + for (int i = 0; i < partialLength; i++) + { + hOutputs[outputIdx + i] = hInputs[inputIdx + partialAverageLength * mCpRank + i]; + } + inputIdx += ctxLength; + outputIdx += partialAverageLength; + } + else if (reqTypes[seqIdx] == RequestType::kGENERATION) + { + auto const& genLength = nbSeq - seqIdx; + int32_t partialAverageLength = (genLength + mCpSize - 1) / mCpSize; + int32_t partialLength + = mCpRank == mCpSize - 1 ? genLength - partialAverageLength * (mCpSize - 1) : partialAverageLength; + for (int i = 0; i < partialLength; i++) + { + hOutputs[outputIdx + i] = hInputs[inputIdx + partialAverageLength * mCpRank + i]; + } + outputIdx += partialAverageLength; + break; + } + } + int32_t hOutputLength = outputIdx; + inputIdx = 0; + outputIdx = 0; + for (int32_t seqIdx = 0; seqIdx < nbSeq; ++seqIdx) + { + if (reqTypes[seqIdx] == RequestType::kCONTEXT) + { + auto const& ctxLength = hContextLengths[seqIdx]; + int32_t partialAverageLength = (ctxLength + mCpSize - 1) / mCpSize; + for (int32_t idx = 0; idx < ctxLength; ++idx) + { + hOutputJoinIdx[inputIdx + idx] + = idx % partialAverageLength + idx / partialAverageLength * hOutputLength + outputIdx; + } + inputIdx += ctxLength; + outputIdx += partialAverageLength; + } + else if (reqTypes[seqIdx] == RequestType::kGENERATION) + { + auto const& genLength = nbSeq - seqIdx; + int32_t partialAverageLength = (genLength + mCpSize - 1) / mCpSize; + for (int32_t idx = 0; idx < genLength; ++idx) + { + hOutputJoinIdx[inputIdx + idx] + = idx % partialAverageLength + idx / partialAverageLength * hOutputLength + outputIdx; + } + break; + } + } + cudaMemcpyAsync(outputIds, hOutputs, sizeof(int32_t) * hOutputLength, cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(outputLength, &hOutputLength, sizeof(int32_t), cudaMemcpyHostToDevice, stream); + cudaMemcpyAsync(outputJoinIdx, hOutputJoinIdx, sizeof(int32_t) * tokenNum, cudaMemcpyHostToDevice, stream); + sync_check_cuda_error(stream); + return 0; +} + +nvinfer1::IPluginV3* CpSplitPlugin::attachToContext(nvinfer1::IPluginResourceContext* context) noexcept +{ + return clone(); +} + +nvinfer1::PluginFieldCollection const* CpSplitPlugin::getFieldsToSerialize() noexcept +{ + return &mFCToSerialize; +} + +CpSplitPluginCreator::CpSplitPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("cp_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("cp_rank", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* CpSplitPluginCreator::getPluginName() const noexcept +{ + return CPSPLIT_PLUGIN_NAME; +} + +char const* CpSplitPluginCreator::getPluginVersion() const noexcept +{ + return CPSPLIT_PLUGIN_VERSION; +} + +nvinfer1::PluginFieldCollection const* CpSplitPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +nvinfer1::IPluginV3* CpSplitPluginCreator::createPlugin( + char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept +{ + PluginField const* fields = fc->fields; + int cp_size{}; + int cp_rank{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "cp_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + cp_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "cp_rank")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + cp_rank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + } + try + { + auto* obj = new CpSplitPlugin(cp_size, cp_rank); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h b/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h new file mode 100644 index 000000000000..1dc8c15b355a --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class CpSplitPlugin : public BasePluginV3 +{ +public: + CpSplitPlugin(); + CpSplitPlugin(int cpSize, int cpRank); + CpSplitPlugin(CpSplitPlugin const& p) = default; + void initFieldsToSerialize(); + + // IPluginV3 methods + nvinfer1::IPluginCapability* getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept override; + nvinfer1::IPluginV3* clone() noexcept override; + + // IPluginV3OneCore methods + char const* getPluginName() const noexcept override; + char const* getPluginVersion() const noexcept override; + + // IPluginV3OneBuild methods + int32_t configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept override; // nochange + int32_t getOutputDataTypes(nvinfer1::DataType* outputTypes, int32_t nbOutputs, nvinfer1::DataType const* inputTypes, + int32_t nbInputs) const noexcept override; // fixed + int32_t getOutputShapes(nvinfer1::DimsExprs const* inputs, int32_t nbInputs, nvinfer1::DimsExprs const* shapeInputs, + int32_t nbShapeInputs, nvinfer1::DimsExprs* outputs, int32_t nbOutputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; // fixed + bool supportsFormatCombination(int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, + int32_t nbOutputs) noexcept override; // fixed + int32_t getNbOutputs() const noexcept override; // fixed + size_t getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept override; // fixed + int32_t getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept override; + int32_t getNbTactics() noexcept override; + char const* getTimingCacheID() noexcept override; + int32_t getFormatCombinationLimit() noexcept override; + char const* getMetadataString() noexcept override; + + // IPluginV3OneRuntime methods + int32_t setTactic(int32_t tactic) noexcept override; + int32_t onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, nvinfer1::PluginTensorDesc const* out, + int32_t nbOutputs) noexcept override; + int32_t enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept override; // fixed + nvinfer1::IPluginV3* attachToContext(nvinfer1::IPluginResourceContext* context) noexcept override; + nvinfer1::PluginFieldCollection const* getFieldsToSerialize() noexcept override; + +private: + int mCpSize; + int mCpRank; + std::vector<nvinfer1::PluginField> mDataToSerialize; + nvinfer1::PluginFieldCollection mFCToSerialize; + + enum IdxEntry + { + INPUT_IDS, + REQUEST_TYPES, + HOST_CONTEXT_LENGTH, + }; + + enum class RequestType : int32_t + { + kCONTEXT = 0, + kGENERATION = 1 + }; +}; + +class CpSplitPluginCreator : public BaseCreatorV3 +{ +public: + CpSplitPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV3* createPlugin( + char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.cpp b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.cpp new file mode 100644 index 000000000000..802e828c9250 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.cpp @@ -0,0 +1,293 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cudaStreamPlugin.h" +#include "tensorrt_llm/runtime/iBuffer.h" + +#include <cassert> + +using namespace nvinfer1; +using tensorrt_llm::plugins::CudaStreamPluginCreator; +using tensorrt_llm::plugins::CudaStreamPlugin; + +static char const* CUDA_STREAM_PLUGIN_VERSION{"1"}; +static char const* CUDA_STREAM_PLUGIN_NAME{"CudaStream"}; +PluginFieldCollection CudaStreamPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> CudaStreamPluginCreator::mPluginAttributes; + +CudaStreamPlugin::CudaStreamPlugin(int sideStreamId, int nbInputs, nvinfer1::DataType type) + : mSideStreamId(sideStreamId) + , mNbInputs(nbInputs) + , mType(type) +{ + init(); +} + +CudaStreamPlugin::CudaStreamPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mSideStreamId); + read(d, mNbInputs); + read(d, mType); + + init(); + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +CudaStreamPlugin::CudaStreamPlugin(CudaStreamPlugin const& other) + : mSideStreamId(other.mSideStreamId) + , mNbInputs(other.mNbInputs) + , mType(other.mType) +{ + init(); +} + +void CudaStreamPlugin::init() +{ + mSideStreamPtr = nullptr; +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* CudaStreamPlugin::clone() const noexcept +{ + auto* plugin = new CudaStreamPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs CudaStreamPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + assert(outputIndex == 0); + return inputs[outputIndex]; +} + +bool CudaStreamPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + TLLM_CHECK_WITH_INFO(nbInputs == mNbInputs, "CudaStreamPlugin only accepts mNbInputs inputs"); + TLLM_CHECK_WITH_INFO(nbOutputs == 1, "CudaStreamPlugin only accepts 1 output"); + + auto const& desc = inOut[pos]; + if (desc.format != TensorFormat::kLINEAR) + { + return false; + } + + if (pos > 0 && pos < nbInputs) + { + return true; + } + return desc.type == mType; +} + +void CudaStreamPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t CudaStreamPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int CudaStreamPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + if (!mSideStreamPtr) + { + auto const resource_name = nvinfer1::pluginInternal::SideStream::getResourceKey(mSideStreamId); + nvinfer1::pluginInternal::SideStream side_stream{}; + mSideStreamPtr = reinterpret_cast<nvinfer1::pluginInternal::SideStream*>( + getPluginRegistry()->acquirePluginResource(resource_name.c_str(), &side_stream)); + } + mSideStreamPtr->waitSideStreamOnMainStream(stream); + size_t count = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) + { + count *= inputDesc[0].dims.d[i]; + } + count *= tensorrt_llm::runtime::BufferDataType(inputDesc[0].type).getSize(); + TLLM_CUDA_CHECK(cudaMemcpyAsync(outputs[0], inputs[0], count, cudaMemcpyDeviceToDevice, stream)); + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType CudaStreamPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* CudaStreamPlugin::getPluginType() const noexcept +{ + return CUDA_STREAM_PLUGIN_NAME; +} + +char const* CudaStreamPlugin::getPluginVersion() const noexcept +{ + return CUDA_STREAM_PLUGIN_VERSION; +} + +int CudaStreamPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int CudaStreamPlugin::initialize() noexcept +{ + return 0; +} + +void CudaStreamPlugin::terminate() noexcept +{ + if (mSideStreamPtr) + { + auto const resource_name = nvinfer1::pluginInternal::SideStream::getResourceKey(mSideStreamId); + getPluginRegistry()->releasePluginResource(resource_name.c_str()); + mSideStreamPtr = nullptr; + } +} + +size_t CudaStreamPlugin::getSerializationSize() const noexcept +{ + return sizeof(mSideStreamId) + sizeof(mNbInputs) + sizeof(mType); +} + +void CudaStreamPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mSideStreamId); + write(d, mNbInputs); + write(d, mType); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void CudaStreamPlugin::destroy() noexcept +{ + delete this; +} + +/////////////// + +CudaStreamPluginCreator::CudaStreamPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("side_stream_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("num_inputs", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* CudaStreamPluginCreator::getPluginName() const noexcept +{ + return CUDA_STREAM_PLUGIN_NAME; +} + +char const* CudaStreamPluginCreator::getPluginVersion() const noexcept +{ + return CUDA_STREAM_PLUGIN_VERSION; +} + +PluginFieldCollection const* CudaStreamPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* CudaStreamPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int sideStreamId; + int nbInputs; + int type; + + // Read configurations from each fields + struct MapPair + { + char const* key; + int& field; + bool optional = false; + bool set = false; + }; + + std::array input_map{ + MapPair{"side_stream_id", std::ref(sideStreamId)}, + MapPair{"num_inputs", std::ref(nbInputs)}, + MapPair{"type_id", std::ref(type)}, + }; + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + for (auto& item : input_map) + { + if (!strcmp(item.key, attrName)) + { + TLLM_CHECK(fields[i].type == nvinfer1::PluginFieldType::kINT32); + TLLM_CHECK_WITH_INFO(!item.set, "Parameter %s was set twice", item.key); + item.field = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + item.set = true; + } + } + } + + for (auto& item : input_map) + { + TLLM_CHECK_WITH_INFO(item.set || item.optional, "Parameter %s is required but not set", item.key); + } + + try + { + auto* obj = new CudaStreamPlugin(sideStreamId, nbInputs, static_cast<nvinfer1::DataType>(type)); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* CudaStreamPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call CudaStreamPlugin::destroy() + try + { + auto* obj = new CudaStreamPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h new file mode 100644 index 000000000000..5b78c3b873bb --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h @@ -0,0 +1,265 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "NvInferPlugin.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/runtime/cudaMemPool.h" +#include "tensorrt_llm/runtime/utils/debugUtils.h" +#include <memory> +#include <string> +#include <vector> + +namespace nvinfer1 +{ +namespace pluginInternal +{ +class SideWorkspace +{ +public: + SideWorkspace(cudaStream_t stream) + : mWorkspaceSize{0} + , mWorkspacePtr{nullptr} + , mStream{stream} + { + } + + ~SideWorkspace() + { + if (mWorkspacePtr) + { + TLLM_CUDA_CHECK(cudaFreeAsync(mWorkspacePtr, mStream)); + } + } + + void* get(size_t workspaceSize) + { + if (mWorkspacePtr && mWorkspaceSize < workspaceSize) + { + TLLM_CUDA_CHECK(cudaFreeAsync(mWorkspacePtr, mStream)); + mWorkspacePtr = nullptr; + } + if (!mWorkspacePtr) + { + mWorkspaceSize = workspaceSize; + auto pool_ptr + = tensorrt_llm::runtime::CudaMemPool::getPrimaryPoolForDevice(tensorrt_llm::common::getDevice()); + TLLM_CUDA_CHECK(cudaMallocFromPoolAsync(&mWorkspacePtr, mWorkspaceSize, pool_ptr->getPool(), mStream)); + } + return mWorkspacePtr; + } + +private: + size_t mWorkspaceSize; + void* mWorkspacePtr; + cudaStream_t mStream; +}; + +class SideStream : public IPluginResource +{ +public: + SideStream(bool init = false) + : mStream{} + , mMainEvent{} + , mSideEvent{} + , mWorkspace{} + , mInit{init} + { + // The object passed to acquirePluginResource should use the default value init=false + if (init) + { + TLLM_CUDA_CHECK(cudaStreamCreate(&mStream)); + TLLM_CUDA_CHECK(cudaEventCreateWithFlags(&mMainEvent, cudaEventDisableTiming)); + TLLM_CUDA_CHECK(cudaEventCreateWithFlags(&mSideEvent, cudaEventDisableTiming)); + mWorkspace = std::make_shared<SideWorkspace>(mStream); + } + } + + void free() + { + if (mInit) + { + mWorkspace = nullptr; + TLLM_CUDA_CHECK(cudaStreamSynchronize(mStream)); + TLLM_CUDA_CHECK(cudaStreamDestroy(mStream)); + TLLM_CUDA_CHECK(cudaEventDestroy(mMainEvent)); + TLLM_CUDA_CHECK(cudaEventDestroy(mSideEvent)); + mInit = false; + } + } + + int32_t release() noexcept override + { + try + { + free(); + } + catch (std::exception const& e) + { + return -1; + } + return 0; + } + + IPluginResource* clone() noexcept override + { + // An object is cloned only when calling acquirePluginResource for the first time for each key + std::unique_ptr<SideStream> cloned{}; + try + { + if (!mInit) + { + cloned = std::make_unique<SideStream>(/* init */ true); + } + else + { + return nullptr; + } + } + catch (std::exception const& e) + { + return nullptr; + } + return cloned.release(); + } + + ~SideStream() override + { + free(); + } + + void* getWorkspacePtr(size_t workspaceSize) + { + return mWorkspace->get(workspaceSize); + } + + cudaStream_t getStream() const + { + return mStream; + } + + void waitMainStreamOnSideStream(cudaStream_t const stream) const + { + TLLM_CUDA_CHECK(cudaEventRecord(mMainEvent, stream)); + TLLM_CUDA_CHECK(cudaStreamWaitEvent(mStream, mMainEvent)); + } + + void waitSideStreamOnMainStream(cudaStream_t const stream) const + { + TLLM_CUDA_CHECK(cudaEventRecord(mSideEvent, mStream)); + TLLM_CUDA_CHECK(cudaStreamWaitEvent(stream, mSideEvent)); + } + + void stallMainStream(char const* name, cudaStream_t const stream, std::optional<int> delay = std::nullopt) const + { + tensorrt_llm::runtime::utils::stallStream(name, stream, delay); + } + + void stallSideStream(char const* name, std::optional<int> delay = std::nullopt) const + { + tensorrt_llm::runtime::utils::stallStream(name, mStream, delay); + } + + static std::string getResourceKey(int const stream_id) + { + return "side_stream_" + std::to_string(stream_id); + } + +private: + cudaStream_t mStream; + cudaEvent_t mMainEvent; + cudaEvent_t mSideEvent; + std::shared_ptr<SideWorkspace> mWorkspace; + bool mInit; +}; + +} // namespace pluginInternal +} // namespace nvinfer1 + +namespace tensorrt_llm::plugins +{ + +class CudaStreamPlugin : public BasePlugin +{ +public: + CudaStreamPlugin(int sideStreamId, int nbInputs, nvinfer1::DataType type); + + CudaStreamPlugin(void const* data, size_t length); + + CudaStreamPlugin(CudaStreamPlugin const&); + + void init(); + + ~CudaStreamPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + const std::string mLayerName; + int mSideStreamId; + int mNbInputs; + nvinfer1::DataType mType; + nvinfer1::pluginInternal::SideStream* mSideStreamPtr; +}; + +class CudaStreamPluginCreator : public BaseCreator +{ +public: + CudaStreamPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/CMakeLists.txt new file mode 100644 index 000000000000..ea25de075f34 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/CMakeLists.txt @@ -0,0 +1,22 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.cpp b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.cpp new file mode 100644 index 000000000000..927a42ebac2f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.cpp @@ -0,0 +1,299 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cumsumLastDimPlugin.h" +#include "tensorrt_llm/common/assert.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::CumsumLastDimPluginCreator; +using tensorrt_llm::plugins::CumsumLastDimPlugin; + +static char const* CUMSUM_LAST_DIM_PLUGIN_VERSION{"1"}; +static char const* CUMSUM_LAST_DIM_PLUGIN_NAME{"CumsumLastDim"}; +PluginFieldCollection CumsumLastDimPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> CumsumLastDimPluginCreator::mPluginAttributes; + +static constexpr SizeType32 LENGTH_LIMIT_FOR_BLOCKSCAN = 4096; + +CumsumLastDimPlugin::CumsumLastDimPlugin(SizeType32 inputLength, nvinfer1::DataType type, size_t temp_storage_bytes) + : mInputLength(inputLength) + , mTempStorageBytes(temp_storage_bytes) + , mType(type) +{ + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF) + || (mType == DataType::kINT32), + "Only support int, float, half, and bfloat16."); + if (mTempStorageBytes == 0) + { + mTempStorageBytes = getWorkspaceSizeNeeded(inputLength, type); + } +} + +// Parameterized constructor +CumsumLastDimPlugin::CumsumLastDimPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mInputLength); + read(d, mTempStorageBytes); + read(d, mType); + TLLM_CHECK(d == a + length); + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF) + || (mType == DataType::kINT32), + "Only support int, float, half, and bfloat16."); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* CumsumLastDimPlugin::clone() const noexcept +{ + auto* plugin = new CumsumLastDimPlugin(mInputLength, mType, mTempStorageBytes); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +// Outputs +// output_tensor: [batch_size, inputLength] +nvinfer1::DimsExprs CumsumLastDimPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + TLLM_CHECK_WITH_INFO(outputIndex == 0, "Only one output."); + return inputs[getInputTensorIdx()]; +} + +bool CumsumLastDimPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); +} + +void CumsumLastDimPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t CumsumLastDimPlugin::getWorkspaceSizeNeeded(SizeType32 inputLength, nvinfer1::DataType type) +{ + size_t tempStorageBytes{0}; + if (inputLength < LENGTH_LIMIT_FOR_BLOCKSCAN) // last dim unknown or small, use BlockScan + { + tempStorageBytes = 0; + } + else if (type == DataType::kINT32) + { + tempStorageBytes = invokeComputeCumsumLastDimWorkspaceSize<int>(inputLength); + } + else if (type == DataType::kHALF) + { + tempStorageBytes = invokeComputeCumsumLastDimWorkspaceSize<half>(inputLength); + } + else if (type == DataType::kFLOAT) + { + tempStorageBytes = invokeComputeCumsumLastDimWorkspaceSize<float>(inputLength); + } +#ifdef ENABLE_BF16 + else if (type == DataType::kBF16) + { + tempStorageBytes = invokeComputeCumsumLastDimWorkspaceSize<__nv_bfloat16>(inputLength); + } +#endif + return tempStorageBytes; +} + +size_t CumsumLastDimPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return mTempStorageBytes; +} + +template <typename T> +int CumsumLastDimPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) +{ + // inputs + // 0. input_tensor [batch_size, inputLength] + // outputs + // 0. output_tensor [batch_size, inputLength] + auto const batchSize = inputDesc[getInputTensorIdx()].dims.d[0]; + auto const inputLength = inputDesc[getInputTensorIdx()].dims.d[1]; + /* + Two cases where we should use BlockScan: + 1. inputLength is small + 2. batchSize is large (since DeviceScan causes kernel launch per row) + */ + void* wp = inputLength < LENGTH_LIMIT_FOR_BLOCKSCAN || batchSize > 2 ? nullptr : workspace; + invokeCumsumLastDim<T>( + batchSize, inputLength, inputs[getInputTensorIdx()], outputs[0], wp, mTempStorageBytes, stream); + + sync_check_cuda_error(stream); + return 0; +} + +int CumsumLastDimPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + if (mType == DataType::kINT32) + { + return enqueueImpl<int>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kHALF) + { + return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kFLOAT) + { + return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16) + { + return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType CumsumLastDimPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK_WITH_INFO(index == 0, "Only one output."); + return inputTypes[getInputTensorIdx()]; +} + +// IPluginV2 Methods + +char const* CumsumLastDimPlugin::getPluginType() const noexcept +{ + return CUMSUM_LAST_DIM_PLUGIN_NAME; +} + +char const* CumsumLastDimPlugin::getPluginVersion() const noexcept +{ + return CUMSUM_LAST_DIM_PLUGIN_VERSION; +} + +int CumsumLastDimPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int CumsumLastDimPlugin::initialize() noexcept +{ + return 0; +} + +void CumsumLastDimPlugin::terminate() noexcept {} + +size_t CumsumLastDimPlugin::getSerializationSize() const noexcept +{ + return sizeof(mInputLength) + sizeof(mTempStorageBytes) + sizeof(mType); +} + +void CumsumLastDimPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mInputLength); + write(d, mTempStorageBytes); + write(d, mType); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void CumsumLastDimPlugin::destroy() noexcept +{ + delete this; +} + +/////////////// + +CumsumLastDimPluginCreator::CumsumLastDimPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("input_length", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* CumsumLastDimPluginCreator::getPluginName() const noexcept +{ + return CUMSUM_LAST_DIM_PLUGIN_NAME; +} + +char const* CumsumLastDimPluginCreator::getPluginVersion() const noexcept +{ + return CUMSUM_LAST_DIM_PLUGIN_VERSION; +} + +PluginFieldCollection const* CumsumLastDimPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* CumsumLastDimPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int inputLength{}; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "input_length")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + inputLength = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + try + { + auto* obj = new CumsumLastDimPlugin(inputLength, type); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* CumsumLastDimPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call CumsumLastDimPlugin::destroy() + try + { + auto* obj = new CumsumLastDimPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.h b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.h new file mode 100644 index 000000000000..3cbf4e2356dd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/cumsumLastDimPlugin/cumsumLastDimPlugin.h @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRT_CUMSUM_LAST_DIM_PLUGIN_H +#define TRT_CUMSUM_LAST_DIM_PLUGIN_H + +#include "tensorrt_llm/kernels/cumsumLastDim.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> + +namespace tensorrt_llm::plugins +{ +class CumsumLastDimPlugin : public BasePlugin +{ +public: + using SizeType32 = tensorrt_llm::kernels::SizeType32; + + CumsumLastDimPlugin(SizeType32 inputLength, nvinfer1::DataType type, size_t tempStorageBytes = 0); + CumsumLastDimPlugin(void const* data, size_t length); + ~CumsumLastDimPlugin() override = default; + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + template <typename T> + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + size_t getWorkspaceSizeNeeded(SizeType32 inputLength, nvinfer1::DataType type); + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + using IndexType = std::int32_t; + + IndexType getInputTensorIdx() const + { + return 0; + }; + +private: + SizeType32 mInputLength; + size_t mTempStorageBytes; + nvinfer1::DataType mType; +}; + +class CumsumLastDimPluginCreator : public BaseCreator +{ +public: + CumsumLastDimPluginCreator(); + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins + +#endif diff --git a/cpp/tensorrt_llm/plugins/doraPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/doraPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/doraPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.cpp b/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.cpp new file mode 100644 index 000000000000..7c980f079ceb --- /dev/null +++ b/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.cpp @@ -0,0 +1,392 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "doraPlugin.h" + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/runtime/iBuffer.h" + +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::DoraPlugin; +using tensorrt_llm::plugins::DoraPluginCreator; + +static char const* DORA_PLUGIN_VERSION{"1"}; +static char const* DORA_PLUGIN_NAME{"Dora"}; +PluginFieldCollection DoraPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> DoraPluginCreator::mPluginAttributes; + +DoraPlugin::DoraPlugin(std::vector<int32_t> const& outHiddenSizes, nvinfer1::DataType type, bool removeInputPadding) + : mType(type) + , mRemoveInputPadding(removeInputPadding) + , mDoraImpl(outHiddenSizes, type) +{ + mOutHiddenSizes.resize(outHiddenSizes.size()); + mOutHiddenSizes.assign(outHiddenSizes.cbegin(), outHiddenSizes.cend()); + init(); +} + +void DoraPlugin::init() +{ + // initialize data to serialize + mDataToSerialize.clear(); + mDataToSerialize.emplace_back( + "out_hidden_sizes", mOutHiddenSizes.data(), PluginFieldType::kINT32, mOutHiddenSizes.size()); + mDataToSerialize.emplace_back("type", &mType, PluginFieldType::kINT32, 1); + mDataToSerialize.emplace_back("remove_input_padding", &mRemoveInputPadding, PluginFieldType::kINT8, 1); + mFieldsToSerialize.nbFields = static_cast<int32_t>(mDataToSerialize.size()); + mFieldsToSerialize.fields = mDataToSerialize.data(); +} + +// IPluginV3 methods +nvinfer1::IPluginCapability* DoraPlugin::getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept +{ + switch (type) + { + case PluginCapabilityType::kBUILD: return static_cast<IPluginV3OneBuild*>(this); + case PluginCapabilityType::kRUNTIME: return static_cast<IPluginV3OneRuntime*>(this); + case PluginCapabilityType::kCORE: return static_cast<IPluginV3OneCore*>(this); + } + return nullptr; +} + +nvinfer1::IPluginV3* DoraPlugin::clone() noexcept +{ + std::unique_ptr<DoraPlugin> plugin{std::make_unique<DoraPlugin>(mOutHiddenSizes, mType, mRemoveInputPadding)}; + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin.release(); +} + +// IPluginV3OneCore methods +char const* DoraPlugin::getPluginName() const noexcept +{ + return DORA_PLUGIN_NAME; +} + +char const* DoraPlugin::getPluginVersion() const noexcept +{ + return DORA_PLUGIN_VERSION; +} + +// IPluginV3OneBuild methods +int32_t DoraPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept +{ + return 0; +} + +int32_t DoraPlugin::getOutputDataTypes( + DataType* outputTypes, int32_t nbOutputs, DataType const* inputTypes, int32_t nbInputs) const noexcept +{ + try + { + TLLM_CHECK(nbOutputs == 1); + TLLM_CHECK(nbInputs == 2 + static_cast<int32_t>(mOutHiddenSizes.size()) + (mRemoveInputPadding ? 1 : 0)); + TLLM_CHECK(inputTypes[IdxEntry::kINPUT_TENSOR] == mType); + // output has the same dtype as the input, the plugin just applies scaling + outputTypes[0] = inputTypes[IdxEntry::kINPUT_TENSOR]; + } + catch (std::exception const& e) + { + caughtError(e); + } + return 0; +} + +int32_t DoraPlugin::getOutputShapes(DimsExprs const* inputs, int32_t nbInputs, DimsExprs const* shapeInputs, + int32_t nbShapeInputs, DimsExprs* outputs, int32_t nbOutputs, IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbOutputs == 1); + TLLM_CHECK(nbShapeInputs == 0); + TLLM_CHECK(nbInputs == 2 + static_cast<int32_t>(mOutHiddenSizes.size()) + (mRemoveInputPadding ? 1 : 0)); + + auto const inputTensorDims = inputs[IdxEntry::kINPUT_TENSOR]; + TLLM_CHECK(inputTensorDims.nbDims == (mRemoveInputPadding ? 2 : 3)); + + auto const lastDim = inputTensorDims.d[inputTensorDims.nbDims - 1]; + TLLM_CHECK(lastDim->isConstant()); + TLLM_CHECK(lastDim->getConstantValue() == std::accumulate(mOutHiddenSizes.cbegin(), mOutHiddenSizes.cend(), 0)); + + outputs[0].nbDims = inputTensorDims.nbDims; + for (auto dim = 0; dim < inputTensorDims.nbDims; ++dim) + { + outputs[0].d[dim] = inputTensorDims.d[dim]; + } + } + catch (std::exception const& e) + { + caughtError(e); + } + return 0; +} + +bool DoraPlugin::supportsFormatCombination( + int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + auto const numModules = static_cast<int32_t>(mOutHiddenSizes.size()); + if (nbInputs != 2 + numModules + (mRemoveInputPadding ? 1 : 0)) + { + return false; + } + + bool const isInput = pos < nbInputs; + if (pos == IdxEntry::kHOST_REQUEST_TYPES) + { + return (inOut[pos].desc.type == nvinfer1::DataType::kINT32); + } + // optional host_context_lens after lora pointers + else if (pos == IdxEntry::kLORA_WEIGHTS_PTRS_START + numModules and isInput) + { + return (inOut[pos].desc.type == nvinfer1::DataType::kINT32 and mRemoveInputPadding); + } + // lora weight pointers + else if (pos >= IdxEntry::kLORA_WEIGHTS_PTRS_START and pos < IdxEntry::kLORA_WEIGHTS_PTRS_START + numModules) + { + return (inOut[pos].desc.type == nvinfer1::DataType::kINT64); + } + else if (pos != 0 and isInput) + { + TLLM_LOG_WARNING("%s: got an unexpected input at position %d", __PRETTY_FUNCTION__, pos); + return false; + } + + return (inOut[pos].desc.type == mType) and (inOut[pos].desc.format == TensorFormat::kLINEAR); +} + +int32_t DoraPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +size_t DoraPlugin::getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept +{ + auto const inputTensorMax = inputs[IdxEntry::kINPUT_TENSOR].max; + auto const maxNumTokens = mRemoveInputPadding ? inputTensorMax.d[0] : inputTensorMax.d[0] * inputTensorMax.d[1]; + auto const size = mDoraImpl.getWorkspaceSize(maxNumTokens); + return size; +} + +int32_t DoraPlugin::getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept +{ + return 0; +} + +int32_t DoraPlugin::getNbTactics() noexcept +{ + return 0; +} + +char const* DoraPlugin::getTimingCacheID() noexcept +{ + return nullptr; +} + +int32_t DoraPlugin::getFormatCombinationLimit() noexcept +{ + return 1; +} + +char const* DoraPlugin::getMetadataString() noexcept +{ + return nullptr; +} + +// IPluginV3OneRuntime methods +int32_t DoraPlugin::setTactic(int32_t tactic) noexcept +{ + return 0; +} + +int32_t DoraPlugin::onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::PluginTensorDesc const* out, int32_t nbOutputs) noexcept +{ + return 0; +} + +int32_t DoraPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + + auto const numModules = static_cast<int32_t>(mOutHiddenSizes.size()); + auto const numReqs = inputDesc[IdxEntry::kHOST_REQUEST_TYPES].dims.d[0]; + + auto const inputTensorDesc = inputDesc[IdxEntry::kINPUT_TENSOR]; + auto const numTokens + = mRemoveInputPadding ? inputTensorDesc.dims.d[0] : inputTensorDesc.dims.d[0] * inputTensorDesc.dims.d[1]; + auto const seqLen = mRemoveInputPadding ? 0 : inputTensorDesc.dims.d[1]; + + void const* inputTensor = inputs[IdxEntry::kINPUT_TENSOR]; + auto const* hostRequestTypes = static_cast<int32_t const*>(inputs[IdxEntry::kHOST_REQUEST_TYPES]); + void const* const* loraWeightsPtrs = &inputs[IdxEntry::kLORA_WEIGHTS_PTRS_START]; + + int32_t const* hostContextLengths = mRemoveInputPadding + ? static_cast<int32_t const*>(inputs[IdxEntry::kLORA_WEIGHTS_PTRS_START + numModules]) + : nullptr; + + mExpandDoraWeightPtrs.clear(); + mExpandDoraWeightPtrs.reserve(numModules * numTokens); + + bool hasAnyDora = false; + + for (auto moduleIdx = 0; moduleIdx < numModules; moduleIdx++) + { + auto const loraWeightModulePtrs = static_cast<int64_t const*>(loraWeightsPtrs[moduleIdx]); + + int idx = 0; + for (int reqId = 0; reqId < numReqs; reqId++) + { + // loraWeightModulePtrs has 3 pointers for each module: A,B, and an optional DoRA magnitude + // the current DoRA plugin does not apply LoRA, so A and B are ignored. + RequestType const reqType = static_cast<RequestType const>(hostRequestTypes[reqId]); + auto const* modulePtr = reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3 + 2]); + hasAnyDora = hasAnyDora or modulePtr != nullptr; + + if (reqType == RequestType::kGENERATION) + { + mExpandDoraWeightPtrs.push_back(modulePtr); + idx += 1; + } + else + { + int contextLen = (mRemoveInputPadding ? hostContextLengths[reqId] : seqLen); + + for (int contextId = 0; contextId < contextLen; contextId++) + { + mExpandDoraWeightPtrs.push_back(modulePtr); + idx += 1; + } + } + } + if (idx != numTokens) + { + TLLM_LOG_ERROR("LoraParams and input dims don't match, lora tokens %d input tokens %d", idx, numTokens); + return -1; + } + } + + if (hasAnyDora) + { + mDoraImpl.run(numTokens, inputTensor, mExpandDoraWeightPtrs.data(), outputs, workspace, stream); + } + else + { + // skip dora scaling if all requests are pure-lora + auto const inputRank = inputTensorDesc.dims.nbDims; + auto const numel + = std::accumulate(inputTensorDesc.dims.d, inputTensorDesc.dims.d + inputRank, 1, std::multiplies()); + auto const elemSize = tensorrt_llm::common::getDTypeSize(mType); + tensorrt_llm::common::cudaAutoCpy((int8_t*) outputs[0], (int8_t*) inputTensor, numel * elemSize, stream); + } + + sync_check_cuda_error(stream); + return 0; +} + +nvinfer1::IPluginV3* DoraPlugin::attachToContext(nvinfer1::IPluginResourceContext* context) noexcept +{ + return clone(); +} + +nvinfer1::PluginFieldCollection const* DoraPlugin::getFieldsToSerialize() noexcept +{ + return &mFieldsToSerialize; +} + +DoraPluginCreator::DoraPluginCreator() +{ + mPluginAttributes.clear(); + mPluginAttributes.emplace_back("num_modules", nullptr, PluginFieldType::kINT32, 1); + mPluginAttributes.emplace_back("type", nullptr, PluginFieldType::kINT32, 1); + mPluginAttributes.emplace_back("remove_input_padding", nullptr, PluginFieldType::kINT8, 1); + mFC.nbFields = static_cast<int32_t>(mPluginAttributes.size()); + mFC.fields = mPluginAttributes.data(); +} + +char const* DoraPluginCreator::getPluginName() const noexcept +{ + return DORA_PLUGIN_NAME; +} + +char const* DoraPluginCreator::getPluginVersion() const noexcept +{ + return DORA_PLUGIN_VERSION; +} + +nvinfer1::PluginFieldCollection const* DoraPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +nvinfer1::IPluginV3* DoraPluginCreator::createPlugin( + char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept +{ + PluginField const* fields = fc->fields; + nvinfer1::DataType type{}; + bool removeInputPadding{}; + std::vector<int32_t> outHiddenSizes; + + // Read configurations from each field + for (int i = 0; i < fc->nbFields; ++i) + { + auto const field = fields[i]; + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type")) + { + TLLM_CHECK(field.type == PluginFieldType::kINT32 and field.length == 1); + type = *static_cast<nvinfer1::DataType const*>(field.data); + } + else if (!strcmp(attrName, "remove_input_padding")) + { + TLLM_CHECK(field.type == PluginFieldType::kINT8 and field.length == 1); + removeInputPadding = *static_cast<bool const*>(field.data); + } + else if (!strcmp(attrName, "out_hidden_sizes")) + { + TLLM_CHECK(field.type == PluginFieldType::kINT32); + auto const* outHiddenSizesPtr = static_cast<int32_t const*>(field.data); + outHiddenSizes.resize(field.length); + outHiddenSizes.assign(outHiddenSizesPtr, outHiddenSizesPtr + field.length); + } + else + { + TLLM_LOG_WARNING("%s: got an unexpected attribute: %s", __PRETTY_FUNCTION__, attrName); + } + } + + try + { + auto* obj = new DoraPlugin(outHiddenSizes, type, removeInputPadding); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.h b/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.h new file mode 100644 index 000000000000..dfee11fdc90e --- /dev/null +++ b/cpp/tensorrt_llm/plugins/doraPlugin/doraPlugin.h @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "tensorrt_llm/kernels/lora/dora.h" +#include "tensorrt_llm/plugins/common/plugin.h" + +namespace tensorrt_llm::plugins +{ + +class DoraPlugin : public BasePluginV3 +{ +public: + DoraPlugin() = delete; + DoraPlugin(std::vector<int32_t> const& outHiddenSizes, nvinfer1::DataType type, bool removeInputPadding); + DoraPlugin(DoraPlugin const& p) = default; + + // IPluginV3 methods + nvinfer1::IPluginCapability* getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept override; + nvinfer1::IPluginV3* clone() noexcept override; + + // IPluginV3OneCore methods + char const* getPluginName() const noexcept override; + char const* getPluginVersion() const noexcept override; + + // IPluginV3OneBuild methods + int32_t configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept override; + int32_t getOutputDataTypes(nvinfer1::DataType* outputTypes, int32_t nbOutputs, nvinfer1::DataType const* inputTypes, + int32_t nbInputs) const noexcept override; + int32_t getOutputShapes(nvinfer1::DimsExprs const* inputs, int32_t nbInputs, nvinfer1::DimsExprs const* shapeInputs, + int32_t nbShapeInputs, nvinfer1::DimsExprs* outputs, int32_t nbOutputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination(int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, + int32_t nbOutputs) noexcept override; + int32_t getNbOutputs() const noexcept override; + size_t getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept override; + int32_t getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept override; + int32_t getNbTactics() noexcept override; + char const* getTimingCacheID() noexcept override; + int32_t getFormatCombinationLimit() noexcept override; + char const* getMetadataString() noexcept override; + + // IPluginV3OneRuntime methods + int32_t setTactic(int32_t tactic) noexcept override; + int32_t onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, nvinfer1::PluginTensorDesc const* out, + int32_t nbOutputs) noexcept override; + int32_t enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept override; // fixed + nvinfer1::IPluginV3* attachToContext(nvinfer1::IPluginResourceContext* context) noexcept override; + nvinfer1::PluginFieldCollection const* getFieldsToSerialize() noexcept override; + +private: + void init(); + + std::vector<nvinfer1::PluginField> mDataToSerialize; + nvinfer1::PluginFieldCollection mFieldsToSerialize; + + enum IdxEntry + { + kINPUT_TENSOR = 0, + kHOST_REQUEST_TYPES = 1, + kLORA_WEIGHTS_PTRS_START = 2 + }; + + // TODO(oargov) this is shared with the LoRA plugin, put it somewhere else + enum class RequestType : int32_t + { + kCONTEXT = 0, + kGENERATION = 1 + }; + + std::vector<int32_t> mOutHiddenSizes; + nvinfer1::DataType mType; + bool mRemoveInputPadding; + tensorrt_llm::kernels::DoraImpl mDoraImpl; + + std::vector<void const*> mExpandDoraWeightPtrs{}; +}; + +class DoraPluginCreator : public BaseCreatorV3 +{ +public: + DoraPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV3* createPlugin( + char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +}; // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/eaglePlugin/CMakeLists.txt new file mode 100644 index 000000000000..b6bd0439cc0c --- /dev/null +++ b/cpp/tensorrt_llm/plugins/eaglePlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.cpp b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.cpp new file mode 100644 index 000000000000..899c93855b9f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.cpp @@ -0,0 +1,945 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "eagleDecodeDraftTokensPlugin.h" + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/kernels/samplingTopKKernels.h" +#include "tensorrt_llm/kernels/speculativeDecoding/eagleDecodingKernels.h" +#include "tensorrt_llm/kernels/speculativeDecoding/medusaDecodingKernels.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iTensor.h" + +using namespace nvinfer1; +using tensorrt_llm::plugins::EagleDecodeDraftTokensPluginCreator; +using tensorrt_llm::plugins::EagleDecodeDraftTokensPlugin; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::kernels::speculative_decoding; +using namespace tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; + +static char const* EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_VERSION{"1"}; +static char const* EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_NAME{"EagleDecodeDraftTokens"}; +PluginFieldCollection EagleDecodeDraftTokensPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> EagleDecodeDraftTokensPluginCreator::mPluginAttributes; + +EagleDecodeDraftTokensPlugin::EagleDecodeDraftTokensPlugin( + nvinfer1::DataType type, int32_t layerIdx, int32_t numEagleLayers, bool topKSampling) + : mDtype(type) + , mLayerIdx(layerIdx) + , mNumEagleLayers(numEagleLayers) + , mTopKSampling(topKSampling) +{ + TLLM_CHECK_WITH_INFO(mTopKSampling, "Multinomial sampling is not supported yet."); +} + +// Parameterized constructor +EagleDecodeDraftTokensPlugin::EagleDecodeDraftTokensPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mDtype); + read(d, mLayerIdx); + read(d, mNumEagleLayers); + read(d, mTopKSampling); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + static_cast<int>(length), static_cast<int>(d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* EagleDecodeDraftTokensPlugin::clone() const noexcept +{ + auto* plugin = new EagleDecodeDraftTokensPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs EagleDecodeDraftTokensPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + TLLM_CHECK(outputIndex < getNbOutputs()); + TLLM_CHECK(nbInputs == 12); + auto const batchSizeExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[0]; + auto const maxDecodingTokensExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[1]; + auto const maxPathLengthExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[2]; + auto const maxDecodingDraftTokensExpr + = exprBuilder.operation(DimensionOperation::kSUB, *maxDecodingTokensExpr, *exprBuilder.constant(1)); + + auto const numEagleLayersExpr + = exprBuilder.operation(DimensionOperation::kSUB, *maxPathLengthExpr, *exprBuilder.constant(1)); + auto const maxDecodingDraftTokensSquareExpr + = exprBuilder.operation(DimensionOperation::kPROD, *maxDecodingDraftTokensExpr, + *maxDecodingDraftTokensExpr); // maxDecodingDraftTokensExpr * maxDecodingDraftTokensExpr + + nvinfer1::DimsExprs ret; + if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_DRAFT_TOKEN_IDS)) + { + // output_draft_token_ids: [batch_size, max_decoding_draft_tokens] + ret.nbDims = 2; + ret.d[0] = batchSizeExpr; + ret.d[1] = maxDecodingDraftTokensExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_DRAFT_LENS)) + { + // output_draft_lens: [batch_size] + ret.nbDims = 1; + ret.d[0] = batchSizeExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_PATHS)) + { + // output_path: [batch_size, max_decoding_tokens, max_path_len] + ret.nbDims = 3; + ret.d[0] = batchSizeExpr; + ret.d[1] = maxDecodingTokensExpr; + ret.d[2] = maxPathLengthExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_CURRENT_SCORES)) + { + // output_current_scores: [batch_size, max_decoding_draft_tokens] + ret.nbDims = 2; + ret.d[0] = batchSizeExpr; + ret.d[1] = maxDecodingDraftTokensExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_NEXT_EXPAND_INDICES)) + { + // output_next_expand_index + ret.nbDims = 2; + ret.d[0] = batchSizeExpr; + ret.d[1] = maxDecodingDraftTokensExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_SCORES)) + { + // output_all_layers_scores: + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + ret.nbDims = 3; + ret.d[0] = batchSizeExpr; + ret.d[1] = numEagleLayersExpr; + ret.d[2] = maxDecodingDraftTokensSquareExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS)) + { + // output_all_layers_draft_token_ids: + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + ret.nbDims = 3; + ret.d[0] = batchSizeExpr; + ret.d[1] = numEagleLayersExpr; + ret.d[2] = maxDecodingDraftTokensSquareExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR)) + { + // output_all_layers_draft_token_ids_predecessor + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + ret.nbDims = 3; + ret.d[0] = batchSizeExpr; + ret.d[1] = numEagleLayersExpr; + ret.d[2] = maxDecodingDraftTokensSquareExpr; + } + else + { + TLLM_CHECK_WITH_INFO( + false, "Wrong outputIndex %d in EagleDecodeDraftTokensPlugin::getOutputDimensions", outputIndex); + } + return ret; +} + +bool EagleDecodeDraftTokensPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + TLLM_CHECK(nbInputs == 12 && nbOutputs == getNbOutputs()); + TLLM_CHECK(pos < nbInputs + nbOutputs); + + if (pos == getIdx(InputIdxEntry::LOGITS)) + { + // input: logits + // output: output_all_layers_scores + return (inOut[pos].type == mDtype) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == getIdx(InputIdxEntry::INPUT_ALL_LAYERS_SCORES) || pos == getIdx(InputIdxEntry::INPUT_PREV_SCORES) + || pos == nbInputs + getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_SCORES) + || pos == nbInputs + getIdx(OutputIdxEntry::OUTPUT_CURRENT_SCORES)) + { + // input: rand_sample, input_all_layers_scores, input_prev_scores + // output: output_all_layers_scores, output_current_scores + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else + { + // input: path, num_valid_logits, use_dynamic_tree, dynamic_tree_max_topK, input_draft_token_ids, + // input_draft_lens, input_current_expand_index, input_all_layers_draft_token_ids + // output: output_draft_token_ids, output_draft_lens, output_path, output_next_expand_index + // output_all_layers_draft_token_ids, output_all_alyers_draft_token_predecessor + return (inOut[pos].type == nvinfer1::DataType::kINT32) && (inOut[pos].format == TensorFormat::kLINEAR); + } +} + +void EagleDecodeDraftTokensPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +template <typename T> +size_t EagleDecodeDraftTokensPlugin::getWorkspaceSizeType(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + size_t workspaceSize{0}; + auto const numInputLogits = inputs[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; + auto const batchSize = inputs[getIdx(InputIdxEntry::PATHS)].dims.d[0]; + auto const vocabSizePadded = inputs[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; + auto const maxDecodingTokens = inputs[getIdx(InputIdxEntry::PATHS)].dims.d[1]; + auto const maxDecodingDraftTokens = maxDecodingTokens - 1; + auto const maxTopK = maxDecodingDraftTokens; + auto const mNumEagleLayers = inputs[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_SCORES)].dims.d[1]; + + // Greedy sampling + if (mTopKSampling) + { + // 0. The first topK sampling workspace + auto const draftTokenSamplingWorkspaceSize + = getTopKWorkspaceSize<T>(numInputLogits, /* maxTokensPerStep */ 1, /* maxTopK */ maxTopK, vocabSizePadded); + + // 1. The first TopKs [numInputLogits] + auto const topKsSize = numInputLogits * sizeof(SizeType32); + + // 2. Topks offset [batchSize] + // Each request will have different number of logits that need to be sampled + // This tensor will record the start offset of the topK for each request + auto const topKOffsetSize = batchSize * sizeof(SizeType32); + + // 3. Logits ptrs [numInputLogits] + auto const logitsPtrsSize = numInputLogits * sizeof(T*); + + // 4. The first topK sampling's output ids ptrs [numInputLogits][maxDecodingDraftTokens] + auto const firstTopKOutputIdsPtrsSize = numInputLogits * sizeof(TokenIdType*); + + // 5. The first topK sampling's output ids (temporary buffer) [numInputLogits * maxDecodingDraftTokens] + auto const firstTopKOutputIdsSize = numInputLogits * maxDecodingDraftTokens * sizeof(TokenIdType); + + // 6. Number of successors for each nodes, extract from the paths and layerId + // [batchSize * maxDecodingTokens] + auto const numSuccessorsForEachNodeSize = batchSize * maxDecodingTokens * sizeof(SizeType32); + + // 7. Flag whether to do decoding or not. SamplingTopK is done for numInputLogits tokens. + // But only sum(numValidLogitsPerRequest[:]) of them are valid. + // [batchSize * maxDecodingTokens] + auto const skipDecodeSize = numInputLogits * sizeof(bool); + + // 8. The first topK sampling's logprobs [batchSize * maxDecodingDraftTokens] + auto const firstTopKOutputLogProbsSize = numInputLogits * maxDecodingDraftTokens * sizeof(float); + + // 9. Eagle-2, the second topK sampling workspace + // Sampling from [batchSize, maxTopK * maxTopK] to [batchSize, maxTopK] + auto const secondTopKSamplingWorkspaceSize = getTopKWorkspaceSize<float>( + batchSize, /* maxTokensPerStep */ 1, /* maxTopK */ maxTopK, maxTopK * maxTopK); + + // 10. Eagle-2, the outputIds of the second topK sampling, shape [batchSize, maxDecodingTokens] + auto const secondTopKOutputIdsSize = batchSize * maxDecodingTokens * sizeof(TokenIdType); + // 11. Eagle-2, the outputIdsPtr of the second topK sampling, shape [batchSize] + auto const secondTopKOutputIdsPtrSize = batchSize * sizeof(TokenIdType*); + // 12. Eagle-2, the inputScoresPtrs of the second topK sampling, shape [batchSize] + auto const secondTopKInputScoresPtrsSize = batchSize * sizeof(float*); + // 13. Eagle-2, the outpuLogProbs of the second topK samplig, shape [batchSize, maxDecodingDraftTokens] + auto const secondTopKOutputLogProbsSize = batchSize * maxDecodingDraftTokens * sizeof(float); + + // 14. Eagle-2, the input scores pointers of the third topK sampling, shape [batchSize] + // Each points to a vocabSize = '(mNumEagleLayers - 1) * dynamicTreeMaxTopK * dynamicTreeMaxTopK + + // dynamicTreeMaxTopK' + auto const thirdTopKInputScoresPtrsSize = batchSize * sizeof(float*); + // 15. Eagle-2, the output of the third topK sampling, shape [batchSize, maxDecodingDraftTokens] + auto const thirdTopKOutputIdsSize = batchSize * maxDecodingDraftTokens * sizeof(TokenIdType); + // 16. Eagle-2, the output pointers of the third topK sampling, shape [batchSize] + auto const thirdTopKOutputIdsPtrsSize = batchSize * sizeof(TokenIdType*); + // 17. Eagle-2, the workspace of the third topK sampling + // Sampling from [batchSize, '(mNumEagleLayers - 1) * dynamicTreeMaxTopK * dynamicTreeMaxTopK + + // dynamicTreeMaxTopK'] to [batchSize, maxDecodingDraftTokens] We over-set the vocabsize here. + auto const thridTopKSamplingWorkspaceSize = getTopKWorkspaceSize<float>(batchSize, /* maxTokensPerStep */ 1, + /* maxTopK */ maxDecodingDraftTokens, mNumEagleLayers * maxDecodingDraftTokens * maxDecodingDraftTokens); + + // 18. Eagle-2, the topKs for each request in the third topK sampling + // The real topK value is min(maxDecodingDraftTokens, totalNumDraftTokensForAllLayers) + auto const thirdTopKsSize = batchSize * sizeof(SizeType32); + + SizeType32 constexpr NUM_BUFFERS{19}; + size_t workspaces[NUM_BUFFERS]; + workspaces[0] = draftTokenSamplingWorkspaceSize; + workspaces[1] = topKsSize; + workspaces[2] = topKOffsetSize; + workspaces[3] = logitsPtrsSize; + workspaces[4] = firstTopKOutputIdsPtrsSize; + workspaces[5] = firstTopKOutputIdsSize; + workspaces[6] = numSuccessorsForEachNodeSize; + workspaces[7] = skipDecodeSize; + workspaces[8] = firstTopKOutputLogProbsSize; + workspaces[9] = secondTopKSamplingWorkspaceSize; + workspaces[10] = secondTopKOutputIdsSize; + workspaces[11] = secondTopKOutputIdsPtrSize; + workspaces[12] = secondTopKInputScoresPtrsSize; + workspaces[13] = secondTopKOutputLogProbsSize; + workspaces[14] = thirdTopKInputScoresPtrsSize; + workspaces[15] = thirdTopKOutputIdsSize; + workspaces[16] = thirdTopKOutputIdsPtrsSize; + workspaces[17] = thridTopKSamplingWorkspaceSize; + workspaces[18] = thirdTopKsSize; + workspaceSize = tc::calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); + } + else + { + // TODO fill me + // Multinomial sampling + TLLM_CHECK_WITH_INFO(false, "Multinomial sampling is not supported yet."); + } + + return workspaceSize; +} + +size_t EagleDecodeDraftTokensPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + auto const logitsType = inputs[getIdx(InputIdxEntry::LOGITS)].type; + if (logitsType == nvinfer1::DataType::kFLOAT) + { + return getWorkspaceSizeType<float>(inputs, nbInputs, outputs, nbOutputs); + } + else if (logitsType == nvinfer1::DataType::kHALF) + { + return getWorkspaceSizeType<__half>(inputs, nbInputs, outputs, nbOutputs); + } + else + { + TLLM_CHECK_WITH_INFO(false, "Unsupported logits type"); + } + return 0; +} + +template <typename T> +void EagleDecodeDraftTokensPlugin::doTopKSampling(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // We allocate many buffers with 'numInputLogits' size, but the input logits will include some padding logits. + // So only 'batchSize' or 'numValidLogits' size will be actually used. + auto const numInputLogits = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; + auto const vocabSizePadded = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; + auto const batchSize = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[0]; + auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[1]; + auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[2]; + auto const maxDecodingDraftTokens = maxDecodingTokens - 1; + auto const maxTopK = maxDecodingDraftTokens; + + ////////////////////////////////////////// Get plugin inputs ////////////////////////////////////////// + // Plugin inputs + // Input logits for sampling, shape: [numInputLogits, vocabSizePadded] + auto pluginInputLogits = static_cast<T const*>(inputs[getIdx(InputIdxEntry::LOGITS)]); + // Input paths, shape: [batchSize, maxDecodingTokens, maxPathLen] + auto pluginInputPaths = static_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::PATHS)]); + auto numValidLogits = static_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::NUM_VALID_LOGITS)]); + // For Eagle-2 + // Whether to use dynamic tree (i.e., Eagle-2) + auto useDynamicTree = *(static_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::USE_DYNAMIC_TREE)])); + // The max topK for dynamic tree. All the requests have the same expand topK. + // In Eagle-2, dynamicTreeMaxTopK is equal to maxNonLeavesPerLayer in the internal EagleNets. + auto dynamicTreeMaxTopK = *(static_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::DYNAMIC_TREE_MAX_TOPK)])); + // All layer's draft tokenIds, shape: [batchSize, maxDecodingDraftTokens] + auto pluginInputDraftTokenIds + = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::INPUT_DRAFT_TOKEN_IDS)]); + // The number of all layer's draft tokenIds, shape: [batchSize] + auto pluginInputDraftLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::INPUT_DRAFT_LENS)]); + // The previous EagleNet's scores, shape: [batchSize, maxDecodingDraftTokens] + auto pluginInputPrevScores = static_cast<float const*>(inputs[getIdx(InputIdxEntry::INPUT_PREV_SCORES)]); + // The indices of the nodes that will be expand in this layer, shape: [batchSize, maxDecodingDraftTokens] + // The index is related to the final output tree, which has max_decoding_draft_tokens draft tokens. + auto pluginInputCurrentExpandIndices + = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::INPUT_CURRENT_EXPAND_INDICES)]); + // The scores from all previous EagleNets, + // shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] + auto pluginInputAllLayersScores = static_cast<float const*>(inputs[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_SCORES)]); + // The draft tokens from all previous EagleNets, + // shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] + auto pluginInputAllLayersDraftTokenIds + = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_DRAFT_TOKEN_IDS)]); + // The predecessor of all the draft tokens, + // shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] + auto pluginInputAllLayersDraftTokenIdsPredecessor = reinterpret_cast<SizeType32 const*>( + inputs[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR)]); + + ////////////////////////////////////////// Get plugin outputs ////////////////////////////////////////// + // Plugin outputs + // All layer's draft tokenIds, shape: [batchSize, maxDecodingDraftTokens] + auto pluginOutputDraftTokenIds + = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_DRAFT_TOKEN_IDS)]); + // The number of all layer's draft tokenIds, shape: [batchSize] + auto pluginOutputDraftLens = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::OUTPUT_DRAFT_LENS)]); + // For Eagle-2 + // Updated paths base on this layer's sampling result, shape: [batchSize, maxDecodingTokens, maxPathLen] + auto pluginOutputPaths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::OUTPUT_PATHS)]); + // This layer's scores, which will be used in next layers [batchSize, maxDecodingDraftTokens] + auto pluginOutputCurrentScores = static_cast<float*>(outputs[getIdx(OutputIdxEntry::OUTPUT_CURRENT_SCORES)]); + // The indices of the nodes that will be expand in next layer, shape: [batchSize, maxDecodingDraftTokens] + // The index is related to the final output tree, which has max_decoding_draft_tokens draft tokens. + auto pluginOutputNextExpandIndices + = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_NEXT_EXPAND_INDICES)]); + // Updated scores, shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] + auto pluginOutputAllLayersScores = static_cast<float*>(outputs[getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_SCORES)]); + // Updated draft tokens, shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x maxDecodingDraftTokens] + auto pluginOutputAllLayersDraftTokenIds + = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS)]); + // Update the predecessor of the draft tokens, shape: [batchSize, mNumEagleLayers, maxDecodingDraftTokens x + // maxDecodingDraftTokens] + auto pluginOutputAllLayersDraftTokenIdsPredecessor + = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR)]); + + ////////////////////////////////////////// Get workspaces ////////////////////////////////////////// + int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); + size_t offset{0}; + // Workspace 0: Sampling workspace. + // Treat numInputLogits as batchSize + auto const samplingWorkspaceSize + = getTopKWorkspaceSize<T>(numInputLogits, /* maxTokensPerStep */ 1, /* maxTopK */ maxTopK, vocabSizePadded); + void* workspaceSampling + = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, samplingWorkspaceSize)); + + // Workspace 1: Topks tensor: shape [numInputLogits] + SizeType32* topKs = reinterpret_cast<SizeType32*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * sizeof(SizeType32))); + + // Workspace 2: topKOffset tensor: shape: [batchSize], number of nodes that have successors for each requests + SizeType32* topKOffset + = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); + + // Workspace 3: logits pointers tensor: shape: [numInputLogits] + T const** logitsPtrs + = reinterpret_cast<T const**>(tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * sizeof(T*))); + + // Workspace 4: outputIds pointers tensor: shape [numInputLogits], each points to a [maxDecodingDraftTokens] buffer + TokenIdType** firstTopKOutputIdsPtrs = reinterpret_cast<TokenIdType**>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * sizeof(TokenIdType*))); + + // Workspace 5: outputIds tensor: flatten outputIds, shape [numInputLogits * maxDecodingDraftTokens] + TokenIdType* firstTopKOutputIdsFlatten = reinterpret_cast<TokenIdType*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * maxDecodingDraftTokens * sizeof(TokenIdType))); + + // Workspace 6: number of successors for each nodes tensor: shape [batchSize * maxDecodingTokens] + SizeType32* numSuccessorsForEachNode = reinterpret_cast<SizeType32*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(SizeType32))); + + // Workspace 7: skip decoding mask [numInputLogits] + bool* skipDecode + = reinterpret_cast<bool*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * sizeof(bool))); + + // In Eagle-1, we do not need to return logProbs + float* firstTopKOutputLogProbs = nullptr; + if (useDynamicTree) + { + // Workspace 8. The output logProbs of the first topK sampling. + // Which will be updated with the previous layer's scores (i.e., pluginInputPrevScores), and will be treat as + // the input of the second topK sampling. For mLayerIdx == 0, shape: [numInputLogits(batchSize), + // maxDecodingDraftTokens] For mLayerIdx > 0, shape: [numInputLogits(batchSize * dynamicTreeMaxTopK), + // maxDecodingDraftTokens] + firstTopKOutputLogProbs = reinterpret_cast<float*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, numInputLogits * maxDecodingDraftTokens * sizeof(float))); + } + + SizeType32 const secondTopKVocabSize = dynamicTreeMaxTopK * maxDecodingDraftTokens; + // Workspace 9: Sampling from [batchSize, dynamicTreeMaxTopK * maxDecodingDraftTokens] to [batchSize, + // dynamicTreeMaxTopK] + auto const secondTopKSamplingWorkspaceSize + = getTopKWorkspaceSize<float>(batchSize, /* maxTokensPerStep */ 1, /* maxTopK */ maxTopK, secondTopKVocabSize); + void* workspaceScoresSampling + = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, secondTopKSamplingWorkspaceSize)); + + // Workspace 10: the second (scores) sampling's outputIds, shape: [batchSize, maxDecodingDraftTokens] + TokenIdType* secondTopKOutputIdsFlatten = reinterpret_cast<TokenIdType*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingDraftTokens * sizeof(TokenIdType))); + + // Workspace 11: the second (scores) sampling's outputIdsPtrs + TokenIdType** secondTopKOutputIdsPtrs = reinterpret_cast<TokenIdType**>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(TokenIdType*))); + + // Workspace 12: input scores pointers + float** secondTopKInputScoresPtrs + = reinterpret_cast<float**>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(float*))); + + // Workspace 13: the second sampling's outputLogProbs + float* secondTopKOutputLogProbs = reinterpret_cast<float*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingDraftTokens * sizeof(float))); + + // Workspace 14: The input scores pointers of the third topK sampling, shape [batchSize] + float** thirdTopKInputScoresPtrs + = reinterpret_cast<float**>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(float*))); + + // Workspace 15: The output of the third topK sampling, shape [batchSize, maxDecodingDraftTokens] + TokenIdType* thirdTopKOutputIds = reinterpret_cast<TokenIdType*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingDraftTokens * sizeof(TokenIdType))); + + // Workspace 16: The output pointers of the third topK sampling, shape [batchSize] + TokenIdType** thirdTopKOutputIdsPtrs = reinterpret_cast<TokenIdType**>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(TokenIdType*))); + + // The number of draft tokens among all layers + long const totalNumDraftTokensForAllLayers + = (mNumEagleLayers - 1) * dynamicTreeMaxTopK * dynamicTreeMaxTopK + dynamicTreeMaxTopK; + + auto const thridTopKSamplingWorkspaceSize = getTopKWorkspaceSize<float>( + batchSize, /* maxTokensPerStep */ 1, /* maxTopK */ maxDecodingDraftTokens, totalNumDraftTokensForAllLayers); + // Workspace 17: The workspace of the third topK sampling + void* workspaceThirdTopKSampling + = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, thridTopKSamplingWorkspaceSize)); + + // Workspace 18. Eagle-2, the topKs for each request in the third topK sampling, shape [batchSize] + // The real topK value is min(maxDecodingDraftTokens, totalNumDraftTokensForAllLayers) + SizeType32* thirdTopKs + = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); + + ////////////////////////////////////////// Main logic ////////////////////////////////////////// + // Fill logitsPtrs from plugin input logits + // And fill firstTopKOutputIdsPtrs from firstTopKOutputIdsFlatten + invokeAssembleDraftLogitsOffsets(logitsPtrs, pluginInputLogits, firstTopKOutputIdsPtrs, firstTopKOutputIdsFlatten, + skipDecode, numValidLogits, numInputLogits, batchSize, maxDecodingDraftTokens, vocabSizePadded, stream); + sync_check_cuda_error(stream); + + if (useDynamicTree) + { + // For Eagle-2, the topK value between different requests are the same, all set to 'dynamicTreeMaxTopK'. + invokeSetTopKsFromDyanmicTreeMaxTopK( + mLayerIdx, batchSize, numInputLogits, topKs, topKOffset, dynamicTreeMaxTopK, numValidLogits, stream); + sync_check_cuda_error(stream); + + // Do softmax for the input logits + // We set the 'batchSize' and 'maxBatchSize' to 'numInputLogits', while 'numInputLogits' logits may contain + // some padding logits, which do not need to be calculated. + // We use 'skipDecode' list to skip these padding logits. This could avoid redundant calculations. + BiasSoftmaxParams<T> biasSoftmaxParams; + biasSoftmaxParams.logits = const_cast<T*>(pluginInputLogits); + biasSoftmaxParams.logitsPtrs = nullptr; + biasSoftmaxParams.probs = const_cast<T*>(pluginInputLogits); + biasSoftmaxParams.maxBeamWidth = 1; + biasSoftmaxParams.batchSlots = nullptr; + biasSoftmaxParams.batchSize = numInputLogits; + biasSoftmaxParams.maxBatchSize = numInputLogits; + biasSoftmaxParams.vocabSize = vocabSizePadded; + biasSoftmaxParams.vocabSizePadded = vocabSizePadded; + biasSoftmaxParams.skipSoftMax = false; + biasSoftmaxParams.batchSlotsLogits = false; + biasSoftmaxParams.skipDecode = skipDecode; + biasSoftmaxParams.checkParams(); + + invokeAddBiasSoftMax(biasSoftmaxParams, stream); + sync_check_cuda_error(stream); + } + else + { + // For Eagle-1, extract topK value from input path. + invokeExtractTopKsFromPath(pluginInputPaths, topKs, topKOffset, numSuccessorsForEachNode, mLayerIdx, batchSize, + maxDecodingTokens, maxPathLen, stream); + sync_check_cuda_error(stream); + } + + TopKSamplingKernelParams<T> params{}; + params.logProbsPtrs = logitsPtrs; // [numInputLogits][vocabSizePadded] + params.outputIdsPtrs = firstTopKOutputIdsPtrs; // [numInputLogits][maxDecodingDraftTokens] + params.workspace = workspaceSampling; + params.maxTopK = maxTopK; + params.topKs = topKs; // [numInputLogits] + params.batchSize = numInputLogits; + params.maxBatchSize = numInputLogits; + params.maxTokensPerStep = 1; + params.vocabSizePadded = vocabSizePadded; + params.returnAllSelectedTokens = true; + params.strictTopPBoundary = false; + params.skipDecode = skipDecode; + params.outputLogProbs = firstTopKOutputLogProbs; // [numInputLogits * maxDecodingDraftTokens] + params.logitsHasProbs = true; + + invokeBatchTopKSampling(params, stream); + sync_check_cuda_error(stream); + + if (useDynamicTree) + { + // When mLayerIdx == 0, we do not need to update scores. + // We take the outputLogProbs of the first topK sampling as the scores directly. + if (mLayerIdx != 0) + { + // Update firstTopKOutputLogProbs with pluginInputPrevScores, which is the scores from the previous layer + invokeUpdateScores(batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, firstTopKOutputLogProbs, + pluginInputPrevScores, stream); + sync_check_cuda_error(stream); + + // Do the second top-dynamicTreeMaxTopK sampling among this dynamicTreeMaxTopK x dynamicTreeMaxTopK draft + // tokens. Through the second topK sampling, we obtain the dynamicTreeMaxTopK output draft tokens of this + // layer. + + // Although theoretically we only need to select 'dynamicTreeMaxTopK' draft tokens from 'dynamicTreeMaxTopK + // * dynamicTreeMaxTopK' draft tokens, we over-set vocabSize here. This is because when we write the scores + // into firstTopKOutputLogProbs, we store it in the form of [batchSize * dynamicTreeMaxTopK, + // maxDecodingDraftTokens]. For each request, these 'dynamicTreeMaxTopK * dynamicTreeMaxTopK' scores are not + // saved continuously, but in the format of [dynamicTreeMaxTopK, maxDecodingDraftTokens]. For unused + // positions, we set '-inf' to ensure that they will not be sampled. Examples: For a request, + // dynamicTreeMaxTopK == 3, the scores in its buffer ([dynamicTreeMaxTopK, maxDecodingDraftTokens]) are as + // follow: + // [[1.1, 2.2, 3.3, -inf, -inf, ...], + // [4.4, 5.5, 6.6, -inf, -inf, ...], + // [7.7, 8.8, 9.9, -inf, -inf, ...]] + + // Prepare the input of the second topK sampling. + invokeAssembleSecondTopKSamplingInputs(batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, + firstTopKOutputLogProbs, secondTopKInputScoresPtrs, secondTopKOutputIdsFlatten, secondTopKOutputIdsPtrs, + stream); + sync_check_cuda_error(stream); + + TopKSamplingKernelParams<float> params{}; + params.logProbsPtrs = secondTopKInputScoresPtrs; + params.outputIdsPtrs = secondTopKOutputIdsPtrs; + params.workspace = workspaceScoresSampling; + params.maxTopK = maxTopK; // Same to maxDecodingTokens + params.topKs = topKs; // [batchSize], all set to dynamicTreeMaxTopK + params.batchSize = batchSize; + params.maxBatchSize = batchSize; + params.maxTokensPerStep = 1; + params.vocabSizePadded = secondTopKVocabSize; + params.returnAllSelectedTokens = true; + params.strictTopPBoundary = false; + + invokeBatchTopKSampling(params, stream); + sync_check_cuda_error(stream); + } + + // Copy this layer's scores and draft tokensId: + // 1) Copy this layer's scores to pluginOutputAllLayersScores + // 2) Copy dynamicTreeMaxTopK (or dynamicTreeMaxTopK * dynamicTreeMaxTopK) draft tokens to + // pluginOutputAllLayersDraftTokenIds 3) Set the predecessors of these draft tokens and save to + // pluginOutputAllLayersDraftTokenIdsPredecessor, + // which will be used to reconstruct the final output tree at the last layer + invokeCopyScoresAndDraftTokenIds(mLayerIdx, mNumEagleLayers, maxDecodingDraftTokens, batchSize, + dynamicTreeMaxTopK, + pluginInputCurrentExpandIndices, // The indices of the nodes that expand in this layer (i.e., the input + // logits). The index is related to the final tree. + pluginInputAllLayersScores, pluginInputAllLayersDraftTokenIds, pluginInputAllLayersDraftTokenIdsPredecessor, + pluginOutputAllLayersScores, pluginOutputAllLayersDraftTokenIds, + pluginOutputAllLayersDraftTokenIdsPredecessor, + firstTopKOutputLogProbs, // This layer's scores + firstTopKOutputIdsFlatten, // This layer's draft tokens + stream); + sync_check_cuda_error(stream); + + // Update Path + // For mLayerIdx == 0, the output of the first topK sampling are the output draft tokens of this layers. The + // update logic is simple. For mLayerIdx > 0, the output of the second topK sampling are the output draft tokens + // of this layers. 'secondTopKOutputIdsPtrs' contains the top-dynamicTreeMaxTopK selected from the second topK + // sampling. 'pluginOutputNextExpandIndices' record the selected the top-dynamicTreeMaxTopK draft token's Id of + // this layer, + // which will be used in the next layer to compute the predecessors. + // The last layer will completely reconstruct the paths, so there is no need to update the paths here. + if (mLayerIdx != mNumEagleLayers - 1) + { + invokeUpdatePath(mLayerIdx, batchSize, dynamicTreeMaxTopK, maxDecodingTokens, maxPathLen, pluginInputPaths, + pluginOutputPaths, + secondTopKOutputIdsPtrs, // if mLayerIdx == 0, secondTopKOutputIdsPtrs == nullptr, and it's useless + // during update paths + pluginOutputNextExpandIndices, stream); + sync_check_cuda_error(stream); + } + + if (mLayerIdx != 0) + { + // We will extract the real draft tokenIds and scores from 'firstTopKOutputIdsFlatten' and + // 'secondTopKInputScoresPtrs' according to the 'secondTopKOutputIdsPtrs'. And store them into + // 'secondTopKOutputIdsPtrs' and 'secondTopKOutputLogProbs' (reuse these buffers). + // secondTopKInputScoresPtrs: shape [batchSize * dynamicTreeMaxTopK, maxDecodingDraftTokens] + // The original scores, which were used to do the second TopK sampling + // secondTopKOutputIdsPtrs: shape [batchSize], each points to a [maxDecodingDraftTokens] buffer + // The output of the second TopK sampling, which are the indices of the top-dynamicTreeMaxTopK among + // 'dynamicTreeMaxTopK * dynamicTreeMaxTopK'. We need to figure out what these top-dynamicTreeMaxTopK + // draft tokens' real tokenIds. + // firstTopKOutputIdsFlatten: shape [batchSize * dynamicTreeMaxTopK, maxDecodingDraftTokens] + // The value are related to the vocabSize, which is the real tokenIds. + invokeExtractScoresAndRealDraftTokensIds(batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, + secondTopKInputScoresPtrs, secondTopKOutputIdsPtrs, firstTopKOutputIdsFlatten, secondTopKOutputLogProbs, + stream); + sync_check_cuda_error(stream); + } + + // Copy this layer's output draft tokens and scores. + // This layer's output scores is next layer's previous scores. + // if mLayerIdx == 0, directly use the first topK's outputIds / logProbs as this layer's output draft tokens / + // scores if mLayerIdx > 0, we use the second topK's outputIds / logProbs, + // which is updated with the real draft tokenIds / logprobs in 'invokeExtractScoresAndRealDraftTokensIds' + invokeUpdateDraftTokensAndLensAndCurScores(mLayerIdx, batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, + mLayerIdx == 0 ? firstTopKOutputIdsPtrs : secondTopKOutputIdsPtrs, pluginInputDraftTokenIds, + pluginInputDraftLens, pluginOutputDraftTokenIds, pluginOutputDraftLens, + mLayerIdx == 0 ? firstTopKOutputLogProbs : secondTopKOutputLogProbs, pluginOutputCurrentScores, stream); + sync_check_cuda_error(stream); + + if (mLayerIdx == mNumEagleLayers - 1) + { + // The maximum number of nodes on the final tree (exclude the root node) + auto const maxNodesOnFinalTree = std::min(maxDecodingDraftTokens, totalNumDraftTokensForAllLayers); + + // When reach the last EagleNet, we need to do the third sampling, which take all layers' draft tokens and + // scores as input, and then select top-maxDecodingDraftTokens draft tokens among them. We need to + // reconstruct the path/tree after the third topK sampling. + invokeAssembleThridTopKSamplingInputs(batchSize, maxDecodingDraftTokens, mNumEagleLayers, + maxNodesOnFinalTree, thirdTopKs, pluginOutputAllLayersScores, thirdTopKInputScoresPtrs, + thirdTopKOutputIds, thirdTopKOutputIdsPtrs, stream); + sync_check_cuda_error(stream); + + // 1) Do topK sampling among all previous draft tokens + TopKSamplingKernelParams<float> params{}; + params.logProbsPtrs = thirdTopKInputScoresPtrs; + params.outputIdsPtrs = thirdTopKOutputIdsPtrs; + params.workspace = workspaceThirdTopKSampling; + params.topKs = thirdTopKs; // All set to 'maxNodesOnFinalTree' + params.maxTopK = maxDecodingDraftTokens; // We set maxTopK to 'maxDecodingDraftTokens' to align the + // outputIdsPtrs offsets when written back. + params.batchSize = batchSize; + params.maxBatchSize = batchSize; + params.maxTokensPerStep = 1; + params.vocabSizePadded = totalNumDraftTokensForAllLayers; + params.returnAllSelectedTokens = true; + params.strictTopPBoundary = false; // Make sure to select topK tokens. + + invokeBatchTopKSampling(params, stream); + sync_check_cuda_error(stream); + + // 2) Reconstruct the Path + invokeReconstructFinalPath(batchSize, dynamicTreeMaxTopK, maxDecodingDraftTokens, maxDecodingTokens, + maxPathLen, mNumEagleLayers, maxNodesOnFinalTree, thirdTopKOutputIdsPtrs, + pluginOutputAllLayersDraftTokenIdsPredecessor, pluginOutputPaths, stream); + sync_check_cuda_error(stream); + + // 3) Copy this layer's outputIds to outputDraftTokenIds + invokeCopyFinalDraftTokens(batchSize, maxDecodingDraftTokens, mNumEagleLayers, maxNodesOnFinalTree, + thirdTopKOutputIdsPtrs, pluginOutputAllLayersDraftTokenIds, pluginOutputDraftTokenIds, + pluginOutputDraftLens, stream); + sync_check_cuda_error(stream); + } + } + else + { + // Eagle-1: Copy output token id from outputIdsPtrs to the plugin output buffer + invokeCopyOutputTokensIds(firstTopKOutputIdsPtrs, topKs, topKOffset, pluginInputDraftTokenIds, + pluginInputDraftLens, numValidLogits, pluginOutputDraftTokenIds, pluginOutputDraftLens, mLayerIdx, + batchSize, maxDecodingDraftTokens, pluginInputPaths, pluginOutputPaths, maxPathLen, stream); + sync_check_cuda_error(stream); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +template <typename T> +void EagleDecodeDraftTokensPlugin::enqueueType(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // TODO split batch into greedy and non-greedy and execute both paths + if (mTopKSampling) + { + doTopKSampling<T>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else + { + // TODO fill me + TLLM_CHECK_WITH_INFO(false, "Multinomial sampling is not supported yet"); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +int EagleDecodeDraftTokensPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + auto const logitsType = inputDesc[getIdx(InputIdxEntry::LOGITS)].type; + if (logitsType == nvinfer1::DataType::kFLOAT) + { + enqueueType<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (logitsType == nvinfer1::DataType::kHALF) + { + enqueueType<__half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else + { + TLLM_CHECK_WITH_INFO(false, "Unsupported logits type"); + } + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType EagleDecodeDraftTokensPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index < getNbOutputs()); + TLLM_CHECK(index < getNbOutputs()); + if (index == getIdx(OutputIdxEntry::OUTPUT_ALL_LAYERS_SCORES) + || index == getIdx(OutputIdxEntry::OUTPUT_CURRENT_SCORES)) + { + // Only output_prev_socres are float + return inputTypes[getIdx(InputIdxEntry::INPUT_ALL_LAYERS_SCORES)]; + } + else + { + // output_draft_token_ids, output_draft_lens, output_paths, output_next_expand_index, + // output_all_layers_draft_token_ids, output_all_layers_draft_token_ids_predecessor + // are all int32 type, same as path + return inputTypes[getIdx(InputIdxEntry::PATHS)]; + } +} + +// IPluginV2 Methods + +char const* EagleDecodeDraftTokensPlugin::getPluginType() const noexcept +{ + return EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_NAME; +} + +char const* EagleDecodeDraftTokensPlugin::getPluginVersion() const noexcept +{ + return EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_VERSION; +} + +int EagleDecodeDraftTokensPlugin::getNbOutputs() const noexcept +{ + return 8; +} + +int EagleDecodeDraftTokensPlugin::initialize() noexcept +{ + return 0; +} + +void EagleDecodeDraftTokensPlugin::terminate() noexcept {} + +size_t EagleDecodeDraftTokensPlugin::getSerializationSize() const noexcept +{ + return sizeof(mDtype) + sizeof(mLayerIdx) + sizeof(mNumEagleLayers) + sizeof(mTopKSampling); +} + +void EagleDecodeDraftTokensPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mDtype); + write(d, mLayerIdx); + write(d, mNumEagleLayers); + write(d, mTopKSampling); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void EagleDecodeDraftTokensPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +EagleDecodeDraftTokensPluginCreator::EagleDecodeDraftTokensPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("layer_idx", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("num_eagle_layers", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("top_k_sampling", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* EagleDecodeDraftTokensPluginCreator::getPluginName() const noexcept +{ + return EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_NAME; +} + +char const* EagleDecodeDraftTokensPluginCreator::getPluginVersion() const noexcept +{ + return EAGLE_DECODE_DRAFT_TOKENS_PLUGIN_VERSION; +} + +PluginFieldCollection const* EagleDecodeDraftTokensPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* EagleDecodeDraftTokensPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int32_t layerIdx{}; + int32_t numEagleLayers{}; + nvinfer1::DataType type{}; + bool topKSampling{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "layer_idx")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + layerIdx = *static_cast<int32_t const*>(fields[i].data); + } + else if (!strcmp(attrName, "num_eagle_layers")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + numEagleLayers = *static_cast<int32_t const*>(fields[i].data); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "top_k_sampling")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + topKSampling = static_cast<bool>(*static_cast<int32_t const*>(fields[i].data)); + } + } + + try + { + auto* obj = new EagleDecodeDraftTokensPlugin(type, layerIdx, numEagleLayers, topKSampling); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* EagleDecodeDraftTokensPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call EagleDecodeDraftTokensPlugin::destroy() + try + { + auto* obj = new EagleDecodeDraftTokensPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.h b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.h new file mode 100644 index 000000000000..8c144a1bc073 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleDecodeDraftTokensPlugin.h @@ -0,0 +1,174 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class EagleDecodeDraftTokensPlugin : public BasePlugin +{ +public: + EagleDecodeDraftTokensPlugin(nvinfer1::DataType type, int32_t layerIdx, int32_t numEagleLayers, bool topKSampling); + + EagleDecodeDraftTokensPlugin(void const* data, size_t length); + + ~EagleDecodeDraftTokensPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + enum class InputIdxEntry : int32_t + { + // 12 inputs + // [num_input_logits, vocab_size_padded] + LOGITS = 0, + // [batch_size, max_decoding_tokens, max_path_len] + PATHS, + // [1] + NUM_VALID_LOGITS, + // [1] + USE_DYNAMIC_TREE, + // [1] + DYNAMIC_TREE_MAX_TOPK, + + // [batch_size, max_decoding_draft_tokens] + INPUT_DRAFT_TOKEN_IDS, + // [batch_size] + INPUT_DRAFT_LENS, + + // [batch_size, max_decoding_draft_tokens] + INPUT_PREV_SCORES, + + // [batch_size, max_decoding_draft_tokens] + INPUT_CURRENT_EXPAND_INDICES, + + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + INPUT_ALL_LAYERS_SCORES, + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + INPUT_ALL_LAYERS_DRAFT_TOKEN_IDS, + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + INPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR + }; + + enum class OutputIdxEntry : int32_t + { + // 8 outputs + // [batch_size, max_decoding_draft_tokens] + OUTPUT_DRAFT_TOKEN_IDS = 0, + // [batch_size] + OUTPUT_DRAFT_LENS, + + // [batch_size, max_decoding_tokens, max_path_len] + OUTPUT_PATHS, + + // [batch_size, max_decoding_draft_tokens] + OUTPUT_CURRENT_SCORES, + + // [batch_size, max_decoding_draft_tokens] + OUTPUT_NEXT_EXPAND_INDICES, + + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + OUTPUT_ALL_LAYERS_SCORES, + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS, + // [batch_size, num_eagle_layers, max_decoding_draft_tokens x max_decoding_draft_tokens] + OUTPUT_ALL_LAYERS_DRAFT_TOKEN_IDS_PREDECESSOR + }; + + int32_t getIdx(InputIdxEntry idx) const + { + return static_cast<int32_t>(idx); + } + + int32_t getIdx(OutputIdxEntry idx) const + { + return static_cast<int32_t>(idx); + } + +private: + template <typename T> + size_t getWorkspaceSizeType(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept; + + template <typename T> + void enqueueType(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; + + template <typename T> + void doTopKSampling(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; + +private: + nvinfer1::DataType mDtype; // Logit datatype + int32_t mLayerIdx{-1}; // Index of eagle layer + int32_t mNumEagleLayers{-1}; // Number of eagle layers + bool mTopKSampling; // Use TopK sampling or multinomial sampling +}; + +class EagleDecodeDraftTokensPluginCreator : public BaseCreator +{ +public: + EagleDecodeDraftTokensPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.cpp b/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.cpp new file mode 100644 index 000000000000..2cd8c695e296 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.cpp @@ -0,0 +1,548 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "eaglePrepareDrafterInputsPlugin.h" + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/kernels/speculativeDecoding/eagleDecodingKernels.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iTensor.h" + +using namespace nvinfer1; +using tensorrt_llm::plugins::EaglePrepareDrafterInputsPluginCreator; +using tensorrt_llm::plugins::EaglePrepareDrafterInputsPlugin; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::kernels::speculative_decoding; +using namespace tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; + +static char const* EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_VERSION{"1"}; +static char const* EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_NAME{"EaglePrepareDrafterInputs"}; +PluginFieldCollection EaglePrepareDrafterInputsPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> EaglePrepareDrafterInputsPluginCreator::mPluginAttributes; + +EaglePrepareDrafterInputsPlugin::EaglePrepareDrafterInputsPlugin( + int32_t layerIdx, int32_t numLayers, int32_t maxNonLeavesPerLayer) + : mLayerIdx(layerIdx) + , mNumLayers(numLayers) + , mMaxNonLeavesPerLayer(maxNonLeavesPerLayer) +{ +} + +void EaglePrepareDrafterInputsPlugin::initFieldsToSerialize() +{ + mDataToSerialize.clear(); + mDataToSerialize.emplace_back(PluginField("layer_idx", &mLayerIdx, PluginFieldType::kINT32, 1)); + mDataToSerialize.emplace_back(PluginField("num_layers", &mNumLayers, PluginFieldType::kINT32, 1)); + mDataToSerialize.emplace_back( + PluginField("max_non_leaves_per_layer", &mMaxNonLeavesPerLayer, PluginFieldType::kINT32, 1)); + mFCToSerialize.nbFields = mDataToSerialize.size(); + mFCToSerialize.fields = mDataToSerialize.data(); +} + +nvinfer1::IPluginCapability* EaglePrepareDrafterInputsPlugin::getCapabilityInterface( + nvinfer1::PluginCapabilityType type) noexcept +{ + try + { + if (type == nvinfer1::PluginCapabilityType::kBUILD) + { + return static_cast<nvinfer1::IPluginV3OneBuild*>(this); + } + if (type == nvinfer1::PluginCapabilityType::kRUNTIME) + { + return static_cast<nvinfer1::IPluginV3OneRuntime*>(this); + } + TLLM_CHECK(type == nvinfer1::PluginCapabilityType::kCORE); + return static_cast<nvinfer1::IPluginV3OneCore*>(this); + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +// IPluginV3 methods +nvinfer1::IPluginV3* EaglePrepareDrafterInputsPlugin::clone() noexcept +{ + auto clone = std::make_unique<EaglePrepareDrafterInputsPlugin>(*this); + clone->initFieldsToSerialize(); + return clone.release(); +} + +// IPluginV3OneCore methods +char const* EaglePrepareDrafterInputsPlugin::getPluginName() const noexcept +{ + return EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_NAME; +} + +char const* EaglePrepareDrafterInputsPlugin::getPluginVersion() const noexcept +{ + return EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_VERSION; +} + +char const* EaglePrepareDrafterInputsPlugin::getPluginNamespace() const noexcept +{ + return tensorrt_llm::plugins::api::kDefaultNamespace; +} + +// IPluginV3OneBuild methods +int32_t EaglePrepareDrafterInputsPlugin::getNbOutputs() const noexcept +{ + return 11; +} + +int32_t EaglePrepareDrafterInputsPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept +{ + return 0; +} + +bool EaglePrepareDrafterInputsPlugin::supportsFormatCombination( + int32_t pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept +{ + return (inOut[pos].desc.type == nvinfer1::DataType::kINT32) && (inOut[pos].desc.format == TensorFormat::kLINEAR); +} + +int32_t EaglePrepareDrafterInputsPlugin::getOutputDataTypes(nvinfer1::DataType* outputTypes, int32_t nbOutputs, + nvinfer1::DataType const* inputTypes, int32_t nbInputs) const noexcept +{ + outputTypes[0] = nvinfer1::DataType::kINT32; + outputTypes[1] = nvinfer1::DataType::kINT32; + outputTypes[2] = nvinfer1::DataType::kINT32; + outputTypes[3] = nvinfer1::DataType::kINT32; + outputTypes[4] = nvinfer1::DataType::kINT32; + outputTypes[5] = nvinfer1::DataType::kINT32; + outputTypes[6] = nvinfer1::DataType::kINT32; + outputTypes[7] = nvinfer1::DataType::kINT32; + outputTypes[8] = nvinfer1::DataType::kINT32; + outputTypes[9] = nvinfer1::DataType::kINT32; + outputTypes[10] = nvinfer1::DataType::kINT32; + outputTypes[11] = nvinfer1::DataType::kINT32; + return 0; +} + +int32_t EaglePrepareDrafterInputsPlugin::getOutputShapes(nvinfer1::DimsExprs const* inputs, int32_t nbInputs, + nvinfer1::DimsExprs const* shapeInputs, int32_t nbShapeInputs, nvinfer1::DimsExprs* outputs, int32_t nbOutputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + TLLM_CHECK(nbOutputs == 11); + TLLM_CHECK(nbInputs == 15); + TLLM_CHECK(nbShapeInputs == 0); + auto const numTokens = inputs[getIdx(InputIdxEntry::INPUT_IDS)].d[0]; + auto const batchSizeExpr = inputs[getIdx(InputIdxEntry::PREV_DRAFT_PATHS)].d[0]; + auto const numGenRequestsExpr = inputs[getIdx(InputIdxEntry::SPEC_DECODING_GENERATION_LENGTHS)].d[0]; + auto const numInputGenTokensExpr = inputs[getIdx(InputIdxEntry::INPUT_GEN_TOKENS)].d[0]; + auto const maxDecodingLenExpr = inputs[getIdx(InputIdxEntry::PREV_DRAFT_PATHS)].d[1]; + auto const maxPathLenExpr = inputs[getIdx(InputIdxEntry::PREV_DRAFT_PATHS)].d[2]; + + for (SizeType32 outputIndex = 0; outputIndex < nbOutputs; ++outputIndex) + { + if (outputIndex == getIdx(OutputIdxEntry::SEQUENCE_LENGTHS) + || outputIndex == getIdx(OutputIdxEntry::CONTEXT_LENGTHS) + || outputIndex == getIdx(OutputIdxEntry::SPEC_DECODING_GENERATION_LENGTHS)) + { + outputs[outputIndex] = inputs[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)]; + } + else if (outputIndex == getIdx(OutputIdxEntry::SPEC_DECODING_PACKED_MASK)) + { + outputs[outputIndex].nbDims = 3; + outputs[outputIndex].d[0] = batchSizeExpr; + outputs[outputIndex].d[1] = maxDecodingLenExpr; + outputs[outputIndex].d[2] + = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *maxDecodingLenExpr, *exprBuilder.constant(32)); + } + else if (outputIndex == getIdx(OutputIdxEntry::SPEC_DECODING_POSITION_OFFSETS)) + { + outputs[outputIndex].nbDims = 2; + outputs[outputIndex].d[0] = batchSizeExpr; + outputs[outputIndex].d[1] = maxDecodingLenExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::OUTPUT_IDS) + || outputIndex == getIdx(OutputIdxEntry::HIDDEN_STATES_INDICES) + || (mLayerIdx == 0 && outputIndex == getIdx(OutputIdxEntry::POSITION_IDS))) + { + if (mLayerIdx == 0) + { + // We have at most numGenRequests * (mNumLayers + 1) accepted tokens per step for gen requests and + // input_ids - numGenTokens tokens for context requests. + auto numOutputGenTokensExpr = exprBuilder.operation( + DimensionOperation::kPROD, *numGenRequestsExpr, *exprBuilder.constant(mNumLayers + 1)); + auto numInputCtxTokensExpr + = exprBuilder.operation(DimensionOperation::kSUB, *numTokens, *numInputGenTokensExpr); + outputs[outputIndex].nbDims = 1; + outputs[outputIndex].d[0] = exprBuilder.operation(DimensionOperation::kMAX, *exprBuilder.constant(1), + *exprBuilder.operation(DimensionOperation::kSUM, *numOutputGenTokensExpr, *numInputCtxTokensExpr)); + } + else + { + // At most we have mMaxNonLeavesPerLayer non-leaves at this layer. + // And in total we pass all non-leaves + all their preceding nodes. + // batchSize * mMaxNonLeavesPerLayer * layerIdx + outputs[outputIndex].nbDims = 1; + outputs[outputIndex].d[0] = exprBuilder.operation(DimensionOperation::kPROD, + *exprBuilder.operation(DimensionOperation::kPROD, *exprBuilder.constant(mLayerIdx), + *exprBuilder.constant(mMaxNonLeavesPerLayer)), + *batchSizeExpr); + } + } + else if (mLayerIdx > 0 && outputIndex == getIdx(OutputIdxEntry::POSITION_IDS)) + { + outputs[outputIndex].nbDims = 1; + outputs[outputIndex].d[0] = batchSizeExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::LAST_TOKEN_INDICES)) + { + outputs[outputIndex].nbDims = 1; + outputs[outputIndex].d[0] = exprBuilder.operation( + DimensionOperation::kPROD, *exprBuilder.constant(mMaxNonLeavesPerLayer), *batchSizeExpr); + } + else if (outputIndex == getIdx(OutputIdxEntry::NUM_LAST_TOKEN_INDICES)) + { + outputs[outputIndex].nbDims = 1; + outputs[outputIndex].d[0] = exprBuilder.constant(1); + } + else if (outputIndex == getIdx(OutputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)) + { + // batchSize * (maxPathLen - 1) + 1 + outputs[outputIndex].nbDims = 1; + outputs[outputIndex].d[0] = exprBuilder.operation(DimensionOperation::kSUM, *exprBuilder.constant(1), + *exprBuilder.operation(DimensionOperation::kPROD, *batchSizeExpr, + *exprBuilder.operation(DimensionOperation::kSUB, *maxPathLenExpr, *exprBuilder.constant(1)))); + } + } + return 0; +} + +int32_t EaglePrepareDrafterInputsPlugin::onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::PluginTensorDesc const* out, int32_t nbOutputs) noexcept +{ + return 0; +} + +nvinfer1::IPluginV3* EaglePrepareDrafterInputsPlugin::attachToContext( + nvinfer1::IPluginResourceContext* context) noexcept +{ + return clone(); +} + +PluginFieldCollection const* EaglePrepareDrafterInputsPlugin::getFieldsToSerialize() noexcept +{ + return &mFCToSerialize; +} + +size_t EaglePrepareDrafterInputsPlugin::getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + size_t workspaceSize{0}; + + auto const batchSize = inputs[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].max.d[0]; + auto const maxDecodingTokens = inputs[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].max.d[1]; + + if (mLayerIdx > 0) + { + SizeType32 constexpr NUM_BUFFERS{9}; + size_t workspaces[NUM_BUFFERS]; + workspaces[0] = batchSize * maxDecodingTokens * sizeof(int8_t); // isLeafMask + workspaces[1] = batchSize * maxDecodingTokens * sizeof(SizeType32); // selectedDraftIndices + workspaces[2] = batchSize * maxDecodingTokens * sizeof(SizeType32); // selectedDraftPosOffsets + workspaces[3] = batchSize * sizeof(SizeType32); // numSelectedDraftIndices + workspaces[4] = batchSize * maxDecodingTokens * maxDecodingTokens * sizeof(int8_t); // selectedMasks + workspaces[5] = (batchSize + 1) * sizeof(SizeType32); // cumSumGenerationLengths + workspaces[6] = batchSize * maxDecodingTokens * sizeof(SizeType32); // nonLeavesInLevelOffsets + workspaces[7] = batchSize * maxDecodingTokens * sizeof(SizeType32); // parentNonLeafInLevelOffset + workspaces[8] = 1 * sizeof(SizeType32); // maxGenerationLength + workspaceSize = tc::calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); + } + + return workspaceSize; +} + +void EaglePrepareDrafterInputsPlugin::prepareCtxEagleNetData(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const batchSize = inputDesc[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)].dims.d[0]; + + auto const numTokens = inputDesc[getIdx(InputIdxEntry::INPUT_IDS)].dims.d[0]; + auto const numGenRequests = inputDesc[getIdx(InputIdxEntry::SPEC_DECODING_GENERATION_LENGTHS)].dims.d[0]; + auto const numInputGenTokens = inputDesc[getIdx(InputIdxEntry::INPUT_GEN_TOKENS)].dims.d[0]; + + auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::ACCEPTED_TOKENS)].dims.d[1]; + auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].dims.d[1]; + + auto eagleNetSequenceLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SEQUENCE_LENGTHS)]); + auto eagleNetContextLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::CONTEXT_LENGTHS)]); + auto outputIds = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_IDS)]); + auto positionIds = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::POSITION_IDS)]); + auto hiddenStatesIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::HIDDEN_STATES_INDICES)]); + auto lastTokenIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::LAST_TOKEN_INDICES)]); + auto numLastTokenIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::NUM_LAST_TOKEN_INDICES)]); + auto hiddenSizeBatchLevelStarts + = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)]); + + auto inputIds = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::INPUT_IDS)]); + auto chunkedContextNextTokens + = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::CHUNKED_CONTEXT_NEXT_TOKENS)]); + auto baseNetSequenceLengths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)]); + auto baseNetContextLengths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::CONTEXT_LENGTHS)]); + auto acceptedTokens = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::ACCEPTED_TOKENS)]); + auto acceptedLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::ACCEPTED_LENS)]); + auto prevDraftLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::PREV_DRAFT_LENS)]); + auto prevPaths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::PREV_DRAFT_PATHS)]); + auto bestPathIds = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::ACCEPTED_PATHS)]); + + auto const numOutputTokens = (numTokens - numInputGenTokens) + (numGenRequests * (mNumLayers + 1)); + cudaMemsetAsync(positionIds, 0, numOutputTokens * sizeof(SizeType32), stream); + cudaMemsetAsync(hiddenStatesIndices, 0, numOutputTokens * sizeof(SizeType32), stream); + + invokePrepareCtxEagleNetInputs(eagleNetSequenceLengths, eagleNetContextLengths, outputIds, positionIds, + hiddenStatesIndices, lastTokenIndices, numLastTokenIndices, hiddenSizeBatchLevelStarts, inputIds, + chunkedContextNextTokens, baseNetSequenceLengths, baseNetContextLengths, acceptedTokens, acceptedLens, + prevDraftLens, prevPaths, bestPathIds, batchSize, maxPathLen, maxDecodingTokens, mMaxNonLeavesPerLayer, stream); + + sync_check_cuda_error(stream); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void EaglePrepareDrafterInputsPlugin::prepareGenEagleNetData(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const batchSize = inputDesc[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)].dims.d[0]; + auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].dims.d[1]; + auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)].dims.d[2]; + + auto eagleNetSequenceLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SEQUENCE_LENGTHS)]); + auto eagleNetContextLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::CONTEXT_LENGTHS)]); + auto outputIds = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::OUTPUT_IDS)]); + auto positionIds = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::POSITION_IDS)]); + auto specDecodingGenLengths + = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SPEC_DECODING_GENERATION_LENGTHS)]); + auto specDecodingPositionOffsets + = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SPEC_DECODING_POSITION_OFFSETS)]); + auto specDecodingPackedMasks + = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::SPEC_DECODING_PACKED_MASK)]); + auto hiddenStatesIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::HIDDEN_STATES_INDICES)]); + auto lastTokenIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::LAST_TOKEN_INDICES)]); + auto numLastTokenIndices = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::NUM_LAST_TOKEN_INDICES)]); + auto outputHiddenSizeBatchStartsPerLevel + = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)]); + + auto eagleNet0SequenceLengths + = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::SEQUENCE_LENGTHS)]); + auto eagleNet0ContextLength = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::CONTEXT_LENGTHS)]); + auto nextDraftPaths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::NEXT_DRAFT_PATHS)]); + auto nextDraftIds = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::NEXT_DRAFT_TOKENS)]); + auto inputHiddenSizeBatchStartsPerLevel + = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)]); + + int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); + size_t offset{0}; + + int8_t* isLeafMask = reinterpret_cast<int8_t*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(int8_t))); + TokenIdType* selectedDraftIndices = reinterpret_cast<TokenIdType*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(TokenIdType))); + SizeType32* selectedDraftPosOffsets = reinterpret_cast<SizeType32*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(SizeType32))); + SizeType32* numSelectedDraftIndices + = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); + bool* selectedMasks = reinterpret_cast<bool*>(tc::nextWorkspacePtr( + workspaceBytePtr, offset, batchSize * maxDecodingTokens * maxDecodingTokens * sizeof(int8_t))); + SizeType32* cumSumGenerationLengths = reinterpret_cast<SizeType32*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, (batchSize + 1) * sizeof(SizeType32))); + SizeType32* nonLeavesInLevelOffsets = reinterpret_cast<SizeType32*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(SizeType32))); + SizeType32* parentNonLeafInLevelOffset = reinterpret_cast<SizeType32*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(SizeType32))); + SizeType32* maxGenerationLength + = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, 1 * sizeof(SizeType32))); + + cudaMemsetAsync(hiddenStatesIndices, 0, batchSize * mMaxNonLeavesPerLayer * mLayerIdx * sizeof(SizeType32), stream); + cudaMemsetAsync(selectedMasks, 0, batchSize * maxDecodingTokens * maxDecodingTokens * sizeof(int8_t), stream); + // Prefill mask setting all to leaves. + cudaMemsetAsync(isLeafMask, 1, batchSize * maxDecodingTokens * sizeof(int8_t), stream); + + PrepareGenEagleNetInputsParams params; + params.nextSequenceLengths = eagleNetSequenceLengths; + params.nextContextLengths = eagleNetContextLengths; + params.outputIds = outputIds; + params.positionIds = positionIds; + params.specDecodingGenLengths = specDecodingGenLengths; + params.specDecodingPositionOffsets = specDecodingPositionOffsets; + params.specDecodingPackedMasks = specDecodingPackedMasks; + params.hiddenStatesIndices = hiddenStatesIndices; + params.lastTokenIndices = lastTokenIndices; + params.numLastTokenIndices = numLastTokenIndices; + params.outputHiddenSizeBatchStartsPerLevel = outputHiddenSizeBatchStartsPerLevel; + + // tmp data + params.isLeafMask = isLeafMask; + params.selectedDraftIndices = selectedDraftIndices; + params.selectedDraftPosOffsets = selectedDraftPosOffsets; + params.numSelectedDraftIndices = numSelectedDraftIndices; + params.selectedMasks = selectedMasks; + params.cumSumGenerationLengths = cumSumGenerationLengths; + params.maxGenerationLength = maxGenerationLength; + params.nonLeavesInLevelOffsets = nonLeavesInLevelOffsets; + params.parentNonLeafInLevelOffset = parentNonLeafInLevelOffset; + + params.nextDraftIds = nextDraftIds; + params.eagleNet0SequenceLengths = eagleNet0SequenceLengths; + params.prevContextLengths = eagleNet0ContextLength; + params.nextPaths = nextDraftPaths; + params.inputHiddenSizeBatchStartsPerLevel = inputHiddenSizeBatchStartsPerLevel; + params.levelIdx = mLayerIdx; + params.batchSize = batchSize; + params.maxPathLen = maxPathLen; + params.maxDecodingTokens = maxDecodingTokens; + params.maxNonLeavesPerLayer = mMaxNonLeavesPerLayer; + params.stream = stream; + + params.checkParams(); + + invokePrepareGenEagleNetInputs(params); + + sync_check_cuda_error(stream); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +int EaglePrepareDrafterInputsPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // First EagleNet instance (EagleNet0) is always chunked context attn, + // where we process either context tokens or newly accepted tokens and append them to EagleNet KV cache. + + // For all following EagleNetX (X > 0) instances there is need for masked spec decoding attn. + // Ideally with mask for context. + // Let's say we have prompt ABCD and two variants of tokens spec decoding tokens E and F + // predicted by EagleNet0. If we draw full attn mask, it becomes: + // |A|B|C|D|E|F + // E|1|1|1|1|1|0 + // F|1|1|1|1|0|1 + // + // In the next step we predict token G from ABCDE branch and token H from ABCDF branch -- like beam search. + // And we'd need spec decoding mask that includes kv cache: + // |A|B|C|D|E|F|G|H + // G|1|1|1|1|1|0|1|0 + // H|1|1|1|1|0|1|0|1 + // + // But TRT-LLM does not support such mask for now. We can only provide + // |G|H + // G|1|0 + // H|0|1 + // , which is wrong mask. + // + // For now we WAR this by passing EFGH for the EagleNet1 with right mask + // and using only G and H logits for sampling, but that's redundant compute: + // |E|F|G|H + // E|1|0|0|0 + // F|0|1|0|0 + // G|1|0|1|0 + // H|0|1|0|1 + + if (mLayerIdx == 0) + { + prepareCtxEagleNetData(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else + { + prepareGenEagleNetData(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + + return 0; +} + +/////////////// + +EaglePrepareDrafterInputsPluginCreator::EaglePrepareDrafterInputsPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("layer_idx", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("num_layers", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("max_non_leaves_per_layer", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* EaglePrepareDrafterInputsPluginCreator::getPluginName() const noexcept +{ + return EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_NAME; +} + +char const* EaglePrepareDrafterInputsPluginCreator::getPluginVersion() const noexcept +{ + return EAGLE_PREPARE_DRAFTER_INPUTS_PLUGIN_VERSION; +} + +PluginFieldCollection const* EaglePrepareDrafterInputsPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +nvinfer1::IPluginV3* EaglePrepareDrafterInputsPluginCreator::createPlugin( + char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept +{ + try + { + int32_t layerIdx{0}; + int32_t numLayers{0}; + int32_t maxNonLeavesPerLayer{0}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fc->fields[i].name; + if (!strcmp(attrName, "layer_idx")) + { + TLLM_CHECK(fc->fields[i].type == PluginFieldType::kINT32); + layerIdx = *static_cast<int32_t const*>(fc->fields[i].data); + } + else if (!strcmp(attrName, "num_layers")) + { + TLLM_CHECK(fc->fields[i].type == PluginFieldType::kINT32); + numLayers = *static_cast<int32_t const*>(fc->fields[i].data); + } + else if (!strcmp(attrName, "max_non_leaves_per_layer")) + { + TLLM_CHECK(fc->fields[i].type == PluginFieldType::kINT32); + maxNonLeavesPerLayer = *static_cast<int32_t const*>(fc->fields[i].data); + } + } + return new EaglePrepareDrafterInputsPlugin(layerIdx, numLayers, maxNonLeavesPerLayer); + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +char const* EaglePrepareDrafterInputsPluginCreator::getPluginNamespace() const noexcept +{ + return tensorrt_llm::plugins::api::kDefaultNamespace; +} diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.h b/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.h new file mode 100644 index 000000000000..0059c46f6c8d --- /dev/null +++ b/cpp/tensorrt_llm/plugins/eaglePlugin/eaglePrepareDrafterInputsPlugin.h @@ -0,0 +1,186 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class EaglePrepareDrafterInputsPlugin : public nvinfer1::IPluginV3, + public nvinfer1::IPluginV3OneCore, + public nvinfer1::IPluginV3OneBuild, + public nvinfer1::IPluginV3OneRuntime +{ +public: + EaglePrepareDrafterInputsPlugin(EaglePrepareDrafterInputsPlugin const& p) = default; + + EaglePrepareDrafterInputsPlugin(int32_t layerIdx, int32_t numLayers, int32_t maxNonLeavesPerLayer); + + nvinfer1::IPluginV3* clone() noexcept override; + + nvinfer1::IPluginCapability* getCapabilityInterface(nvinfer1::PluginCapabilityType type) noexcept override; + + void initFieldsToSerialize(); + + char const* getPluginName() const noexcept override; + char const* getPluginVersion() const noexcept override; + char const* getPluginNamespace() const noexcept override; + + int32_t getNbOutputs() const noexcept override; + + bool supportsFormatCombination( + int pos, nvinfer1::DynamicPluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept override; + int32_t configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int32_t nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept override; + + int32_t getOutputDataTypes(nvinfer1::DataType* outputTypes, int32_t nbOutputs, nvinfer1::DataType const* inputTypes, + int32_t nbInputs) const noexcept override; + + int32_t getOutputShapes(nvinfer1::DimsExprs const* inputs, int32_t nbInputs, nvinfer1::DimsExprs const* shapeInputs, + int32_t nbShapeInputs, nvinfer1::DimsExprs* outputs, int32_t nbOutputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + + int32_t onShapeChange(nvinfer1::PluginTensorDesc const* in, int32_t nbInputs, nvinfer1::PluginTensorDesc const* out, + int32_t nbOutputs) noexcept override; + + nvinfer1::IPluginV3* attachToContext(nvinfer1::IPluginResourceContext* context) noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldsToSerialize() noexcept override; + + size_t getWorkspaceSize(nvinfer1::DynamicPluginTensorDesc const* inputs, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + +private: + enum class InputIdxEntry : int32_t + { + //! [batch_size] + SEQUENCE_LENGTHS = 0, + //! [batch_size] + CONTEXT_LENGTHS, + //! [num_tokens] + INPUT_IDS, + //! [batch_size] + CHUNKED_CONTEXT_NEXT_TOKENS, + //! [batch_size, max_path_len] + ACCEPTED_TOKENS, + //! [batch_size] + ACCEPTED_LENS, + //! [batch_size] + ACCEPTED_PATHS, + //! [batch_size, max_decoding_draft_tokens] + NEXT_DRAFT_TOKENS, + //! [batch_size] + NEXT_DRAFT_LENS, + //! [batch_size, max_decoding_tokens, max_path_len] + NEXT_DRAFT_PATHS, + //! [batch_size] + PREV_DRAFT_LENS, + //! [batch_size, max_decoding_tokens, max_path_len] + PREV_DRAFT_PATHS, + //! [(max_path_len - 1) * batch_size + 1] + HIDDEN_SIZE_BATCH_LEVEL_STARTS, + //! [num_gen_tokens] + INPUT_GEN_TOKENS, + //! [num_gen_requests] + SPEC_DECODING_GENERATION_LENGTHS, + }; + + enum class OutputIdxEntry : int32_t + { + //! [batch_size] + SEQUENCE_LENGTHS = 0, + //! [batch_size] + CONTEXT_LENGTHS, + //! [batch_size] + SPEC_DECODING_GENERATION_LENGTHS, + //! [batch_size, max_decoding_tokens] + SPEC_DECODING_POSITION_OFFSETS, + //! [batchSize, maxDecodingTokens, ceil(maxDecodingTokens / 32)] + SPEC_DECODING_PACKED_MASK, + //! [batchSize * mMaxNonLeavesPerLayer * layerIdx] for layerIdx > 0 + //! [num_tokens - numGenTokens + numGenRequests * (mNumLayers + 1)] for layerIdx == 0 + OUTPUT_IDS, + //! [batchSize] for layerIdx > 0 + //! [num_tokens - numGenTokens + numGenRequests * (mNumLayers + 1)] for layerIdx == 0 + POSITION_IDS, + //! [batchSize * mMaxNonLeavesPerLayer * layerIdx] for layerIdx > 0 + //! [num_tokens - numGenTokens + numGenRequests * (mNumLayers + 1)] for layerIdx == 0 + HIDDEN_STATES_INDICES, + //! [batchSize * mMaxNonLeavesPerLayer] + LAST_TOKEN_INDICES, + //! [1] + NUM_LAST_TOKEN_INDICES, + //! [(max_path_len - 1) * batch_size + 1] + HIDDEN_SIZE_BATCH_LEVEL_STARTS, + }; + + int32_t getIdx(InputIdxEntry idx) const + { + return static_cast<int32_t>(idx); + } + + int32_t getIdx(OutputIdxEntry idx) const + { + return static_cast<int32_t>(idx); + } + +private: + void prepareCtxEagleNetData(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept; + + void prepareGenEagleNetData(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept; + +private: + int32_t mLayerIdx{0}; + int32_t mNumLayers{0}; + int32_t mMaxNonLeavesPerLayer{0}; + std::vector<nvinfer1::PluginField> mDataToSerialize; + nvinfer1::PluginFieldCollection mFCToSerialize; +}; + +class EaglePrepareDrafterInputsPluginCreator : public nvinfer1::IPluginCreatorV3One +{ +public: + EaglePrepareDrafterInputsPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + char const* getPluginNamespace() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV3* createPlugin( + char const* name, nvinfer1::PluginFieldCollection const* fc, nvinfer1::TensorRTPhase phase) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.cpp b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.cpp new file mode 100644 index 000000000000..5fb30f583712 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.cpp @@ -0,0 +1,565 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "eagleSampleAndAcceptDraftTokensPlugin.h" + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/kernels/samplingTopKKernels.h" +#include "tensorrt_llm/kernels/speculativeDecoding/common.h" +#include "tensorrt_llm/kernels/speculativeDecoding/eagleDecodingKernels.h" +#include "tensorrt_llm/kernels/speculativeDecoding/medusaDecodingKernels.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iTensor.h" + +using namespace nvinfer1; +using tensorrt_llm::plugins::EagleSampleAndAcceptDraftTokensPluginCreator; +using tensorrt_llm::plugins::EagleSampleAndAcceptDraftTokensPlugin; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::kernels::speculative_decoding; +using namespace tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; + +static char const* EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_VERSION{"1"}; +static char const* EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_NAME{"EagleSampleAndAcceptDraftTokens"}; +PluginFieldCollection EagleSampleAndAcceptDraftTokensPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> EagleSampleAndAcceptDraftTokensPluginCreator::mPluginAttributes; + +EagleSampleAndAcceptDraftTokensPlugin::EagleSampleAndAcceptDraftTokensPlugin(nvinfer1::DataType type) + : mDtype(type) +{ +} + +// Parameterized constructor +EagleSampleAndAcceptDraftTokensPlugin::EagleSampleAndAcceptDraftTokensPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mDtype); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* EagleSampleAndAcceptDraftTokensPlugin::clone() const noexcept +{ + auto* plugin = new EagleSampleAndAcceptDraftTokensPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs EagleSampleAndAcceptDraftTokensPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + TLLM_CHECK(nbInputs == 10); + TLLM_CHECK(outputIndex < 7); + auto const batchSizeExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[0]; + auto const maxDecodingDraftTokensExpr = inputs[getIdx(InputIdxEntry::DRAFT_TOKEN_IDS)].d[1]; + auto const maxDecodingTokensExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[1]; + auto const maxPathLenExpr = inputs[getIdx(InputIdxEntry::PATHS)].d[2]; + + nvinfer1::DimsExprs ret; + if (outputIndex == getIdx(OutputIdxEntry::ACCEPTED_TOKENS)) + { + ret.nbDims = 2; + ret.d[0] = batchSizeExpr; + ret.d[1] = maxPathLenExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::ACCEPTED_LENS)) + { + ret.nbDims = 1; + ret.d[0] = batchSizeExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::BEST_ACCEPTED_PATHS)) + { + ret.nbDims = 1; + ret.d[0] = batchSizeExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::NEXT_DRAFT_TOKEN_IDS)) + { + ret.nbDims = 2; + ret.d[0] = batchSizeExpr; + ret.d[1] = maxDecodingDraftTokensExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::NEXT_DRAFT_LENS)) + { + ret.nbDims = 1; + ret.d[0] = batchSizeExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::NEXT_DRAFT_PATHS)) + { + ret.nbDims = 3; + ret.d[0] = batchSizeExpr; + ret.d[1] = maxDecodingTokensExpr; + ret.d[2] = maxPathLenExpr; + } + else if (outputIndex == getIdx(OutputIdxEntry::HIDDEN_SIZE_BATCH_LEVEL_STARTS)) + { + ret.nbDims = 1; + ret.d[0] = exprBuilder.operation(DimensionOperation::kSUM, *exprBuilder.constant(1), + *exprBuilder.operation(DimensionOperation::kPROD, + *exprBuilder.operation(DimensionOperation::kSUB, *maxPathLenExpr, *exprBuilder.constant(1)), + *batchSizeExpr)); + } + return ret; +} + +bool EagleSampleAndAcceptDraftTokensPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + if (pos == getIdx(InputIdxEntry::LOGITS)) // logits + { + return (inOut[pos].type == mDtype) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == getIdx(InputIdxEntry::TEMPERATURE) || pos == getIdx(InputIdxEntry::RAND_VALIDATION) + || pos == getIdx(InputIdxEntry::POSTERIOR_ALPHA) + || pos == getIdx(InputIdxEntry::POSTERIOR_THRESHOLD)) // temperature, rand_validation + { + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else // everything else + { + return (inOut[pos].type == nvinfer1::DataType::kINT32) && (inOut[pos].format == TensorFormat::kLINEAR); + } +} + +void EagleSampleAndAcceptDraftTokensPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +template <typename T> +size_t EagleSampleAndAcceptDraftTokensPlugin::getWorkspaceSizeType(nvinfer1::PluginTensorDesc const* inputs, + int nbInputs, nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + size_t workspaceSize{0}; + + auto const vocabSizePadded = inputs[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; + auto const batchSize = inputs[getIdx(InputIdxEntry::PATHS)].dims.d[0]; + auto const maxDecodingTokens = inputs[getIdx(InputIdxEntry::PATHS)].dims.d[1]; + + // Greedy sampling + // Top1 sampling workspace + auto const greedySamplingWorkspaceSize + = getTopKWorkspaceSize<T>(batchSize, maxDecodingTokens, /* maxTopK */ 1, vocabSizePadded); + + // Multinomial sampling + auto const typicalSamplingWorkspaceSize + = getTypicalAcceptanceWorkspaceSize<T>(batchSize, maxDecodingTokens, vocabSizePadded); + + auto const primarySamplingWorkspaceSize = std::max(greedySamplingWorkspaceSize, typicalSamplingWorkspaceSize); + + // Target output ids + auto const targetOutputIdsSize = batchSize * maxDecodingTokens * sizeof(TokenIdType); + // Logits ptrs + auto const logitsPtrsSize = batchSize * maxDecodingTokens * sizeof(T*); + SizeType32 constexpr NUM_BUFFERS{4}; + size_t workspaces[NUM_BUFFERS]; + workspaces[0] = targetOutputIdsSize; + workspaces[1] = primarySamplingWorkspaceSize; + workspaces[2] = logitsPtrsSize; + workspaces[3] = batchSize * sizeof(SizeType32); + workspaceSize = tc::calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); + + return workspaceSize; +} + +size_t EagleSampleAndAcceptDraftTokensPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + auto const logitsType = inputs[getIdx(InputIdxEntry::LOGITS)].type; + if (logitsType == nvinfer1::DataType::kFLOAT) + { + return getWorkspaceSizeType<float>(inputs, nbInputs, outputs, nbOutputs); + } + else if (logitsType == nvinfer1::DataType::kHALF) + { + return getWorkspaceSizeType<__half>(inputs, nbInputs, outputs, nbOutputs); + } + else + { + TLLM_CHECK_WITH_INFO(false, "Unsupported logits type"); + } + return 0; +} + +template <typename T> +void EagleSampleAndAcceptDraftTokensPlugin::samplePrimeHeadTokens(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // auto const maxNumTokens = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; + auto const vocabSizePadded = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; + auto const batchSize = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[0]; + auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[1]; + + auto logits = static_cast<T const*>(inputs[getIdx(InputIdxEntry::LOGITS)]); + auto prevDraftLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::DRAFT_LENS)]); + + int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); + size_t offset{0}; + + auto const samplingWorkspaceSize + = getTopKWorkspaceSize<T>(batchSize, maxDecodingTokens, /* maxTopK */ 1, vocabSizePadded); + + TokenIdType* outputIds = reinterpret_cast<TokenIdType*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(TokenIdType))); + void* workspaceSampling + = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, samplingWorkspaceSize)); + T const** logitsPtrs = reinterpret_cast<T const**>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(T*))); + SizeType32* decodingTokens + = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); + + // Assemble pointers to logits + invokeAssembleTargetLogitsOffsets( + logitsPtrs, decodingTokens, logits, prevDraftLens, batchSize, maxDecodingTokens, vocabSizePadded, stream); + + sync_check_cuda_error(stream); + + TopKSamplingKernelParams<T> params; + params.logProbsPtrs = logitsPtrs; + params.outputIds = outputIds; + params.workspace = workspaceSampling; + params.maxTopK = 1; + params.batchSize = batchSize; + params.maxBatchSize = batchSize; + params.tokensPerStep = decodingTokens; + params.maxTokensPerStep = maxDecodingTokens; + params.maxSeqLen = maxDecodingTokens; + params.vocabSizePadded = vocabSizePadded; + + invokeBatchTopKSampling(params, stream); + + sync_check_cuda_error(stream); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +template <typename T> +void EagleSampleAndAcceptDraftTokensPlugin::doTypicalAcceptance(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // auto const maxNumTokens = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; + auto const vocabSizePadded = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; + + auto const batchSize = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[0]; + auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[1]; + // auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[2]; + // auto const maxDraftPathLen = maxPathLen - 1; + + auto logits = static_cast<T const*>(inputs[getIdx(InputIdxEntry::LOGITS)]); + auto prevDraftLens = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::DRAFT_LENS)]); + + int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); + size_t offset{0}; + + // Multinomial sampling + auto const primarySamplingWorkspaceSize + = getTypicalAcceptanceWorkspaceSize<T>(batchSize, maxDecodingTokens, vocabSizePadded); + + TokenIdType* outputIds = reinterpret_cast<TokenIdType*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(TokenIdType))); + void* workspaceSampling + = reinterpret_cast<void*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, primarySamplingWorkspaceSize)); + T** logitsPtrs = reinterpret_cast<T**>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(T*))); + SizeType32* decodingTokens + = reinterpret_cast<SizeType32*>(tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * sizeof(SizeType32))); + + // Assemble pointers to logits + invokeAssembleTargetLogitsOffsets(const_cast<T const**>(logitsPtrs), decodingTokens, logits, prevDraftLens, + batchSize, maxDecodingTokens, vocabSizePadded, stream); + + sync_check_cuda_error(stream); + + TypicalAcceptanceSampling<T> params; + params.logitsPtrs = logitsPtrs; + params.generationLengths = decodingTokens; + params.temperatures = reinterpret_cast<float const*>(inputs[getIdx(InputIdxEntry::TEMPERATURE)]); + params.posteriorThresholds = reinterpret_cast<float const*>(inputs[getIdx(InputIdxEntry::POSTERIOR_THRESHOLD)]); + params.posteriorAlphas = reinterpret_cast<float const*>(inputs[getIdx(InputIdxEntry::POSTERIOR_ALPHA)]); + params.outputIds = outputIds; + params.workspace = reinterpret_cast<int8_t*>(workspaceSampling); + params.randomVals = reinterpret_cast<float const*>(inputs[getIdx(InputIdxEntry::RAND_VALIDATION)]); + + params.batchSize = batchSize; + params.maxBatchSize = batchSize; + params.maxDecodingTokens = maxDecodingTokens; + params.vocabSize = vocabSizePadded; + + if (mSmCnt <= 0) + { + auto const deviceId = tensorrt_llm::common::getDevice(); + cudaDeviceProp prop{}; + TLLM_CUDA_CHECK(cudaGetDeviceProperties(&prop, deviceId)); + mSmCnt = prop.multiProcessorCount; + } + params.smCnt = mSmCnt; + + params.checkParams(); + + typicalAcceptanceSampling(params, stream); + + sync_check_cuda_error(stream); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +template <typename T> +void EagleSampleAndAcceptDraftTokensPlugin::acceptDraftTokens(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + // auto const maxNumTokens = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[0]; + auto const vocabSizePadded = inputDesc[getIdx(InputIdxEntry::LOGITS)].dims.d[1]; + + auto const batchSize = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[0]; + auto const maxDecodingTokens = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[1]; + auto const maxPathLen = inputDesc[getIdx(InputIdxEntry::PATHS)].dims.d[2]; + auto const maxDraftPathLen = maxPathLen - 1; + + auto const useDynamicTree = *(reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::USE_DYNAMIC_TREE)])); + + int8_t* workspaceBytePtr = reinterpret_cast<int8_t*>(workspace); + size_t offset{0}; + + // auto const samplingWorkspaceSize + // = getTopKWorkspaceSize<T>(batchSize, maxDecodingTokens, /* maxTopK */ 1, vocabSizePadded); + + TokenIdType* outputIds = reinterpret_cast<TokenIdType*>( + tc::nextWorkspacePtr(workspaceBytePtr, offset, batchSize * maxDecodingTokens * sizeof(TokenIdType))); + + AcceptDraftTokensByIdsWithPathsParams<T> params; + params.outputIds = reinterpret_cast<TokenIdType*>(outputs[getIdx(OutputIdxEntry::ACCEPTED_TOKENS)]); + params.draftIds = reinterpret_cast<TokenIdType const*>(inputs[getIdx(InputIdxEntry::DRAFT_TOKEN_IDS)]); + params.targetIds = outputIds; + params.acceptedLengths = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::ACCEPTED_LENS)]); + params.paths = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::PATHS)]); + params.bestPathIds = reinterpret_cast<SizeType32*>(outputs[getIdx(OutputIdxEntry::BEST_ACCEPTED_PATHS)]); + params.batchSize = batchSize; + params.maxBatchSize = batchSize; + params.vocabSize = vocabSizePadded; + params.maxSeqLen = maxPathLen; + params.maxDraftPathLen = maxDraftPathLen; + params.maxDecodingTokens = maxDecodingTokens; + params.stream = stream; + + params.checkParams(); + + acceptDraftTokensByIdsWithPaths(params); + + if (useDynamicTree) + { + // For Eagle-2, after verification and acceptance, the original path becomes useless. + // All set to '-1' + cudaMemsetAsync(outputs[getIdx(OutputIdxEntry::NEXT_DRAFT_PATHS)], -1, + batchSize * maxDecodingTokens * maxPathLen * sizeof(SizeType32), stream); + } + else + { + // For Eagle-1 + // Copy input paths to the output + cudaMemcpyAsync(outputs[getIdx(OutputIdxEntry::NEXT_DRAFT_PATHS)], inputs[getIdx(InputIdxEntry::PATHS)], + batchSize * maxDecodingTokens * maxPathLen * sizeof(SizeType32), cudaMemcpyDeviceToDevice, stream); + } + + sync_check_cuda_error(stream); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +template <typename T> +void EagleSampleAndAcceptDraftTokensPlugin::enqueueType(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const greedySampling = reinterpret_cast<SizeType32 const*>(inputs[getIdx(InputIdxEntry::GREEDY_SAMPLING)])[0]; + // TODO split batch into greedy and non-greedy and execute both paths + if (greedySampling) + { + // Sample all main head tokens with Top-1. + samplePrimeHeadTokens<T>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else + { + // Typical sampling for typical acceptance. + doTypicalAcceptance<T>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + + // Accept tokens based on token ids, write the best path and best token id. + acceptDraftTokens<T>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +int EagleSampleAndAcceptDraftTokensPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + auto const logitsType = inputDesc[getIdx(InputIdxEntry::LOGITS)].type; + if (logitsType == nvinfer1::DataType::kFLOAT) + { + enqueueType<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (logitsType == nvinfer1::DataType::kHALF) + { + enqueueType<__half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else + { + TLLM_CHECK_WITH_INFO(false, "Unsupported logits type"); + } + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType EagleSampleAndAcceptDraftTokensPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index < 7); + // input 1 is draft tokens now of int32 type. All outputs are int32_t as well. + return inputTypes[getIdx(InputIdxEntry::DRAFT_TOKEN_IDS)]; +} + +// IPluginV2 Methods + +char const* EagleSampleAndAcceptDraftTokensPlugin::getPluginType() const noexcept +{ + return EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_NAME; +} + +char const* EagleSampleAndAcceptDraftTokensPlugin::getPluginVersion() const noexcept +{ + return EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_VERSION; +} + +int EagleSampleAndAcceptDraftTokensPlugin::getNbOutputs() const noexcept +{ + return 7; +} + +int EagleSampleAndAcceptDraftTokensPlugin::initialize() noexcept +{ + return 0; +} + +void EagleSampleAndAcceptDraftTokensPlugin::terminate() noexcept {} + +size_t EagleSampleAndAcceptDraftTokensPlugin::getSerializationSize() const noexcept +{ + return sizeof(mDtype); +} + +void EagleSampleAndAcceptDraftTokensPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mDtype); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void EagleSampleAndAcceptDraftTokensPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +EagleSampleAndAcceptDraftTokensPluginCreator::EagleSampleAndAcceptDraftTokensPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* EagleSampleAndAcceptDraftTokensPluginCreator::getPluginName() const noexcept +{ + return EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_NAME; +} + +char const* EagleSampleAndAcceptDraftTokensPluginCreator::getPluginVersion() const noexcept +{ + return EAGLE_SAMPLE_AND_ACCEPT_DRAFT_TOKENS_PLUGIN_VERSION; +} + +PluginFieldCollection const* EagleSampleAndAcceptDraftTokensPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* EagleSampleAndAcceptDraftTokensPluginCreator::createPlugin( + char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + + try + { + auto* obj = new EagleSampleAndAcceptDraftTokensPlugin(type); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* EagleSampleAndAcceptDraftTokensPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call EagleSampleAndAcceptDraftTokensPlugin::destroy() + try + { + auto* obj = new EagleSampleAndAcceptDraftTokensPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.h b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.h new file mode 100644 index 000000000000..3b14bab83170 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/eaglePlugin/eagleSampleAndAcceptDraftTokensPlugin.h @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" + +#include <cassert> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class EagleSampleAndAcceptDraftTokensPlugin : public BasePlugin +{ +public: + EagleSampleAndAcceptDraftTokensPlugin(nvinfer1::DataType type); + + EagleSampleAndAcceptDraftTokensPlugin(void const* data, size_t length); + + ~EagleSampleAndAcceptDraftTokensPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + enum class InputIdxEntry : int32_t + { + //! [num_tokens, vocab_size_padded] + LOGITS = 0, + //! [batch_size, max_decoding_draft_tokens] + DRAFT_TOKEN_IDS, + //! [batch_size] + DRAFT_LENS, + //! [batch_size] + TEMPERATURE, + //! [batch_size, max_decoding_tokens] + RAND_VALIDATION, + //! [batch_size] + POSTERIOR_ALPHA, + //! [batch_size] + POSTERIOR_THRESHOLD, + //! [batch_size, max_decoding_tokens, max_path_len] + PATHS, + //! [1] + GREEDY_SAMPLING, + //! [1] + USE_DYNAMIC_TREE + }; + + enum class OutputIdxEntry : int32_t + { + //! [batch_size, max_path_len] + ACCEPTED_TOKENS = 0, + //! [batch_size] + ACCEPTED_LENS, + //! [batch_size] + BEST_ACCEPTED_PATHS, + //! [batch_size, max_decoding_draft_tokens] + NEXT_DRAFT_TOKEN_IDS, + //! [batch_size] + NEXT_DRAFT_LENS, + //! [batch_size, max_decoding_tokens, max_path_len] + NEXT_DRAFT_PATHS, + //! [max_draft_path_len * batch_size] + HIDDEN_SIZE_BATCH_LEVEL_STARTS, + }; + + int32_t getIdx(InputIdxEntry idx) const + { + return static_cast<int32_t>(idx); + } + + int32_t getIdx(OutputIdxEntry idx) const + { + return static_cast<int32_t>(idx); + } + +private: + template <typename T> + size_t getWorkspaceSizeType(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept; + + template <typename T> + void samplePrimeHeadTokens(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept; + + template <typename T> + void doTypicalAcceptance(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; + + template <typename T> + void acceptDraftTokens(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; + + template <typename T> + void enqueueType(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept; + +private: + nvinfer1::DataType mDtype; + int32_t mSmCnt{0}; +}; + +class EagleSampleAndAcceptDraftTokensPluginCreator : public BaseCreator +{ +public: + EagleSampleAndAcceptDraftTokensPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/exports.def b/cpp/tensorrt_llm/plugins/exports.def new file mode 100644 index 000000000000..5d4ac9e3e793 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/exports.def @@ -0,0 +1,19 @@ +; SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +; SPDX-License-Identifier: Apache-2.0 +; +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. + +LIBRARY nvinfer_plugin_tensorrt_llm +EXPORTS +getPluginRegistry +initLibNvInferPlugins diff --git a/cpp/tensorrt_llm/plugins/exports.map b/cpp/tensorrt_llm/plugins/exports.map new file mode 100644 index 000000000000..c6c949775079 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/exports.map @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Hides all symbols except those specified in the global section */ +{ + global: + initTrtLlmPlugins; + setLoggerFinder; + getPluginCreators; + getCreators; + extern "C++" { + nvinfer1::IPluginCreator::*; + nvinfer1::IPluginV2Ext::*; + nvinfer1::IPluginV2IOExt::*; + nvinfer1::PluginRegistrar*; + tensorrt_llm::plugins::api::*; + tensorrt_llm::plugins::*; + }; + local: *; +}; diff --git a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.cpp b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.cpp new file mode 100644 index 000000000000..05f06ae38feb --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.cpp @@ -0,0 +1,434 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <numeric> + +#include "fp4GemmPlugin.h" +#include "tensorrt_llm/common/assert.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::Fp4GemmPluginCreator; +using tensorrt_llm::plugins::Fp4GemmPlugin; +using tensorrt_llm::plugins::Fp4GemmPluginProfiler; +#if defined(USING_OSS_CUTLASS_FP4_GEMM) +using namespace tensorrt_llm::kernels::cutlass_kernels; +#else +using namespace tensorrt_llm::kernels::internal_cutlass_kernels; +#endif + +constexpr nvinfer1::DataType FP4_DTYPE = nvinfer1::DataType::kFP4; +constexpr nvinfer1::DataType FP8_DTYPE = nvinfer1::DataType::kFP8; + +static char const* FP4_GEMM_PLUGIN_VERSION{"1"}; +static char const* FP4_GEMM_PLUGIN_NAME{"Fp4Gemm"}; +PluginFieldCollection Fp4GemmPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> Fp4GemmPluginCreator::mPluginAttributes; + +void Fp4GemmPluginProfiler::runTactic( + int m, int n, int k, Fp4GemmPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) +{ + // Workspace size required by gemm runner + // NB: this function will throw exception when selected tactic exceeds SMEM, which is then + // caught by gemmPluginProfiler and it will register this tactic as invalid + size_t wsSizeRunner = mRunner->getWorkspaceSize(m, n, k, /* batch_count */ 1); + + // Workspace size required by profiling + size_t wsByteOffset = 0; + int8_t* wsBytePointer = reinterpret_cast<int8_t*>(workspace); + void* aTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, (m * k) / 2)); + void* bTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, (n * k) / 2)); + void* dTmp = reinterpret_cast<void*>( + nextWorkspacePtr(wsBytePointer, wsByteOffset, m * n * (mType == nvinfer1::DataType::kFLOAT ? 4u : 2u))); + // SF M/N is padded along 128 and K is padded along 4. + int vector_size = 16; + int sf_round_m = ((m + 127) / 128) * 128; + int sf_round_n = ((n + 127) / 128) * 128; + int sf_round_k = ((k / vector_size + 3) / 4) * 4; + float* a_sf = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, sf_round_m * sf_round_k)); + float* b_sf = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, sf_round_n * sf_round_k)); + float* global_sf = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, sizeof(float))); + char* workspaceTmp = reinterpret_cast<char*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, wsSizeRunner)); + + // Run profiling + mRunner->gemm(dTmp, aTmp, bTmp, a_sf, b_sf, global_sf, m, n, k, /* batch_count */ 1, tactic, workspaceTmp, + wsSizeRunner, stream); + sync_check_cuda_error(stream); +} + +void Fp4GemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + size_t vector_size = 16; + size_t sf_round_m = ((maxM + 127) / 128) * 128; + size_t sf_round_n = ((n + 127) / 128) * 128; + size_t sf_round_k = ((k / vector_size + 3) / 4) * 4; + std::vector<size_t> workspaces = { + (size_t) (maxM * k / 2), // A + (size_t) (n * k / 2), // B + maxM * n * (mType == nvinfer1::DataType::kFLOAT ? 4u : 2u), // D + (size_t) (sf_round_m * sf_round_k), // A_SF + (size_t) (sf_round_n * sf_round_k), // B_SF + sizeof(float), // Global_SF + mRunner->getWorkspaceSize(maxM, n, k, /* batch_count */ 1) // workspace + }; + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<Fp4GemmPluginProfiler::Config> Fp4GemmPluginProfiler::getTactics(int m, int n, int k) const +{ + return mRunner->getConfigs(); +} + +Fp4GemmPlugin::Fp4GemmPlugin( + int sfVecSize, nvinfer1::DataType OutputType, Fp4GemmPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) + , mSfVecSize(sfVecSize) + , mOutputType(OutputType) +{ + init(OutputType); +} + +Fp4GemmPlugin::Fp4GemmPlugin(void const* data, size_t length, Fp4GemmPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mSfVecSize); + read(d, mOutputType); + read(d, mDims); + + init(mOutputType); + mPluginProfiler->deserialize(d, mDims, mGemmId); + + TLLM_CHECK(d == a + length); +} + +void Fp4GemmPlugin::init(nvinfer1::DataType type) +{ + TLLM_CHECK_WITH_INFO((getSMVersion() >= 100), "FP4 Gemm not supported before Blackwell"); + TLLM_CHECK_WITH_INFO( + (mOutputType == DataType::kBF16) || (mOutputType == DataType::kFLOAT) || (mOutputType == DataType::kHALF), + "Only support float, half, bfloat16, got %d.", (int) mOutputType); + mOutputType = type; + if (mOutputType == nvinfer1::DataType::kHALF) + { + mGemmRunner = std::make_shared<CutlassFp4GemmRunner<half>>(); + } + else if (mOutputType == nvinfer1::DataType::kFLOAT) + { + mGemmRunner = std::make_shared<CutlassFp4GemmRunner<float>>(); + } +#ifdef ENABLE_BF16 + else if (mOutputType == nvinfer1::DataType::kBF16) + { + mGemmRunner = std::make_shared<CutlassFp4GemmRunner<__nv_bfloat16>>(); + } +#endif + + mGemmId = GemmIdCore(mDims.n, mDims.k, mOutputType); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* Fp4GemmPlugin::clone() const noexcept +{ + auto* plugin = new Fp4GemmPlugin(*this); + return plugin; +} + +nvinfer1::DimsExprs Fp4GemmPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + TLLM_CHECK_WITH_INFO(outputIndex == 0, "Only support one output"); + auto const& dimsInput = inputs[getInputTensorIdx()]; + auto const& dimsWeights = inputs[getWeightsTensorIdx()]; + TLLM_CHECK_WITH_INFO(dimsInput.nbDims >= 2 && dimsWeights.nbDims == 2, "Fp4GemmPlugin input dim=%d, weights dim=%d", + dimsInput.nbDims, dimsWeights.nbDims); + nvinfer1::DimsExprs ret; + if (outputIndex == 0) + { + ret.nbDims = dimsInput.nbDims; + for (int i = 0; i < dimsInput.nbDims - 1; ++i) + { + ret.d[i] = dimsInput.d[i]; + } + ret.d[dimsInput.nbDims - 1] = dimsWeights.d[0]; + } + else + { + TLLM_CHECK_WITH_INFO(outputIndex == 0, "output fp4 not supported now."); + ret.nbDims = 1; + auto vecCount = dimsInput.d[0]; + int numDim = dimsInput.nbDims; + for (int idx = 1; idx < numDim - 1; ++idx) + { + vecCount = exprBuilder.operation(nvinfer1::DimensionOperation::kPROD, *vecCount, *dimsInput.d[idx]); + } + auto constant128 = exprBuilder.constant(128); + auto alignedRowCount = exprBuilder.operation(nvinfer1::DimensionOperation::kCEIL_DIV, *vecCount, *constant128); + alignedRowCount = exprBuilder.operation(nvinfer1::DimensionOperation::kPROD, *alignedRowCount, *constant128); + auto constant4 = exprBuilder.constant(4); + auto constantSFSize = exprBuilder.constant(mSfVecSize); + auto sfColumn + = exprBuilder.operation(nvinfer1::DimensionOperation::kCEIL_DIV, *dimsInput.d[numDim - 1], *constantSFSize); + auto alignedColumnCount = exprBuilder.operation(nvinfer1::DimensionOperation::kCEIL_DIV, *sfColumn, *constant4); + alignedColumnCount + = exprBuilder.operation(nvinfer1::DimensionOperation::kPROD, *alignedColumnCount, *constant4); + auto totalSize + = exprBuilder.operation(nvinfer1::DimensionOperation::kPROD, *alignedColumnCount, *alignedRowCount); + ret.d[0] = totalSize; + } + return ret; +} + +bool Fp4GemmPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + if (inOut[pos].format != TensorFormat::kLINEAR) + { + return false; + } + if (pos == getInputTensorIdx()) + { + return (inOut[pos].type == FP4_DTYPE); + } + else if (pos == getWeightsTensorIdx()) + { + return (inOut[pos].type == FP4_DTYPE); + } + else if (pos == getInputSFTensorIdx() || pos == getWeightsSFTensorIdx()) + { + return (inOut[pos].type == FP8_DTYPE); + } + else if (pos == getGlobalSFTensorIdx()) + { + return (inOut[pos].type == DataType::kFLOAT); + } + else if (pos == nbInputs) + { + // Output + return (inOut[pos].type == DataType::kFLOAT || inOut[pos].type == DataType::kBF16 + || inOut[pos].type == DataType::kHALF); + } + return false; +} + +void Fp4GemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + int const maxN = in[2].max.d[0]; + int const minK = in[0].min.d[in[0].min.nbDims - 1]; + int const minN = in[2].min.d[0]; + + TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); + TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, maxN, maxK}; + } + mGemmId = {maxN, maxK, mOutputType}; + m_workspaceMaxSize = mGemmRunner->getWorkspaceSize(maxM, maxN, maxK, /* batch_count */ 1); +} + +size_t Fp4GemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return m_workspaceMaxSize; +} + +int Fp4GemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + // inputs + // 0. input_tensor [num_tokens, dim] + // 1. input_block_scale [num_tokens, dim / SFVecSize] (padded) + // 2. weights_tensor [out_dim, dim] + // 3. weights_block_scale [out_dim, dim / SFVecSize] (padded) + // 4. alpha (global scaling factor) [1] + // outputs + // 0. output_tensor [num_tokens, out_dim] + int64_t m = 1; + for (int i = 0; i < inputDesc[getInputTensorIdx()].dims.nbDims - 1; ++i) + { + m *= inputDesc[getInputTensorIdx()].dims.d[i]; + } + int const n = inputDesc[getWeightsTensorIdx()].dims.d[0]; + int const k = inputDesc[getWeightsTensorIdx()].dims.d[1]; + TLLM_CHECK_WITH_INFO(k % 32 == 0, "K dim should be aligned to 16 Bytes"); + int N_align = mOutputType == nvinfer1::DataType::kFLOAT ? 4u : 8u; + TLLM_CHECK_WITH_INFO(n % N_align == 0, "N dim should be aligned to 16 Bytes"); + size_t const wsSize = mGemmRunner->getWorkspaceSize(m, n, k, /* batch_count */ 1); + auto const bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); + TLLM_CHECK_WITH_INFO(bestTactic, "No valid FP4 GEMM tactic"); + if (m >= 1) + { + mGemmRunner->gemm(outputs[0], inputs[0], inputs[2], inputs[1], inputs[3], + reinterpret_cast<float const*>(inputs[4]), m, n, k, /* batch_count */ 1, *bestTactic, + reinterpret_cast<char*>(workspace), wsSize, stream); + } + sync_check_cuda_error(stream); + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType Fp4GemmPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK_WITH_INFO(index == 0, "Only support one output"); + return mOutputType; +} + +// IPluginV2 Methods + +char const* Fp4GemmPlugin::getPluginType() const noexcept +{ + return FP4_GEMM_PLUGIN_NAME; +} + +char const* Fp4GemmPlugin::getPluginVersion() const noexcept +{ + return FP4_GEMM_PLUGIN_VERSION; +} + +int Fp4GemmPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int Fp4GemmPlugin::initialize() noexcept +{ + configGemm(); + return 0; +} + +void Fp4GemmPlugin::terminate() noexcept {} + +size_t Fp4GemmPlugin::getSerializationSize() const noexcept +{ + return sizeof(mSfVecSize) + // mSfVecSize + sizeof(nvinfer1::DataType) + // dtype + sizeof(mDims) + // Dimensions + mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size +} + +void Fp4GemmPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mSfVecSize); + write(d, mOutputType); + write(d, mDims); + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void Fp4GemmPlugin::destroy() noexcept +{ + delete this; +} + +void Fp4GemmPlugin::configGemm() +{ + mPluginProfiler->profileTactics(mGemmRunner, mOutputType, mDims, mGemmId); +} + +/////////////// + +Fp4GemmPluginCreator::Fp4GemmPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("sv_vec_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("output_type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* Fp4GemmPluginCreator::getPluginName() const noexcept +{ + return FP4_GEMM_PLUGIN_NAME; +} + +char const* Fp4GemmPluginCreator::getPluginVersion() const noexcept +{ + return FP4_GEMM_PLUGIN_VERSION; +} + +PluginFieldCollection const* Fp4GemmPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* Fp4GemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + TLLM_CHECK(fc->nbFields == 2); + int sf_vec_size{}; + nvinfer1::DataType output_type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "sf_vec_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + sf_vec_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "output_type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + output_type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + try + { + // Fp4GemmPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); + auto* obj = new Fp4GemmPlugin(sf_vec_size, output_type, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* Fp4GemmPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call CumsumLastDimPlugin::destroy() + try + { + // Create plugin profiler with private tactics map which is read from the serialized engine + auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); + auto* obj = new Fp4GemmPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.h b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.h new file mode 100644 index 000000000000..9947e849d84f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fp4GemmPlugin/fp4GemmPlugin.h @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#if defined(USING_OSS_CUTLASS_FP4_GEMM) +#include "tensorrt_llm/kernels/cutlass_kernels/include/fp4_gemm.h" +#else +#include "fp4_gemm.h" +#endif + +#include <cassert> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +#if defined(USING_OSS_CUTLASS_FP4_GEMM) +using Fp4GemmRunnerPtr = std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFp4GemmRunnerInterface>; +#else +using Fp4GemmRunnerPtr + = std::shared_ptr<tensorrt_llm::kernels::internal_cutlass_kernels::CutlassFp4GemmRunnerInterface>; +#endif + +class Fp4GemmPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + Fp4GemmRunnerPtr, GemmIdCore, GemmIdCoreHash> +{ +public: + using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + std::vector<Config> getTactics(int m, int n, int k) const override; + +private: + tensorrt_llm::common::QuantMode mQuantMode; +}; + +class Fp4GemmPlugin : public BasePlugin +{ +public: + using PluginProfilerPtr = std::shared_ptr<Fp4GemmPluginProfiler>; + + Fp4GemmPlugin() = delete; + + Fp4GemmPlugin(int sfVecSize, nvinfer1::DataType OutputType, PluginProfilerPtr const& pluginProfiler); + + Fp4GemmPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler); + + ~Fp4GemmPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + using IndexType = std::int32_t; + + IndexType getInputTensorIdx() const + { + return 0; + }; + + IndexType getInputSFTensorIdx() const + { + return 1; + }; + + IndexType getWeightsTensorIdx() const + { + return 2; + }; + + IndexType getWeightsSFTensorIdx() const + { + return 3; + }; + + IndexType getGlobalSFTensorIdx() const + { + return 4; + } + + void init(nvinfer1::DataType type); + void configGemm(); + + Fp4GemmRunnerPtr mGemmRunner; + PluginProfilerPtr mPluginProfiler; + + int mSfVecSize; + nvinfer1::DataType mOutputType; + size_t m_workspaceMaxSize; + GemmDims mDims{}; + GemmIdCore mGemmId{}; +}; + +class Fp4GemmPluginCreator : public BaseCreator +{ +public: + Fp4GemmPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<Fp4GemmPluginProfiler> mGemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/CMakeLists.txt new file mode 100644 index 000000000000..3b714a3928fb --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp *.cu) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.cpp b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.cpp new file mode 100644 index 000000000000..84963df50a21 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.cpp @@ -0,0 +1,422 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fp8RowwiseGemmPlugin.h" +#include "cutlass_extensions/gemm_configs.h" + +#include <NvInferRuntimeBase.h> +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels::cutlass_kernels; +using tensorrt_llm::plugins::Fp8RowwiseGemmPluginCreator; +using tensorrt_llm::plugins::Fp8RowwiseGemmPlugin; +using tensorrt_llm::plugins::Fp8RowwiseGemmPluginProfiler; + +static char const* FP8_ROWWISE_GEMM_PLUGIN_VERSION{"1"}; +static char const* FP8_ROWWISE_GEMM_PLUGIN_NAME{"Fp8RowwiseGemm"}; +PluginFieldCollection Fp8RowwiseGemmPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> Fp8RowwiseGemmPluginCreator::mPluginAttributes; + +size_t Fp8RowwiseGemmPluginProfiler::getBytePerElement(nvinfer1::DataType type) +{ + size_t bpe; + if (type == nvinfer1::DataType::kHALF || type == nvinfer1::DataType::kBF16) + { + bpe = 2; + } + else if (type == nvinfer1::DataType::kINT8 || type == nvinfer1::DataType::kFP8) + { + bpe = 1; + } + else + { + TLLM_THROW("Not recognized/implemented"); + } + return bpe; +} + +void Fp8RowwiseGemmPluginProfiler::setQuantMode(tensorrt_llm::common::QuantMode const& quantMode) +{ + mQuantMode = quantMode; +} + +void Fp8RowwiseGemmPluginProfiler::runTactic(int m, int n, int k, Fp8RowwiseGemmPluginProfiler::Config const& tactic, + char* workspace, cudaStream_t const& stream) +{ + size_t bpeIn = getBytePerElement(nvinfer1::DataType::kFP8); + size_t bpeOut = getBytePerElement(mType); + + // Workspace size required by gemm runner + // NB: this function will throw exception when selected tactic exceeds SMEM, which is then + // caught by gemmPluginProfiler and it will register this tactic as invalid + size_t wsSizeRunner = mRunner->getWorkspaceSize(m, n, k); + + // Workspace size required by profiling + size_t wsByteOffset = 0; + int8_t* wsBytePointer = reinterpret_cast<int8_t*>(workspace); + void* aTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * k * bpeIn)); + void* bTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, n * k * bpeIn)); + // void* cTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, n * bpeOut)); + void* dTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * n * bpeOut)); + float* scaleD0Tmp = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * sizeof(float))); + float* scaleD1Tmp = reinterpret_cast<float*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, n * sizeof(float))); + char* workspaceTmp = reinterpret_cast<char*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, wsSizeRunner)); + + // Run profiling + mRunner->gemm(dTmp, aTmp, bTmp, nullptr, mQuantMode, m, n, k, scaleD0Tmp, scaleD1Tmp, tactic, workspaceTmp, + wsSizeRunner, stream); + sync_check_cuda_error(stream); +} + +int Fp8RowwiseGemmPluginProfiler::getMaxProfileM() const +{ + // Max_num_tokens are not suggested to be set larger than 16k. + return 16384; +} + +void Fp8RowwiseGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + std::vector<size_t> workspaces = { + maxM * k * getBytePerElement(nvinfer1::DataType::kFP8), // A + n * k * getBytePerElement(nvinfer1::DataType::kFP8), // B + // n * getBytePerElement(mType), // C_bias + maxM * n * getBytePerElement(mType), // D + maxM * sizeof(float), // alphaRow + n * sizeof(float), // alphaCol + maxM * sizeof(float), // alphaOutput + mRunner->getWorkspaceSize(maxM, n, k) // workspace + }; + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<Fp8RowwiseGemmPluginProfiler::Config> Fp8RowwiseGemmPluginProfiler::getTactics(int m, int n, int k) const +{ + return mRunner->getConfigs(); +} + +Fp8RowwiseGemmPlugin::Fp8RowwiseGemmPlugin( + QuantMode quantMode, nvinfer1::DataType type, Fp8RowwiseGemmPlugin::PluginProfilerPtr const& pluginProfiler) + : mQuantMode(quantMode) + , mPluginProfiler(pluginProfiler) +{ + init(type); +} + +// Parameterized constructor +Fp8RowwiseGemmPlugin::Fp8RowwiseGemmPlugin( + void const* data, size_t length, Fp8RowwiseGemmPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + nvinfer1::DataType type; + unsigned int quantMode; + read(d, quantMode); + read(d, type); + read(d, mDims); + + mQuantMode = QuantMode(quantMode); + + init(type); + + mPluginProfiler->deserialize(d, mDims, mGemmId); + + TLLM_CHECK(d == a + length); +} + +void Fp8RowwiseGemmPlugin::init(nvinfer1::DataType type) +{ + mType = type; + if (mType == nvinfer1::DataType::kHALF) + { + mGemmRunner = std::make_shared<CutlassFp8RowwiseGemmRunner<half>>(); + } +#ifdef ENABLE_BF16 + else if (mType == nvinfer1::DataType::kBF16) + { + mGemmRunner = std::make_shared<CutlassFp8RowwiseGemmRunner<__nv_bfloat16>>(); + } +#endif + else + { + TLLM_THROW("Fp8 Rowwise Gemm plugin doesn't support this type now"); + } + + mPluginProfiler->setQuantMode(mQuantMode); + + mGemmId = GemmIdCore(mDims.n, mDims.k, mType); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* Fp8RowwiseGemmPlugin::clone() const noexcept +{ + auto* plugin = new Fp8RowwiseGemmPlugin(*this); + return plugin; +} + +nvinfer1::DimsExprs Fp8RowwiseGemmPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 4); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + TLLM_CHECK(nbDimsA >= 2); + DimsExprs ret; + ret.nbDims = nbDimsA; + for (int ii = 0; ii < nbDimsA - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + ret.d[nbDimsA - 1] = inputs[1].d[0]; + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool Fp8RowwiseGemmPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + switch (pos) + { + case 0: + // activation + return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; + case 1: + // weights + // Weights stored in checkpoint must have fp8 type + return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; + case 2: + // scales channels + case 3: + // scales tokens + return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + case 4: + // out + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + default: + // All other format combinations are unsupported. + return false; + } +} + +void Fp8RowwiseGemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + int const maxN = in[1].max.d[0]; + int const minK = in[0].min.d[in[0].min.nbDims - 1]; + int const minN = in[1].min.d[0]; + + TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); + TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, maxN, maxK}; + } + mGemmId = {maxN, maxK, mType}; + + mWorkspaceMaxSize = mGemmRunner->getWorkspaceSize(maxM, maxN, maxK); +} + +size_t Fp8RowwiseGemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return mWorkspaceMaxSize; +} + +int Fp8RowwiseGemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // mat1 [M(*), K] + // mat2 [N, K] + // scale_tokens [M, 1] if has_per_token_scaling else [1, 1] + // scale_channels [1, N] if has_per_channel_scaling else [1, 1] + // outputs + // mat [M(*), N] + int m = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m *= inputDesc[0].dims.d[ii]; + } + int const n = inputDesc[1].dims.d[0]; + int const k = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; + size_t const wsSize = mGemmRunner->getWorkspaceSize(m, n, k); + + auto const bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); + TLLM_CHECK_WITH_INFO(bestTactic, "No valid GEMM tactic"); + mGemmRunner->gemm(outputs[0], inputs[0], inputs[1], nullptr, mQuantMode, m, n, k, + reinterpret_cast<float const*>(inputs[2]), reinterpret_cast<float const*>(inputs[3]), *bestTactic, + reinterpret_cast<char*>(workspace), wsSize, stream); + sync_check_cuda_error(stream); + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType Fp8RowwiseGemmPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* Fp8RowwiseGemmPlugin::getPluginType() const noexcept +{ + return FP8_ROWWISE_GEMM_PLUGIN_NAME; +} + +char const* Fp8RowwiseGemmPlugin::getPluginVersion() const noexcept +{ + return FP8_ROWWISE_GEMM_PLUGIN_VERSION; +} + +int Fp8RowwiseGemmPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int Fp8RowwiseGemmPlugin::initialize() noexcept +{ + configGemm(); // gemm profiler in action + return 0; +} + +void Fp8RowwiseGemmPlugin::terminate() noexcept {} + +size_t Fp8RowwiseGemmPlugin::getSerializationSize() const noexcept +{ + return sizeof(unsigned int) + // QuantMode + sizeof(nvinfer1::DataType) + // dtype + sizeof(mDims) + // Dimensions + mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size +} + +void Fp8RowwiseGemmPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mQuantMode.value()); + write(d, mType); + write(d, mDims); + + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void Fp8RowwiseGemmPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +void Fp8RowwiseGemmPlugin::configGemm() +{ + mPluginProfiler->profileTactics(mGemmRunner, mType, mDims, mGemmId); +} + +Fp8RowwiseGemmPluginCreator::Fp8RowwiseGemmPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("has_per_channel_scaling", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("has_per_token_scaling", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* Fp8RowwiseGemmPluginCreator::getPluginName() const noexcept +{ + return FP8_ROWWISE_GEMM_PLUGIN_NAME; +} + +char const* Fp8RowwiseGemmPluginCreator::getPluginVersion() const noexcept +{ + return FP8_ROWWISE_GEMM_PLUGIN_VERSION; +} + +PluginFieldCollection const* Fp8RowwiseGemmPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* Fp8RowwiseGemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + TLLM_CHECK(fc->nbFields == 3); + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + try + { + // Fp8RowwiseGemmPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); + QuantMode quantMode = QuantMode{}; + auto* obj = new Fp8RowwiseGemmPlugin(quantMode, type, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* Fp8RowwiseGemmPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call Fp8RowwiseGemmPlugin::destroy() + try + { + // Create plugin profiler with private tactics map which is read from the serialized engine + auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); + auto* obj = new Fp8RowwiseGemmPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.h b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.h new file mode 100644 index 000000000000..36f22ad5885d --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fp8RowwiseGemmPlugin/fp8RowwiseGemmPlugin.h @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +using Fp8RowwiseGemmRunnerPtr + = std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFp8RowwiseGemmRunnerInterface>; + +class Fp8RowwiseGemmPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + Fp8RowwiseGemmRunnerPtr, GemmIdCore, GemmIdCoreHash> + +{ +public: + using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; + + void setQuantMode(tensorrt_llm::common::QuantMode const& quantMode); + + virtual int getMaxProfileM() const override; + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + std::vector<Config> getTactics(int m, int n, int k) const override; + +private: + size_t getBytePerElement(nvinfer1::DataType type); + + tensorrt_llm::common::QuantMode mQuantMode; +}; + +class Fp8RowwiseGemmPlugin : public BasePlugin +{ +public: + using PluginProfilerPtr = std::shared_ptr<Fp8RowwiseGemmPluginProfiler>; + + Fp8RowwiseGemmPlugin() = delete; + + Fp8RowwiseGemmPlugin( + tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, PluginProfilerPtr const& pluginProfiler); + + Fp8RowwiseGemmPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); + + ~Fp8RowwiseGemmPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + void init(nvinfer1::DataType type); + + void configGemm(); + +private: + const std::string mLayerName; + + Fp8RowwiseGemmRunnerPtr mGemmRunner; + tensorrt_llm::common::QuantMode mQuantMode; // not configurable yet + size_t mWorkspaceMaxSize; + + GemmDims mDims{}; + GemmIdCore mGemmId{}; + + PluginProfilerPtr mPluginProfiler; + + nvinfer1::DataType mType; +}; + +class Fp8RowwiseGemmPluginCreator : public BaseCreator +{ +public: + Fp8RowwiseGemmPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<Fp8RowwiseGemmPluginProfiler> mGemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/CMakeLists.txt new file mode 100755 index 000000000000..7cc985b60b7a --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.cpp b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.cpp new file mode 100644 index 000000000000..541afdadc4c8 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.cpp @@ -0,0 +1,388 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "fusedLayernormPlugin.h" +#include "pluginUtils.h" +#include "tensorrt_llm/common/assert.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::FusedLayernormPluginCreator; +using tensorrt_llm::plugins::FusedLayernormPlugin; + +static char const* FUSED_LAYERNORM_PLUGIN_VERSION{"1"}; +static char const* FUSED_LAYERNORM_PLUGIN_NAME{"FusedLayernorm"}; +PluginFieldCollection FusedLayernormPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> FusedLayernormPluginCreator::mPluginAttributes; + +FusedLayernormPlugin::FusedLayernormPlugin(float eps, bool needFP32Output, bool needQuantize, nvinfer1::DataType type) + : mEps(eps) + , mNeedFP32Output(needFP32Output) + , mNeedQuantize(needQuantize) + , mType(type) +{ +} + +// Parameterized constructor +FusedLayernormPlugin::FusedLayernormPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mEps); + read(d, mNeedFP32Output); + read(d, mNeedQuantize); + read(d, mType); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* FusedLayernormPlugin::clone() const noexcept +{ + auto* plugin = new FusedLayernormPlugin(mEps, mNeedFP32Output, mNeedQuantize, mType); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs FusedLayernormPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + // Dim should be the same as input hidden states + if (!mNeedQuantize) + { + return inputs[0]; + } + + if (outputIndex == 1) // un-normed output fp16 + { + return inputs[0]; + } + if (outputIndex == 0) // quantized normed output + { + // Quantized output with int64_t data type (16 FP4 values per element). + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int di = 0; di < ret.nbDims; ++di) + { + ret.d[di] = inputs[0].d[di]; + } + return ret; + } + + // Scaling Factors. + try + { + TLLM_CHECK(outputIndex == 2); + + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int di = 0; di < ret.nbDims; ++di) + { + ret.d[di] = inputs[0].d[di]; + } + // Sequence dimension or token dimension. + // Pad to multiple of 128. + auto dimM + = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 2], *exprBuilder.constant(128)); + ret.d[ret.nbDims - 2] = exprBuilder.operation(DimensionOperation::kPROD, *dimM, *exprBuilder.constant(128)); + // Hidden size dimension. + ret.d[ret.nbDims - 1] + = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 1], *exprBuilder.constant(16)); + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool FusedLayernormPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + int const totalPoses = 5 + 2 * static_cast<int>(mNeedQuantize); + TLLM_CHECK(0 <= pos && pos < totalPoses); + TLLM_CHECK(nbInputs == 3 + static_cast<int>(mNeedQuantize)); + if (pos < nbInputs) + { + switch (pos) + { + case 0: + case 1: + case 2: return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + case 3: return (inOut[pos].type == nvinfer1::DataType::kFLOAT); + } + } + if (pos == nbInputs) // Normed output + { + if (mNeedQuantize) + { + // fp4 quantized output -- fp4 padded tp int64 + return (inOut[pos].type == nvinfer1::DataType::kFP4) && (inOut[pos].format == TensorFormat::kLINEAR); + } + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == nbInputs + 1) // Un-normed output + { + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + // fp4 act_per_block_scale -- fp8 padded to int32 + return (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); +} + +void FusedLayernormPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t FusedLayernormPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return sizeof(WarpSpecializedCounters); +} + +int FusedLayernormPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // input [M(*), N] + // residual [M(*), N] + // weight [N, ] + // scale [1, ] - if needQuantize + // outputs + // output [M(*), N] - fp4 padded to int64 / fp16 + // un-normed output [M(*), N] - fp16 + // act_per_block_scale - fp8 padded to int32 - if needQuantize + +#define SETUP_PARAM \ + Param param; \ + int64_t m64 = 1; \ + for (int i = 0; i < inputDesc[0].dims.nbDims - 1; ++i) \ + { \ + m64 *= inputDesc[0].dims.d[i]; \ + } \ + int const m = TLLM_INT32_CAST(m64); \ + int const n = TLLM_INT32_CAST(inputDesc[2].dims.d[0]); \ + param.m = m; \ + param.n = n; \ + param.layernorm_eps = mEps; \ + param.input = const_cast<Input*>(reinterpret_cast<Input const*>(inputs[0])); \ + param.residual = const_cast<Input*>(reinterpret_cast<Input const*>(inputs[1])); \ + param.gamma = const_cast<Input*>(reinterpret_cast<Input const*>(inputs[2])); \ + if (mNeedQuantize) \ + { \ + param.sf_scale = const_cast<float*>(reinterpret_cast<float const*>(inputs[3])); \ + } \ + param.counters = reinterpret_cast<WarpSpecializedCounters*>(workspace); \ + param.stream = stream; \ + param.normed_output = reinterpret_cast<uint32_t*>(outputs[0]); \ + param.output = reinterpret_cast<Input*>(outputs[1]); \ + param.sf_out = reinterpret_cast<uint32_t*>(outputs[2]); + +#define CLEANUP_AND_INVOKE \ + TLLM_CUDA_CHECK(cudaMemsetAsync(workspace, 0, sizeof(WarpSpecializedCounters), stream)); \ + invokeWSLayerNorm(param, true, num_sms); + + int num_sms = tensorrt_llm::common::getMultiProcessorCount(); + + if (mType == DataType::kHALF) + { + using Input = half; + using Param = WarpSpecializedParam<GeneralFP4AddBiasResidualPreLayerNormParam<Input>>; + SETUP_PARAM + CLEANUP_AND_INVOKE + } +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16) + { + using Input = __nv_bfloat16; + using Param = WarpSpecializedParam<GeneralFP4AddBiasResidualPreLayerNormParam<Input>>; + SETUP_PARAM + CLEANUP_AND_INVOKE + } +#endif + else + { + TLLM_LOG_ERROR("Unsupported data type"); + return 1; + } + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType FusedLayernormPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + // assert((mNeedFP32Output && index < 3) || (!mNeedFP32Output && index < 2)); + assert((mNeedQuantize && index < 3) || (!mNeedQuantize && index < 2)); + if (index == 0) + { + // Output 0 quantized output of layernorm - fp4 padded to int64 + if (mNeedQuantize) + { + return nvinfer1::DataType::kFP4; + } + return mType; + } + else if (index == 1) + { + // Output 1 un-normed output + return mType; + } + // Output 2 act_per_block_scale - fp8 padded to int32 + return nvinfer1::DataType::kFP8; +} + +// IPluginV2 Methods + +char const* FusedLayernormPlugin::getPluginType() const noexcept +{ + return FUSED_LAYERNORM_PLUGIN_NAME; +} + +char const* FusedLayernormPlugin::getPluginVersion() const noexcept +{ + return FUSED_LAYERNORM_PLUGIN_VERSION; +} + +int FusedLayernormPlugin::getNbOutputs() const noexcept +{ + return 2 + static_cast<int>(mNeedQuantize); +} + +int FusedLayernormPlugin::initialize() noexcept +{ + return 0; +} + +void FusedLayernormPlugin::terminate() noexcept {} + +size_t FusedLayernormPlugin::getSerializationSize() const noexcept +{ + return sizeof(mEps) + sizeof(mNeedFP32Output) + sizeof(mNeedQuantize) + sizeof(mType); +} + +void FusedLayernormPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mEps); + write(d, mNeedFP32Output); + write(d, mNeedQuantize); + write(d, mType); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void FusedLayernormPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +FusedLayernormPluginCreator::FusedLayernormPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("eps", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("need_fp32_output", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("need_quantize", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* FusedLayernormPluginCreator::getPluginName() const noexcept +{ + return FUSED_LAYERNORM_PLUGIN_NAME; +} + +char const* FusedLayernormPluginCreator::getPluginVersion() const noexcept +{ + return FUSED_LAYERNORM_PLUGIN_VERSION; +} + +PluginFieldCollection const* FusedLayernormPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* FusedLayernormPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + float eps{}; + nvinfer1::DataType type{}; + bool needFP32Output{}; + bool needQuantize{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "eps")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + eps = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "need_fp32_output")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + needFP32Output = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "need_quantize")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + needQuantize = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + } + try + { + auto* obj = new FusedLayernormPlugin(eps, needFP32Output, needQuantize, type); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* FusedLayernormPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call FusedLayernormPlugin::destroy() + try + { + auto* obj = new FusedLayernormPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.h b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.h new file mode 100755 index 000000000000..c6c899950fdc --- /dev/null +++ b/cpp/tensorrt_llm/plugins/fusedLayernormPlugin/fusedLayernormPlugin.h @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/kernels/fusedLayernormKernels/layernorm_param.h" +#include "tensorrt_llm/kernels/fusedLayernormKernels/ws_layernorm.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class FusedLayernormPlugin : public BasePlugin +{ +public: + FusedLayernormPlugin() = delete; + + FusedLayernormPlugin(float eps, bool needFP32Output, bool needQuantize, nvinfer1::DataType type); + + FusedLayernormPlugin(void const* data, size_t length); + + ~FusedLayernormPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + float mEps; + bool mNeedFP32Output; + bool mNeedQuantize; + nvinfer1::DataType mType; + + const std::string mLayerName; +}; + +class FusedLayernormPluginCreator : public BaseCreator +{ +public: + FusedLayernormPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/CMakeLists.txt new file mode 100644 index 000000000000..1d1fa98f4132 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.cpp b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.cpp new file mode 100644 index 000000000000..08ee2af55406 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.cpp @@ -0,0 +1,721 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "gemmAllReducePlugin.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h" +#include "tensorrt_llm/plugins/common/pluginUtils.h" + +#include <unistd.h> + +static char const* GEMM_ALLREDUCE_PLUGIN_VERSION = "1"; +static char const* GEMM_ALLREDUCE_PLUGIN_NAME = "GemmAllReduce"; +template <nvinfer1::DataType T> +using CutlassType = ::tensorrt_llm::kernels::cutlass_kernels::CutlassType<T>; + +namespace tensorrt_llm::plugins +{ +template <typename K, typename V, DataType ElementA, DataType ElementB, DataType ElementD> +static std::pair<K, V> makeEntry() +{ + return {std::make_tuple(ElementA, ElementB, ElementD), + [&]() + { + using GemmTraits + = cutlass_kernels::GemmTypes<typename CutlassType<ElementA>::type, typename CutlassType<ElementB>::type, + typename CutlassType<ElementD>::type, // C, unused + typename CutlassType<ElementD>::type, + std::conditional_t<ElementA == DataType::kFP4, cutlass::float_ue4m3_t, void>, // SFA + std::conditional_t<ElementB == DataType::kFP4, cutlass::float_ue4m3_t, void>, // SFB + cutlass::layout::RowMajor, cutlass::layout::ColumnMajor, + cutlass::layout::RowMajor, // C, unused + cutlass::layout::RowMajor>; + return new cutlass_kernels::GemmAllReduceImplRunner<GemmTraits>(); + }}; +} + +template <typename K, typename V> +static std::map<K, V> getTypedInstantiators() +{ + return std::map<K, V>({makeEntry<K, V, DataType::kHALF, DataType::kHALF, DataType::kHALF>(), + makeEntry<K, V, DataType::kBF16, DataType::kBF16, DataType::kBF16>(), + makeEntry<K, V, DataType::kFP8, DataType::kFP8, DataType::kHALF>(), + makeEntry<K, V, DataType::kFP8, DataType::kFP8, DataType::kBF16>(), + makeEntry<K, V, DataType::kFP4, DataType::kFP4, DataType::kHALF>(), + makeEntry<K, V, DataType::kFP4, DataType::kFP4, DataType::kBF16>()}); +} + +//////////////////////////////////////////////////////////// +// GemmAllReducePlugin Methods +//////////////////////////////////////////////////////////// +GemmAllReducePlugin::GemmAllReducePlugin(GemmAllReducePluginOptions const& options) + : mOptions(options) + , mGemmId(GemmIdCore(options.maxProblemShape.n, options.maxProblemShape.k, options.typeD)) + , mProfiler(mGemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/options.deserialize)) +{ + // construct mapping of input/output pos to argument + int argIdx = 0; + // inputs + mArgMap[argIdx++] = TensorArg::IN_ACTIVATION; + mArgMap[argIdx++] = TensorArg::IN_WEIGHT; + if (mOptions.hasSFA) + { + mArgMap[argIdx++] = TensorArg::IN_ACTIVATION_SF; + } + if (mOptions.hasSFB) + { + mArgMap[argIdx++] = TensorArg::IN_WEIGHT_SF; + } + if (mOptions.alphaIsPtr) + { + mArgMap[argIdx++] = TensorArg::IN_ALPHA; + } + mNbInputs = argIdx; + // outputs + mArgMap[argIdx++] = TensorArg::OUT_D_UC; + mArgMap[argIdx++] = TensorArg::OUT_D_MC; + mArgMap[argIdx++] = TensorArg::OUT_D_IPC; + mNbOutputs = argIdx - mNbInputs; + + // Create mapping of argument to tensor pos + for (auto const& pair : mArgMap) + { + mArgInvMap[pair.second] = pair.first; + } + + // Use map instead of huge switch case + mTypedInstantiators = getTypedInstantiators<KeyType, ValueType>(); + + auto key = std::make_tuple(mOptions.typeA, mOptions.typeB, mOptions.typeD); + + TLLM_CHECK_WITH_INFO(mTypedInstantiators.count(key) > 0, "No cutlass gemm for impl."); + mGemm = std::shared_ptr<cutlass_kernels::GemmAllReduceImplInterface>(mTypedInstantiators[key]()); +} + +void GemmAllReducePlugin::allocatePersistentWorkspace() +{ + TLLM_CHECK(mOptions.maxProblemShape.isInitialized()); + + mWorkspaceKey = "gemm_allreduce_workspace_m" + std::to_string(mOptions.maxProblemShape.maxM); + + cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig smallest_tile_config + = mGemm->getSupportedLaunchConfigs()[0]; + cutlass_kernels::GemmAllReduceImplInterface::ProblemArgs args; + args.argProblemShape(mOptions.maxProblemShape.maxM, mOptions.maxProblemShape.n, mOptions.maxProblemShape.k, 1) + .argRanks(mRank, mOptions.group) + .argLaunchConfig(smallest_tile_config); + + TLLM_CHECK(mWorkspace == nullptr); + + // Wrap persistent workspace in IPluginResource type + // so that clone() can be called to allocate memory + GemmAllReducePersistentWorkspace unallocated_resource(mGemm->getPersistentWorkspace(args)); + + // Register and allocate workspace + mWorkspace = static_cast<GemmAllReducePersistentWorkspace*>( + getPluginRegistry()->acquirePluginResource(mWorkspaceKey.c_str(), &unallocated_resource)); + TLLM_CHECK(mWorkspace != nullptr); +} + +LaunchConfig GemmAllReducePlugin::getStaticHeuristicLaunchConfig(int M) const +{ + using namespace tensorrt_llm::cutlass_extensions; + // This is only applicable when we swap and transpose A & B. + // When M is small we want to select tile that best fits it to maximize MMA efficiency. + auto filterByM = [&](std::vector<LaunchConfig> candidateConfigs) + { + std::vector<LaunchConfig> result; + if (M <= 16) + { + std::copy_if(candidateConfigs.begin(), candidateConfigs.end(), std::back_inserter(result), + [](const LaunchConfig& config) + { return config.tile_shape == TileShape::TileShape_128x16x128 and config.transposed; }); + } + else if (M <= 32) + { + std::copy_if(candidateConfigs.begin(), candidateConfigs.end(), std::back_inserter(result), + [](const LaunchConfig& config) + { return config.tile_shape == TileShape::TileShape_128x32x128 and config.transposed; }); + } + else if (M <= 64) + { + std::copy_if(candidateConfigs.begin(), candidateConfigs.end(), std::back_inserter(result), + [](const LaunchConfig& config) + { return config.tile_shape == TileShape::TileShape_128x64x128 and config.transposed; }); + } + else + { + std::copy_if(candidateConfigs.begin(), candidateConfigs.end(), std::back_inserter(result), + [](const LaunchConfig& config) + { return config.tile_shape == TileShape::TileShape_128x128x128 and config.transposed; }); + } + // If result empty then use any. + if (result.empty()) + { + result = candidateConfigs; + } + return result; + }; + + auto bestLaunchConfigs = mGemm->getSupportedLaunchConfigs(); + bestLaunchConfigs = filterByM(bestLaunchConfigs); + TLLM_CHECK(!bestLaunchConfigs.empty()); + // Return first one, because who knows which is best. + return bestLaunchConfigs.front(); +} + +static GemmAllReducePluginOptions deserializeOptions(void const*& data, size_t length) +{ + char const* begin = reinterpret_cast<char const*>(data); + char const*& end = reinterpret_cast<char const*&>(data); + GemmAllReducePluginOptions options; + options.deserialize = true; + + read(end, options.typeA); + read(end, options.typeB); + read(end, options.typeD); + read(end, options.transA); + read(end, options.transB); + read(end, options.alpha); + read(end, options.maxProblemShape); + read(end, options.groupSize); + for (int i = 0; i < options.groupSize; ++i) + { + int rank = -1; + read(end, rank); + options.group.insert(rank); + } + read(end, options.hasSFA); + read(end, options.hasSFB); + read(end, options.alphaIsPtr); + + TLLM_CHECK_WITH_INFO(end == begin + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (end - begin)); + + return options; +} + +GemmAllReducePlugin::GemmAllReducePlugin(void const* data, size_t length) + : GemmAllReducePlugin(deserializeOptions(std::ref(data), length)) +{ + if (mProfiler->useProfiler()) + { + mProfiler->deserializeFromOwnFile(mGemmId, mOptions.maxProblemShape); + } +} + +////////////////////////////////// +// IPluginV2DynamicExt Methods +////////////////////////////////// +IPluginV2DynamicExt* GemmAllReducePlugin::clone() const noexcept +{ + return new GemmAllReducePlugin(*this); +} + +DimsExprs GemmAllReducePlugin::getOutputDimensions( + int outputIndex, DimsExprs const* inputs, int nbInputs, IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == mNbInputs); // number of input tensors + TLLM_CHECK(inputs[0].nbDims == inputs[1].nbDims); + TLLM_CHECK(outputIndex < getNbOutputs()); + + // List of pointers to D on each rank + if ((nbInputs + outputIndex) == TensorArg::OUT_D_IPC) + { + DimsExprs out_dims; + out_dims.nbDims = 1; + out_dims.d[0] = exprBuilder.constant(mOptions.groupSize); + return out_dims; + } + + TLLM_CHECK(mOptions.transA == false); + TLLM_CHECK(mOptions.transB == true); + + int const nbDimsA = inputs[0].nbDims; // number of dims + int const nbDimsB = inputs[1].nbDims; + + DimsExprs out_dims; + // subtract 2 -> K from each input + out_dims.nbDims = nbDimsA + nbDimsB - 2; + + if (mOptions.transA) + { + for (int i = 1; i < nbDimsA; ++i) + { + out_dims.d[i - 1] = inputs[0].d[i]; + } + } + else + { + for (int i = 0; i < nbDimsA - 1; ++i) + { + out_dims.d[i] = inputs[0].d[i]; + } + } + if (mOptions.transB) + { + for (int i = 0; i < nbDimsB - 1; ++i) + { + out_dims.d[nbDimsA - 1 + i] = inputs[1].d[i]; + } + } + else + { + for (int i = 1; i < nbDimsB; ++i) + { + out_dims.d[nbDimsA - 2 + i] = inputs[1].d[i]; + } + } + return out_dims; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool GemmAllReducePlugin::supportsFormatCombination( + int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept +{ + // inOut[0] -> activation + // inOut[1] -> weight + // inOut[1+hasInputSF] -> activation_sf + // inOut[1+hasInputSF*2] -> weight_sf + // inOut[2+hasInputSF*2] -> output[0] = D_uc + // inOut[3+hasInputSF*2] -> output[1] = D_mc + + TLLM_CHECK_WITH_INFO(pos < mNbInputs + mNbOutputs, "Unexpected pos: %d", pos); + auto const& desc = inOut[pos]; + + TLLM_CHECK_WITH_INFO(mArgMap.count(pos) > 0, "pos %d not found in mArgMap.", pos); + TensorArg arg = mArgMap[pos]; + + auto typeExists = [&](DataType dtype, auto idx) -> bool + { + for (const auto& [key, value] : mTypedInstantiators) + { + // key format: <ActivationType, WeightType, OutputType> + if (std::get<decltype(idx)::value>(key) == dtype) + { + return true; + } + } + return false; + }; + + switch (arg) + { + case TensorArg::IN_ACTIVATION: return typeExists(desc.type, std::integral_constant<size_t, 0>{}); + case TensorArg::IN_WEIGHT: return typeExists(desc.type, std::integral_constant<size_t, 1>{}); + case TensorArg::IN_ACTIVATION_SF: + case TensorArg::IN_WEIGHT_SF: + // Assumed SF for only FP4 at the moment + return desc.type == DataType::kFP8; + case TensorArg::IN_ALPHA: return desc.type == DataType::kFLOAT; + case TensorArg::OUT_D_UC: + case TensorArg::OUT_D_MC: + case TensorArg::OUT_D_IPC: return typeExists(desc.type, std::integral_constant<size_t, 2>{}); + default: return false; + } +} + +void GemmAllReducePlugin::configurePlugin( + DynamicPluginTensorDesc const* in, int32_t nbInputs, DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept +{ + // Get problem shape + int const nbDimsA = in[0].max.nbDims; + int const minM = utils::computeMDimension(mOptions.transA, in[0].min); + int const maxM = utils::computeMDimension(mOptions.transA, in[0].max); + int const N = utils::computeNDimension(mOptions.transB, in[1].max); + int const K = mOptions.transA ? in[0].max.d[0] : in[0].max.d[nbDimsA - 1]; + + TLLM_CHECK_WITH_INFO(out[0].desc.type == mOptions.typeD, "Output type mismatch."); + + // Ensure call from execution phase does + // not override call from build phase + if (!mOptions.maxProblemShape.isInitialized()) + { + mOptions.maxProblemShape = {minM, maxM, N, K}; + mGemmId = {N, K, mOptions.typeD}; + } + + // Build phase doesn't have COMM_SESSION (i.e built on single rank) + // so do not allocate persistent workspace + if (!isBuilding()) + { + auto getTPRank = [&]() + { + int rank = COMM_SESSION.getRank(); + auto it = std::find(mOptions.group.begin(), mOptions.group.end(), rank); + TLLM_CHECK_WITH_INFO(it != mOptions.group.end(), + "Incorrect group specified - rank " + std::to_string(rank) + " not found in group"); + return std::distance(mOptions.group.begin(), it); + }; + + mRank = getTPRank(); + + if (mWorkspace == nullptr) + { + allocatePersistentWorkspace(); + } + } +} + +size_t GemmAllReducePlugin::getWorkspaceSize( + PluginTensorDesc const* inputs, int32_t nbInputs, PluginTensorDesc const* outputs, int32_t nbOutputs) const noexcept +{ + return 0; +} + +int GemmAllReducePlugin::enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + // inputs[0] -> [M(*), K] + // inputs[1] -> [K, N] + // outputs[0] -> [M(*), N] unicast ptr + // outputs[1] -> [M(*), N] multicast ptr + auto const nbDimsA = inputDesc[0].dims.nbDims; + auto const M = utils::computeMDimension(mOptions.transA, inputDesc[0].dims); + auto const N = utils::computeNDimension(mOptions.transB, inputDesc[1].dims); + auto const K = mOptions.transA ? inputDesc[0].dims.d[0] : inputDesc[0].dims.d[nbDimsA - 1]; + + TLLM_CHECK_WITH_INFO(M <= mOptions.maxProblemShape.maxM, "GemmAllReducePlugin M > maxM."); + TLLM_CHECK_WITH_INFO(M > 0, "GemmAllReducePlugin M is 0."); + TLLM_CHECK_WITH_INFO(N > 0, "GemmAllReducePlugin N is 0."); + TLLM_CHECK_WITH_INFO(K > 0, "GemmAllReducePlugin K is 0."); + TLLM_CHECK_WITH_INFO(mWorkspace != nullptr, "GemmAllReducePlugin workspace is null."); + + LaunchConfig bestLaunchConfig; + if (mProfiler->useProfiler()) + { + bestLaunchConfig = mProfiler->getBestConfig(M, mGemmId).value(); + } + else + { + bestLaunchConfig = getStaticHeuristicLaunchConfig(M); + } + + void const* activation = inputs[mArgInvMap[TensorArg::IN_ACTIVATION]]; + void const* weight = inputs[mArgInvMap[TensorArg::IN_WEIGHT]]; + void* D_out_uc = outputs[mArgInvMap[TensorArg::OUT_D_UC] - mNbInputs]; + void* D_out_mc = outputs[mArgInvMap[TensorArg::OUT_D_MC] - mNbInputs]; + void* D_out_ipc = outputs[mArgInvMap[TensorArg::OUT_D_IPC] - mNbInputs]; + + TLLM_CHECK_WITH_INFO(activation != nullptr, "GemmAllReducePlugin activation is NULL"); + TLLM_CHECK_WITH_INFO(weight != nullptr, "GemmAllReducePlugin weight is NULL"); + TLLM_CHECK_WITH_INFO(D_out_uc != nullptr, "GemmAllReducePlugin out_uc is NULL"); + TLLM_CHECK_WITH_INFO(D_out_mc != nullptr, "GemmAllReducePlugin out_mc is NULL"); + TLLM_CHECK_WITH_INFO(D_out_ipc != nullptr, "GemmAllReducePlugin out_ipc is NULL"); + + cutlass_kernels::GemmAllReduceImplInterface::ProblemArgs args; + args.argProblemShape(M, N, K, 1) + .argA(activation) + .argB(weight) + .argC(nullptr) + .argD(D_out_uc, D_out_mc, (void**) D_out_ipc) + .argRanks(mRank, mOptions.group) + .argBeta(0.f) // no bias + .argLaunchConfig(bestLaunchConfig) + .argWorkspace(mWorkspace->mWorkspace.get()); + // tensor for scaling input A + if (mOptions.hasSFA) + { + void const* activation_sf = inputs[mArgInvMap[TensorArg::IN_ACTIVATION_SF]]; + TLLM_CHECK_WITH_INFO(activation_sf != nullptr, "GemmAllReducePlugin activation_sf is NULL"); + args.argAScale(activation_sf); + } + // tensor for scaling input B + if (mOptions.hasSFB) + { + void const* weight_sf = inputs[mArgInvMap[TensorArg::IN_WEIGHT_SF]]; + TLLM_CHECK_WITH_INFO(weight_sf != nullptr, "GemmAllReducePlugin weight_sf is NULL"); + args.argBScale(weight_sf); + } + // tensor for scaling output D + if (mOptions.alphaIsPtr) + { + void const* alpha_vec = inputs[mArgInvMap[TensorArg::IN_ALPHA]]; + TLLM_CHECK_WITH_INFO(alpha_vec != nullptr, "GemmAllReducePlugin alpha_vec is NULL"); + args.argAlphaPtr(reinterpret_cast<float const*>(alpha_vec)); + } + else + { + args.argAlpha(mOptions.alpha); + } + + mGemm->run(args, stream); + + return 0; +} + +////////////////////////////////// +// IPluginV2Ext Methods +////////////////////////////////// +DataType GemmAllReducePlugin::getOutputDataType(int index, DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK_WITH_INFO(index < getNbOutputs(), "Output index out of bounds: %d", index); + return mOptions.typeD; +} + +////////////////////////////////// +// IPluginV2 Methods +////////////////////////////////// +char const* GemmAllReducePlugin::getPluginType() const noexcept +{ + return GEMM_ALLREDUCE_PLUGIN_NAME; +} + +char const* GemmAllReducePlugin::getPluginVersion() const noexcept +{ + return GEMM_ALLREDUCE_PLUGIN_VERSION; +} + +int GemmAllReducePlugin::getNbOutputs() const noexcept +{ + return mNbOutputs; +} + +int GemmAllReducePlugin::initialize() noexcept +{ + if (isBuilding() && mProfiler->useProfiler()) + { + // TODO (xsimmons): interfaces between GemmPluginProfiler and Plugin + // needs to be relooked at - current interface implicitly assigns runner to profiler + // object in profileTactics() + assert(mOptions.maxProblemShape.isInitialized()); + mProfiler->profileTactics(mGemm, mOptions.typeD, mOptions.maxProblemShape, mGemmId); + } + return 0; +} + +void GemmAllReducePlugin::terminate() noexcept +{ + if (isBuilding()) // need this otherwise getComm will crash during build phase + { + return; + } + + // free mWorkspace + if (mWorkspace) + { + getPluginRegistry()->releasePluginResource(mWorkspaceKey.c_str()); + mWorkspace = nullptr; + } +} + +size_t GemmAllReducePlugin::getSerializationSize() const noexcept +{ + // cannot use sizeof(GemmAllReducePluginOptions) + // becaused need packed attribute which doesn't work on enum + // without making the enum also packed + size_t size = 0; + size += sizeof(mOptions.typeA); + size += sizeof(mOptions.typeB); + size += sizeof(mOptions.typeD); + size += sizeof(mOptions.transA); + size += sizeof(mOptions.transB); + size += sizeof(mOptions.alpha); + size += sizeof(mOptions.maxProblemShape); + size += sizeof(mOptions.groupSize); + size += mOptions.group.size() * sizeof(int); + size += sizeof(mOptions.hasSFA); + size += sizeof(mOptions.hasSFB); + size += sizeof(mOptions.alphaIsPtr); + return size; +} + +void GemmAllReducePlugin::serialize(void* buffer) const noexcept +{ + char* begin = reinterpret_cast<char*>(buffer); + char* end = reinterpret_cast<char*>(buffer); + + write(end, mOptions.typeA); + write(end, mOptions.typeB); + write(end, mOptions.typeD); + write(end, mOptions.transA); + write(end, mOptions.transB); + write(end, mOptions.alpha); + write(end, mOptions.maxProblemShape); + write(end, mOptions.groupSize); + for (auto const& rank : mOptions.group) + { + write(end, rank); + } + write(end, mOptions.hasSFA); + write(end, mOptions.hasSFB); + write(end, mOptions.alphaIsPtr); + TLLM_CHECK(end == begin + getSerializationSize()); + + // Profiler MNK->kernel mappings need to be deterministic and consistent across ranks + // to ensure correct functionality (unlike standalone GEMMs). + // Since by default each rank will generate and serialize its own profiler mapping + // this can lead to different mappings between ranks which will result in fatal + // error. Therefore only generate and use profiler mapping for single rank. + if (mProfiler->useProfiler() && COMM_SESSION.getRank() == 0) + { + mProfiler->serializeToOwnFile(mGemmId); + } +} + +void GemmAllReducePlugin::destroy() noexcept +{ + delete this; +} + +//////////////////////////////////////////////////////////// +// GemmAllReducePluginCreator Methods +//////////////////////////////////////////////////////////// +PluginFieldCollection GemmAllReducePluginCreator::mFC; +std::vector<PluginField> GemmAllReducePluginCreator::mPluginAttributes; + +GemmAllReducePluginCreator::GemmAllReducePluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back("type_a", nullptr, PluginFieldType::kINT32, 1); + mPluginAttributes.emplace_back("type_b", nullptr, PluginFieldType::kINT32, 1); + mPluginAttributes.emplace_back("type_d", nullptr, PluginFieldType::kINT32, 1); + mPluginAttributes.emplace_back("transa", nullptr, PluginFieldType::kINT32, 1); + mPluginAttributes.emplace_back("transb", nullptr, PluginFieldType::kINT32, 1); + mPluginAttributes.emplace_back("alpha", nullptr, PluginFieldType::kFLOAT32, 1); + mPluginAttributes.emplace_back("group", nullptr, PluginFieldType::kINT32, 1); + mPluginAttributes.emplace_back("has_sfa", nullptr, PluginFieldType::kINT8, 1); + mPluginAttributes.emplace_back("has_sfb", nullptr, PluginFieldType::kINT8, 1); + mPluginAttributes.emplace_back("alpha_is_ptr", nullptr, PluginFieldType::kINT8, 1); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* GemmAllReducePluginCreator::getPluginName() const noexcept +{ + return GEMM_ALLREDUCE_PLUGIN_NAME; +} + +char const* GemmAllReducePluginCreator::getPluginVersion() const noexcept +{ + return GEMM_ALLREDUCE_PLUGIN_VERSION; +} + +PluginFieldCollection const* GemmAllReducePluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* GemmAllReducePluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + GemmAllReducePluginOptions options; + options.deserialize = false; + + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type_a")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + options.typeA = *static_cast<DataType const*>(fields[i].data); + } + else if (!strcmp(attrName, "type_b")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + options.typeB = *static_cast<DataType const*>(fields[i].data); + } + else if (!strcmp(attrName, "type_d")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + options.typeD = *static_cast<DataType const*>(fields[i].data); + } + else if (!strcmp(attrName, "transa")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + options.transA = *static_cast<int const*>(fields[i].data); + } + else if (!strcmp(attrName, "transb")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + options.transB = *static_cast<int const*>(fields[i].data); + } + else if (!strcmp(attrName, "alpha")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + options.alpha = *static_cast<float const*>(fields[i].data); + } + else if (!strcmp(attrName, "group")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + auto const* ranks = static_cast<int const*>(fields[i].data); + for (int j = 0; j < fields[i].length; ++j) + { + options.group.insert(ranks[j]); + } + options.groupSize = options.group.size(); + } + else if (!strcmp(attrName, "has_sfa")) // passed in as input tensor + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + options.hasSFA = *static_cast<int8_t const*>(fields[i].data); + } + else if (!strcmp(attrName, "has_sfb")) // passed in as input tensor + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + options.hasSFB = *static_cast<int8_t const*>(fields[i].data); + } + else if (!strcmp(attrName, "alpha_is_ptr")) // passed in as input tensor + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + options.alphaIsPtr = *static_cast<int8_t const*>(fields[i].data); + } + } + + try + { + // GemmAllReducePluginCreator is unique and shared for an engine generation + auto* obj = new GemmAllReducePlugin(options); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + return nullptr; + } +} + +IPluginV2* GemmAllReducePluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call GemmAllReducePlugin::destroy() + try + { + auto* obj = new GemmAllReducePlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h new file mode 100644 index 000000000000..457926246002 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePlugin.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) +#include "tensorrt_llm/kernels/cutlass_kernels/include/allreduce_gemm_runner.h" +#else +#include "allreduce_gemm_runner.h" +#endif + +#include "gemmAllReducePluginProfiler.h" +#include "gemmAllReducePluginResource.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +using namespace nvinfer1; + +using nvinfer1::DataType; +#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) +namespace cutlass_kernels = ::tensorrt_llm::kernels::opened_cutlass_kernels; +#else +namespace cutlass_kernels = ::tensorrt_llm::kernels::cutlass_kernels; +#endif + +using LaunchConfig = typename cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig; + +namespace tensorrt_llm::plugins +{ +struct GemmAllReducePluginOptions +{ + // Don't need to specify problem shape, this + // is specified in configurePlugin + DataType typeA; + DataType typeB; + DataType typeD; + int transA; + int transB; + float alpha; + // ranks participating in collective + std::set<int> group; + int groupSize; + // Set in configurePlugin during build phase + GemmDims maxProblemShape; + bool deserialize; // used for profiler instantiation + int8_t hasSFA = 0; + int8_t hasSFB = 0; + int8_t alphaIsPtr = 0; +}; + +class GemmAllReducePlugin : public BasePlugin +{ + friend class GemmAllReducePluginCreator; + +public: + ~GemmAllReducePlugin() override = default; + + ////////////////////////////////// + // IPluginV2DynamicExt Methods + ////////////////////////////////// + IPluginV2DynamicExt* clone() const noexcept override; + + DimsExprs getOutputDimensions( + int outputIndex, DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept override; + + // inOut[0] -> activation + // inOut[1] -> weight + // inOut[2] -> result + bool supportsFormatCombination( + int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept override; + + // in[0] -> activation + // in[1] -> weight + // no bias needed + void configurePlugin(DynamicPluginTensorDesc const* in, int32_t nbInputs, DynamicPluginTensorDesc const* out, + int32_t nbOutputs) noexcept override; + + size_t getWorkspaceSize(PluginTensorDesc const* inputs, int32_t nbInputs, PluginTensorDesc const* outputs, + int32_t nbOutputs) const noexcept override; + + // in[0] -> activation + // in[1] -> weight + // out[0] -> result_uc + // out[1] -> result_mc + int enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc, void const* const* inputs, + void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + ////////////////////////////////// + // IPluginV2Ext Methods + ////////////////////////////////// + DataType getOutputDataType(int index, DataType const* inputTypes, int nbInputs) const noexcept override; + + ////////////////////////////////// + // IPluginV2 Methods + ////////////////////////////////// + char const* getPluginType() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + int getNbOutputs() const noexcept override; + + int initialize() noexcept override; + + void terminate() noexcept override; + + size_t getSerializationSize() const noexcept override; + + void serialize(void* buffer) const noexcept override; + + void destroy() noexcept override; + +private: + explicit GemmAllReducePlugin(GemmAllReducePluginOptions const& options); + // Parameterized constructor + explicit GemmAllReducePlugin(void const* data, size_t length); + + void allocatePersistentWorkspace(); + + LaunchConfig getStaticHeuristicLaunchConfig(int M) const; + + // Params that are initialized during constructor + using KeyType = std::tuple<DataType, DataType, DataType>; + using ValueType = std::function<cutlass_kernels::GemmAllReduceImplInterface*()>; + GemmAllReducePluginOptions mOptions; + int mRank = 0; + + enum TensorArg + { + IN_ACTIVATION, + IN_ACTIVATION_SF, + IN_WEIGHT, + IN_WEIGHT_SF, + IN_ALPHA, + OUT_D_UC, + OUT_D_MC, + OUT_D_IPC + }; + + std::unordered_map<int, TensorArg> mArgMap; + std::unordered_map<TensorArg, int> mArgInvMap; + int mNbInputs = 0; + int mNbOutputs = 0; + + std::map<KeyType, ValueType> mTypedInstantiators; + std::string mWorkspaceKey; + std::shared_ptr<cutlass_kernels::GemmAllReduceImplInterface> mGemm; + // Params that are initialized during configurePlugin() + GemmAllReducePersistentWorkspace* mWorkspace = nullptr; + + // Used for selecting best GEMM for given problem shapes + GemmIdCore mGemmId{}; + GemmPluginProfilerManager<GemmAllReducePluginProfiler> mGemmPluginProfileManager; + std::shared_ptr<GemmAllReducePluginProfiler> mProfiler; +}; + +class GemmAllReducePluginCreator : public BaseCreator +{ +public: + GemmAllReducePluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.cpp b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.cpp new file mode 100644 index 000000000000..a6f7ca2615df --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.cpp @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "gemmAllReducePlugin.h" +#include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h" +#include "tensorrt_llm/plugins/common/pluginUtils.h" + +namespace tc = tensorrt_llm::common; + +namespace tensorrt_llm::plugins +{ +void GemmAllReducePluginProfiler::serializeToOwnFile(GemmIdCore gemmId) +{ + std::vector<char> file_buf(getSerializationSize(gemmId)); + char* begin = file_buf.data(); + char* end = file_buf.data(); + serialize(end, gemmId); + assert(end == begin + file_buf.size()); + + auto fileName = getCacheFileName(gemmId); + std::ofstream file(fileName, std::ios::binary); + TLLM_CHECK(file.is_open()); + file.write(begin, file_buf.size()); + file.flush(); + file.close(); +} + +void GemmAllReducePluginProfiler::deserializeFromOwnFile(GemmIdCore gemmId, GemmDims problemShape) +{ + auto fileName = getCacheFileName(gemmId); + std::ifstream file(fileName, std::ios::binary); + TLLM_CHECK(file.is_open()); + file.seekg(0, std::ios::end); + std::streamsize size = file.tellg(); + TLLM_CHECK(size > 0); + file.seekg(0, std::ios::beg); + + std::vector<char> file_buf(size); + file.read(file_buf.data(), size); + file.close(); + + char const* begin = const_cast<char const*>(file_buf.data()); + char const* end = begin; + deserialize(end, problemShape, gemmId); + assert(end == begin + size); +} + +bool GemmAllReducePluginProfiler::useProfiler() +{ + // char const* envDir = getenv("GEMM_AR_PLUGIN_PROFILE_DIR"); + // return envDir != nullptr; + // TODO(xsimmons): currently the profiler does not add any perf gain + // due to static heuristics being sufficient. We can re-enable this + // when we need more configurations. + return false; +} + +std::string GemmAllReducePluginProfiler::getCacheFileName(GemmIdCore gemmId) +{ + std::stringstream fileName; + char const* envDir = getenv("GEMM_AR_PLUGIN_PROFILE_DIR"); + std::string directory = envDir ? std::string(envDir) : "/tmp/"; + fileName << directory + "/gemm-AR"; + fileName << "-n" << std::to_string(gemmId.n); + fileName << "-k" << std::to_string(gemmId.k); + fileName << "-" << tc::getDtypeString(gemmId.dtype); + fileName << ".prof_cache"; + return fileName.str(); +} + +void GemmAllReducePluginProfiler::runTactic(int m, int n, int k, + cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig const& tactic, char* workspace, + cudaStream_t const& stream) +{ + const size_t dtype_size = tc::getDTypeSize(mType); + char* inputA = workspace; + char* inputB = inputA + m * k * dtype_size; + char* outputD = inputB + n * k * dtype_size; + char* inputSFA = outputD + m * n * dtype_size; + char* inputSFB = inputSFA + m * k * dtype_size; + std::set<int> tpGroup = {0}; + + // Run on single-GPU + cutlass_kernels::GemmAllReduceImplInterface::ProblemArgs args; + args.argProblemShape(m, n, k, 1) + .argA((void*) inputA) + .argB((void*) inputB) + .argD((void*) outputD, /*output_mc=*/nullptr) + .argAScale((void*) inputSFA) + .argBScale((void*) inputSFB) + .argRanks(0, tpGroup) + .argAlpha(1.f) + .argBeta(0.f) // no bias + .argLaunchConfig(tactic); + + TLLM_CHECK(mRunner != nullptr); + mRunner->run(args, stream); +} + +void GemmAllReducePluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + TLLM_CHECK(maxM != 0); + TLLM_CHECK(n != 0); + TLLM_CHECK(k != 0); + // mType refers to the output data type + // WARNING: This code assumes that the output precision is >= to input precision + const size_t dtype_size = tc::getDTypeSize(mType); + size_t bytes = 0; + bytes += maxM * k * dtype_size; // A + bytes += n * k * dtype_size; // B + // No C + // Note that D is typically IPC, however, when tuning GEMM we need it to run on single GPU + bytes += maxM * n * dtype_size; // D + // scale tensors for A & B - will at most be same size as A/B + bytes += maxM * k * dtype_size; // A + bytes += n * k * dtype_size; // B + + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig> GemmAllReducePluginProfiler::getTactics( + int m, int n, int k) const +{ + TLLM_CHECK(mRunner != nullptr); + return mRunner->getSupportedLaunchConfigs(); +} +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.h b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.h new file mode 100644 index 000000000000..faacbb3b8c0f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginProfiler.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) +#include "tensorrt_llm/kernels/cutlass_kernels/include/allreduce_gemm_runner.h" +#else +#include "allreduce_gemm_runner.h" +#endif +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" + +namespace tensorrt_llm::plugins +{ +/* + * Used for tuning to find best GEMM configs for different problem shapes. + * WARNING: Tuning GEMM+AR kernel may not be fully representable of real + * multi-GPU workloads as tuning only runs on single-GPU. + * IMPORTANT: TRT-LLM does not support deterministic tuning across ranks. + * Because of this, we have to serialize/deserialize our own configuration file. + */ + +#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) +namespace cutlass_kernels = ::tensorrt_llm::kernels::opened_cutlass_kernels; +#else +namespace cutlass_kernels = ::tensorrt_llm::kernels::cutlass_kernels; +#endif +class GemmAllReducePluginProfiler + : public GemmPluginProfiler<cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig, + std::shared_ptr<cutlass_kernels::GemmAllReduceImplInterface>, GemmIdCore, GemmIdCoreHash> +{ +public: + void serializeToOwnFile(GemmIdCore gemmId); + + void deserializeFromOwnFile(GemmIdCore gemmId, GemmDims problemShape); + + bool useProfiler(); + +protected: + //////////////////////////////////// + // GemmPluginProfiler methods + //////////////////////////////////// + void runTactic(int m, int n, int k, cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig const& tactic, + char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + std::vector<cutlass_kernels::GemmAllReduceImplInterface::LaunchConfig> getTactics( + int m, int n, int k) const override; + +private: + static std::string getCacheFileName(GemmIdCore gemmId); +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginResource.h b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginResource.h new file mode 100644 index 000000000000..8136bd363bd7 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmAllReducePlugin/gemmAllReducePluginResource.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "NvInferPlugin.h" + +#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) +#include "tensorrt_llm/kernels/cutlass_kernels/include/allreduce_gemm_runner.h" +#else +#include "allreduce_gemm_runner.h" +#endif +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/plugins/common/plugin.h" + +using namespace nvinfer1; + +namespace tensorrt_llm::plugins +{ + +#if defined(USING_OSS_CUTLASS_ALLREDUCE_GEMM) +namespace cutlass_kernels = ::tensorrt_llm::kernels::opened_cutlass_kernels; +#else +namespace cutlass_kernels = ::tensorrt_llm::kernels::cutlass_kernels; +#endif +class GemmAllReducePersistentWorkspace : public IPluginResource +{ +public: + GemmAllReducePersistentWorkspace(std::shared_ptr<cutlass_kernels::PersistentWorkspaceInterface> workspace) + : mWorkspace(workspace) + { + } + + ////////////////////////////////// + // IPluginResource Methods + ////////////////////////////////// + IPluginResource* clone() noexcept override + { + auto copy = new GemmAllReducePersistentWorkspace(mWorkspace); + // Resource initialization (if any) may be skipped for non-cloned objects + // since only clones will be registered by TensorRT. + try + { + copy->mWorkspace->allocate(); + return copy; + } + catch (std::exception const& e) + { + TLLM_LOG_ERROR(e.what()); + return nullptr; + } + } + + int32_t release() noexcept override + { + try + { + return mWorkspace->free(); + } + catch (std::exception const& e) + { + TLLM_LOG_ERROR(e.what()); + return -1; + } + } + + std::shared_ptr<cutlass_kernels::PersistentWorkspaceInterface> mWorkspace; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gemmPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.cpp b/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.cpp new file mode 100644 index 000000000000..9e06ad01d10f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.cpp @@ -0,0 +1,614 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gemmPlugin.h" + +#include "gemmPluginProfiler.h" +#include "plugin.h" +#include "pluginUtils.h" +#include "tensorrt_llm/kernels/weightOnlyBatchedGemv/cudaCoreGemm.h" +#include "tensorrt_llm/runtime/utils/debugUtils.h" + +#include <NvInferRuntime.h> + +#include <cassert> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::GemmDims; +using tensorrt_llm::plugins::GemmPluginCreator; +using tensorrt_llm::plugins::GemmPlugin; +using tensorrt_llm::plugins::CublasLtGemmPluginProfiler; +using tensorrt_llm::plugins::CublasGemmWrapperPtr; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +static char const* GEMM_PLUGIN_VERSION{"1"}; +static char const* GEMM_PLUGIN_NAME{"Gemm"}; +PluginFieldCollection GemmPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> GemmPluginCreator::mPluginAttributes; + +void getProblemParams(cublasOperation_t& transa, cublasOperation_t& transb, int& m, int& n, int& k, int& lda, int& ldb, + int& ldc, bool transA, bool transB, int M, int N, int K, int padLda, int padLdb, int padLdc) +{ + transa = transB ? CUBLAS_OP_T : CUBLAS_OP_N; + transb = transA ? CUBLAS_OP_T : CUBLAS_OP_N; + m = N; + n = M; + k = K; + lda = transB ? K + padLdb : N + padLdb; + ldb = transA ? M + padLda : K + padLda; + ldc = N + padLdc; +} + +void runGemm(int const M, int const N, int const K, bool const transA, bool const transB, int const padLda, + int const padLdb, int const padLdc, nvinfer1::DataType const type, CublasGemmWrapperPtr const& cublasWrapperPtr, + void const* act, void const* weight, float const alpha, void* output, + std::optional<cublasLtMatmulHeuristicResult_t> const& heuristic, void* workspace, cudaStream_t stream) +{ + if (M == 0 || N == 0 || K == 0) + return; + + cublasWrapperPtr->setStream(stream); + cublasWrapperPtr->setWorkspace(workspace); + + cublasOperation_t transa, transb; + int m, n, k; + int lda, ldb, ldc; + getProblemParams(transa, transb, m, n, k, lda, ldb, ldc, transA, transB, M, N, K, padLda, padLdb, padLdc); + + cublasWrapperPtr->createDescriptors(transa, transb, m, n, k, lda, ldb, ldc); + cublasWrapperPtr->Gemm(transa, transb, m, n, k, weight, lda, act, ldb, output, ldc, alpha, 0.0f, heuristic); + cublasWrapperPtr->destroyDescriptors(); +} + +void CublasLtGemmPluginProfiler::runTactic( + int m, int n, int k, CublasLtGemmPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) +{ + size_t dataSize = sizeof(half); + if (mType == nvinfer1::DataType::kFLOAT) + { + dataSize = sizeof(float); + } + + void* actPtr = reinterpret_cast<void*>(workspace); + void* weightPtr = reinterpret_cast<void*>( + nextWorkspacePtrWithAlignment(reinterpret_cast<int8_t*>(actPtr), m * k * dataSize, ALIGNMENT)); + void* outputPtr = reinterpret_cast<void*>( + nextWorkspacePtrWithAlignment(reinterpret_cast<int8_t*>(weightPtr), n * k * dataSize, ALIGNMENT)); + char* workspacePtr = reinterpret_cast<char*>( + nextWorkspacePtrWithAlignment(reinterpret_cast<int8_t*>(outputPtr), m * (n + mPadLdc) * dataSize, ALIGNMENT)); + runGemm(m, n, k, mTransA, mTransB, mPadLda, mPadLdb, mPadLdc, mType, mRunner, actPtr, weightPtr, 1.0f, outputPtr, + {tactic}, workspacePtr, stream); +} + +bool CublasLtGemmPluginProfiler::checkTactic(int m, int n, int k, Config const& tactic) const +{ + cublasOperation_t transa, transb; + int M = m, N = n, K = k; + int lda, ldb, ldc; + getProblemParams(transa, transb, m, n, k, lda, ldb, ldc, mTransA, mTransB, M, N, K, mPadLda, mPadLdb, mPadLdc); + + mRunner->createDescriptors(transa, transb, m, n, k, lda, ldb, ldc); + + auto const checkResult = mRunner->checkTactic(transa, transb, m, n, k, lda, ldb, ldc, tactic.algo); + + mRunner->destroyDescriptors(); + + return checkResult; +} + +void CublasLtGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + size_t dataSize = getDTypeSize(mType); + size_t outputDataSize = getDTypeSize(mOutputType); + + std::vector<size_t> workspaces = { + maxM * k * dataSize, // A + n * k * dataSize, // B + maxM * (n + mPadLdc) * outputDataSize, // C + CUBLAS_WORKSPACE_SIZE // workspace + }; + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size(), ALIGNMENT); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<CublasLtGemmPluginProfiler::Config> CublasLtGemmPluginProfiler::getTactics(int M, int N, int K) const +{ + cublasOperation_t transa, transb; + int m, n, k; + int lda, ldb, ldc; + getProblemParams(transa, transb, m, n, k, lda, ldb, ldc, mTransA, mTransB, M, N, K, mPadLda, mPadLdb, mPadLdc); + + mRunner->createDescriptors(transa, transb, m, n, k, lda, ldb, ldc); + auto const heruistics = mRunner->getTactics(transa, transb, m, n, k, lda, ldb, ldc); + mRunner->destroyDescriptors(); + + return heruistics; +} + +GemmPlugin::GemmPlugin(int transA, int transB, int padLda, int padLdb, int padLdc, nvinfer1::DataType type, bool useFp8, + float alpha, GemmPlugin::PluginProfilerPtr const& pluginProfiler) + : mTransA(transA) + , mTransB(transB) + , mPadLda(padLda) + , mPadLdb(padLdb) + , mPadLdc(padLdc) + , mType(type) + , mOutputType(type) + , mUseFp8(useFp8) + , mAlpha(alpha) + , mPluginProfiler(pluginProfiler) +{ + init(); +} + +// Parameterized constructor +GemmPlugin::GemmPlugin(void const* data, size_t length, GemmPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mTransA); + read(d, mTransB); + read(d, mPadLda); + read(d, mPadLdb); + read(d, mPadLdc); + read(d, mType); + read(d, mUseFp8); + read(d, mAlpha); + read(d, mDims); + read(d, mOutputType); + + init(); + + mPluginProfiler->deserialize(d, mDims, mGemmId); + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +thread_local CublasGemmWrapperPtr GemmPlugin::mCublasWrapper = nullptr; + +void GemmPlugin::init() +{ + auto cublasHandle = getCublasHandle(); + auto cublasLtHandle = getCublasLtHandle(); + mCublasWrapper = std::make_shared<CublasMMWrapper>(cublasHandle, cublasLtHandle, nullptr, nullptr); + + mPluginProfiler->setTranspose(mTransA, mTransB); + mPluginProfiler->setOutputType(mOutputType); + mPluginProfiler->setPadLd(mPadLda, mPadLdb, mPadLdc); + + mGemmId = GemmIdCublas(mDims.n, mDims.k, mType, mTransA, mTransB, mOutputType); + + mArch = tensorrt_llm::common::getSMVersion(); +} + +void GemmPlugin::setGemmConfig() +{ + if (mType == nvinfer1::DataType::kHALF) + { + mCublasWrapper->setFP16GemmConfig(trtToCublasDtype(mOutputType)); + } + else if (mType == nvinfer1::DataType::kFLOAT) + { + mCublasWrapper->setFP32GemmConfig(); + } +#ifdef ENABLE_BF16 + else if (mType == nvinfer1::DataType::kBF16) + { + mCublasWrapper->setBF16GemmConfig(trtToCublasDtype(mOutputType)); + } +#endif + +#ifdef ENABLE_FP8 + if (mUseFp8) + { + mCublasWrapper->setFP8GemmConfig(trtToCublasDtype(mOutputType)); + } +#endif +} + +void GemmPlugin::configGemm() +{ + if (!mDims.isInitialized()) + { + return; + } + + setGemmConfig(); + + mPluginProfiler->profileTactics(mCublasWrapper, mType, mDims, mGemmId); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* GemmPlugin::clone() const noexcept +{ + auto* plugin = new GemmPlugin(*this); + return plugin; +} + +nvinfer1::DimsExprs GemmPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 2); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + int const nbDimsB = inputs[1].nbDims; + DimsExprs ret; + ret.nbDims = nbDimsA + nbDimsB - 2; + + if (mTransA) + { + for (int i = 1; i < nbDimsA; ++i) + { + ret.d[i - 1] = inputs[0].d[i]; + } + } + else + { + for (int i = 0; i < nbDimsA - 1; ++i) + { + ret.d[i] = inputs[0].d[i]; + } + } + if (mTransB) + { + for (int i = 0; i < nbDimsB - 1; ++i) + { + ret.d[nbDimsA - 1 + i] = exprBuilder.constant(inputs[1].d[i]->getConstantValue() + mPadLdc); + } + } + else + { + for (int i = 1; i < nbDimsB; ++i) + { + ret.d[nbDimsA - 2 + i] = exprBuilder.constant(inputs[1].d[i]->getConstantValue() + mPadLdc); + } + } + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool GemmPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + auto const& desc = inOut[pos]; + if (desc.format != TensorFormat::kLINEAR) + { + return false; + } + + if (pos < nbInputs) + { + // If use FP8, act/weight dtype should be kFP8 + if (mUseFp8) + { + return desc.type == nvinfer1::DataType::kFP8; + } + else + { + return desc.type == mType; + } + } + + return desc.type == mType || desc.type == nvinfer1::DataType::kFLOAT; +} + +void GemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const nbDimsA = in[0].max.nbDims; + + auto const minM = utils::computeMDimension(mTransA, in[0].min); + auto const maxM = utils::computeMDimension(mTransA, in[0].max); + auto const N = utils::computeNDimension(mTransB, in[1].max); + auto const K = static_cast<utils::DimType64>(mTransA ? in[0].max.d[0] : in[0].max.d[nbDimsA - 1]); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, N, K}; + } + mGemmId.n = N; + mGemmId.k = K; + + mOutputType = out[0].desc.type; +} + +size_t GemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return CUBLAS_WORKSPACE_SIZE; +} + +int GemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + // inputs + // mat1 [M, K] (mTransA = False) + // mat2 [K, N] (mTransB = False) + // outputs + // mat [M, N] + if (mCublasWrapper == nullptr) + { + auto cublasHandle = getCublasHandle(); + auto cublasLtHandle = getCublasLtHandle(); + mCublasWrapper = std::make_shared<CublasMMWrapper>(cublasHandle, cublasLtHandle, nullptr, nullptr); + } + setGemmConfig(); + + int const nbDimsA = inputDesc[0].dims.nbDims; + int const padM = mTransA ? mPadLda : 0; + int const padN = mTransB ? 0 : mPadLdb; + int const padK = mTransA ? 0 : mPadLda; + auto const M = utils::computeMDimension(mTransA, inputDesc[0].dims) - padM; + auto const N = utils::computeNDimension(mTransB, inputDesc[1].dims) - padN; + int const K = static_cast<utils::DimType64>( + mTransA ? inputDesc[0].dims.d[0] - padK : inputDesc[0].dims.d[nbDimsA - 1] - padK); + + bool noPadDim = padM == 0 && padN == 0 && padK == 0 && mPadLdc == 0; + bool cudaKernelSupportType = mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kFLOAT + || mType == nvinfer1::DataType::kBF16; + + // skip computation for a TRT empty tensor + if (M == 0) + { + return 0; + } + + std::string mnkStr = "MNK={" + std::to_string(M) + ", " + std::to_string(N) + ", " + std::to_string(K) + "}"; + { + std::string const activationStr = "GEMM layer's activation before GEMM with " + mnkStr; + TLLM_CHECK_DEBUG_WITH_INFO( + tensorrt_llm::runtime::utils::tensorHasInvalid(M, K, mType, inputs[0], stream, activationStr) == false, + "Found invalid number (NaN or Inf) in " + activationStr); + } + + bool cudaKernelFinished = false; + bool isArch90or100 = mArch >= 90 && mArch < 120; + // TODO: sub tensor matmul is not supported in fp8 gemm cuda kernel + if (!isArch90or100 && M <= 4 && N <= 128000 && mUseFp8 && noPadDim && cudaKernelSupportType) + { + tensorrt_llm::kernels::cuda_core_gemm::Params params(reinterpret_cast<void const*>(inputs[0]), + reinterpret_cast<void const*>(inputs[1]), mAlpha, reinterpret_cast<void*>(outputs[0]), M, N, K, + CUDA_R_8F_E4M3, trtToCublasDtype(mOutputType)); + cudaKernelFinished = tensorrt_llm::kernels::cuda_core_gemm::cudaCoreGemmDispatcher(params, stream); + } + else if (!isArch90or100 && ((mArch < 90 && M <= 6) || (isArch90or100 && M <= 2)) && N <= 128000 && !mUseFp8 + && noPadDim && cudaKernelSupportType) + { + tensorrt_llm::kernels::cuda_core_gemm::Params params(reinterpret_cast<void const*>(inputs[0]), + reinterpret_cast<void const*>(inputs[1]), mAlpha, reinterpret_cast<void*>(outputs[0]), M, N, K, + trtToCublasDtype(mType), trtToCublasDtype(mOutputType)); + cudaKernelFinished = tensorrt_llm::kernels::cuda_core_gemm::cudaCoreGemmDispatcher(params, stream); + } + + if (!cudaKernelFinished) + { + auto bestTactic = mPluginProfiler->getBestConfig(M, mGemmId); + runGemm(M, N, K, mTransA, mTransB, mPadLda, mPadLdb, mPadLdc, mType, mCublasWrapper, inputs[0], inputs[1], + mAlpha, outputs[0], bestTactic, workspace, stream); + } + + { + std::string const outputStr = "GEMM layer's output after GEMM with " + mnkStr; + TLLM_CHECK_DEBUG_WITH_INFO( + tensorrt_llm::runtime::utils::tensorHasInvalid(M, N + mPadLdc, mType, outputs[0], stream, outputStr) + == false, + "Found invalid number (NaN or Inf) in " + outputStr); + } + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType GemmPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* GemmPlugin::getPluginType() const noexcept +{ + return GEMM_PLUGIN_NAME; +} + +char const* GemmPlugin::getPluginVersion() const noexcept +{ + return GEMM_PLUGIN_VERSION; +} + +int GemmPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int GemmPlugin::initialize() noexcept +{ + configGemm(); + return 0; +} + +void GemmPlugin::destroy() noexcept +{ + delete this; +} + +size_t GemmPlugin::getSerializationSize() const noexcept +{ + return sizeof(mTransA) + sizeof(mTransB) + sizeof(mPadLda) + sizeof(mPadLdb) + sizeof(mPadLdc) + sizeof(mType) + + sizeof(mDims) + sizeof(mUseFp8) + sizeof(mAlpha) + mPluginProfiler->getSerializationSize(mGemmId) + + sizeof(mOutputType); // selected tactics container size +} + +void GemmPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mTransA); + write(d, mTransB); + write(d, mPadLda); + write(d, mPadLdb); + write(d, mPadLdc); + write(d, mType); + write(d, mUseFp8); + write(d, mAlpha); + write(d, mDims); + write(d, mOutputType); + mPluginProfiler->serialize(d, mGemmId); + + TLLM_CHECK(d == a + getSerializationSize()); +} + +void GemmPlugin::terminate() noexcept {} + +/////////////// + +GemmPluginCreator::GemmPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("transA", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("transB", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("padLda", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("padLdb", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("padLdc", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("use_fp8", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* GemmPluginCreator::getPluginName() const noexcept +{ + return GEMM_PLUGIN_NAME; +} + +char const* GemmPluginCreator::getPluginVersion() const noexcept +{ + return GEMM_PLUGIN_VERSION; +} + +PluginFieldCollection const* GemmPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* GemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int transA{}; + int transB{}; + int padLda{}; + int padLdb{}; + int padLdc{}; + nvinfer1::DataType type{}; + int useFp8{}; + float alpha = 1.F; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "transa")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + transA = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "transb")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + transB = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "pad_lda")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + padLda = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "pad_ldb")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + padLdb = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "pad_ldc")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + padLdc = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "use_fp8")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + useFp8 = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "alpha")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + alpha = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + } + try + { + // GemmPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + // FIXME enable tactic profiler + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false, /* skip */ true); + auto* obj = new GemmPlugin(transA, transB, padLda, padLdb, padLdc, type, useFp8, alpha, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* GemmPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call GemmPlugin::destroy() + try + { + // GemmPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + // FIXME enable tactic profiler + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true, /* skip */ true); + auto* obj = new GemmPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h b/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h new file mode 100644 index 000000000000..1ba553c23d4b --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h @@ -0,0 +1,169 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TRT_GEMM_PLUGIN_H +#define TRT_GEMM_PLUGIN_H + +#include "tensorrt_llm/common/cublasMMWrapper.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" + +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +using CublasGemmWrapper = tensorrt_llm::common::CublasMMWrapper; +using CublasGemmWrapperPtr = std::shared_ptr<CublasGemmWrapper>; + +class CublasLtGemmPluginProfiler + : public GemmPluginProfiler<cublasLtMatmulHeuristicResult_t, CublasGemmWrapperPtr, GemmIdCublas, GemmIdCublasHash> +{ +public: + using Config = cublasLtMatmulHeuristicResult_t; + + void setTranspose(bool transposeA, bool transposeB) + { + mTransA = transposeA; + mTransB = transposeB; + } + + void setPadLd(int padLda, int padLdb, int padLdc) + { + mPadLda = padLda; + mPadLdb = padLdb; + mPadLdc = padLdc; + } + + void setOutputType(nvinfer1::DataType type) + { + mOutputType = type; + } + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + bool checkTactic(int m, int n, int k, Config const& tactic) const override; + + std::vector<Config> getTactics(int m, int n, int k) const override; + +private: + bool mTransA; + bool mTransB; + int mPadLda; + int mPadLdb; + int mPadLdc; + nvinfer1::DataType mOutputType; + + static constexpr size_t ALIGNMENT = 256; +}; + +class GemmPlugin : public BasePlugin +{ +public: + using PluginProfilerPtr = std::shared_ptr<CublasLtGemmPluginProfiler>; + + GemmPlugin() = delete; + + GemmPlugin(int transA, int transB, int padLda, int padLdb, int padLdc, nvinfer1::DataType type, bool useFp8, + float alpha, PluginProfilerPtr const& profiler); + + GemmPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); + + ~GemmPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + void init(); + void configGemm(); + void setGemmConfig(); + +private: + const std::string mLayerName; + + int mTransA; + int mTransB; + int mPadLda; + int mPadLdb; + int mPadLdc; + int mArch; + nvinfer1::DataType mType; + nvinfer1::DataType mOutputType; + + static thread_local CublasGemmWrapperPtr mCublasWrapper; + + GemmDims mDims{}; + GemmIdCublas mGemmId{}; + bool mUseFp8{false}; + float mAlpha{1.f}; + + PluginProfilerPtr mPluginProfiler; +}; + +class GemmPluginCreator : public BaseCreator +{ +public: + GemmPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<CublasLtGemmPluginProfiler> gemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins + +#endif // TRT_GEMM_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/CMakeLists.txt new file mode 100644 index 000000000000..3b714a3928fb --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp *.cu) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cpp b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cpp new file mode 100644 index 000000000000..ed964ace695f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cpp @@ -0,0 +1,446 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gemmSwigluPlugin.h" +#include "cutlass_extensions/gemm_configs.h" + +#include <NvInferRuntimeBase.h> +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels::cutlass_kernels; +using tensorrt_llm::plugins::GemmSwigluPluginCreator; +using tensorrt_llm::plugins::GemmSwigluPlugin; +using tensorrt_llm::plugins::GemmSwigluPluginProfiler; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +static char const* GEMM_SWIGLU_PLUGIN_VERSION{"1"}; +static char const* GEMM_SWIGLU_PLUGIN_NAME{"GemmSwiglu"}; +PluginFieldCollection GemmSwigluPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> GemmSwigluPluginCreator::mPluginAttributes; + +size_t GemmSwigluPluginProfiler::getBytePerElement(nvinfer1::DataType type) +{ + size_t bpe; + if (type == nvinfer1::DataType::kHALF || type == nvinfer1::DataType::kBF16) + { + bpe = 2; + } + else if (type == nvinfer1::DataType::kINT8 || type == nvinfer1::DataType::kFP8) + { + bpe = 1; + } + else + { + TLLM_THROW("Not recognized/implemented"); + } + return bpe; +} + +void GemmSwigluPluginProfiler::setQuantMode(tensorrt_llm::common::QuantMode const& quantMode) +{ + mQuantMode = quantMode; +} + +void GemmSwigluPluginProfiler::runTactic( + int m, int n, int k, GemmSwigluPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) +{ + size_t bpe = getBytePerElement(mType); + + // Workspace size required by gemm runner + // NB: this function will throw exception when selected tactic exceeds SMEM, which is then + // caught by gemmPluginProfiler and it will register this tactic as invalid + size_t wsSizeRunner = mRunner->getWorkspaceSize(m, n, k); + + // Workspace size required by profiling + size_t wsByteOffset = 0; + int8_t* wsBytePointer = reinterpret_cast<int8_t*>(workspace); + void* aTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * k * bpe)); + void* bTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, n * k * bpe)); + void* cTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, 1 * n * bpe)); + void* dTmp = reinterpret_cast<void*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, m * (n / 2) * bpe)); + char* workspaceTmp = reinterpret_cast<char*>(nextWorkspacePtr(wsBytePointer, wsByteOffset, wsSizeRunner)); + + // Run profiling + mRunner->gemm( + dTmp, aTmp, bTmp, cTmp, mQuantMode, m, n, k, 1.0, 1.0, 1.0, tactic, workspaceTmp, wsSizeRunner, stream); +} + +int GemmSwigluPluginProfiler::getMaxProfileM() const +{ + return 32768; +} + +void GemmSwigluPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + std::vector<size_t> workspaces = { + maxM * k * getBytePerElement(mType), // A + n * k * getBytePerElement(mType), // B + 1 * n * getBytePerElement(mType), // C_bias + maxM * (n / 2) * getBytePerElement(mType), // D + mRunner->getWorkspaceSize(maxM, n, k) // workspace + }; + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<GemmSwigluPluginProfiler::Config> GemmSwigluPluginProfiler::getTactics(int m, int n, int k) const +{ + return mRunner->getConfigs(); +} + +GemmSwigluPlugin::GemmSwigluPlugin(QuantMode quantMode, nvinfer1::DataType type, bool hasBias, float scale_d0, + float scale_d1, float scale_output, GemmSwigluPlugin::PluginProfilerPtr const& pluginProfiler) + : mQuantMode(quantMode) + , mPluginProfiler(pluginProfiler) + , mHasBias(hasBias) + , mScaleD0(scale_d0) + , mScaleD1(scale_d1) + , mScaleOutput(scale_output) +{ + init(type); +} + +// Parameterized constructor +GemmSwigluPlugin::GemmSwigluPlugin( + void const* data, size_t length, GemmSwigluPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + nvinfer1::DataType type; + unsigned int quantMode; + read(d, quantMode); + read(d, type); + read(d, mHasBias); + read(d, mScaleD0); + read(d, mScaleD1); + read(d, mScaleOutput); + read(d, mDims); + + mQuantMode = QuantMode(quantMode); + + init(type); + + mPluginProfiler->deserialize(d, mDims, mGemmId); + + TLLM_CHECK(d == a + length); +} + +void GemmSwigluPlugin::init(nvinfer1::DataType type) +{ + mType = type; + if (mType == nvinfer1::DataType::kFP8) + { + mGemmRunner = std::make_shared<CutlassFusedGatedGemmRunner<__nv_fp8_e4m3>>(); + } + else + { + TLLM_THROW("Gemm Swiglu plugin only supports fp8 now"); + } + + mPluginProfiler->setQuantMode(mQuantMode); + + mGemmId = GemmIdCore(mDims.n, mDims.k, mType); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* GemmSwigluPlugin::clone() const noexcept +{ + auto* plugin = new GemmSwigluPlugin(*this); + return plugin; +} + +nvinfer1::DimsExprs GemmSwigluPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 3); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + TLLM_CHECK(nbDimsA >= 2); + DimsExprs ret; + ret.nbDims = nbDimsA; + for (int ii = 0; ii < nbDimsA - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[1].d[1]->getConstantValue() / 2); + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool GemmSwigluPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + switch (pos) + { + case 0: + // activation + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + case 1: + // weights + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + case 2: + // bias + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + case 3: + // out + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + default: + // Never should be here + TLLM_CHECK(false); + return false; + } +} + +void GemmSwigluPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + int const maxN = in[1].max.d[1]; + int const minK = in[0].min.d[in[0].min.nbDims - 1]; + int const minN = in[1].min.d[1]; + + TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); + TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, maxN, maxK}; + } + mGemmId = {maxN, maxK, mType}; + + mWorkspaceMaxSize = mGemmRunner->getWorkspaceSize(maxM, maxN, maxK); +} + +size_t GemmSwigluPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return mWorkspaceMaxSize; +} + +int GemmSwigluPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + // inputs + // mat1 [M(*), K] + // mat2 [K, N] + // bias [1, N] + // outputs + // mat [M(*), N / 2] + int m = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m *= inputDesc[0].dims.d[ii]; + } + int const n = inputDesc[1].dims.d[1]; + int const k = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; + size_t const wsSize = mGemmRunner->getWorkspaceSize(m, n, k); + + auto const bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); + TLLM_CHECK_WITH_INFO(bestTactic, "No valid GEMM tactic"); + mGemmRunner->gemm(outputs[0], inputs[0], inputs[1], inputs[2], mQuantMode, m, n, k, mScaleD0, mScaleD1, + mScaleOutput, *bestTactic, reinterpret_cast<char*>(workspace), wsSize, stream); + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType GemmSwigluPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* GemmSwigluPlugin::getPluginType() const noexcept +{ + return GEMM_SWIGLU_PLUGIN_NAME; +} + +char const* GemmSwigluPlugin::getPluginVersion() const noexcept +{ + return GEMM_SWIGLU_PLUGIN_VERSION; +} + +int GemmSwigluPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int GemmSwigluPlugin::initialize() noexcept +{ + configGemm(); // gemm profiler in action + return 0; +} + +void GemmSwigluPlugin::terminate() noexcept {} + +size_t GemmSwigluPlugin::getSerializationSize() const noexcept +{ + return sizeof(unsigned int) + // QuantMode + sizeof(nvinfer1::DataType) + // dtype + sizeof(bool) + // hasBias + sizeof(float) * 3 + // scales + sizeof(mDims) + // Dimensions + mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size +} + +void GemmSwigluPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mQuantMode.value()); + write(d, mType); + write(d, mHasBias); + write(d, mScaleD0); + write(d, mScaleD1); + write(d, mScaleOutput); + write(d, mDims); + + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void GemmSwigluPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +void GemmSwigluPlugin::configGemm() +{ + mPluginProfiler->profileTactics(mGemmRunner, mType, mDims, mGemmId); +} + +/////////////// + +GemmSwigluPluginCreator::GemmSwigluPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("has_bias", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("scale_d0", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("scale_d1", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("scale_output", nullptr, PluginFieldType::kFLOAT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* GemmSwigluPluginCreator::getPluginName() const noexcept +{ + return GEMM_SWIGLU_PLUGIN_NAME; +} + +char const* GemmSwigluPluginCreator::getPluginVersion() const noexcept +{ + return GEMM_SWIGLU_PLUGIN_VERSION; +} + +PluginFieldCollection const* GemmSwigluPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* GemmSwigluPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + TLLM_CHECK(fc->nbFields == 5); + nvinfer1::DataType type{}; + bool hasBias{}; + float scale_d0{}; + float scale_d1{}; + float scale_output{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "has_bias")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + hasBias = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); + } + else if (!strcmp(attrName, "scale_d0")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + scale_d0 = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + else if (!strcmp(attrName, "scale_d1")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + scale_d1 = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + else if (!strcmp(attrName, "scale_output")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + scale_output = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + } + try + { + // GemmSwigluPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); + QuantMode quantMode = QuantMode{}; + auto* obj = new GemmSwigluPlugin(quantMode, type, hasBias, scale_d0, scale_d1, scale_output, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* GemmSwigluPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call GemmSwigluPlugin::destroy() + try + { + // Create plugin profiler with private tactics map which is read from the serialized engine + auto pluginProfiler = mGemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); + auto* obj = new GemmSwigluPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cu b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cu new file mode 100644 index 000000000000..339c432b1113 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.cu @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gemmSwigluPlugin.h" + +#include "cutlass/util/reference/device/tensor_fill.h" +#include "cutlass_extensions/gemm_configs.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels::cutlass_kernels; +using tensorrt_llm::plugins::GemmSwigluPluginCreator; +using tensorrt_llm::plugins::GemmSwigluPlugin; +using tensorrt_llm::plugins::GemmSwigluPluginProfiler; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +void GemmSwigluPluginProfiler::initTmpData(int m, int n, int k, char* workspace, size_t size, cudaStream_t stream) +{ + size_t bpe = getBytePerElement(mType); + + if (mType == nvinfer1::DataType::kFP8) + { + cutlass::reference::device::BlockFillRandomUniform(reinterpret_cast<cutlass::float_e4m3_t*>(workspace), + m * k + n * k + 1 * n, 42, cutlass::float_e4m3_t{128}, -cutlass::float_e4m3_t{128}, -1, 0, stream); + } +} diff --git a/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.h b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.h new file mode 100644 index 000000000000..766e59aad258 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gemmSwigluPlugin/gemmSwigluPlugin.h @@ -0,0 +1,150 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/kernels/cutlass_kernels/fused_gated_gemm/fused_gated_gemm.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +using GemmSwigluRunnerPtr + = std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassFusedGatedGemmRunnerInterface>; + +class GemmSwigluPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + GemmSwigluRunnerPtr, GemmIdCore, GemmIdCoreHash> + +{ +public: + using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; + + void setQuantMode(tensorrt_llm::common::QuantMode const& quantMode); + + virtual int getMaxProfileM() const override; + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + // TODO(anchengc) implement checkTactic + // bool checkTactic(int m, int n, int k, const Config& tactic) const override; + + std::vector<Config> getTactics(int m, int n, int k) const override; + + void initTmpData(int m, int n, int k, char* workspace, size_t size, cudaStream_t stream) override; + +private: + size_t getBytePerElement(nvinfer1::DataType type); + + tensorrt_llm::common::QuantMode mQuantMode; +}; + +class GemmSwigluPlugin : public BasePlugin +{ +public: + using PluginProfilerPtr = std::shared_ptr<GemmSwigluPluginProfiler>; + + GemmSwigluPlugin() = delete; + + GemmSwigluPlugin(tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, bool hasBias, float scale_d0, + float scale_d1, float scale_output, PluginProfilerPtr const& pluginProfiler); + + GemmSwigluPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); + + ~GemmSwigluPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + void init(nvinfer1::DataType type); + + void configGemm(); + // void setGemmConfig(); + +private: + const std::string mLayerName; + + GemmSwigluRunnerPtr mGemmRunner; + tensorrt_llm::common::QuantMode mQuantMode; // not configurable yet + size_t mWorkspaceMaxSize; + + GemmDims mDims{}; + GemmIdCore mGemmId{}; + + PluginProfilerPtr mPluginProfiler; + + nvinfer1::DataType mType; + bool mHasBias; + float mScaleD0; + float mScaleD1; + float mScaleOutput; +}; + +class GemmSwigluPluginCreator : public BaseCreator +{ +public: + GemmSwigluPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<GemmSwigluPluginProfiler> mGemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gptAttentionCommon/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gptAttentionCommon/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gptAttentionCommon/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.cpp b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.cpp new file mode 100644 index 000000000000..717ab3083e5f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.cpp @@ -0,0 +1,380 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "gptAttentionCommon.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderXQARunner.h" +#include "tensorrt_llm/kernels/gptKernels.h" +#include <NvInferRuntimePlugin.h> +#include <cstdint> + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +namespace tc = tensorrt_llm::common; +using tensorrt_llm::plugins::GPTAttentionPluginCreatorCommon; +using tensorrt_llm::plugins::GPTAttentionPluginCommon; + +GPTAttentionPluginCommon::GPTAttentionPluginCommon(int layer_idx, int num_heads, int vision_start, int vision_length, + int num_kv_heads, int num_kv_heads_origin, int head_size, int unidirectional, float q_scaling, + float attn_logit_softcapping_scale, tensorrt_llm::kernels::PositionEmbeddingType position_embedding_type, + int rotary_embedding_dim, // for RoPE. Use 0 for non-RoPE + float rotary_embedding_base, tensorrt_llm::kernels::RotaryScalingType rotary_embedding_scale_type, + float rotary_embedding_scale, float rotary_embedding_short_m_scale, float rotary_embedding_long_m_scale, + int rotary_embedding_max_positions, int rotary_embedding_original_max_positions, int tp_size, + int tp_rank, // for ALiBi + bool unfuse_qkv_gemm, // for AutoPP + bool use_logn_scaling, // for LognScaling + tensorrt_llm::kernels::ContextFMHAType context_fmha_type, int kv_cache_quant_mode, bool remove_input_padding, + tensorrt_llm::kernels::AttentionMaskType mask_type, tensorrt_llm::kernels::BlockSparseParams block_sparse_params, + bool paged_kv_cache, int tokens_per_block, nvinfer1::DataType type, int32_t max_context_length, + bool qkv_bias_enabled, bool cross_attention, int max_distance, bool pos_shift_enabled, bool dense_context_fmha, + bool use_paged_context_fmha, bool use_fp8_context_fmha, bool has_full_attention_mask, bool use_cache, + bool is_spec_decoding_enabled, bool spec_decoding_is_generation_length_variable, + int32_t spec_decoding_max_generation_length, bool is_mla_enabled, int q_lora_rank, int kv_lora_rank, + int qk_nope_head_dim, int qk_rope_head_dim, int v_head_dim, bool fuse_fp4_quant, bool skip_attn, int cp_size, + int cp_rank, std::set<int32_t> cp_group) + : mResource{DecoderXQARunner::getResourceGlobal()} +{ + mLayerIdx = layer_idx; + mNumHeads = num_heads; + mVisionStart = vision_start; + mVisionLength = vision_length; + mNumKVHeads = num_kv_heads; + mNumKVHeadsOrigin = num_kv_heads_origin; + mHeadSize = head_size; + mUnidirectional = unidirectional; + mQScaling = q_scaling; + mAttnLogitSoftcappingScale = attn_logit_softcapping_scale; + mRotaryEmbeddingDim = rotary_embedding_dim; + mRotaryEmbeddingBase = rotary_embedding_base; + mRotaryEmbeddingScaleType = rotary_embedding_scale_type; + mRotaryEmbeddingScale = rotary_embedding_scale; + mRotaryEmbeddingShortMscale = rotary_embedding_short_m_scale; + mRotaryEmbeddingLongMscale = rotary_embedding_long_m_scale; + mRotaryEmbeddingMaxPositions = rotary_embedding_max_positions; + mRotaryEmbeddingOriginalMaxPositions = rotary_embedding_original_max_positions; + mPositionEmbeddingType = position_embedding_type; + mEnableContextFMHA = context_fmha_type != ContextFMHAType::DISABLED; + mFMHAForceFP32Acc = type == nvinfer1::DataType::kBF16; + mMaskType = mask_type; + mBlockSparseParams = block_sparse_params; + mType = type; + mMultiBlockMode = true; + mEnableXQA = true; + mKVCacheQuantMode = tc::QuantMode(kv_cache_quant_mode); + mRemovePadding = remove_input_padding; + mPagedKVCache = paged_kv_cache; + mTokensPerBlock = tokens_per_block; + mTpSize = tp_size; + mTpRank = tp_rank; + mUnfuseQkvGemm = unfuse_qkv_gemm; + mUseLognScaling = use_logn_scaling; + mMaxContextLength = max_context_length; + mQKVBiasEnabled = qkv_bias_enabled; + mCrossAttention = cross_attention; + mMaxDistance = max_distance; + mPosShiftEnabled = pos_shift_enabled; + mDenseContextFMHA = dense_context_fmha; + mPagedContextFMHA = use_paged_context_fmha; + mFP8ContextFMHA = use_fp8_context_fmha; + mFP8AttenOutput = use_fp8_context_fmha; + mHasFullAttentionMask = has_full_attention_mask; + mUseKVCache = use_cache; + mIsSpecDecodingEnabled = is_spec_decoding_enabled; + mSpecDecodingIsGenerationLengthVariable = spec_decoding_is_generation_length_variable; + mSpecDecodingMaxGenerationLength = spec_decoding_max_generation_length; + mIsMLAEnabled = is_mla_enabled; + mMLAParams = {q_lora_rank, kv_lora_rank, qk_nope_head_dim, qk_rope_head_dim, v_head_dim}; + mCpSize = cp_size; + mCpRank = cp_rank; + mCpGroup = std::move(cp_group); + mFuseFp4Quant = fuse_fp4_quant; + mSkipAttn = skip_attn; +} + +// Parameterized constructor +GPTAttentionPluginCommon::GPTAttentionPluginCommon(void const* data, size_t length) + : mResource{DecoderXQARunner::getResourceGlobal()} +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + unsigned int kvCacheQuantMode; + + read(d, mLayerIdx); + read(d, mNumHeads); + read(d, mVisionStart); + read(d, mVisionLength); + read(d, mNumKVHeads); + read(d, mNumKVHeadsOrigin); + read(d, mHeadSize); + read(d, mUnidirectional); + read(d, mQScaling); + read(d, mAttnLogitSoftcappingScale); + read(d, mPositionEmbeddingType); + read(d, mRotaryEmbeddingDim); + read(d, mRotaryEmbeddingBase); + read(d, mRotaryEmbeddingScaleType); + read(d, mRotaryEmbeddingScale); + read(d, mRotaryEmbeddingShortMscale); + read(d, mRotaryEmbeddingLongMscale); + read(d, mRotaryEmbeddingMaxPositions); + read(d, mRotaryEmbeddingOriginalMaxPositions); + read(d, mTpSize); + read(d, mTpRank); + read(d, mUnfuseQkvGemm); + read(d, mUseLognScaling); + read(d, mEnableContextFMHA); + read(d, mFMHAForceFP32Acc); + read(d, mMultiBlockMode); + read(d, mEnableXQA); + read(d, kvCacheQuantMode); + read(d, mRemovePadding); + read(d, mMaskType); + read(d, mBlockSparseParams); + read(d, mPagedKVCache); + read(d, mTokensPerBlock); + read(d, mType); + read(d, mMaxContextLength); + read(d, mQKVBiasEnabled); + read(d, mCrossAttention); + read(d, mMaxDistance); + read(d, mPosShiftEnabled); + read(d, mDenseContextFMHA); + read(d, mPagedContextFMHA); + read(d, mFP8ContextFMHA); + read(d, mFP8AttenOutput); + read(d, mHasFullAttentionMask); + read(d, mUseKVCache); + read(d, mIsSpecDecodingEnabled); + read(d, mUseSpecDecoding); + read(d, mSpecDecodingIsGenerationLengthVariable); + read(d, mSpecDecodingMaxGenerationLength); + read(d, mIsMLAEnabled); + read(d, mMLAParams); + read(d, mNbMultiBlockSemaphores); + read(d, mFuseFp4Quant); + read(d, mSkipAttn); + read(d, mCpSize); + read(d, mCpRank); + + mKVCacheQuantMode = tc::QuantMode(kvCacheQuantMode); + + uint32_t decoderXQARunnerResourceSerializedSize; + read(d, decoderXQARunnerResourceSerializedSize); + mResource->merge(DecoderXQARunnerResource(d, decoderXQARunnerResourceSerializedSize), /*initialize=*/true); + d += decoderXQARunnerResourceSerializedSize; + + mCpGroup.clear(); + int32_t groupItem = 0; + while (d != a + length) + { + read(d, groupItem); + mCpGroup.insert(groupItem); + } + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); + TLLM_CHECK_WITH_INFO((smVersion() >= 80) || (mType != nvinfer1::DataType::kBF16), + "Unsupported data type, pre SM 80 GPUs do not support bfloat16"); +} + +int GPTAttentionPluginCommon::initialize() noexcept +{ + return AttentionOp::initialize(); +} + +void GPTAttentionPluginCommon::destroy() noexcept +{ + delete this; +} + +size_t GPTAttentionPluginCommon::getCommonSerializationSize() const noexcept +{ + return sizeof(mLayerIdx) + sizeof(mNumHeads) + +sizeof(mVisionStart) + sizeof(mVisionLength) + sizeof(mNumKVHeads) + + sizeof(mNumKVHeadsOrigin) + sizeof(mHeadSize) + sizeof(mUnidirectional) + sizeof(mQScaling) + + sizeof(mAttnLogitSoftcappingScale) + sizeof(mPositionEmbeddingType) + sizeof(mRotaryEmbeddingDim) + + sizeof(mRotaryEmbeddingBase) + sizeof(mRotaryEmbeddingScaleType) + sizeof(mRotaryEmbeddingScale) + + sizeof(mRotaryEmbeddingShortMscale) + sizeof(mRotaryEmbeddingLongMscale) + + sizeof(mRotaryEmbeddingMaxPositions) + sizeof(mRotaryEmbeddingOriginalMaxPositions) + sizeof(mTpSize) + + sizeof(mTpRank) + sizeof(mEnableContextFMHA) + sizeof(mFMHAForceFP32Acc) + sizeof(mMultiBlockMode) + + sizeof(mEnableXQA) + sizeof(unsigned int) // mKVCacheQuantMode + + sizeof(mRemovePadding) + sizeof(mMaskType) + sizeof(mBlockSparseParams) + sizeof(mPagedKVCache) + + sizeof(mTokensPerBlock) + sizeof(mType) + sizeof(mMaxContextLength) + sizeof(mQKVBiasEnabled) + + sizeof(mCrossAttention) + sizeof(mMaxDistance) + sizeof(mPosShiftEnabled) + sizeof(mDenseContextFMHA) + + sizeof(mPagedContextFMHA) + sizeof(mFP8ContextFMHA) + sizeof(mFP8AttenOutput) + sizeof(mHasFullAttentionMask) + + sizeof(mUseKVCache) + sizeof(mUnfuseQkvGemm) + sizeof(mUseLognScaling) + sizeof(mIsSpecDecodingEnabled) + + sizeof(mUseSpecDecoding) + sizeof(mSpecDecodingIsGenerationLengthVariable) + + sizeof(mSpecDecodingMaxGenerationLength) + sizeof(mNbMultiBlockSemaphores) + sizeof(mIsMLAEnabled) + + sizeof(mMLAParams) + sizeof(mFuseFp4Quant) + sizeof(mSkipAttn) + + sizeof(uint32_t) // size of DecoderXQARunnerResource buffer. + + sizeof(mCpSize) + sizeof(mCpRank) + sizeof(int32_t) * mCpGroup.size() + mResource->getSerializationSize(); +} + +void GPTAttentionPluginCommon::serializeCommon(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mLayerIdx); + write(d, mNumHeads); + write(d, mVisionStart); + write(d, mVisionLength); + write(d, mNumKVHeads); + write(d, mNumKVHeadsOrigin); + write(d, mHeadSize); + write(d, mUnidirectional); + write(d, mQScaling); + write(d, mAttnLogitSoftcappingScale); + write(d, mPositionEmbeddingType); + write(d, mRotaryEmbeddingDim); + write(d, mRotaryEmbeddingBase); + write(d, mRotaryEmbeddingScaleType); + write(d, mRotaryEmbeddingScale); + write(d, mRotaryEmbeddingShortMscale); + write(d, mRotaryEmbeddingLongMscale); + write(d, mRotaryEmbeddingMaxPositions); + write(d, mRotaryEmbeddingOriginalMaxPositions); + write(d, mTpSize); + write(d, mTpRank); + write(d, mUnfuseQkvGemm); + write(d, mUseLognScaling); + write(d, mEnableContextFMHA); + write(d, mFMHAForceFP32Acc); + write(d, mMultiBlockMode); + write(d, mEnableXQA); + write(d, mKVCacheQuantMode.value()); + write(d, mRemovePadding); + write(d, mMaskType); + write(d, mBlockSparseParams); + write(d, mPagedKVCache); + write(d, mTokensPerBlock); + write(d, mType); + write(d, mMaxContextLength); + write(d, mQKVBiasEnabled); + write(d, mCrossAttention); + write(d, mMaxDistance); + write(d, mPosShiftEnabled); + write(d, mDenseContextFMHA); + write(d, mPagedContextFMHA); + write(d, mFP8ContextFMHA); + write(d, mFP8AttenOutput); + write(d, mHasFullAttentionMask); + write(d, mUseKVCache); + write(d, mIsSpecDecodingEnabled); + write(d, mUseSpecDecoding); + write(d, mSpecDecodingIsGenerationLengthVariable); + write(d, mSpecDecodingMaxGenerationLength); + write(d, mIsMLAEnabled); + write(d, mMLAParams); + write(d, mNbMultiBlockSemaphores); + write(d, mFuseFp4Quant); + write(d, mSkipAttn); + write(d, mCpSize); + write(d, mCpRank); + + // An uint32_t that specifies the size of the serialized buffer, followed by the actual content. + uint32_t decoderXQARunnerResourceSerializedSize = mResource->getSerializationSize(); + write(d, decoderXQARunnerResourceSerializedSize); + mResource->serialize(d, decoderXQARunnerResourceSerializedSize); + d += decoderXQARunnerResourceSerializedSize; + + for (auto it = mCpGroup.begin(); it != mCpGroup.end(); ++it) + { + write(d, *it); + } + TLLM_CHECK(d == a + getCommonSerializationSize()); +} + +void GPTAttentionPluginCommon::terminate() noexcept +{ + // Do nothing, destroy will always be called, so release the resources there. +} + +/////////////// + +GPTAttentionPluginCreatorCommon::GPTAttentionPluginCreatorCommon() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("layer_idx", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("num_heads", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("vision_start", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("vision_length", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("num_kv_heads", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("num_kv_heads_origin", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("layer_idx_in_cache_pool", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("head_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("unidirectional", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("q_scaling", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("attn_logit_softcapping_scale", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("position_embedding_type", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("rotary_embedding_dim", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("rotary_embedding_base", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("rotary_embedding_scale_type", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("rotary_embedding_scale", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("rotary_embedding_short_m_scale", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("rotary_embedding_long_m_scale", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("rotary_embedding_max_positions", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back( + PluginField("rotary_embedding_original_max_positions", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("tp_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("tp_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("unfuse_qkv_gemm", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("use_logn_scaling", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("context_fmha_type", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("kv_cache_quant_mode", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("remove_input_padding", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("mask_type", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("block_sparse_block_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("block_sparse_homo_head_pattern", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("block_sparse_num_local_blocks", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("block_sparse_vertical_stride", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("paged_kv_cache", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("tokens_per_block", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("max_context_length", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("qkv_bias_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("do_cross_attention", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("max_distance", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("pos_shift_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("dense_context_fmha", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("use_paged_context_fmha", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("use_fp8_context_fmha", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("has_full_attention_mask", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("use_cache", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("is_spec_decoding_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back( + PluginField("spec_decoding_is_generation_length_variable", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back( + PluginField("spec_decoding_max_generation_length", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("is_mla_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("q_lora_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("kv_lora_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("qk_nope_head_dim", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("qk_rope_head_dim", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("v_head_dim", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("fuse_fp4_quant", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("skip_attn", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("cp_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("cp_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("cp_group", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +PluginFieldCollection const* GPTAttentionPluginCreatorCommon::getFieldNames() noexcept +{ + return &mFC; +} diff --git a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h new file mode 100644 index 000000000000..dd87d67aab9e --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/attentionOp.h" +#include "tensorrt_llm/common/cublasMMWrapper.h" +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/kernels/gptKernels.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::kernels +{ +class DecoderXQARunnerResource; +} + +namespace tensorrt_llm::plugins +{ + +class GPTAttentionPluginCommon : public BasePlugin, public tensorrt_llm::common::op::AttentionOp +{ +public: + GPTAttentionPluginCommon() = delete; + + GPTAttentionPluginCommon(int layer_idx, int num_heads, int vision_start, int vision_length, int num_kv_heads, + int num_kv_heads_origin, int head_size, int unidirectional, float q_scaling, float attn_logit_softcapping_scale, + tensorrt_llm::kernels::PositionEmbeddingType position_embedding_type, + int rotary_embedding_dim, // for RoPE. Use 0 for non-RoPE + float rotary_embedding_base, tensorrt_llm::kernels::RotaryScalingType rotary_embedding_scale_type, + float rotary_embedding_scale, float rotary_embedding_short_m_scale, float rotary_embedding_long_m_scale, + int rotary_embedding_max_positions, int rotary_embedding_original_max_positions, int tp_size, + int tp_rank, // for ALiBi + bool unfuse_qkv_gemm, // for AutoPP + bool use_logn_scaling, // for LognScaling + tensorrt_llm::kernels::ContextFMHAType context_fmha_type, int kv_cache_quant_mode, bool remove_input_padding, + tensorrt_llm::kernels::AttentionMaskType mask_type, + tensorrt_llm::kernels::BlockSparseParams block_sparse_params, bool paged_kv_cache, int tokens_per_block, + nvinfer1::DataType type, int32_t max_context_length, bool qkv_bias_enabled, bool cross_attention = false, + int max_distance = 0, bool pos_shift_enabled = false, bool dense_context_fmha = false, + bool use_paged_context_fmha = true, bool use_fp8_context_fmha = true, bool has_full_attention_mask = false, + bool use_cache = true, bool is_spec_decoding_enabled = false, + bool spec_decoding_is_generation_length_variable = false, int32_t spec_decoding_max_generation_length = 1, + bool is_mla_enabled = false, int q_lora_rank = 0, int kv_lora_rank = 0, int qk_nope_head_dim = 0, + int qk_rope_head_dim = 0, int v_head_dim = 0, bool fuse_fp4_quant = false, bool skip_attn = false, + int cp_size = 1, int cp_rank = 0, std::set<int32_t> cp_group = {}); + + GPTAttentionPluginCommon(void const* data, size_t length); + + ~GPTAttentionPluginCommon() override = default; + + template <typename T> + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + //! This is called on every trt Engine creation + int initialize() noexcept override; + //! This is called on every trt Engine destroy + void terminate() noexcept override; + + //! This is called on every trt ExecutionContext creation by TRT + //! Note TRT does not call the initialize on cloned plugin, so clone internally should do initialization. + template <typename T> + T* cloneImpl() const noexcept; + + //! This is called on evert trt Engine or ExecutionContext destroy. + //! None-cloned plugins will call terminate and then call destroy, while the cloned plugins will call destroy only + //! So plugin should put the resource release inside destroy. + void destroy() noexcept override; + + size_t getCommonSerializationSize() const noexcept; + void serializeCommon(void* buffer) const noexcept; + +protected: + std::string const mLayerName; + +private: + std::shared_ptr<tensorrt_llm::kernels::DecoderXQARunnerResource> mResource; +}; + +class GPTAttentionPluginCreatorCommon : public BaseCreator +{ +public: + GPTAttentionPluginCreatorCommon(); + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + template <typename T> + T* deserializePluginImpl(char const* name, void const* serialData, size_t serialLength) noexcept; + +protected: + std::vector<nvinfer1::PluginField> mPluginAttributes; + nvinfer1::PluginFieldCollection mFC{}; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommonImpl.h b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommonImpl.h new file mode 100644 index 000000000000..51462cee6f40 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommonImpl.h @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "gptAttentionCommon.h" + +namespace tensorrt_llm::plugins +{ +template <typename T> +T* GPTAttentionPluginCommon::cloneImpl() const noexcept +{ + static_assert(std::is_base_of_v<GPTAttentionPluginCommon, T>); + auto* plugin = new T(static_cast<T const&>(*this)); + plugin->setPluginNamespace(mNamespace.c_str()); + + // Cloned plugins should be in initialized state with correct resources ready to be enqueued. + plugin->initialize(); + return plugin; +} + +template <typename T> +T* GPTAttentionPluginCreatorCommon::deserializePluginImpl( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call GPTAttentionPluginCommon::destroy() + try + { + auto* obj = new T(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.cpp b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.cpp new file mode 100644 index 000000000000..6f8c41c94131 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.cpp @@ -0,0 +1,1387 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gptAttentionPlugin.h" + +#include "tensorrt_llm/batch_manager/contextProgress.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/kernels/decoderMaskedMultiheadAttention.h" +#include "tensorrt_llm/kernels/gptKernels.h" +#include "tensorrt_llm/kernels/unfusedAttentionKernels.h" +#include "tensorrt_llm/plugins/common/checkMacrosPlugin.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h" +#include "tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommonImpl.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/utils/debugUtils.h" + +#include <NvInferRuntimeBase.h> +#include <algorithm> +#include <cstdint> +#include <functional> +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::GPTAttentionPluginCreator; +using tensorrt_llm::plugins::GPTAttentionPlugin; + +static char const* GPT_ATTENTION_PLUGIN_VERSION{"1"}; +static char const* GPT_ATTENTION_PLUGIN_NAME{"GPTAttention"}; + +GPTAttentionPlugin::GPTAttentionPlugin(int layer_idx, int num_heads, int vision_start, int vision_length, + int num_kv_heads, int num_kv_heads_origin, int head_size, int unidirectional, float q_scaling, + float attn_logit_softcapping_scale, tensorrt_llm::kernels::PositionEmbeddingType position_embedding_type, + int rotary_embedding_dim, // for RoPE. 0 for non-RoPE + float rotary_embedding_base, tensorrt_llm::kernels::RotaryScalingType rotary_embedding_scale_type, + float rotary_embedding_scale, float rotary_embedding_short_m_scale, + float rotary_embedding_long_m_scale, // magnitude scaling factors for Phi-3 long RoPE + int rotary_embedding_max_positions, int rotary_embedding_original_max_positions, int tp_size, + int tp_rank, // for ALiBi + bool unfuse_qkv_gemm, // for AutoPP + bool use_logn_scaling, // for LognScaling + tensorrt_llm::kernels::ContextFMHAType context_fmha_type, int kv_cache_quant_mode, bool remove_input_padding, + tensorrt_llm::kernels::AttentionMaskType mask_type, tensorrt_llm::kernels::BlockSparseParams block_sparse_params, + bool paged_kv_cache, int tokens_per_block, nvinfer1::DataType type, int32_t max_context_length, + bool qkv_bias_enabled, bool cross_attention, int max_distance, bool pos_shift_enabled, bool dense_context_fmha, + bool use_paged_context_fmha, bool use_fp8_context_fmha, bool has_full_attention_mask, bool use_cache, + bool is_spec_decoding_enabled, bool spec_decoding_is_generation_length_variable, + int spec_decoding_max_generation_length, bool is_mla_enabled, int q_lora_rank, int kv_lora_rank, + int qk_nope_head_dim, int qk_rope_head_dim, int v_head_dim, bool fuse_fp4_quant, bool skip_attn, int cp_size, + int cp_rank, std::set<int32_t> cp_group) + : GPTAttentionPluginCommon(layer_idx, num_heads, vision_start, vision_length, num_kv_heads, num_kv_heads_origin, + head_size, unidirectional, q_scaling, attn_logit_softcapping_scale, position_embedding_type, + rotary_embedding_dim, rotary_embedding_base, rotary_embedding_scale_type, rotary_embedding_scale, + rotary_embedding_short_m_scale, rotary_embedding_long_m_scale, rotary_embedding_max_positions, + rotary_embedding_original_max_positions, tp_size, tp_rank, unfuse_qkv_gemm, use_logn_scaling, context_fmha_type, + kv_cache_quant_mode, remove_input_padding, mask_type, block_sparse_params, paged_kv_cache, tokens_per_block, + type, max_context_length, qkv_bias_enabled, cross_attention, max_distance, pos_shift_enabled, + dense_context_fmha, use_paged_context_fmha, use_fp8_context_fmha, has_full_attention_mask, use_cache, + is_spec_decoding_enabled, spec_decoding_is_generation_length_variable, spec_decoding_max_generation_length, + is_mla_enabled, q_lora_rank, kv_lora_rank, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, fuse_fp4_quant, + skip_attn, cp_size, cp_rank, cp_group) +{ + TLLM_CHECK_WITH_INFO( + !is_mla_enabled, "GPTAttentionPlugin no longer supports MLA. Please use the PyTorch workflow instead."); + initEntryIdx(); +} + +GPTAttentionPlugin::GPTAttentionPlugin(void const* data, size_t length) + : GPTAttentionPluginCommon(data, length) +{ + initEntryIdx(); +} + +std::string GPTAttentionPlugin::toString(IdxEntry const& entry) const +{ +#define TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(name) \ + case IdxEntry::name: return #name + + switch (entry) + { + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(QKV_TENSOR); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(K_TENSOR); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(V_TENSOR); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ATTENTION_MASK); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ATTENTION_PACKED_MASK); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SEQUENCE_LENGTH); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_PAST_KEY_VALUE_LENGTHS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_MAX_ATTENTION_WINDOW); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_SINK_TOKEN_LENGTH); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(CONTEXT_LENGTHS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(CACHE_INDIR); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(REQUEST_TYPES); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(KV_CACHE_BLOCK_OFFSETS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_KV_CACHE_BLOCK_OFFSETS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_KV_CACHE_POOL_POINTERS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_KV_CACHE_POOL_MAPPING); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(PAST_KEY_VALUE); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(KV_CACHE_QUANTIZATION_SCALE); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(KV_CACHE_DEQUANTIZATION_SCALE); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ATTENTION_OUTPUT_QUANTIZATION_SCALE); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ATTENTION_OUTPUT_SF_SCALE); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ROTARY_INV_FREQ); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ROTARY_COS_SIN); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ALIBI_SLOPES); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(RELATIVE_ATTENTION_BIAS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(CROSS_KV); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(CROSS_KV_LENGTH); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ENCODER_INPUT_LENGTH); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_CONTEXT_LENGTH); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(QKV_BIAS_TENSOR); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SPEC_DECODING_GENERATION_LENGTHS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SPEC_DECODING_PACKED_MASK); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SPEC_DECODING_POSITION_OFFSETS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SPEC_DECODING_USE); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(LONG_ROPE_ROTARY_INV_FREQ); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(LONG_ROPE_ROTARY_COS_SIN); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MROPE_ROTARY_COS_SIN); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MROPE_POSITION_DELTAS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_RUNTIME_PERF_KNOBS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(HOST_CONTEXT_PROGRESS); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MLA_Q_B_PROJ_TENSOR); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MLA_KV_B_PROJ_TENSOR); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(MLA_K_B_PROJ_TRANS_TENSOR); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(SKIP_ATTN); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(LOGN_SCALING); + TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING(ENUM_SIZE); + } +#undef TLLM_GPT_ATTN_IDX_ENTRY_TO_STRING + TLLM_LOG_TRACE(common::fmtstr("Missing string description for IdxEntry enum %lu.\n", static_cast<size_t>(entry))); + return ""; +} + +bool GPTAttentionPlugin::isEntryUsed(IdxEntry const& entry) const +{ + switch (entry) + { + case IdxEntry::QKV_TENSOR: return true; + case IdxEntry::K_TENSOR: return mUnfuseQkvGemm; + case IdxEntry::V_TENSOR: return mUnfuseQkvGemm; + case IdxEntry::ATTENTION_MASK: return useFullCustomMask(); + case IdxEntry::ATTENTION_PACKED_MASK: return useCustomMask(); + case IdxEntry::SEQUENCE_LENGTH: return useKVCache(); + case IdxEntry::HOST_PAST_KEY_VALUE_LENGTHS: return useKVCache(); + case IdxEntry::HOST_MAX_ATTENTION_WINDOW: return true; + case IdxEntry::HOST_SINK_TOKEN_LENGTH: return true; + case IdxEntry::CONTEXT_LENGTHS: return true; + case IdxEntry::CACHE_INDIR: return useKVCache(); + case IdxEntry::REQUEST_TYPES: return true; + case IdxEntry::KV_CACHE_BLOCK_OFFSETS: return useKVCache() && mPagedKVCache; + case IdxEntry::HOST_KV_CACHE_BLOCK_OFFSETS: return useKVCache() && mPagedKVCache; + case IdxEntry::HOST_KV_CACHE_POOL_POINTERS: return useKVCache() && mPagedKVCache; + case IdxEntry::HOST_KV_CACHE_POOL_MAPPING: return useKVCache() && mPagedKVCache; + case IdxEntry::PAST_KEY_VALUE: return useKVCache() && !mPagedKVCache; + case IdxEntry::KV_CACHE_QUANTIZATION_SCALE: return useKVCache() && mKVCacheQuantMode.hasKvCacheQuant(); + case IdxEntry::KV_CACHE_DEQUANTIZATION_SCALE: return useKVCache() && mKVCacheQuantMode.hasKvCacheQuant(); + case IdxEntry::ATTENTION_OUTPUT_QUANTIZATION_SCALE: return mFP8ContextFMHA; + case IdxEntry::ATTENTION_OUTPUT_SF_SCALE: return mFuseFp4Quant; + case IdxEntry::ROTARY_INV_FREQ: return isRoPE(); + case IdxEntry::ROTARY_COS_SIN: return isRoPE(); + case IdxEntry::ALIBI_SLOPES: return isALiBi(); + case IdxEntry::RELATIVE_ATTENTION_BIAS: return isRelativePosition(); + case IdxEntry::CROSS_KV: return isCrossAttention(); + case IdxEntry::CROSS_KV_LENGTH: return isCrossAttention(); + case IdxEntry::LOGN_SCALING: return isLognScaling(); + case IdxEntry::ENCODER_INPUT_LENGTH: return isCrossAttention(); + case IdxEntry::HOST_CONTEXT_LENGTH: return mRemovePadding; + case IdxEntry::QKV_BIAS_TENSOR: return mQKVBiasEnabled; + case IdxEntry::SPEC_DECODING_GENERATION_LENGTHS: return mIsSpecDecodingEnabled; + case IdxEntry::SPEC_DECODING_PACKED_MASK: return mIsSpecDecodingEnabled; + case IdxEntry::SPEC_DECODING_POSITION_OFFSETS: return mIsSpecDecodingEnabled; + case IdxEntry::SPEC_DECODING_USE: return mIsSpecDecodingEnabled; + case IdxEntry::LONG_ROPE_ROTARY_INV_FREQ: return isLongRoPE(); + case IdxEntry::LONG_ROPE_ROTARY_COS_SIN: return isLongRoPE(); + case IdxEntry::MROPE_ROTARY_COS_SIN: return isMRoPE(); + case IdxEntry::MROPE_POSITION_DELTAS: return isMRoPE(); + case IdxEntry::HOST_RUNTIME_PERF_KNOBS: return true; + case IdxEntry::HOST_CONTEXT_PROGRESS: return true; + case IdxEntry::MLA_Q_B_PROJ_TENSOR: return mIsMLAEnabled; + case IdxEntry::MLA_KV_B_PROJ_TENSOR: return mIsMLAEnabled; + case IdxEntry::MLA_K_B_PROJ_TRANS_TENSOR: return mIsMLAEnabled; + case IdxEntry::SKIP_ATTN: return mSkipAttn; + default: return false; + } +} + +void GPTAttentionPlugin::initEntryIdx() +{ + mEntryIdx.resize(static_cast<size_t>(IdxEntry::ENUM_SIZE)); + size_t entryIdx = 0; + for (size_t i = 0; i < static_cast<size_t>(IdxEntry::ENUM_SIZE); i++) + { + mEntryIdx[i] = entryIdx; + entryIdx += isEntryUsed(static_cast<IdxEntry>(i)); + } +} + +GPTAttentionPlugin::IndexType GPTAttentionPlugin::getIdx(IdxEntry const& entry) const +{ + TLLM_CHECK_WITH_INFO( + isEntryUsed(entry), common::fmtstr("getIdx() should not be used with entry %s.\n", toString(entry).data())); + return mEntryIdx[static_cast<size_t>(entry)]; +} + +// IPluginV2DynamicExt Methods +GPTAttentionPlugin* GPTAttentionPlugin::clone() const noexcept +{ + return dynamic_cast<GPTAttentionPlugin*>(this->cloneImpl<GPTAttentionPlugin>()); +} + +static int getPackedTensorHiddenDimIndex(bool removePadding) +{ + return removePadding ? 1 : 2; +} + +// NOTE: generation input length might be larger than one in the spec decoding mode. +int GPTAttentionPlugin::getGenerationInputSequenceLength( + nvinfer1::PluginTensorDesc const* inputDesc, int32_t localNbSeq, int32_t localNbTokens) const +{ + if (mRemovePadding) + { + // Speculative decoding mode might need variable generation input sequence length. + if (mIsSpecDecodingEnabled && mUseSpecDecoding) + { + TLLM_CHECK_WITH_INFO(mCpSize <= 1, "Context Parallel does not support speculative decoding mode for now"); + // SPEC_DECODING_POSITION_OFFSETS: [batch_size, max_generation_input_length]. + return inputDesc[getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)].dims.d[1]; + } + else + { + if (mCpSize > 1) + { + // Given that localNbTokens == (beamSize * localNbSeq + mCpSize - 1) / mCpSize, but when mCpSize - 1 > + // localNbSeq, there are multiple choices for beamSize. Assume beamSize == 1 here. + TLLM_CHECK_WITH_INFO(localNbTokens == (localNbSeq + mCpSize - 1) / mCpSize, + "Context Parallel does not support beamSize > 1 for non-speculative decoding mode, " + "localNbTokens=%d, localNbSeq=%d", + localNbTokens, localNbSeq); + return 1; + } + // [num_tokens, local_hidden_size] where num_tokens = batch_size * generation_input_length + TLLM_CHECK_WITH_INFO(localNbTokens % localNbSeq == 0, + "seq_len should be same for all generation requests, localNbTokens=%d, localNbSeq=%d", localNbTokens, + localNbSeq); + return localNbTokens / localNbSeq; + } + } + else + { + // We don't have IFB without mRemovePadding, so just take it out from inputDesc + // [batch_size, seq_len, local_hidden_size] + return inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1]; + } +} + +// outputs +// output_tensor [batch_size, seq_len, local_hidden_size] or [num_tokens, local_hidden_size] +// present_key_value_pool (optional if mPagedKVCache is false) [batch_size, 2, local_num_kv_heads, max_seq_len, +// head_size] +nvinfer1::DimsExprs GPTAttentionPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + if (mFuseFp4Quant) + { + TLLM_CHECK(outputIndex == 0 || outputIndex == 1 || (!mPagedKVCache && useKVCache() && outputIndex == 2)); + // Compute the output dimension for FP4 quantized tensor. Consistent with QuantizeToFP4Plugin. + if (outputIndex == 0) + { + auto ret = inputs[getIdx(IdxEntry::QKV_TENSOR)]; + return ret; + } + // Compute the output dimension for output scaling factor tensor. Consistent with QuantizeToFP4Plugin. + if (outputIndex == 1) + { + auto ret = inputs[getIdx(IdxEntry::QKV_TENSOR)]; + // Sequence dimension or token dimension. + // Pad to multiple of 128. + auto dimM = exprBuilder.operation(DimensionOperation::kCEIL_DIV, + *ret.d[getPackedTensorHiddenDimIndex(mRemovePadding) - 1], *exprBuilder.constant(128)); + ret.d[getPackedTensorHiddenDimIndex(mRemovePadding) - 1] + = exprBuilder.operation(DimensionOperation::kPROD, *dimM, *exprBuilder.constant(128)); + // Hidden size dimension. + // Div (rounding up) by 16 since 16 elements share one SF and SF padded to k%4==0. + ret.d[getPackedTensorHiddenDimIndex(mRemovePadding)] = exprBuilder.operation(DimensionOperation::kCEIL_DIV, + *ret.d[getPackedTensorHiddenDimIndex(mRemovePadding)], *exprBuilder.constant(16)); + return ret; + } + } + else + { + TLLM_CHECK(outputIndex == 0 || (!mPagedKVCache && useKVCache() && outputIndex == 1)); + if (outputIndex == 0) + { + auto ret = inputs[getIdx(IdxEntry::QKV_TENSOR)]; + // In MLA, the output dim is v_head_dim + auto const head_size = mHeadSize; + ret.d[getPackedTensorHiddenDimIndex(mRemovePadding)] = exprBuilder.operation( + DimensionOperation::kPROD, *exprBuilder.constant(head_size), *exprBuilder.constant(mNumHeads)); + return ret; + } + } + return inputs[getIdx(IdxEntry::PAST_KEY_VALUE)]; +} + +bool GPTAttentionPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + bool result = false; + int posCaseLine = -1; + if (pos == getIdx(IdxEntry::CONTEXT_LENGTHS) || pos == getIdx(IdxEntry::REQUEST_TYPES) + || pos == getIdx(IdxEntry::HOST_MAX_ATTENTION_WINDOW) || pos == getIdx(IdxEntry::HOST_SINK_TOKEN_LENGTH) + || (isEntryUsed(IdxEntry::SPEC_DECODING_PACKED_MASK) && pos == getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)) + || (isEntryUsed(IdxEntry::SPEC_DECODING_POSITION_OFFSETS) + && pos == getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)) + || (isEntryUsed(IdxEntry::SPEC_DECODING_GENERATION_LENGTHS) + && pos == getIdx(IdxEntry::SPEC_DECODING_GENERATION_LENGTHS)) + || (isEntryUsed(IdxEntry::SPEC_DECODING_USE) && pos == getIdx(IdxEntry::SPEC_DECODING_USE))) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (isMRoPE() && (pos == getIdx(IdxEntry::MROPE_ROTARY_COS_SIN))) + { + return inOut[pos].type == nvinfer1::DataType::kFLOAT; + } + else if (isMRoPE() && (pos == getIdx(IdxEntry::MROPE_POSITION_DELTAS))) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (pos == getIdx(IdxEntry::HOST_RUNTIME_PERF_KNOBS) || pos == getIdx(IdxEntry::HOST_CONTEXT_PROGRESS)) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT64; + } + else if (useKVCache() + && (pos == getIdx(IdxEntry::SEQUENCE_LENGTH) || pos == getIdx(IdxEntry::HOST_PAST_KEY_VALUE_LENGTHS) + || pos == getIdx(IdxEntry::CACHE_INDIR))) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (isRoPE() && (pos == getIdx(IdxEntry::ROTARY_INV_FREQ) || pos == getIdx(IdxEntry::ROTARY_COS_SIN))) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kFLOAT; + } + else if (isLongRoPE() + && (pos == getIdx(IdxEntry::LONG_ROPE_ROTARY_INV_FREQ) || pos == getIdx(IdxEntry::LONG_ROPE_ROTARY_COS_SIN))) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kFLOAT; + } + else if (useKVCache() && mKVCacheQuantMode.hasKvCacheQuant() + && (pos == getIdx(IdxEntry::KV_CACHE_DEQUANTIZATION_SCALE) + || pos == getIdx(IdxEntry::KV_CACHE_QUANTIZATION_SCALE))) + { + // kv_scale for mType->int8/fp8 and int8/fp8->mType conversion + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (mFP8ContextFMHA && pos == getIdx(IdxEntry::ATTENTION_OUTPUT_QUANTIZATION_SCALE)) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (mFuseFp4Quant && pos == getIdx(IdxEntry::ATTENTION_OUTPUT_SF_SCALE)) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (useFullCustomMask() && pos == getIdx(IdxEntry::ATTENTION_MASK)) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kBOOL && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (useCustomMask() && pos == getIdx(IdxEntry::ATTENTION_PACKED_MASK)) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT32 && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (useKVCache() && mPagedKVCache + && (pos == getIdx(IdxEntry::KV_CACHE_BLOCK_OFFSETS) || pos == getIdx(IdxEntry::HOST_KV_CACHE_BLOCK_OFFSETS))) + { + // kv cache block offsets + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT32 && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (useKVCache() && mPagedKVCache && (pos == getIdx(IdxEntry::HOST_KV_CACHE_POOL_POINTERS))) + { + // kv cache pool pointers + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT64 && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (useKVCache() && mPagedKVCache && (pos == getIdx(IdxEntry::HOST_KV_CACHE_POOL_MAPPING))) + { + // kv cache pool mapping + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT32 && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (useKVCache() && mKVCacheQuantMode.hasInt8KvCache() + && (!mPagedKVCache && (pos == getIdx(IdxEntry::PAST_KEY_VALUE) || pos == nbInputs + 1))) + { + // If use Int8 K/V cache we require I/O KV values to int8 + posCaseLine = __LINE__; + result = (inOut[pos].type == nvinfer1::DataType::kINT8) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (useKVCache() && mKVCacheQuantMode.hasFp8KvCache() + && (!mPagedKVCache && (pos == getIdx(IdxEntry::PAST_KEY_VALUE) || pos == nbInputs + 1))) + { + // If use FP8 K/V cache we require I/O KV values to FP8 + posCaseLine = __LINE__; + result = (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (mRemovePadding && (pos == getIdx(IdxEntry::HOST_CONTEXT_LENGTH))) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT32 && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (mCrossAttention + && (pos == getIdx(IdxEntry::CROSS_KV_LENGTH) || pos == getIdx(IdxEntry::ENCODER_INPUT_LENGTH))) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (isLognScaling() && pos == getIdx(IdxEntry::LOGN_SCALING)) + { + return inOut[pos].type == nvinfer1::DataType::kFLOAT; + } + else if (pos == nbInputs && mFuseFp4Quant) + { + // Set dtype for output FP4 quantized tensor. + posCaseLine = __LINE__; + result = (inOut[pos].type == nvinfer1::DataType::kFP4) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == nbInputs + 1 && mFuseFp4Quant) + { + // Set dtype for output scaling factor tensor. Use kINT32 as storage type (same as QuantizeToFP4Plugin). + posCaseLine = __LINE__; + result = (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == nbInputs && mFP8ContextFMHA) + { + // Output tensor now supports fp8 data type. + posCaseLine = __LINE__; + result = (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (mSkipAttn && pos == getIdx(IdxEntry::SKIP_ATTN)) + { + posCaseLine = __LINE__; + result = inOut[pos].type == nvinfer1::DataType::kBOOL && inOut[pos].format == TensorFormat::kLINEAR; + } + else + { + posCaseLine = __LINE__; + result = (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + TLLM_LOG_DEBUG( + "%s: pos: %d, result: %d, posCaseLine: %d", __PRETTY_FUNCTION__, pos, static_cast<int>(result), posCaseLine); + return result; +} + +template <typename T, typename KVCacheBuffer> +void GPTAttentionPlugin::configurePluginImpl(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + TLLM_CHECK(mHeadSize > 0); + + int beamWidth = -1; + if (!isCrossAttention() && useKVCache()) + { + // desc_val == -1 means beam_width is not static, we should look at min/max/opt. + // + // In prepareEnqueueGeneration, we'll prepare for all cases where beam_width doesn't exceed max. + // TODO: pass min AND max to prepareEnqueueGeneration instead of max only. + int desc_val = in[getIdx(IdxEntry::CACHE_INDIR)].desc.dims.d[1]; + int max_val = in[getIdx(IdxEntry::CACHE_INDIR)].max.d[1]; + beamWidth = desc_val == -1 ? max_val : desc_val; + } + else + { + beamWidth = 1; + } + TLLM_CHECK(beamWidth != -1); + + // Commonly, cyclic_attention_window_size, and max_attention_window_size will be the same + // unless each layer has different attention window sizes. + // the kv_cache capacity. + int max_encoder_context_len = isCrossAttention() ? in[getIdx(IdxEntry::CROSS_KV_LENGTH)].desc.dims.d[0] : 0; + int const max_attention_window_size = isCrossAttention() + ? max_encoder_context_len + : (useKVCache() ? in[getIdx(IdxEntry::CACHE_INDIR)].desc.dims.d[2] : 0); + int const cyclic_attention_window_size = max_attention_window_size; + + int const num_requests = 256; + int const sink_token_length = 0; + + EnqueueGenerationParams<T> enqueueParams; + enqueueParams.max_attention_window_size = max_attention_window_size; + enqueueParams.cyclic_attention_window_size = cyclic_attention_window_size; + enqueueParams.max_cyclic_attention_window_size = cyclic_attention_window_size; + enqueueParams.sink_token_length = sink_token_length; + enqueueParams.beam_width = beamWidth; + enqueueParams.num_requests = num_requests; + + prepareEnqueueGeneration<T, KVCacheBuffer>(enqueueParams); + + // Always reserve SemaphoreArray (for multi-block mode) as MMHA may enable multi-block mode when shared memory is + // not enough. + auto const& ctxLenTensor = in[getIdx(IdxEntry::CONTEXT_LENGTHS)]; + TLLM_CHECK_DEBUG(ctxLenTensor.max.nbDims == 1); + int32_t const max_batch_beam = in[getIdx(IdxEntry::CONTEXT_LENGTHS)].max.d[0]; + reserveSemaphoreArray(mNumHeads * max_batch_beam); +} + +template <typename T> +void GPTAttentionPlugin::configurePluginDispatchKVCacheType(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + if (mPagedKVCache) + { + configurePluginImpl<T, KVBlockArray>(in, nbInputs, out, nbOutputs); + } + else + { + configurePluginImpl<T, KVLinearBuffer>(in, nbInputs, out, nbOutputs); + } +} + +void GPTAttentionPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + if (mType == nvinfer1::DataType::kHALF) + { + configurePluginDispatchKVCacheType<half>(in, nbInputs, out, nbOutputs); + } + else if (mType == nvinfer1::DataType::kFLOAT) + { + configurePluginDispatchKVCacheType<float>(in, nbInputs, out, nbOutputs); + } +#ifdef ENABLE_BF16 + else if (mType == nvinfer1::DataType::kBF16) + { + configurePluginDispatchKVCacheType<__nv_bfloat16>(in, nbInputs, out, nbOutputs); + } +#endif +} + +size_t GPTAttentionPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + int const max_context_length = mMaxContextLength; + int const cross_kv_length = isCrossAttention() ? inputs[getIdx(IdxEntry::CROSS_KV_LENGTH)].dims.d[0] : 0; + int const max_num_seq = inputs[getIdx(IdxEntry::CONTEXT_LENGTHS)].dims.d[0]; + auto const type = inputs[getIdx(IdxEntry::QKV_TENSOR)].type; + int const max_kv_cache_length + = isCrossAttention() ? cross_kv_length : (useKVCache() ? inputs[getIdx(IdxEntry::CACHE_INDIR)].dims.d[2] : 0); + int const max_num_tokens + = mRemovePadding ? inputs[getIdx(IdxEntry::QKV_TENSOR)].dims.d[0] : max_num_seq * max_context_length; + int const max_blocks_per_sequence + = (useKVCache() && mPagedKVCache) ? inputs[getIdx(IdxEntry::KV_CACHE_BLOCK_OFFSETS)].dims.d[3] : 0; + + size_t const context_workspace_size + = getWorkspaceSizeForContext(type, max_num_seq, max_context_length, cross_kv_length, max_num_tokens); + + size_t const generation_workspace_size = getWorkspaceSizeForGeneration( + type, max_num_seq, max_kv_cache_length, max_num_tokens, max_blocks_per_sequence); + + size_t attention_input_workspace_size = 0; + + if (mUnfuseQkvGemm) + { + int const local_hidden_units_q + = inputs[getIdx(IdxEntry::QKV_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)]; + int const local_hidden_units_kv + = inputs[getIdx(IdxEntry::K_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)]; + size_t const size = tensorrt_llm::runtime::BufferDataType(type).getSize(); + size_t const attention_input_size = size * max_num_tokens * (local_hidden_units_q + 2 * local_hidden_units_kv); + size_t workspaces[1]; + workspaces[0] = attention_input_size; + attention_input_workspace_size = tensorrt_llm::common::calculateTotalWorkspaceSize(workspaces, 1); + } + + return std::max(context_workspace_size, generation_workspace_size) + attention_input_workspace_size; +} + +static size_t getStride(nvinfer1::Dims const& dims, int n) +{ + TLLM_CHECK(n >= 0 && n < dims.nbDims); + return std::accumulate(dims.d + n + 1, dims.d + dims.nbDims, 1, std::multiplies<size_t>{}); +} + +template <typename T, typename AttentionOutT, typename KVCacheBuffer> +int GPTAttentionPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) +{ + TLLM_LOG_TRACE("Attention plugin start at layer %d", mLayerIdx); + + using runtime::RequestType; + + int32_t const nbSeq = inputDesc[getIdx(IdxEntry::CONTEXT_LENGTHS)].dims.d[0]; + RequestType const* reqTypes = static_cast<RequestType const*>(inputs[getIdx(IdxEntry::REQUEST_TYPES)]); + + int32_t nbContextRequests = 0; + int32_t contextTokenIdxEnd = 0; + int32_t contextTokenIdxEndForCp = 0; + // count context requests + for (int32_t seqIdx = 0; seqIdx < nbSeq; seqIdx++) + { + if (reqTypes[seqIdx] != RequestType::kCONTEXT) + { + break; + } + ++nbContextRequests; + contextTokenIdxEnd += mRemovePadding + ? static_cast<int32_t const*>(inputs[getIdx(IdxEntry::HOST_CONTEXT_LENGTH)])[seqIdx] + : inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1]; + contextTokenIdxEndForCp += mRemovePadding + ? (static_cast<int32_t const*>(inputs[getIdx(IdxEntry::HOST_CONTEXT_LENGTH)])[seqIdx] + mCpSize - 1) + / mCpSize + : (inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1] + mCpSize - 1) / mCpSize; + } + + for (int32_t seqIdx = nbContextRequests; seqIdx < nbSeq; seqIdx++) + { + TLLM_CHECK(reqTypes[seqIdx] == RequestType::kGENERATION); + } + + // mixed requests require mRemovePadding and mPagedKVCache + if (nbContextRequests != 0 && nbContextRequests != nbSeq) + { + TLLM_CHECK(mRemovePadding && mPagedKVCache); + } + + if (nbContextRequests > 0) + { + auto seqIdxBeg = 0; + auto tokenIdxBeg = 0; + auto localNbTokens = contextTokenIdxEnd; + enqueueSome<T, AttentionOutT, KVCacheBuffer>(seqIdxBeg, nbContextRequests, tokenIdxBeg, localNbTokens, + inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + + if (auto nbGenerationSeq = nbSeq - nbContextRequests; nbGenerationSeq > 0) + { + auto seqIdxBeg = nbContextRequests; + auto tokenIdxBeg = mCpSize > 1 ? contextTokenIdxEndForCp : contextTokenIdxEnd; + // if mRemovePadding is true, we may have IFB, and need to remove context tokens. + // if mRemovePadding is false, it is only generation requests, so just multiply batch_beam and seq_len (May not + // 1 for Parallel Decoding) + auto localNbTokens = mRemovePadding + ? inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[0] - tokenIdxBeg + : inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[0] * inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1]; + enqueueSome<T, AttentionOutT, KVCacheBuffer>(seqIdxBeg, nbGenerationSeq, tokenIdxBeg, localNbTokens, inputDesc, + outputDesc, inputs, outputs, workspace, stream); + } + + sync_check_cuda_error(stream); + TLLM_LOG_TRACE("Attention plugin stop at layer %d", mLayerIdx); + + return 0; +} + +template <typename T, typename AttentionOutT, typename KVCacheBuffer> +int GPTAttentionPlugin::enqueueSome(int32_t seqIdxBeg, int32_t localNbSeq, int32_t tokenIdxBeg, int32_t localNbTokens, + nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) +{ + // relative_attention_bias [head_num, max_seq_len, max_seq_len] (optional in relative position) + // or [head_num, num_buckets] (optional in implicit relative attention) + // cross_kv [batch_size, seq_len, 2 * local_hidden_size] or [num_tokens, 2 * local_hidden_size] + // when enable remove_input_padding (optional in cross attention mode) + // cross_kv_length [int] max encoder input context length (optional in cross attention mode) + // encoder_input_lengths [batch_size] raw sequence lengths (optional in cross attention mode) + + using runtime::RequestType; + + auto const* const reqTypeInBatchPtr + = static_cast<RequestType const*>(inputs[getIdx(IdxEntry::REQUEST_TYPES)]) + seqIdxBeg; + bool const is_context = (reqTypeInBatchPtr[0] == RequestType::kCONTEXT); + + T const* attention_input = static_cast<T const*>(inputs[getIdx(IdxEntry::QKV_TENSOR)]) + + inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)] + * size_t(tokenIdxBeg); + + bool changeSpecDecodingMode = false; + if (mIsSpecDecodingEnabled) + { + bool useSpecDecoding + = static_cast<bool>(reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_USE)])[0]); + changeSpecDecodingMode = mUseSpecDecoding != useSpecDecoding; + mUseSpecDecoding = useSpecDecoding; + } + + [[maybe_unused]] MlaParams<T> mla_params; + + T const* qkv_bias = nullptr; + if (mQKVBiasEnabled) + { + qkv_bias = reinterpret_cast<T const*>(inputs[getIdx(IdxEntry::QKV_BIAS_TENSOR)]); + } + + // Note we still need context length during generation for MMHA optimization. + int32_t const max_context_q_len = [&]() + { + if (!mRemovePadding) + { + return static_cast<int>(inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[1]); + } + auto const host_context_lengths + = static_cast<int32_t const*>(inputs[getIdx(IdxEntry::HOST_CONTEXT_LENGTH)]) + seqIdxBeg; + return *std::max_element(host_context_lengths, host_context_lengths + localNbSeq); + }(); + + // Rotary inv_freq, cos_sin cache to avoid re-computing. + float const* rotary_inv_freq = nullptr; + float2 const* rotary_cos_sin = nullptr; + + bool const useLongRoPECache = isLongRoPE() && max_context_q_len > mRotaryEmbeddingOriginalMaxPositions; + if (isRoPE()) + { + auto inputName = useLongRoPECache ? IdxEntry::LONG_ROPE_ROTARY_INV_FREQ : IdxEntry::ROTARY_INV_FREQ; + rotary_inv_freq = reinterpret_cast<float const*>(inputs[getIdx(inputName)]); + } + if (isRoPE()) + { + auto inputName = useLongRoPECache ? IdxEntry::LONG_ROPE_ROTARY_COS_SIN : IdxEntry::ROTARY_COS_SIN; + rotary_cos_sin = reinterpret_cast<float2 const*>(inputs[getIdx(inputName)]); + } + + auto const mrope_rotary_cos_sin + = isMRoPE() ? reinterpret_cast<float2 const*>(inputs[getIdx(IdxEntry::MROPE_ROTARY_COS_SIN)]) : nullptr; + + auto const mrope_position_deltas + = isMRoPE() ? reinterpret_cast<int32_t const*>(inputs[getIdx(IdxEntry::MROPE_POSITION_DELTAS)]) : nullptr; + + if (mUnfuseQkvGemm) + { + int const max_seqlen = inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[mRemovePadding ? 0 : 1]; + int const batch_size = mRemovePadding ? 1 : inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[0]; + + T const* attention_input_q = static_cast<T const*>(inputs[getIdx(IdxEntry::QKV_TENSOR)]); + T const* attention_input_k = static_cast<T const*>(inputs[getIdx(IdxEntry::K_TENSOR)]); + T const* attention_input_v = static_cast<T const*>(inputs[getIdx(IdxEntry::V_TENSOR)]); + size_t const hidden_units_q + = inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)]; + size_t const hidden_units_kv + = inputDesc[getIdx(IdxEntry::K_TENSOR)].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)]; + size_t const hidden_units = hidden_units_q + 2 * hidden_units_kv; + size_t const size_qkv = sizeof(T) * hidden_units; + size_t const size_q = sizeof(T) * hidden_units_q; + size_t const size_kv = sizeof(T) * hidden_units_kv; + size_t const total_size = size_qkv * batch_size * max_seqlen; + int8_t* workspace_byte_ptr = reinterpret_cast<int8_t*>(workspace); + size_t offset = 0; + T* attention_input_qkv = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, total_size)); + workspace = reinterpret_cast<void*>(workspace_byte_ptr + offset); + + cudaMemcpy2DAsync(attention_input_qkv, size_qkv, attention_input_q, size_q, size_q, batch_size * max_seqlen, + cudaMemcpyDeviceToDevice, stream); + cudaMemcpy2DAsync(attention_input_qkv + hidden_units_q, size_qkv, attention_input_k, size_kv, size_kv, + batch_size * max_seqlen, cudaMemcpyDeviceToDevice, stream); + cudaMemcpy2DAsync(attention_input_qkv + hidden_units_q + hidden_units_kv, size_qkv, attention_input_v, size_kv, + size_kv, batch_size * max_seqlen, cudaMemcpyDeviceToDevice, stream); + + attention_input = attention_input_qkv + hidden_units * tokenIdxBeg; + } + + int const* context_q_lengths = reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::CONTEXT_LENGTHS)]) + seqIdxBeg; + int const* sequence_kv_length = useKVCache() + ? static_cast<int const*>(inputs[getIdx(IdxEntry::SEQUENCE_LENGTH)]) + seqIdxBeg + : context_q_lengths; + + int max_encoder_context_len = isCrossAttention() ? inputDesc[getIdx(IdxEntry::CROSS_KV_LENGTH)].dims.d[0] : 0; + // for enc-dec model, since decoder_input_ids could be longer than 1, + // such model has an encoder context (for cross attn) and an decoder context (for self attn) + // clarify 3 lens: + // -- max_context_q_len: len of decoder input. No "max" concept, it's what it is given. + // Also called (decoder_)input_seq_length, normally 1 for encoder-decoder start token + // -- max_seq_len: max allowed len of decoder output, i.e. final results + // -- max_encoder_context_len: len of encoder input (in cross attn). Also called encoder_input_seq_length + + int const beamWidth + = isCrossAttention() ? 1 : (useKVCache() ? inputDesc[getIdx(IdxEntry::CACHE_INDIR)].dims.d[1] : 1); + + // Commonly, cyclic_attention_window_size, and max_attention_window_size will be the same + // unless each layer has different attention window sizes. + // the kv_cache capacity. + int const max_attention_window_size = isCrossAttention() + ? max_encoder_context_len + : (useKVCache() ? inputDesc[getIdx(IdxEntry::CACHE_INDIR)].dims.d[2] : 0); + // The cyclic_attention_window_size will determine the cyclic kv cache position of new tokens. + // Note that this cyclic_attention_window_size might be smaller than the actual kv cache capactity. + int const* cyclic_attention_window_sizes + = reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::HOST_MAX_ATTENTION_WINDOW)]); + int const cyclic_attention_window_size + = isCrossAttention() ? max_encoder_context_len : cyclic_attention_window_sizes[mLayerIdx]; + int const sink_token_length = reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::HOST_SINK_TOKEN_LENGTH)])[0]; + int const num_attn_layer = inputDesc[getIdx(IdxEntry::HOST_MAX_ATTENTION_WINDOW)].dims.d[0]; + int const max_cyclic_attention_window_size = isCrossAttention() + ? max_encoder_context_len + : *std::max_element(cyclic_attention_window_sizes, cyclic_attention_window_sizes + num_attn_layer); + bool const can_use_one_more_block = beamWidth > 1; + + float const* kv_scale_orig_quant = nullptr; + float const* kv_scale_quant_orig = nullptr; + if (useKVCache() && mKVCacheQuantMode.hasKvCacheQuant()) + { + assert(inputDesc[getIdx(IdxEntry::KV_CACHE_QUANTIZATION_SCALE)].type == nvinfer1::DataType::kFLOAT); + assert(inputDesc[getIdx(IdxEntry::KV_CACHE_DEQUANTIZATION_SCALE)].type == nvinfer1::DataType::kFLOAT); + kv_scale_orig_quant = reinterpret_cast<float const*>(inputs[getIdx(IdxEntry::KV_CACHE_QUANTIZATION_SCALE)]); + kv_scale_quant_orig = reinterpret_cast<float const*>(inputs[getIdx(IdxEntry::KV_CACHE_DEQUANTIZATION_SCALE)]); + } + + float const* attention_output_orig_quant = nullptr; + if (mFP8ContextFMHA) + { + assert(inputDesc[getIdx(IdxEntry::ATTENTION_OUTPUT_QUANTIZATION_SCALE)].type == nvinfer1::DataType::kFLOAT); + attention_output_orig_quant + = reinterpret_cast<float const*>(inputs[getIdx(IdxEntry::ATTENTION_OUTPUT_QUANTIZATION_SCALE)]); + } + float const* attention_output_sf_scale = nullptr; + if (mFuseFp4Quant) + { + assert(inputDesc[getIdx(IdxEntry::ATTENTION_OUTPUT_SF_SCALE)].type == nvinfer1::DataType::kFLOAT); + attention_output_sf_scale = reinterpret_cast<float const*>(inputs[getIdx(IdxEntry::ATTENTION_OUTPUT_SF_SCALE)]); + } + uint32_t const* attention_packed_mask = nullptr; + if (useCustomMask()) + { + assert(inputDesc[getIdx(IdxEntry::ATTENTION_PACKED_MASK)].type == nvinfer1::DataType::kINT32); + attention_packed_mask = reinterpret_cast<uint32_t const*>(inputs[getIdx(IdxEntry::ATTENTION_PACKED_MASK)]); + } + bool const* attention_mask = nullptr; + int attention_mask_stride = 0; + if (useFullCustomMask()) + { + attention_mask_stride = static_cast<int>(inputDesc[getIdx(IdxEntry::ATTENTION_MASK)].dims.d[1]); + attention_mask = reinterpret_cast<bool const*>(inputs[getIdx(IdxEntry::ATTENTION_MASK)]) + + attention_mask_stride * static_cast<size_t>(tokenIdxBeg); + } + + int max_blocks_per_sequence = 0; + kernels::KVBlockArray::DataType* block_offsets = nullptr; + void* host_primary_pool_pointer = nullptr; + void* host_secondary_pool_pointer = nullptr; + if (useKVCache() && mPagedKVCache) + { + auto const& kvCacheBlockOffsetsShape = inputDesc[getIdx(IdxEntry::KV_CACHE_BLOCK_OFFSETS)].dims; + max_blocks_per_sequence = kvCacheBlockOffsetsShape.d[kvCacheBlockOffsetsShape.nbDims - 1]; + + std::int32_t const* host_pool_mapping + = static_cast<std::int32_t const*>(inputs[getIdx(IdxEntry::HOST_KV_CACHE_POOL_MAPPING)]); + + int32_t const layerToPool = host_pool_mapping[mLayerIdx * 2]; + int32_t const layerIdxInCachePool = host_pool_mapping[mLayerIdx * 2 + 1]; + TLLM_LOG_TRACE("Layer%d: LayerCachePoolLocator{.indexOfPool=%d, .layerIdxInCachePool=%d}", mLayerIdx, + layerToPool, layerIdxInCachePool); + auto const seqStride = getStride(kvCacheBlockOffsetsShape, 1); + auto const poolStride = getStride(kvCacheBlockOffsetsShape, 0); + auto const seqOffset = seqIdxBeg * seqStride; + auto const poolOffset = layerToPool * poolStride; + + block_offsets + = reinterpret_cast<kernels::KVBlockArray::DataType*>(inputs[getIdx(IdxEntry::KV_CACHE_BLOCK_OFFSETS)]) + + poolOffset + seqOffset; + + auto const* const typed_host_pool_pointers + = static_cast<char* const*>(inputs[getIdx(IdxEntry::HOST_KV_CACHE_POOL_POINTERS)]); + + auto const cacheElemSize = (mKVCacheQuantMode.hasKvCacheQuant() ? 1 : sizeof(T)); + + auto const kv_cache_head_num = (mNumKVHeads + mCpSize - 1) / mCpSize; + auto const blockSize = mTokensPerBlock * kv_cache_head_num * mHeadSize; + auto const bytesPerBlock = blockSize * cacheElemSize; + auto const layerOffset = layerIdxInCachePool * 2 * bytesPerBlock; + + host_primary_pool_pointer = reinterpret_cast<void*>(typed_host_pool_pointers[layerToPool * 2] + layerOffset); + host_secondary_pool_pointer + = reinterpret_cast<void*>(typed_host_pool_pointers[layerToPool * 2 + 1] + layerOffset); + } + + // The index of kv cache tensor in outputs. If fuse FP4 quant, an additional scaling factor output is added before + // the kv cache tensor. + int const kvCacheIdxInOutputs = mFuseFp4Quant ? 2 : 1; + // The number of elements per storage type. For FP4 output, storage type is uint8_t. + int const numEltsPerStorageType = mFuseFp4Quant ? 2 : 1; + + AttentionOutT* context_buf_ = static_cast<AttentionOutT*>(outputs[0]) + + outputDesc[0].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)] * tokenIdxBeg / numEltsPerStorageType; + + __nv_fp8_e4m3* context_buf_sf_ = nullptr; + if (mFuseFp4Quant) + { + // The output address for FP4 scaling factor. + context_buf_sf_ = static_cast<__nv_fp8_e4m3*>(outputs[1]); + } + + void* key_value_cache = nullptr; + if (useKVCache() && !mPagedKVCache) + { + auto const cacheElemSize = (mKVCacheQuantMode.hasKvCacheQuant() ? 1 : sizeof(T)); + key_value_cache = static_cast<std::byte*>(outputs[kvCacheIdxInOutputs]) + + cacheElemSize * getStride(outputDesc[kvCacheIdxInOutputs].dims, 0) * seqIdxBeg; + void const* past_key_value_cache = inputs[getIdx(IdxEntry::PAST_KEY_VALUE)]; + if (past_key_value_cache != outputs[kvCacheIdxInOutputs]) + { + auto shape = outputDesc[kvCacheIdxInOutputs].dims; + auto const size + = cacheElemSize * std::accumulate(shape.d, shape.d + shape.nbDims, 1, std::multiplies<size_t>{}); + cudaMemcpyAsync(outputs[kvCacheIdxInOutputs], past_key_value_cache, size, cudaMemcpyDeviceToDevice, stream); + } + } + + T const* alibi_slopes = isALiBi() ? static_cast<T const*>(inputs[getIdx(IdxEntry::ALIBI_SLOPES)]) : nullptr; + + int const* spec_decoding_packed_mask = nullptr; + int const* spec_decoding_position_offsets = nullptr; + int const* spec_decoding_generation_lengths = nullptr; + int num_decoding_draft_tokens = 0; + if (mIsSpecDecodingEnabled && mUseSpecDecoding) + { + // Second dimension of spec_decoding_position_offsets is num_decoding_draft_tokens + 1. + // [batch_size, num_decoding_draft_tokens + 1] + num_decoding_draft_tokens = inputDesc[getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)].dims.d[1] - 1; + if (num_decoding_draft_tokens > 0) + { + // spec_decoding_* tensors are not filled for context requests. Hence, always strting from 0th index + int32_t constexpr genSeqIdx = 0; + spec_decoding_packed_mask = static_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)]) + + genSeqIdx * getStride(inputDesc[getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)].dims, 0); + // Packed as [num_tokens, packed_mask_size] + // Use seqIdxBeg * (num_decoding_draft_tokens + 1) here as only generation tokens have the packed_mask + // buffer. + // TODO: support variable sequence length based on generationTokenIdxBeg. + spec_decoding_packed_mask = static_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)]) + + genSeqIdx * (num_decoding_draft_tokens + 1) + * getStride(inputDesc[getIdx(IdxEntry::SPEC_DECODING_PACKED_MASK)].dims, 0); + spec_decoding_position_offsets + = static_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)]) + + genSeqIdx * getStride(inputDesc[getIdx(IdxEntry::SPEC_DECODING_POSITION_OFFSETS)].dims, 0); + spec_decoding_generation_lengths + = static_cast<int const*>(inputs[getIdx(IdxEntry::SPEC_DECODING_GENERATION_LENGTHS)]) + genSeqIdx; + } + } + + int32_t const* host_past_kv_len_list = useKVCache() + ? static_cast<int const*>(inputs[getIdx(IdxEntry::HOST_PAST_KEY_VALUE_LENGTHS)]) + seqIdxBeg + : nullptr; + int32_t const max_context_kv_len = useKVCache() + ? *std::max_element(host_past_kv_len_list, host_past_kv_len_list + localNbSeq) + : max_context_q_len; + + int const* host_context_lengths + = mRemovePadding ? reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::HOST_CONTEXT_LENGTH)]) : nullptr; + + int64_t const* runtime_perf_knobs = static_cast<int64_t const*>(inputs[getIdx(IdxEntry::HOST_RUNTIME_PERF_KNOBS)]); + + EnqueueParams<T> common_enqueue_params; + common_enqueue_params.attention_input = attention_input; + common_enqueue_params.qkv_bias = qkv_bias; + common_enqueue_params.attention_mask = attention_mask; + common_enqueue_params.rotary_inv_freq = rotary_inv_freq; + common_enqueue_params.rotary_cos_sin = rotary_cos_sin; + common_enqueue_params.max_attention_window_size = max_attention_window_size; + common_enqueue_params.cyclic_attention_window_size = cyclic_attention_window_size; + common_enqueue_params.max_cyclic_attention_window_size = max_cyclic_attention_window_size; + common_enqueue_params.can_use_one_more_block = can_use_one_more_block; + common_enqueue_params.sink_token_length = sink_token_length; + common_enqueue_params.kv_scale_orig_quant = kv_scale_orig_quant; + common_enqueue_params.kv_scale_quant_orig = kv_scale_quant_orig; + common_enqueue_params.attention_output_orig_quant = attention_output_orig_quant; + common_enqueue_params.attention_output_sf_scale = attention_output_sf_scale; + common_enqueue_params.alibi_slopes = alibi_slopes; + common_enqueue_params.context_buf = context_buf_; + common_enqueue_params.context_buf_sf = context_buf_sf_; + common_enqueue_params.key_value_cache = key_value_cache; + common_enqueue_params.block_offsets = block_offsets; + common_enqueue_params.host_primary_pool_pointer = host_primary_pool_pointer; + common_enqueue_params.host_secondary_pool_pointer = host_secondary_pool_pointer; + common_enqueue_params.num_tokens = localNbTokens; + common_enqueue_params.max_blocks_per_sequence = max_blocks_per_sequence; + common_enqueue_params.sequence_lengths = sequence_kv_length; + common_enqueue_params.context_lengths = context_q_lengths; + common_enqueue_params.host_context_lengths = host_context_lengths; + common_enqueue_params.workspace = workspace; + common_enqueue_params.runtime_perf_knobs = runtime_perf_knobs; + + if (isRelativePosition()) + { + common_enqueue_params.relative_attention_bias + = static_cast<T const*>(inputs[getIdx(IdxEntry::RELATIVE_ATTENTION_BIAS)]); + common_enqueue_params.relative_attention_bias_stride + = inputDesc[getIdx(IdxEntry::RELATIVE_ATTENTION_BIAS)].dims.d[1]; // max_seq_len or num_buckets + } + if (isLognScaling()) + { + common_enqueue_params.logn_scaling_ptr = static_cast<float const*>(inputs[getIdx(IdxEntry::LOGN_SCALING)]); + } + if (isCrossAttention()) + { + common_enqueue_params.encoder_input_lengths + = reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::ENCODER_INPUT_LENGTH)]) + seqIdxBeg; + } + + if (is_context) // context stage + { + int const batch_size = localNbSeq; + int const request_batch_size = batch_size; + // num of total tokens (without paddings when remove paddings). + int num_encoder_tokens = 0; + if (isCrossAttention()) + { + if (!mRemovePadding) + { + num_encoder_tokens = request_batch_size * max_encoder_context_len; + } + else + { + num_encoder_tokens = inputDesc[getIdx(IdxEntry::CROSS_KV)].dims.d[0]; + } + } + + common_enqueue_params.input_seq_length = max_context_q_len; + common_enqueue_params.max_past_kv_length = max_context_kv_len; + EnqueueContextParams<T> enqueue_params{common_enqueue_params}; + enqueue_params.attention_packed_mask = attention_packed_mask; + enqueue_params.batch_size = batch_size; + enqueue_params.mrope_rotary_cos_sin = mrope_rotary_cos_sin; + enqueue_params.total_kv_len = enqueue_params.num_tokens; + + if (isCrossAttention()) + { + enqueue_params.cross_kv = static_cast<T const*>(inputs[getIdx(IdxEntry::CROSS_KV)]); + enqueue_params.cross_kv_length = max_encoder_context_len; + enqueue_params.num_encoder_tokens = num_encoder_tokens; + } + + enqueueContext<T, KVCacheBuffer>(enqueue_params, stream); + + { + std::string const afterContexStr = "ctx attention at layer " + std::to_string(mLayerIdx); + TLLM_LOG_TRACE("GPTAttentionPlugin - %s", afterContexStr.c_str()); + + auto progress = static_cast<batch_manager::ContextProgress* const*>( + inputs[getIdx(IdxEntry::HOST_CONTEXT_PROGRESS)])[0]; + if (progress != nullptr) + { + progress->recordEvent(mLayerIdx, stream); + } + + if (!mFuseFp4Quant) + { + TLLM_CHECK_DEBUG_WITH_INFO( + tensorrt_llm::runtime::utils::tensorHasInvalid(localNbTokens, + outputDesc[0].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)], + mFP8ContextFMHA ? nvinfer1::DataType::kFP8 : mType, context_buf_, stream, afterContexStr) + == false, + "Found invalid number (NaN or Inf) in " + afterContexStr); + } + } + } + else // generation stage; max_context_q_len == input_seq_len == 1 + { + TLLM_CHECK_WITH_INFO(useKVCache(), "KV-cache-less is only supported for context"); + int batch_beam = localNbSeq; + TLLM_CHECK(batch_beam % beamWidth == 0); + int32_t const num_requests = batch_beam / beamWidth; + + int const* cache_indir + = beamWidth == 1 ? nullptr : reinterpret_cast<int const*>(inputs[getIdx(IdxEntry::CACHE_INDIR)]); + + // Medusa: the max input sequence length if variable sequence length is needed. + int const input_seq_length = getGenerationInputSequenceLength(inputDesc, localNbSeq, localNbTokens); + int const max_past_kv_length = isCrossAttention() ? max_encoder_context_len : max_context_kv_len; + auto qkvDims = inputDesc[getIdx(IdxEntry::QKV_TENSOR)].dims; + TLLM_CHECK_WITH_INFO(input_seq_length == 1 || (mIsSpecDecodingEnabled && mUseSpecDecoding), + "Only speculative decoding mode supports input length > 1 in the generation phase, input_seq_length=%d, " + "mIsSpecDecodingEnabled=%s, nDims=%d, (" FMT_DIM ", " FMT_DIM ", " FMT_DIM ")", + input_seq_length, mIsSpecDecodingEnabled ? "true" : "false", qkvDims.nbDims, qkvDims.d[0], qkvDims.d[1], + qkvDims.d[2]); + TLLM_CHECK_WITH_INFO( + input_seq_length == num_decoding_draft_tokens + 1, "The generation input length is not expected."); + common_enqueue_params.input_seq_length = input_seq_length; + common_enqueue_params.max_past_kv_length = max_past_kv_length; + EnqueueGenerationParams<T> enqueue_params{common_enqueue_params}; + enqueue_params.beam_width = beamWidth; + enqueue_params.attention_mask_stride = attention_mask_stride; + enqueue_params.num_requests = num_requests; + enqueue_params.cache_indir = cache_indir; + enqueue_params.semaphores = multiBlockSemaphores(); + enqueue_params.host_past_key_value_lengths = host_past_kv_len_list; + enqueue_params.mrope_position_deltas = mrope_position_deltas; + if (mIsSpecDecodingEnabled && mUseSpecDecoding) + { + enqueue_params.spec_decoding_packed_mask = spec_decoding_packed_mask; + enqueue_params.spec_decoding_position_offsets = spec_decoding_position_offsets; + enqueue_params.spec_decoding_generation_lengths = spec_decoding_generation_lengths; + enqueue_params.spec_decoding_is_generation_length_variable = mSpecDecodingIsGenerationLengthVariable; + enqueue_params.spec_decoding_max_generation_length = mSpecDecodingMaxGenerationLength; + } + if (mFuseFp4Quant) + { + enqueue_params.start_token_idx_sf = tokenIdxBeg; + } + + if (changeSpecDecodingMode) + { + // mUseSpecDecoding is changed, need to re-prepare the DecoderXQARunner + prepareEnqueueGeneration<T, KVCacheBuffer>(enqueue_params); + } + + enqueueGeneration<T, KVCacheBuffer>(enqueue_params, stream); + + { + std::string const afterGenStr = "gen attention at layer " + std::to_string(mLayerIdx); + { + TLLM_CHECK_DEBUG_WITH_INFO( + tensorrt_llm::runtime::utils::tensorHasInvalid(localNbTokens, + outputDesc[0].dims.d[getPackedTensorHiddenDimIndex(mRemovePadding)], + mFP8ContextFMHA ? nvinfer1::DataType::kFP8 : mType, context_buf_, stream, afterGenStr) + == false, + "Found invalid number (NaN or Inf) in " + afterGenStr); + } + } + } + + return 0; +} + +template <typename T, typename AttentionOutT> +int GPTAttentionPlugin::enqueueDispatchKVCacheType(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) +{ + if (mPagedKVCache) + { + return enqueueImpl<T, AttentionOutT, KVBlockArray>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else + { + return enqueueImpl<T, AttentionOutT, KVLinearBuffer>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + return 0; +} + +int GPTAttentionPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + if (mSkipAttn) + { + bool const* SKIP_ATTN = reinterpret_cast<bool const*>(inputs[getIdx(IdxEntry::SKIP_ATTN)]); + if (SKIP_ATTN[0]) + { + return 0; + } + } + + if (mType == nvinfer1::DataType::kHALF) + { + if (mFuseFp4Quant) + { + return enqueueDispatchKVCacheType<half, uint8_t>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_FP8 + if (mFP8ContextFMHA) + { + return enqueueDispatchKVCacheType<half, __nv_fp8_e4m3>( + inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return enqueueDispatchKVCacheType<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == nvinfer1::DataType::kFLOAT) + { + return enqueueDispatchKVCacheType<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_BF16 + else if (mType == nvinfer1::DataType::kBF16) + { + if (mFuseFp4Quant) + { + return enqueueDispatchKVCacheType<__nv_bfloat16, uint8_t>( + inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_FP8 + if (mFP8ContextFMHA) + { + return enqueueDispatchKVCacheType<__nv_bfloat16, __nv_fp8_e4m3>( + inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return enqueueDispatchKVCacheType<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType GPTAttentionPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + if (mFuseFp4Quant) + { + TLLM_CHECK(index == 0 || index == 1 || (!mPagedKVCache && useKVCache() && index == 2)); + } + else + { + TLLM_CHECK(index == 0 || (!mPagedKVCache && useKVCache() && index == 1)); + } + if (index == 0) + { + if (mFuseFp4Quant) + { + return nvinfer1::DataType::kFP4; + } + return mFP8ContextFMHA && mEnableContextFMHA ? nvinfer1::DataType::kFP8 + : inputTypes[getIdx(IdxEntry::QKV_TENSOR)]; + } + if (mFuseFp4Quant && index == 1) + { + return nvinfer1::DataType::kFP8; + } + return inputTypes[getIdx(IdxEntry::PAST_KEY_VALUE)]; +} + +// IPluginV2 Methods + +char const* GPTAttentionPlugin::getPluginType() const noexcept +{ + return GPT_ATTENTION_PLUGIN_NAME; +} + +char const* GPTAttentionPlugin::getPluginVersion() const noexcept +{ + return GPT_ATTENTION_PLUGIN_VERSION; +} + +int GPTAttentionPlugin::getNbOutputs() const noexcept +{ + int nbOutputs = mFuseFp4Quant ? 2 : 1; + if (!mPagedKVCache && useKVCache()) + { + nbOutputs += 1; + } + return nbOutputs; +} + +size_t GPTAttentionPlugin::getSerializationSize() const noexcept +{ + return GPTAttentionPluginCommon::getCommonSerializationSize(); +} + +void GPTAttentionPlugin::serialize(void* buffer) const noexcept +{ + GPTAttentionPluginCommon::serializeCommon(buffer); +} + +/////////////// + +GPTAttentionPluginCreator::GPTAttentionPluginCreator() + : GPTAttentionPluginCreatorCommon() +{ +} + +char const* GPTAttentionPluginCreator::getPluginName() const noexcept +{ + return GPT_ATTENTION_PLUGIN_NAME; +} + +char const* GPTAttentionPluginCreator::getPluginVersion() const noexcept +{ + return GPT_ATTENTION_PLUGIN_VERSION; +} + +PluginFieldCollection const* GPTAttentionPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* GPTAttentionPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginFieldParser p{fc->nbFields, fc->fields}; + + try + { + auto* obj = new GPTAttentionPlugin(p.getScalar<int32_t>("layer_idx").value(), + p.getScalar<int32_t>("num_heads").value(), p.getScalar<int32_t>("vision_start").value(), + p.getScalar<int32_t>("vision_length").value(), p.getScalar<int32_t>("num_kv_heads").value(), + p.getScalar<int32_t>("num_kv_heads_origin").value(), p.getScalar<int32_t>("head_size").value(), + p.getScalar<int32_t>("unidirectional").value(), p.getScalar<float>("q_scaling").value(), + p.getScalar<float>("attn_logit_softcapping_scale").value(), + static_cast<PositionEmbeddingType>(p.getScalar<int8_t>("position_embedding_type").value()), + p.getScalar<int32_t>("rotary_embedding_dim").value(), p.getScalar<float>("rotary_embedding_base").value(), + static_cast<RotaryScalingType>(p.getScalar<int8_t>("rotary_embedding_scale_type").value()), + p.getScalar<float>("rotary_embedding_scale").value(), + p.getScalar<float>("rotary_embedding_short_m_scale").value(), + p.getScalar<float>("rotary_embedding_long_m_scale").value(), + p.getScalar<int32_t>("rotary_embedding_max_positions").value(), + p.getScalar<int32_t>("rotary_embedding_original_max_positions").value(), + static_cast<int32_t>(p.getScalar<int32_t>("tp_size").value()), + static_cast<int32_t>(p.getScalar<int32_t>("tp_rank").value()), + static_cast<bool>(p.getScalar<int8_t>("unfuse_qkv_gemm").value()), + static_cast<bool>(p.getScalar<int8_t>("use_logn_scaling").value()), + static_cast<ContextFMHAType>(p.getScalar<int8_t>("context_fmha_type").value()), + p.getScalar<int32_t>("kv_cache_quant_mode").value(), + static_cast<bool>(p.getScalar<int8_t>("remove_input_padding").value()), + static_cast<AttentionMaskType>(p.getScalar<int32_t>("mask_type").value()), + BlockSparseParams{p.getScalar<int32_t>("block_sparse_block_size").value(), + static_cast<bool>(p.getScalar<int8_t>("block_sparse_homo_head_pattern").value()), + p.getScalar<int32_t>("block_sparse_num_local_blocks").value(), + p.getScalar<int32_t>("block_sparse_vertical_stride").value()}, + static_cast<bool>(p.getScalar<int32_t>("paged_kv_cache").value()), + p.getScalar<int32_t>("tokens_per_block").value(), + static_cast<nvinfer1::DataType>(p.getScalar<int32_t>("type_id").value()), + p.getScalar<int32_t>("max_context_length").value(), + static_cast<bool>(p.getScalar<int8_t>("qkv_bias_enabled").value()), + static_cast<bool>(p.getScalar<int8_t>("do_cross_attention").value()), + static_cast<int32_t>(p.getScalar<int32_t>("max_distance").value()), + static_cast<bool>(p.getScalar<int8_t>("pos_shift_enabled").value()), + static_cast<bool>(p.getScalar<int8_t>("dense_context_fmha").value()), + static_cast<bool>(p.getScalar<int8_t>("use_paged_context_fmha").value()), + static_cast<bool>(p.getScalar<int8_t>("use_fp8_context_fmha").value()), + static_cast<bool>(p.getScalar<int8_t>("has_full_attention_mask").value()), + static_cast<bool>(p.getScalar<int32_t>("use_cache").value()), + static_cast<bool>(p.getScalar<int8_t>("is_spec_decoding_enabled").value()), + static_cast<bool>(p.getScalar<int8_t>("spec_decoding_is_generation_length_variable").value()), + p.getScalar<int32_t>("spec_decoding_max_generation_length").value(), + static_cast<int8_t>(p.getScalar<int8_t>("is_mla_enabled").value()), + static_cast<int32_t>(p.getScalar<int32_t>("q_lora_rank").value()), + static_cast<int32_t>(p.getScalar<int32_t>("kv_lora_rank").value()), + static_cast<int32_t>(p.getScalar<int32_t>("qk_nope_head_dim").value()), + static_cast<int32_t>(p.getScalar<int32_t>("qk_rope_head_dim").value()), + static_cast<int32_t>(p.getScalar<int32_t>("v_head_dim").value()), + static_cast<bool>(p.getScalar<int8_t>("fuse_fp4_quant").value()), + static_cast<bool>(p.getScalar<int8_t>("skip_attn").value()), + static_cast<int32_t>(p.getScalar<int32_t>("cp_size").value()), + static_cast<int32_t>(p.getScalar<int32_t>("cp_rank").value()), + static_cast<std::set<int32_t>>(p.getSet<int32_t>("cp_group").value())); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* GPTAttentionPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call GPTAttentionPlugin::destroy() + try + { + auto* obj = new GPTAttentionPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h new file mode 100644 index 000000000000..3e34703c6221 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h @@ -0,0 +1,259 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "checkMacrosPlugin.h" +#include "tensorrt_llm/common/cublasMMWrapper.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/common/stringUtils.h" +#include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmhaRunner.h" +#include "tensorrt_llm/kernels/contextFusedMultiHeadAttention/fused_multihead_attention_common.h" +#include "tensorrt_llm/kernels/gptKernels.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h" +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ +// batch_size = num_ctx_requests + num_gen_requests * beam_width +// num_ctx_requests = number of context requests (single sequence per request). +// num_gen_requests = number of generation requests (beam_width sequences per request). +// Context sequences have to appear first, generation sequences after + +// inputs (see GPTAttentionPlugin::isEntryUsed for when each tensor is actually used) +// 0. input_tensor [batch_size, seq_len, local_hidden_size + 2 * local_num_kv_heads * head_size] or +// [num_tokens, local_hidden_size + 2 * local_num_kv_heads * head_size] when +// enable_remove_input_padding +// 1. sequence_length [batch_size] (optional) +// 2. host_past_key_value_lengths [batch_size] (int32) (optional) +// 3. host_max_attention_window_sizes [num_layers] (int32) +// 4. host_sink_token_length [1] (int32) +// 5. context_lengths [batch_size] +// 6. cache_indir [num_gen_requests, beam_width, memory_max_len] (required in beamsearch) (optional) +// 7. host_request_types [batch_size] int32. 0: context; 1: generation: 2: none. When not in inflight-batching +// mode, +// all elements must be identical. +// 8. past_key_value_pool [batch_size, 2, local_num_kv_heads, max_seq_len, head_size] or +// block_offsets [batch_size, 2, max_blocks_per_seq] if paged kv cache (optional) +// 8.1 host_pool_pointers [2] if paged kv cache (optional) +// 9. kv_cache_quantization_scale [1] (optional) +// 10. kv_cache_dequantization_scale [1] (optional) +// 11. attention_output_quantization_scale [1] (on device, optional) +// 12. attention_mask [num_tokens, kv_seqlen] (on device, bool, optional) +// 13. attention_packed_mask [num_tokens, kv_seqlen / 32] (on device, uint32_t, optional) +// - pack masks by encoding multiple mask positions into a single 32-bit unsigned integer. +// - see kernels/contextMultiHeadAttention/fmhaPackedMask.cpp for more details. +// 14. rotary_inv_freq [head_size / 2] or [head_size] (longrope type) (float) (on device, optional) +// 15. rotary_cos_sin [max_num_embedding_positions, 2] (float) (on device, optional) +// 16. alibi_slopes [num_heads] (optional for ALiBi position embedding) +// 17. relative_attention_bias [num_heads] (optional for ALiBi position embedding) +// 18. host_context_lengths [batch_size] int32. (optional, required when remove_input_padding is true) +// 19. qkv_bias (optional) [local_hidden_size * 3] +// 20. spec_decoding_generation_lengths (optional, required when medusa is enabled) (int32_t) [batch_size] +// 21. spec_decoding_packed_mask (optional, required when medusa is enabled) (int32_t) [num_tokens, packed_mask_dim] +// packed_mask_dim = divUp(max_num_spec_decoding_tokens + 1, 32) +// 22. spec_decoding_position_offsets (optional, required when medusa is enabled) (int32_t) [batch_size, +// max_num_spec_decoding_tokens + 1] +// 23. spec_decoding_use (optional, bool) [1]: If it is set as true, enable speculative decoding +// 24. long_rope_rotary_inv_freq [head / 2] (float) (on device, optional) +// 25. long_rope_rotary_cos_sin [max_num_embedding_positions, 2] (float) (on device, optional) +// 26. host_runtime_perf_knobs (int64) +// 27. host_context_progress (void*) +// 28. position_id_tensor(MLA) [total_tokens], used for rope embedding in MLA +// 29. q_a_proj_tensor(MLA) [hidden_dim, c_q_dim + c_k_dim + ropd_dim], used to proj compacted QKV +// 30. q_a_layernorm_tensor(MLA) [c_q_dim], rmsnorm weight for compacted q +// 31. q_b_proj_tensor(MLA) [c_q_dim, head_num * head_size], weight for companted q to q in context +// 32. kv_a_proj_with_mqa_tensor(MLA) [c_q_dim, head_num * (c_k_dim + rope_dim)], weight for companted q to kdim in +// generation +// 33. kv_a_layernorm_tensor(MLA) [c_k_dim], rmsnorm weight for compacted kv +// 34. kv_b_proj_tensor(MLA) [c_k_dim, head_num * 2 * (head_size - rope_dim)], weight for compacted kv to kv in +// context +// 35. skip_attn (optional, bool) [1]: If it is set as true, skip the atteniton plugin and return +// directly. +// +// outputs +// output_tensor [batch_size, seq_len, local_hidden_size] +// present_key_value_pool (optional if not paged kv cache) [batch_size, 2, local_num_kv_heads, max_seq_len, +// head_size] + +class GPTAttentionPlugin : public GPTAttentionPluginCommon +{ +public: + GPTAttentionPlugin(int layer_idx, int num_heads, int vision_start, int vision_length, int num_kv_heads, + int num_kv_heads_origin, int head_size, int unidirectional, float q_scaling, float attn_logit_softcapping_scale, + tensorrt_llm::kernels::PositionEmbeddingType position_embedding_type, + int rotary_embedding_dim, // for RoPE. 0 for non-RoPE + float rotary_embedding_base, tensorrt_llm::kernels::RotaryScalingType rotary_embedding_scale_type, + float rotary_embedding_scale, float rotary_embedding_short_m_scale, float rotary_embedding_long_m_scale, + int rotary_embedding_max_positions, int rotary_embedding_original_max_positions, int tp_size, + int tp_rank, // for ALiBi + bool unfuse_qkv_gemm, // for AutoPP + bool use_logn_scaling, // for LognScaling + tensorrt_llm::kernels::ContextFMHAType context_fmha_type, int kv_cache_quant_mode, bool remove_input_padding, + tensorrt_llm::kernels::AttentionMaskType mask_type, + tensorrt_llm::kernels::BlockSparseParams block_sparse_params, bool paged_kv_cache, int tokens_per_block, + nvinfer1::DataType type, int32_t max_context_length, bool qkv_bias_enabled, bool cross_attention = false, + int max_distance = 0, bool pos_shift_enabled = false, bool dense_context_fmha = false, + bool use_paged_context_fmha = true, bool use_fp8_context_fmha = true, bool has_full_attention_mask = false, + bool use_cache = true, bool is_spec_decoding_enabled = false, + bool spec_decoding_is_generation_length_variable = false, int spec_decoding_max_generation_length = 1, + bool is_mla_enabled = false, int q_lora_rank = 0, int kv_lora_rank = 0, int qk_nope_head_dim = 0, + int qk_rope_head_dim = 0, int v_head_dim = 0, bool fuse_fp4_quant = false, bool skip_attn = false, + int cp_size = 1, int cp_rank = 0, std::set<int32_t> cp_group = {}); + + GPTAttentionPlugin(void const* data, size_t length); + + ~GPTAttentionPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + template <typename T, typename AttentionOutT, typename KVCacheBuffer> + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + template <typename T, typename AttentionOutT = T> + int enqueueDispatchKVCacheType(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream); + + template <typename T, typename KVCacheBuffer> + void configurePluginImpl(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept; + template <typename T> + void configurePluginDispatchKVCacheType(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + + //! This is called on every trt ExecutionContext creation by TRT + //! Note TRT does not call the initialize on cloned plugin, so clone internally should do initialization. + GPTAttentionPlugin* clone() const noexcept override; + + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + +private: + template <typename T, typename AttentionOutT, typename KVCacheBuffer> + int enqueueSome(int32_t seqIdxBeg, int32_t localNbSeq, int32_t tokenIdxBeg, int32_t localNbTokens, + nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + using IndexType = std::int32_t; + + std::vector<size_t> mEntryIdx; + enum class IdxEntry : size_t + { + QKV_TENSOR, + K_TENSOR, + V_TENSOR, + ATTENTION_MASK, + ATTENTION_PACKED_MASK, + SEQUENCE_LENGTH, + HOST_PAST_KEY_VALUE_LENGTHS, + HOST_MAX_ATTENTION_WINDOW, + HOST_SINK_TOKEN_LENGTH, + CONTEXT_LENGTHS, + CACHE_INDIR, + REQUEST_TYPES, + KV_CACHE_BLOCK_OFFSETS, + HOST_KV_CACHE_BLOCK_OFFSETS, + HOST_KV_CACHE_POOL_POINTERS, + HOST_KV_CACHE_POOL_MAPPING, + PAST_KEY_VALUE, + KV_CACHE_QUANTIZATION_SCALE, + KV_CACHE_DEQUANTIZATION_SCALE, + ATTENTION_OUTPUT_QUANTIZATION_SCALE, + ATTENTION_OUTPUT_SF_SCALE, + ROTARY_INV_FREQ, + ROTARY_COS_SIN, + ALIBI_SLOPES, + RELATIVE_ATTENTION_BIAS, + CROSS_KV, + CROSS_KV_LENGTH, + ENCODER_INPUT_LENGTH, + HOST_CONTEXT_LENGTH, + QKV_BIAS_TENSOR, + SPEC_DECODING_GENERATION_LENGTHS, + SPEC_DECODING_PACKED_MASK, + SPEC_DECODING_POSITION_OFFSETS, + SPEC_DECODING_USE, + LONG_ROPE_ROTARY_INV_FREQ, + LONG_ROPE_ROTARY_COS_SIN, + MROPE_ROTARY_COS_SIN, + MROPE_POSITION_DELTAS, + HOST_RUNTIME_PERF_KNOBS, + HOST_CONTEXT_PROGRESS, + MLA_Q_B_PROJ_TENSOR, + MLA_KV_B_PROJ_TENSOR, + MLA_K_B_PROJ_TRANS_TENSOR, + SKIP_ATTN, + LOGN_SCALING, + ENUM_SIZE, // Used to count the number of IdxEntry, must put in last + }; + + std::string toString(IdxEntry const& entry) const; + bool isEntryUsed(IdxEntry const& entry) const; + void initEntryIdx(); + IndexType getIdx(IdxEntry const& entry) const; + + // Get generation input sequence length (might be larger than 1 in the speculative decoding mode). + int getGenerationInputSequenceLength( + nvinfer1::PluginTensorDesc const* inputDesc, int32_t localNbSeq, int32_t localNbTokens) const; +}; + +class GPTAttentionPluginCreator : public GPTAttentionPluginCreatorCommon +{ +public: + GPTAttentionPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/identityPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/identityPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/identityPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.cpp b/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.cpp new file mode 100644 index 000000000000..109010e7a933 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.cpp @@ -0,0 +1,199 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "identityPlugin.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" + +using namespace nvinfer1; +using tensorrt_llm::plugins::IdentityPluginCreator; +using tensorrt_llm::plugins::IdentityPlugin; + +static char const* IDENTITY_PLUGIN_VERSION{"1"}; +static char const* IDENTITY_PLUGIN_NAME{"Identity"}; +PluginFieldCollection IdentityPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> IdentityPluginCreator::mPluginAttributes; + +IdentityPlugin::IdentityPlugin() {} + +// Parameterized constructor +IdentityPlugin::IdentityPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* IdentityPlugin::clone() const noexcept +{ + auto* plugin = new IdentityPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs IdentityPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + return inputs[outputIndex]; +} + +bool IdentityPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + assert(0 <= pos && pos < 2); + PluginTensorDesc const& input = inOut[0]; + PluginTensorDesc const& output = inOut[1]; + switch (pos) + { + case 0: return input.format == nvinfer1::TensorFormat::kLINEAR; + case 1: return output.type == input.type && output.format == nvinfer1::TensorFormat::kLINEAR; + } + return false; +} + +void IdentityPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t IdentityPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int IdentityPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + size_t count = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) + { + count *= inputDesc[0].dims.d[i]; + } + count *= tensorrt_llm::runtime::BufferDataType(inputDesc[0].type).getSize(); + + cudaMemcpyAsync(outputs[0], inputs[0], count, cudaMemcpyDeviceToDevice, stream); + + sync_check_cuda_error(stream); + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType IdentityPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + assert(index == 0); + return inputTypes[0]; +} + +// IPluginV2 Methods + +char const* IdentityPlugin::getPluginType() const noexcept +{ + return IDENTITY_PLUGIN_NAME; +} + +char const* IdentityPlugin::getPluginVersion() const noexcept +{ + return IDENTITY_PLUGIN_VERSION; +} + +int IdentityPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int IdentityPlugin::initialize() noexcept +{ + return 0; +} + +void IdentityPlugin::terminate() noexcept {} + +size_t IdentityPlugin::getSerializationSize() const noexcept +{ + return 0; +} + +void IdentityPlugin::serialize(void* buffer) const noexcept {} + +void IdentityPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +IdentityPluginCreator::IdentityPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* IdentityPluginCreator::getPluginName() const noexcept +{ + return IDENTITY_PLUGIN_NAME; +} + +char const* IdentityPluginCreator::getPluginVersion() const noexcept +{ + return IDENTITY_PLUGIN_VERSION; +} + +PluginFieldCollection const* IdentityPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* IdentityPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + try + { + auto* obj = new IdentityPlugin(); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* IdentityPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call IdentityPlugin::destroy() + try + { + auto* obj = new IdentityPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.h b/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.h new file mode 100644 index 000000000000..9ab10601ae59 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/identityPlugin/identityPlugin.h @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class IdentityPlugin : public BasePlugin +{ +public: + IdentityPlugin(); + + IdentityPlugin(void const* data, size_t length); + + ~IdentityPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + const std::string mLayerName; +}; + +class IdentityPluginCreator : public BaseCreator +{ +public: + IdentityPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/CMakeLists.txt new file mode 100755 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.cpp b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.cpp new file mode 100644 index 000000000000..02a40a00c919 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.cpp @@ -0,0 +1,472 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "layernormQuantizationPlugin.h" +#include "pluginUtils.h" +#include "tensorrt_llm/kernels/layernormKernels.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::LayernormQuantizationPluginCreator; +using tensorrt_llm::plugins::LayernormQuantizationPlugin; + +static char const* LAYERNORM_QUANTIZATION_PLUGIN_VERSION{"1"}; +static char const* LAYERNORM_QUANTIZATION_PLUGIN_NAME{"LayernormQuantization"}; +PluginFieldCollection LayernormQuantizationPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> LayernormQuantizationPluginCreator::mPluginAttributes; + +LayernormQuantizationPlugin::LayernormQuantizationPlugin(float eps, bool useDiffOfSquares, + bool dynamicActivationScaling, bool sumPerToken, bool clampValEnabled, tensorrt_llm::common::QuantMode quantMode, + nvinfer1::DataType type, nvinfer1::DataType outputType) + : mEps(eps) + , mUseDiffOfSquares(useDiffOfSquares) + , mDynActScaling(dynamicActivationScaling) + , mType(type) + , mOutputType(outputType) + , mClampValEnabled(clampValEnabled) + , mQuantMode(quantMode) + , mSumPerToken(sumPerToken) +{ + TLLM_CHECK_WITH_INFO(mOutputType == nvinfer1::DataType::kINT8 || mOutputType == nvinfer1::DataType::kFP8, + "Only int8 or fp8 output type is allowed."); + // Check if the quant mode is valid. + TLLM_CHECK_WITH_INFO(mQuantMode.hasPerTokenScaling(), "The quant mode is not valid."); +} + +// Parameterized constructor +LayernormQuantizationPlugin::LayernormQuantizationPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mEps); + read(d, mUseDiffOfSquares); + read(d, mDynActScaling); + read(d, mSumPerToken); + read(d, mClampValEnabled); + read(d, mQuantMode); + read(d, mType); + read(d, mOutputType); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* LayernormQuantizationPlugin::clone() const noexcept +{ + auto* plugin = new LayernormQuantizationPlugin( + mEps, mUseDiffOfSquares, mDynActScaling, mSumPerToken, mClampValEnabled, mQuantMode, mType, mOutputType); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs LayernormQuantizationPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + if (outputIndex == 0) + { + // Quantized output + return inputs[outputIndex]; + } + + // Dynamic scaling or per-token sum if enabled + try + { + if (outputIndex == 1) + { + TLLM_CHECK(mDynActScaling); + } + else if (outputIndex == 2) + { + TLLM_CHECK(mSumPerToken); + } + else + { + TLLM_CHECK(false); + } + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int di = 0; di < ret.nbDims - 1; ++di) + { + ret.d[di] = inputs[0].d[di]; + } + ret.d[ret.nbDims - 1] = exprBuilder.constant(1); + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool LayernormQuantizationPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + int const totalPoses + = 6 + static_cast<int>(mClampValEnabled) + static_cast<int>(mDynActScaling) + static_cast<int>(mSumPerToken); + TLLM_CHECK(0 <= pos && pos < totalPoses); + TLLM_CHECK(nbInputs == 4 + static_cast<int>(mClampValEnabled)); + if (pos < nbInputs) + { + if (pos < 3) + { + // activatation, weight, bias + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == 3) + { + // scale + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == 4 && mClampValEnabled) + { + // clamp_max_v + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + } + else + { + auto const output_pos = pos - nbInputs; + if (output_pos == 0) + { + // Quantized output + return (inOut[pos].type == mOutputType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (output_pos == 1 && mDynActScaling) + { + // Dynamic scaling if enabled + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (output_pos == 2 && static_cast<int>(mClampValEnabled)) + { + // Clamp value + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + } + + // We should never reach this point + TLLM_CHECK_WITH_INFO(false, "The input/output is not supported."); + return false; +} + +void LayernormQuantizationPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t LayernormQuantizationPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int LayernormQuantizationPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // input [M(*), N] + // weight [N, ] + // bias [N, ] + // scale_to_int [1] + // clamp_max_v [2], contains min val, and max val (optional) + // outputs + // output [M(*), N] Normalized activations, potentially with quantization applied. + // dynamic_scaling [M(*), 1] (Optional) Per-token scales if quantization is enabled. + // token_sums [M(*), 1] (Optional) Per-token sums of all the channels (before quantization). + + int64_t m64 = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims - 1; ++i) + { + m64 *= inputDesc[0].dims.d[i]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); + + void const* input = inputs[0]; + void const* weight = inputs[1]; + void const* bias = inputs[2]; + void const* scale = inputs[3]; + void const* clampValPtr = mClampValEnabled ? inputs[4] : nullptr; + void* output = outputs[0]; + void* dynamic_scale = mDynActScaling ? outputs[1] : nullptr; + void* sum_per_token = mSumPerToken ? outputs[2] : nullptr; + + if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kINT8) + { + dispatchDataType<float, int8_t>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, + clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kFP8) + { + dispatchDataType<float, __nv_fp8_e4m3>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, + clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#endif // ENABLE_FP8 + else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kINT8) + { + dispatchDataType<half, int8_t>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, clampValPtr, + scale, dynamic_scale, sum_per_token, output); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kFP8) + { + dispatchDataType<half, __nv_fp8_e4m3>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, + clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#endif // ENABLE_FP8 +#ifdef ENABLE_BF16 + else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kINT8) + { + dispatchDataType<__nv_bfloat16, int8_t>(nullptr, input, weight, bias, mEps, m, n, stream, mUseDiffOfSquares, + clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kFP8) + { + dispatchDataType<__nv_bfloat16, __nv_fp8_e4m3>(nullptr, input, weight, bias, mEps, m, n, stream, + mUseDiffOfSquares, clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#endif // ENABLE_FP8 +#endif // ENABLE_BF16 + sync_check_cuda_error(stream); + return 0; +} + +template <typename T, typename QuantT> +void LayernormQuantizationPlugin::dispatchDataType(void* out, void const* input, void const* gamma, void const* beta, + float const eps, int const tokens, int const hidden_dim, cudaStream_t stream, bool use_diff_of_squares, + void const* clampValPtr, void const* scale, void* dynamic_scale, void* sum_per_token, + void* normed_output_quant) noexcept +{ + // inputs + // activation [dim0(*), dim1] + // clamp_value [2], contains min val, and max val (optional) + // outputs + // quant [dim0(*), dim1] + // scale_tokens [dim0(*), 1] + + invokeGeneralLayerNorm(reinterpret_cast<T*>(out), reinterpret_cast<T const*>(input), + reinterpret_cast<T const*>(gamma), reinterpret_cast<T const*>(beta), eps, tokens, hidden_dim, mQuantMode, + stream, use_diff_of_squares, reinterpret_cast<float const*>(clampValPtr), reinterpret_cast<float const*>(scale), + reinterpret_cast<float*>(dynamic_scale), reinterpret_cast<float*>(sum_per_token), + reinterpret_cast<QuantT*>(normed_output_quant)); +} + +// IPluginV2Ext Methods +nvinfer1::DataType LayernormQuantizationPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + assert(index <= 2); + + if (index == 0) + { + // Output 0 quantized output of layer norm + return mOutputType; + } + else if (index == 1) + { + assert(mDynActScaling); + // Output 1 dynamic act scaling + return nvinfer1::DataType::kFLOAT; + } + else if (index == 2) + { + assert(mDynActScaling && mSumPerToken); + // Output 2 per-token sums + return nvinfer1::DataType::kFLOAT; + } + + // We should never reach this point + TLLM_CHECK_WITH_INFO(false, "The output index is not supported."); + return nvinfer1::DataType::kFLOAT; +} + +// IPluginV2 Methods + +char const* LayernormQuantizationPlugin::getPluginType() const noexcept +{ + return LAYERNORM_QUANTIZATION_PLUGIN_NAME; +} + +char const* LayernormQuantizationPlugin::getPluginVersion() const noexcept +{ + return LAYERNORM_QUANTIZATION_PLUGIN_VERSION; +} + +int LayernormQuantizationPlugin::getNbOutputs() const noexcept +{ + return 1 + static_cast<int>(mDynActScaling) + static_cast<int>(mSumPerToken); +} + +int LayernormQuantizationPlugin::initialize() noexcept +{ + return 0; +} + +void LayernormQuantizationPlugin::terminate() noexcept {} + +size_t LayernormQuantizationPlugin::getSerializationSize() const noexcept +{ + return sizeof(mEps) + sizeof(mUseDiffOfSquares) + sizeof(mDynActScaling) + sizeof(mSumPerToken) + + sizeof(mClampValEnabled) + sizeof(mQuantMode) + sizeof(mType) + sizeof(mOutputType); +} + +void LayernormQuantizationPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mEps); + write(d, mUseDiffOfSquares); + write(d, mDynActScaling); + write(d, mSumPerToken); + write(d, mClampValEnabled); + write(d, mQuantMode); + write(d, mType); + write(d, mOutputType); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void LayernormQuantizationPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +LayernormQuantizationPluginCreator::LayernormQuantizationPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("eps", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("use_diff_of_squares", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("dyn_act_scaling", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("sum_per_token", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("clamp_val_enabled", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("quant_mode", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("out_type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* LayernormQuantizationPluginCreator::getPluginName() const noexcept +{ + return LAYERNORM_QUANTIZATION_PLUGIN_NAME; +} + +char const* LayernormQuantizationPluginCreator::getPluginVersion() const noexcept +{ + return LAYERNORM_QUANTIZATION_PLUGIN_VERSION; +} + +PluginFieldCollection const* LayernormQuantizationPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* LayernormQuantizationPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + tensorrt_llm::common::QuantMode quantMode{}; + float eps{}; + nvinfer1::DataType type{}; + nvinfer1::DataType outputType{}; + bool useDiffOfSquares{}; + bool dynamicActivationScaling{}; + bool sumPerToken{}; + bool clampValEnabled{}; + + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "eps")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + eps = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "dyn_act_scaling")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dynamicActivationScaling = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "use_diff_of_squares")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + useDiffOfSquares = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "sum_per_token")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + sumPerToken = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "clamp_val_enabled")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + clampValEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "quant_mode")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + quantMode = QuantMode(*(static_cast<int32_t const*>(fields[i].data))); + } + else if (!strcmp(attrName, "out_type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + outputType = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + try + { + auto* obj = new LayernormQuantizationPlugin( + eps, useDiffOfSquares, dynamicActivationScaling, sumPerToken, clampValEnabled, quantMode, type, outputType); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* LayernormQuantizationPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call LayernormQuantizationPlugin::destroy() + try + { + auto* obj = new LayernormQuantizationPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.h b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.h new file mode 100644 index 000000000000..5cf3fa7e022f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/layernormQuantizationPlugin/layernormQuantizationPlugin.h @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class LayernormQuantizationPlugin : public BasePlugin +{ +public: + LayernormQuantizationPlugin(float eps, bool useDiffOfSquares, bool dynamicActivationScaling, bool sumPerToken, + bool clampValEnabled, tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, + nvinfer1::DataType outputType); + + LayernormQuantizationPlugin(void const* data, size_t length); + + ~LayernormQuantizationPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + template <typename T, typename QuantT> + void dispatchDataType(void* out, void const* input, void const* gamma, void const* beta, float const eps, + int const tokens, int const hidden_dim, cudaStream_t stream, bool use_diff_of_squares, void const* clampValPtr, + void const* scale, void* dynamic_scale, void* sum_per_token, void* normed_output_quant) noexcept; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + float mEps; + bool mUseDiffOfSquares; + bool mDynActScaling; + nvinfer1::DataType mType; + + const std::string mLayerName; + // The quantized output data type + nvinfer1::DataType mOutputType; + // Do we clamp the input tensor? + bool mClampValEnabled; + // The quantization mode + tensorrt_llm::common::QuantMode mQuantMode; + // Should we output the sum of channels per-token? (Used by QServe GEMM) + bool mSumPerToken; +}; + +class LayernormQuantizationPluginCreator : public BaseCreator +{ +public: + LayernormQuantizationPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/lookupPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/lookupPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lookupPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.cpp b/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.cpp new file mode 100644 index 000000000000..e4d26f9e5ec6 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.cpp @@ -0,0 +1,341 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdio> + +#include "lookupPlugin.h" +#include "tensorrt_llm/kernels/lookupKernels.h" +#include "tensorrt_llm/plugins/common/plugin.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::LookupPluginCreator; +using tensorrt_llm::plugins::LookupPlugin; + +static char const* LOOKUP_PLUGIN_VERSION{"1"}; +static char const* LOOKUP_PLUGIN_NAME{"Lookup"}; +PluginFieldCollection LookupPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> LookupPluginCreator::mPluginAttributes; + +LookupPlugin::LookupPlugin(nvinfer1::DataType type, int rank) + : mType(type) + , mRank(rank) +{ + mArch = tensorrt_llm::common::getSMVersion(); +} + +// Parameterized constructor +LookupPlugin::LookupPlugin(void const* data, size_t length) +{ + mArch = tensorrt_llm::common::getSMVersion(); + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mType); + read(d, mRank); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* LookupPlugin::clone() const noexcept +{ + auto* plugin = new LookupPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + plugin->initialize(); + return plugin; +} + +nvinfer1::DimsExprs LookupPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 2 || nbInputs == 3); + TLLM_CHECK(outputIndex == 0); + DimsExprs ret; + int const nbDimsInput = inputs[0].nbDims; + int const nbDimsWeight = inputs[1].nbDims; + ret.nbDims = nbDimsInput + 1; + + for (int i = 0; i < nbDimsInput; ++i) + { + ret.d[i] = inputs[0].d[i]; + } + ret.d[nbDimsInput] = inputs[1].d[nbDimsWeight - 1]; + + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool LookupPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + bool res = false; + if (nbInputs == 2) + { + switch (pos) + { + case 0: res = ((inOut[0].type == DataType::kINT32) && (inOut[0].format == TensorFormat::kLINEAR)); break; + case 1: res = ((inOut[1].type == mType) && (inOut[1].format == TensorFormat::kLINEAR)); break; + case 2: res = ((inOut[2].type == mType) && (inOut[2].format == TensorFormat::kLINEAR)); break; + default: // should NOT be here! + res = false; + } + } + else + { + TLLM_CHECK_WITH_INFO(mArch == 90, "int8 weight only lookupPlugin is only supported in SM 90 now."); + switch (pos) + { + case 0: res = ((inOut[0].type == DataType::kINT32) && (inOut[0].format == TensorFormat::kLINEAR)); break; + case 1: + res = ((inOut[1].type == DataType::kINT8 || inOut[1].type == mType) + && (inOut[1].format == TensorFormat::kLINEAR)); + break; + case 2: res = ((inOut[2].type == mType) && (inOut[2].format == TensorFormat::kLINEAR)); break; + case 3: res = ((inOut[3].type == mType) && (inOut[3].format == TensorFormat::kLINEAR)); break; + default: // should NOT be here! + res = false; + } + } + return res; +} + +void LookupPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + mNbInputs = nbInputs; +} + +size_t LookupPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int LookupPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + // inputs + // input [tokenNum] + // weight [localVocabSize, hidden] + // per_token_scales [localVocabSize], optional + // outputs + // embedding [tokenNum, hidden] + + int64_t tokenNum = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) + { + tokenNum *= inputDesc[0].dims.d[i]; + } + + int const localVocabSize = inputDesc[1].dims.d[0]; + int const hidden = inputDesc[1].dims.d[inputDesc[1].dims.nbDims - 1]; + int const* input = reinterpret_cast<int const*>(inputs[0]); + + int offset = mRank * localVocabSize; + + if (mNbInputs == 3) + { + int8_t const* weight = reinterpret_cast<int8_t const*>(inputs[1]); + if (mType == DataType::kHALF) + { + half const* per_token_scales = reinterpret_cast<half const*>(inputs[2]); + half* output = reinterpret_cast<half*>(outputs[0]); + invokeLookUp<half, int8_t, int>( + output, input, weight, tokenNum, offset, localVocabSize, hidden, per_token_scales, stream); + } + else if (mType == DataType::kFLOAT) + { + float const* per_token_scales = reinterpret_cast<float const*>(inputs[2]); + float* output = reinterpret_cast<float*>(outputs[0]); + invokeLookUp<float, int8_t, int>( + output, input, weight, tokenNum, offset, localVocabSize, hidden, per_token_scales, stream); + } + else if (mType == DataType::kBF16) + { + __nv_bfloat16 const* per_token_scales = reinterpret_cast<__nv_bfloat16 const*>(inputs[2]); + __nv_bfloat16* output = reinterpret_cast<__nv_bfloat16*>(outputs[0]); + invokeLookUp<__nv_bfloat16, int8_t, int>( + output, input, weight, tokenNum, offset, localVocabSize, hidden, per_token_scales, stream); + } + } + else + { + if (mType == DataType::kHALF) + { + half const* weight = reinterpret_cast<half const*>(inputs[1]); + half* output = reinterpret_cast<half*>(outputs[0]); + invokeLookUp<half, half, int>( + output, input, weight, tokenNum, offset, localVocabSize, hidden, nullptr, stream); + } + else if (mType == DataType::kFLOAT) + { + float const* weight = reinterpret_cast<float const*>(inputs[1]); + float* output = reinterpret_cast<float*>(outputs[0]); + invokeLookUp<float, float, int>( + output, input, weight, tokenNum, offset, localVocabSize, hidden, nullptr, stream); + } + else if (mType == DataType::kBF16) + { + __nv_bfloat16 const* weight = reinterpret_cast<__nv_bfloat16 const*>(inputs[1]); + __nv_bfloat16* output = reinterpret_cast<__nv_bfloat16*>(outputs[0]); + invokeLookUp<__nv_bfloat16, __nv_bfloat16, int>( + output, input, weight, tokenNum, offset, localVocabSize, hidden, nullptr, stream); + } + } + sync_check_cuda_error(stream); + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType LookupPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* LookupPlugin::getPluginType() const noexcept +{ + return LOOKUP_PLUGIN_NAME; +} + +char const* LookupPlugin::getPluginVersion() const noexcept +{ + return LOOKUP_PLUGIN_VERSION; +} + +int LookupPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int LookupPlugin::initialize() noexcept +{ + return 0; +} + +void LookupPlugin::destroy() noexcept +{ + delete this; +} + +size_t LookupPlugin::getSerializationSize() const noexcept +{ + return sizeof(mType) + sizeof(mRank); +} + +void LookupPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mRank); + + TLLM_CHECK(d == a + getSerializationSize()); +} + +void LookupPlugin::terminate() noexcept {} + +/////////////// + +LookupPluginCreator::LookupPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("rank", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* LookupPluginCreator::getPluginName() const noexcept +{ + return LOOKUP_PLUGIN_NAME; +} + +char const* LookupPluginCreator::getPluginVersion() const noexcept +{ + return LOOKUP_PLUGIN_VERSION; +} + +PluginFieldCollection const* LookupPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* LookupPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + nvinfer1::DataType type{}; + int rank{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "rank")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + rank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + } + try + { + auto* obj = new LookupPlugin(type, rank); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* LookupPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call LookupPlugin::destroy() + try + { + auto* obj = new LookupPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.h b/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.h new file mode 100644 index 000000000000..4dddaa1d8bdc --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lookupPlugin/lookupPlugin.h @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class LookupPlugin : public BasePlugin +{ +public: + LookupPlugin() = delete; + + LookupPlugin(nvinfer1::DataType type, int rank); + + LookupPlugin(void const* data, size_t length); + + ~LookupPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + const std::string mLayerName; + + nvinfer1::DataType mType; + int mRank; + int mNbInputs = 0; + int mArch; +}; + +class LookupPluginCreator : public BaseCreator +{ +public: + LookupPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/loraPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/loraPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/loraPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.cpp b/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.cpp new file mode 100644 index 000000000000..7a7d925a74f6 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.cpp @@ -0,0 +1,525 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "loraPlugin.h" + +#include "pluginUtils.h" +#include "tensorrt_llm/common/assert.h" + +#include <vector> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::LoraPluginCreator; +using tensorrt_llm::plugins::LoraPlugin; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +static char const* LORA_PLUGIN_VERSION{"1"}; +static char const* LORA_PLUGIN_NAME{"Lora"}; +PluginFieldCollection LoraPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> LoraPluginCreator::mPluginAttributes; + +LoraPlugin::LoraPlugin(int in_hidden_size, std::vector<int> out_hidden_sizes, int transA, int transB, + int num_lora_modules, nvinfer1::DataType type, LoraPlugin::PluginProfilerPtr const& pluginProfiler, + bool remove_input_padding, int max_low_rank, int weight_index) + : mTransA(transA) + , mTransB(transB) + , mType(type) + , mRemoveInputPadding(remove_input_padding) + , mNumLoraModules(num_lora_modules) + , mInHiddenSize(in_hidden_size) + , mMaxLowRank(max_low_rank) + , mWeightIndex(weight_index) + , mPluginProfiler(pluginProfiler) +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + mOutHiddenSizes.resize(mNumLoraModules); + mOutHiddenSizes.assign(out_hidden_sizes.begin(), out_hidden_sizes.end()); + init(); +} + +// Parameterized constructor +LoraPlugin::LoraPlugin(void const* data, size_t length, LoraPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mInHiddenSize); + read(d, mTransA); + read(d, mTransB); + read(d, mNumLoraModules); + read(d, mType); + read(d, mRemoveInputPadding); + read(d, mMaxLowRank); + read(d, mWeightIndex); + mOutHiddenSizes.resize(mNumLoraModules); + for (int i = 0; i < mNumLoraModules; i++) + { + read(d, mOutHiddenSizes[i]); + } + init(); + + mPluginProfiler->deserialize(d, mDims, mGemmId); + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +void LoraPlugin::init() +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + + auto cublasHandle = getCublasHandle(); + auto cublasLtHandle = getCublasLtHandle(); + auto cublasWraper = std::make_shared<CublasMMWrapper>(cublasHandle, cublasLtHandle, nullptr, nullptr); + + mLoraImpl = std::make_shared<kernels::LoraImpl>( + mInHiddenSize, mOutHiddenSizes, mTransA, mTransB, mNumLoraModules, mType, mMaxLowRank, cublasWraper); + + mPluginProfiler->setTranspose(mTransA, mTransB); + mGemmId = GemmIdCublas(mDims.n, mDims.k, mType, mTransA, mTransB, mType); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* LoraPlugin::clone() const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + auto* plugin = new LoraPlugin(*this); + return plugin; +} + +nvinfer1::DimsExprs LoraPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + try + { + TLLM_CHECK(outputIndex < mNumLoraModules); + int const nbDimsA = inputs[getInputTensorIdx()].nbDims; + DimsExprs ret; + ret.nbDims = nbDimsA; + + for (int i = 0; i < ret.nbDims; ++i) + { + ret.d[0] = 0; + } + + if (mTransA) + { + for (int i = 1; i < nbDimsA; ++i) + { + ret.d[i - 1] = inputs[getInputTensorIdx()].d[i]; + } + } + else + { + for (int i = 0; i < nbDimsA - 1; ++i) + { + ret.d[i] = inputs[getInputTensorIdx()].d[i]; + } + } + + auto const* outHiddenSize = exprBuilder.constant(mOutHiddenSizes.at(outputIndex)); + TLLM_CHECK(outHiddenSize != nullptr); + ret.d[ret.nbDims - 1] = outHiddenSize; + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool LoraPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + if (pos == getHostRequestTypesIdx()) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (pos >= getLoraRanksIdx() && pos < getLoraRanksIdx() + mNumLoraModules) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (pos >= getLoraWeightsPtrsIdx() && pos < getLoraWeightsPtrsIdx() + mNumLoraModules) + { + return inOut[pos].type == nvinfer1::DataType::kINT64; + } + else if (mRemoveInputPadding && pos == getHostContextLengthsIdx()) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else + { + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } +} + +void LoraPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + + auto const input = in[getInputTensorIdx()]; + + int const nbDimsA = input.max.nbDims; + + auto const minM = utils::computeMDimension(mTransA, input.min); + auto const maxM = utils::computeMDimension(mTransA, input.max); + auto const N = utils::computeNDimension(mTransB, in[getHostRequestTypesIdx()].max); + auto const K = static_cast<utils::DimType64>(mTransA ? input.max.d[0] : input.max.d[nbDimsA - 1]); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, N, K}; + } + mGemmId.n = N; + mGemmId.k = K; + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +size_t LoraPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + + int const nbReq = inputs[getLoraRanksIdx()].dims.d[0]; + auto const type = inputs[getInputTensorIdx()].type; + auto const numTokens = getNumTokens(inputs); + return mLoraImpl->getWorkspaceSize(numTokens, nbReq, type); +} + +int64_t LoraPlugin::getNumTokens(nvinfer1::PluginTensorDesc const* input_tensors) const +{ + int ndim = input_tensors[getInputTensorIdx()].dims.nbDims; + TLLM_CHECK_WITH_INFO( + 3 == ndim || 2 == ndim, "hidden_state dimension should be either 2 [numTokens, hidden], or 3 [b, s, hidden]"); + int64_t num_tokens = input_tensors[getInputTensorIdx()].dims.d[0]; + if (ndim == 3) + { + num_tokens *= input_tensors[getInputTensorIdx()].dims.d[1]; + } + return num_tokens; +} + +int LoraPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + if (isBuilding()) + { + return 0; + } + + auto const numReqs = inputDesc[getLoraRanksIdx()].dims.d[0]; + void const* input = inputs[getInputTensorIdx()]; + int const seqLen = mRemoveInputPadding ? 0 : inputDesc[getInputTensorIdx()].dims.d[1]; + int32_t const* reqTypes = static_cast<int32_t const*>(inputs[getHostRequestTypesIdx()]); + void const* const* loraRanks = &inputs[getLoraRanksIdx()]; + void const* const* loraWeightPtrs = &inputs[getLoraWeightsPtrsIdx()]; + int32_t const* hostContextLengths + = mRemoveInputPadding ? static_cast<int32_t const*>(inputs[getHostContextLengthsIdx()]) : nullptr; + + int numTokens = getNumTokens(inputDesc); + mExpandLoraWeightPtrs.clear(); + mExpandLoraRanks.clear(); + mExpandLoraWeightPtrs.reserve(mNumLoraModules * numTokens * 2); + mExpandLoraRanks.reserve(mNumLoraModules * numTokens); + + for (int loraModuleIdx = 0; loraModuleIdx < mNumLoraModules; loraModuleIdx++) + { + auto const loraWeightModulePtrs = static_cast<int64_t const*>(loraWeightPtrs[loraModuleIdx]); + auto const loraRankModule = static_cast<int32_t const*>(loraRanks[loraModuleIdx]); + + int idx = 0; + for (int reqId = 0; reqId < numReqs; reqId++) + { + // loraWeightModulePtrs has 3 pointers for each module: A,B, and an optional DoRA magnitude + // the current LoRA plugin does not apply DoRA scaling, so the magnitude is ignored + RequestType const reqType = static_cast<RequestType>(reqTypes[reqId]); + if (reqType == RequestType::kGENERATION) + { + mExpandLoraWeightPtrs.push_back(reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3])); + mExpandLoraWeightPtrs.push_back(reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3 + 1])); + mExpandLoraRanks.push_back(loraRankModule[reqId]); + idx += 1; + } + else + { + int contextLen = (mRemoveInputPadding ? hostContextLengths[reqId] : seqLen); + + for (int contextId = 0; contextId < contextLen; contextId++) + { + mExpandLoraWeightPtrs.push_back(reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3])); + mExpandLoraWeightPtrs.push_back(reinterpret_cast<void const*>(loraWeightModulePtrs[reqId * 3 + 1])); + mExpandLoraRanks.push_back(loraRankModule[reqId]); + idx += 1; + } + } + } + + // In 1st generation phase cross attention qkv lora, cross qkv is skipped by passing an empty encoder_output + // (passing 0 to dim) getNumTokens() will get in cross qkv_lora. Skipping the check for this case. + if (numTokens > 0) + { + TLLM_CHECK_WITH_INFO(idx == numTokens, + fmtstr("LoraParams and input dims don't match, lora tokens %d input tokens %d", idx, numTokens)); + } + } + + // only used for unified gemm + auto bestTactic = mPluginProfiler->getBestConfig(numTokens, mGemmId); + mLoraImpl->setBestTactic(bestTactic); + mLoraImpl->run(numTokens, numReqs, input, mExpandLoraRanks.data(), mExpandLoraWeightPtrs.data(), mWeightIndex, + outputs, workspace, stream); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType LoraPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + TLLM_CHECK(index < mNumLoraModules); + return mType; +} + +// IPluginV2 Methods + +char const* LoraPlugin::getPluginType() const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + return LORA_PLUGIN_NAME; +} + +char const* LoraPlugin::getPluginVersion() const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + return LORA_PLUGIN_VERSION; +} + +int LoraPlugin::getNbOutputs() const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + return mNumLoraModules; +} + +int LoraPlugin::initialize() noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + if (!mDims.isInitialized()) + { + return 0; + } + + mLoraImpl->setGemmConfig(); + + mPluginProfiler->profileTactics(mLoraImpl->getCublasWrapper(), mType, mDims, mGemmId); + return 0; +} + +void LoraPlugin::destroy() noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + delete this; +} + +size_t LoraPlugin::getSerializationSize() const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + return sizeof(mInHiddenSize) + sizeof(mTransA) + sizeof(mTransB) + sizeof(mNumLoraModules) + sizeof(mType) + + mPluginProfiler->getSerializationSize(mGemmId) + sizeof(mRemoveInputPadding) + sizeof(mMaxLowRank) + + sizeof(mWeightIndex) + sizeof(int) * mNumLoraModules; // selected tactics container size +} + +void LoraPlugin::serialize(void* buffer) const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + char *d = static_cast<char*>(buffer), *a = d; + write(d, mInHiddenSize); + write(d, mTransA); + write(d, mTransB); + write(d, mNumLoraModules); + write(d, mType); + write(d, mRemoveInputPadding); + write(d, mMaxLowRank); + write(d, mWeightIndex); + for (int i = 0; i < mNumLoraModules; i++) + { + write(d, mOutHiddenSizes.at(i)); + } + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void LoraPlugin::terminate() noexcept {} + +/////////////// + +LoraPluginCreator::LoraPluginCreator() +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("transA", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("transB", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("num_lora_modules", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("weight_index", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* LoraPluginCreator::getPluginName() const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + return LORA_PLUGIN_NAME; +} + +char const* LoraPluginCreator::getPluginVersion() const noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + return LORA_PLUGIN_VERSION; +} + +PluginFieldCollection const* LoraPluginCreator::getFieldNames() noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + return &mFC; +} + +IPluginV2* LoraPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + + PluginField const* fields = fc->fields; + nvinfer1::DataType type{}; + int num_lora_modules{}; + int in_hidden_size{}; + int transA{}; + int transB{}; + bool remove_input_padding{}; + int max_low_rank{}; + int weight_index{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "in_hidden_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + in_hidden_size = *(static_cast<int32_t const*>(fields[i].data)); + } + else if (!strcmp(attrName, "transa")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + transA = *(static_cast<int const*>(fields[i].data)); + } + else if (!strcmp(attrName, "transb")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + transB = *(static_cast<int const*>(fields[i].data)); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "remove_input_padding")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + remove_input_padding = static_cast<bool>(*(static_cast<int8_t const*>(fields[i].data))); + } + else if (!strcmp(attrName, "max_low_rank")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + max_low_rank = *(static_cast<int const*>(fields[i].data)); + } + else if (!strcmp(attrName, "num_lora_modules")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + num_lora_modules = *(static_cast<int const*>(fields[i].data)); + } + else if (!strcmp(attrName, "weight_index")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + weight_index = *(static_cast<int const*>(fields[i].data)); + } + } + std::vector<int> out_hidden_sizes; + out_hidden_sizes.resize(num_lora_modules); + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + for (int j = 0; j < num_lora_modules; j++) + { + if (!strcmp(attrName, fmtstr("out_hidden_size_%d", j).c_str())) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + out_hidden_sizes.at(j) = *(static_cast<int const*>(fields[i].data)); + } + } + } + try + { + // LoraPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + // FIXME enable tactic profiler + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false, /* skip */ true); + auto* obj = new LoraPlugin(in_hidden_size, out_hidden_sizes, transA, transB, num_lora_modules, type, + pluginProfiler, remove_input_padding, max_low_rank, weight_index); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* LoraPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept +{ + TLLM_LOG_DEBUG("%s", __PRETTY_FUNCTION__); + // This object will be deleted when the network is destroyed, which will + // call LoraPlugin::destroy() + try + { + // LoraPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + // FIXME enable tactic profiler + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true, /* skip */ true); + auto* obj = new LoraPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.h b/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.h new file mode 100644 index 000000000000..7795f7b7c76d --- /dev/null +++ b/cpp/tensorrt_llm/plugins/loraPlugin/loraPlugin.h @@ -0,0 +1,159 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TRT_LORA_PLUGIN_H +#define TRT_LORA_PLUGIN_H +#include "tensorrt_llm/kernels/lora/lora.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h" +#include <cassert> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class LoraPlugin : public BasePlugin +{ +public: + using PluginProfilerPtr = std::shared_ptr<CublasLtGemmPluginProfiler>; + using ImplPtr = std::shared_ptr<kernels::LoraImpl>; + using Config = cublasLtMatmulHeuristicResult_t; + + LoraPlugin() = delete; + + LoraPlugin(int in_hidden_size, std::vector<int> out_hidden_sizes, int transA, int transB, int num_lora_modules, + nvinfer1::DataType type, PluginProfilerPtr const& profiler, bool remove_input_padding, int max_low_rank, + int weight_index); + + LoraPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); + + ~LoraPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + int64_t getNumTokens(nvinfer1::PluginTensorDesc const* input_tensors) const; + void init(); + + using IndexType = std::int32_t; + + IndexType getInputTensorIdx() const + { + return 0; + } + + IndexType getHostRequestTypesIdx() const + { + return 1; + } + + IndexType getLoraRanksIdx() const + { + return 2; + } + + IndexType getLoraWeightsPtrsIdx() const + { + return 2 + mNumLoraModules; + } + + IndexType getHostContextLengthsIdx() const + { + TLLM_CHECK(mRemoveInputPadding); + return 2 + mNumLoraModules + mNumLoraModules; + } + + enum class RequestType : int32_t + { + kCONTEXT = 0, + kGENERATION = 1 + }; + +private: + const std::string mLayerName; + + std::vector<int> mOutHiddenSizes; + int mTransA; + int mTransB; + nvinfer1::DataType mType; + bool mRemoveInputPadding; + int mNumLoraModules; + int mInHiddenSize; + int mMaxLowRank; + int mWeightIndex; + + std::vector<void const*> mExpandLoraWeightPtrs{}; + std::vector<int32_t> mExpandLoraRanks{}; + + GemmDims mDims{}; + GemmIdCublas mGemmId{}; + + PluginProfilerPtr mPluginProfiler; + ImplPtr mLoraImpl; +}; + +class LoraPluginCreator : public BaseCreator +{ +public: + LoraPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<CublasLtGemmPluginProfiler> gemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins + +#endif // TRT_LORA_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/CMakeLists.txt new file mode 100644 index 000000000000..b6bd0439cc0c --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.cpp b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.cpp new file mode 100644 index 000000000000..6165d6210f29 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.cpp @@ -0,0 +1,425 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lowLatencyGemmPlugin.h" +#include "low_latency_gemm.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaFp8Utils.h" +#include "tensorrt_llm/common/logger.h" +#include <NvInferRuntime.h> +#include <NvInferRuntimeBase.h> +#include <NvInferRuntimePlugin.h> +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <numeric> +#include <optional> +#include <vector> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels::internal_cutlass_kernels; +using tensorrt_llm::plugins::LowLatencyGemmPluginCreator; +using tensorrt_llm::plugins::LowLatencyGemmPlugin; +using tensorrt_llm::plugins::LowLatencyGemmPluginProfiler; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +static char const* LOW_LATENCY_GEMM_PLUGIN_VERSION{"1"}; +static char const* LOW_LATENCY_GEMM_PLUGIN_NAME{"LowLatencyGemm"}; + +PluginFieldCollection LowLatencyGemmPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> LowLatencyGemmPluginCreator::mPluginAttributes; + +using FP8Type = __nv_fp8_e4m3; + +static std::optional<float> getFloatEnv(char const* name) +{ + char const* const env = std::getenv(name); + if (env == nullptr) + { + return std::nullopt; + } + try + { + float value = std::stof(env); + return {value}; + } + catch (std::invalid_argument const& e) + { + return std::nullopt; + } + catch (std::out_of_range const& e) + { + return std::nullopt; + } +}; + +void LowLatencyGemmPluginProfiler::runTactic(int m, int n, int k, LowLatencyGemmPluginProfiler::Config const& tactic, + char* workspace, cudaStream_t const& stream) +{ + + float default_pdl_overlap_ratio = 0.5; + float default_prefetch_ratio = -1.0; + FP8Type* aTmp = reinterpret_cast<FP8Type*>(workspace); + FP8Type* bTmp + = reinterpret_cast<FP8Type*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(aTmp), m * k * sizeof(FP8Type))); + void* cTmp = reinterpret_cast<void*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(bTmp), n * k * sizeof(FP8Type))); + size_t workspaceSize = mRunner->getWorkspaceSize(m, n, k); + char* workspaceTmp = reinterpret_cast<char*>(nextWorkspacePtr( + reinterpret_cast<int8_t*>(cTmp), m * n * (mType == nvinfer1::DataType::kFLOAT ? sizeof(float) : sizeof(half)))); + mRunner->gemm(aTmp, bTmp, 1.0f, 0.0f, nullptr, cTmp, m, n, k, default_pdl_overlap_ratio, default_prefetch_ratio, + tactic, workspaceTmp, workspaceSize, stream); +} + +void LowLatencyGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + + std::vector<size_t> workspaces = {maxM * k * sizeof(FP8Type), n * k * sizeof(FP8Type), + maxM * n * (mType == nvinfer1::DataType::kFLOAT ? sizeof(float) : sizeof(half)), + mRunner->getWorkspaceSize(maxM, n, k)}; + + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<LowLatencyGemmPluginProfiler::Config> LowLatencyGemmPluginProfiler::getTactics(int m, int n, int k) const +{ + return mRunner->getConfigs(); +} + +LowLatencyGemmPlugin::LowLatencyGemmPlugin( + nvinfer1::DataType type, float alpha, PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) + , mAplha(alpha) +{ + init(type); +} + +LowLatencyGemmPlugin::LowLatencyGemmPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + + char const *d = reinterpret_cast<char const*>(data), *a = d; + nvinfer1::DataType type; + read(d, type); + read(d, mAplha); + read(d, mDims); + init(type); + mPluginProfiler->deserialize(d, mDims, mGemmId); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +void LowLatencyGemmPlugin::init(nvinfer1::DataType type) +{ + + mType = type; + + if (mType == nvinfer1::DataType::kFLOAT) + { + m_lowLatencyGemmRunner = std::make_shared<CutlassLowLatencyFp8GemmRunner<float>>(); + } + else if (mType == nvinfer1::DataType::kHALF) + { + m_lowLatencyGemmRunner = std::make_shared<CutlassLowLatencyFp8GemmRunner<half>>(); + } +#ifdef ENABLE_BF16 + + else if (mType == nvinfer1::DataType::kBF16) + { + m_lowLatencyGemmRunner = std::make_shared<CutlassLowLatencyFp8GemmRunner<__nv_bfloat16>>(); + } +#endif + else + { + TLLM_THROW("Unsupported data type"); + } + mGemmId = GemmIdCore(mDims.n, mDims.k, mType); +} + +nvinfer1::DimsExprs LowLatencyGemmPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 2); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + TLLM_CHECK(nbDimsA >= 2); + DimsExprs ret; + ret.nbDims = nbDimsA; + for (int ii = 0; ii < nbDimsA - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + // input[1] , weights [n,k] + ret.d[nbDimsA - 1] = inputs[1].d[0]; + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool LowLatencyGemmPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + switch (pos) + { + case 0: + // activation + return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; + case 1: + // weights + // Weights stored in checkpoint must have fp8 type + return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; + case 2: + // out + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + default: + // Never should be here + assert(false); + return false; + } +} + +void LowLatencyGemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + int const maxN = in[1].max.d[0]; + int const minK = in[0].min.d[in[0].min.nbDims - 1]; + int const minN = in[1].min.d[0]; + + TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); + TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, maxN, maxK}; + } + mGemmId = {maxN, maxK, mType}; + + m_workspaceMaxSize = m_lowLatencyGemmRunner->getWorkspaceSize(maxM, maxN, maxK); +} + +size_t LowLatencyGemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return m_workspaceMaxSize; +} + +int LowLatencyGemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + + // input0 activation [M,K] + // input1 weights [N,K] + // output0 [M,N] + + int64_t m64 = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m64 *= inputDesc[0].dims.d[ii]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); + int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); + int const wsSize = m_lowLatencyGemmRunner->getWorkspaceSize(m, n, k); + auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); + TLLM_CHECK_WITH_INFO(bestTactic, "No valid Low Latency GEMM tactic"); + + auto env_pdl_overlap_ratio = getFloatEnv("TRTLLM_PDL_OVERLAP_RATIO"); + auto env_prefetch_ratio = getFloatEnv("TRTLLM_PREFETCH_RATIO"); + auto valid_ratio = [](std::optional<float>& env_val, float default_val) + { + if (env_val.has_value()) + { + TLLM_CHECK_WITH_INFO(env_val.value() <= 1.0f, "Valid ratio should be less than or equal to 1.0"); + return env_val.value(); + } + return default_val; + }; + float pdl_overlap_ratio = valid_ratio(env_pdl_overlap_ratio, /*default_val=*/0.5); + float prefetch_ratio = valid_ratio(env_prefetch_ratio, /*default_val=*/-1.0); + m_lowLatencyGemmRunner->gemm(const_cast<FP8Type*>(reinterpret_cast<FP8Type const*>(inputs[0])), + const_cast<FP8Type*>(reinterpret_cast<FP8Type const*>(inputs[1])), mAplha, 0.0F, nullptr, outputs[0], m, n, k, + pdl_overlap_ratio, prefetch_ratio, *bestTactic, reinterpret_cast<char*>(workspace), wsSize, stream); + + return 0; +} + +nvinfer1::DataType LowLatencyGemmPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* LowLatencyGemmPlugin::getPluginType() const noexcept +{ + return LOW_LATENCY_GEMM_PLUGIN_NAME; +} + +char const* LowLatencyGemmPlugin::getPluginVersion() const noexcept +{ + return LOW_LATENCY_GEMM_PLUGIN_VERSION; +} + +int LowLatencyGemmPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int LowLatencyGemmPlugin::initialize() noexcept +{ + configGemm(); + return 0; +} + +void LowLatencyGemmPlugin::terminate() noexcept {} + +nvinfer1::IPluginV2DynamicExt* LowLatencyGemmPlugin::clone() const noexcept +{ + auto* plugin = new LowLatencyGemmPlugin(*this); + return plugin; +} + +size_t LowLatencyGemmPlugin::getSerializationSize() const noexcept +{ + return sizeof(nvinfer1::DataType) + // dtype + sizeof(float) * 1 + // alpha + sizeof(mDims) + mPluginProfiler->getSerializationSize(mGemmId); +} + +void LowLatencyGemmPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mAplha); + write(d, mDims); + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void LowLatencyGemmPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +void LowLatencyGemmPlugin::configGemm() +{ + mPluginProfiler->profileTactics(m_lowLatencyGemmRunner, mType, mDims, mGemmId); +} + +LowLatencyGemmPluginCreator::LowLatencyGemmPluginCreator() +{ + + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("alpha", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* LowLatencyGemmPluginCreator::getPluginName() const noexcept +{ + return LOW_LATENCY_GEMM_PLUGIN_NAME; +} + +char const* LowLatencyGemmPluginCreator::getPluginVersion() const noexcept +{ + return LOW_LATENCY_GEMM_PLUGIN_VERSION; +} + +PluginFieldCollection const* LowLatencyGemmPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* LowLatencyGemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + float alpha{}; + nvinfer1::DataType type{}; + for (int i = 0; i < fc->nbFields; i++) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "alpha")) + { + + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + alpha = *(static_cast<float const*>(fields[i].data)); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + + try + { + + // + // GemmPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/false); + auto* obj = new LowLatencyGemmPlugin(type, alpha, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* LowLatencyGemmPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + try + { + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/true); + auto* obj = new LowLatencyGemmPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h new file mode 100644 index 000000000000..98b8f4807174 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lowLatencyGemmPlugin/lowLatencyGemmPlugin.h @@ -0,0 +1,135 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "low_latency_gemm.h" + +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <cstddef> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +using LowLatencyGemmRunnerPtr + = std::shared_ptr<tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmRunnerInterface>; + +class LowLatencyGemmPluginProfiler + : public GemmPluginProfiler< + tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmRunnerInterface::ConfigType, + LowLatencyGemmRunnerPtr, GemmIdCore, GemmIdCoreHash> +{ + +public: + using Config = tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmRunnerInterface::ConfigType; + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + std::vector<Config> getTactics(int m, int n, int k) const override; +}; + +class LowLatencyGemmPlugin : public BasePlugin +{ + +public: + using PluginProfilerPtr = std::shared_ptr<LowLatencyGemmPluginProfiler>; + + LowLatencyGemmPlugin() = delete; + + LowLatencyGemmPlugin(nvinfer1::DataType type, float alpha, PluginProfilerPtr const& pluginProfiler); + + LowLatencyGemmPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler); + ~LowLatencyGemmPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + void init(nvinfer1::DataType type); + void configGemm(); + +private: + std::string const mLayerName; + + LowLatencyGemmRunnerPtr m_lowLatencyGemmRunner; + size_t m_workspaceMaxSize; + + GemmDims mDims{}; + GemmIdCore mGemmId{}; + + PluginProfilerPtr mPluginProfiler; + + nvinfer1::DataType mType; + float mAplha{1.0F}; +}; + +class LowLatencyGemmPluginCreator : public BaseCreator +{ +public: + LowLatencyGemmPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<LowLatencyGemmPluginProfiler> gemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/CMakeLists.txt new file mode 100644 index 000000000000..b6bd0439cc0c --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.cpp b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.cpp new file mode 100644 index 000000000000..a1aa11c2f165 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.cpp @@ -0,0 +1,468 @@ + +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lowLatencyGemmSwigluPlugin.h" +#include "low_latency_gemm_swiglu.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/cudaFp8Utils.h" +#include "tensorrt_llm/common/logger.h" +#include <NvInferRuntime.h> +#include <NvInferRuntimeBase.h> +#include <NvInferRuntimePlugin.h> +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <numeric> +#include <optional> +#include <vector> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels::internal_cutlass_kernels; +using tensorrt_llm::plugins::LowLatencyGemmSwigluPluginCreator; +using tensorrt_llm::plugins::LowLatencyGemmSwigluPlugin; +using tensorrt_llm::plugins::LowLatencyGemmSwigluPluginProfiler; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +static char const* LOW_LATENCY_GEMM_SWIGLU_PLUGIN_VERSION{"1"}; +static char const* LOW_LATENCY_GEMM_SWIGLU_PLUGIN_NAME{"LowLatencyGemmSwiglu"}; + +PluginFieldCollection LowLatencyGemmSwigluPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> LowLatencyGemmSwigluPluginCreator::mPluginAttributes; + +using FP8Type = __nv_fp8_e4m3; + +static std::optional<float> getFloatEnv(char const* name) +{ + char const* const env = std::getenv(name); + if (env == nullptr) + { + return std::nullopt; + } + try + { + float value = std::stof(env); + return {value}; + } + catch (std::invalid_argument const& e) + { + return std::nullopt; + } + catch (std::out_of_range const& e) + { + return std::nullopt; + } +}; + +static size_t getBytePerElement(nvinfer1::DataType type) +{ + size_t bpe; + if (type == nvinfer1::DataType::kFLOAT) + { + bpe = 4; + } + else if (type == nvinfer1::DataType::kHALF || type == nvinfer1::DataType::kBF16) + { + bpe = 2; + } + else if (type == nvinfer1::DataType::kINT8 || type == nvinfer1::DataType::kFP8) + { + bpe = 1; + } + else + { + TLLM_THROW("Not recognized/implemented"); + } + return bpe; +} + +void LowLatencyGemmSwigluPluginProfiler::runTactic(int m, int n, int k, + LowLatencyGemmSwigluPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) +{ + + float default_pdl_overlap_ratio = 0.5; + float default_prefetch_ratio = -1.0; + FP8Type* aTmp = reinterpret_cast<FP8Type*>(workspace); + FP8Type* bTmp + = reinterpret_cast<FP8Type*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(aTmp), m * k * sizeof(FP8Type))); + void* dTmp = reinterpret_cast<void*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(bTmp), n * k * sizeof(FP8Type))); + size_t workspaceSize = mRunner->getWorkspaceSize(m, n, k); + char* workspaceTmp = reinterpret_cast<char*>( + nextWorkspacePtr(reinterpret_cast<int8_t*>(dTmp), (n / 2 * m * getBytePerElement(mType)))); + mRunner->gemm(aTmp, bTmp, 1.0f, 0.0f, 1.0f, 1.0f, nullptr, dTmp, m, n, k, default_pdl_overlap_ratio, + default_prefetch_ratio, tactic, workspaceTmp, workspaceSize, stream); +} + +int LowLatencyGemmSwigluPluginProfiler::getMaxProfileM() const +{ + return 32768; +} + +void LowLatencyGemmSwigluPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + + std::vector<size_t> workspaces = {maxM * k * sizeof(FP8Type), // A + n * k * sizeof(FP8Type), // B + maxM * (n / 2) * getBytePerElement(mType), // D + mRunner->getWorkspaceSize(maxM, n, k)}; // workspace + + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<LowLatencyGemmSwigluPluginProfiler::Config> LowLatencyGemmSwigluPluginProfiler::getTactics( + int m, int n, int k) const +{ + return mRunner->getConfigs(); +} + +LowLatencyGemmSwigluPlugin::LowLatencyGemmSwigluPlugin(nvinfer1::DataType type, float scale_output, float scale_d0, + float scale_d1, PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) + , mScaleOutput(scale_output) + , mScaleD0(scale_d0) + , mScaleD1(scale_d1) +{ + init(type); +} + +LowLatencyGemmSwigluPlugin::LowLatencyGemmSwigluPlugin( + void const* data, size_t length, PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + + char const *d = reinterpret_cast<char const*>(data), *a = d; + nvinfer1::DataType type; + read(d, type); + read(d, mScaleOutput); + read(d, mScaleD0); + read(d, mScaleD1); + read(d, mDims); + + init(type); + mPluginProfiler->deserialize(d, mDims, mGemmId); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +void LowLatencyGemmSwigluPlugin::init(nvinfer1::DataType type) +{ + + mType = type; + + if (mType == nvinfer1::DataType::kFP8) + { + mLowLatencyGemmSwigluRunner = std::make_shared<CutlassLowLatencyFp8GemmSwigluRunner<__nv_fp8_e4m3>>(); + } + else + { + TLLM_THROW("Unsupported data type"); + } + mGemmId = GemmIdCore(mDims.n, mDims.k, mType); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* LowLatencyGemmSwigluPlugin::clone() const noexcept +{ + auto* plugin = new LowLatencyGemmSwigluPlugin(*this); + return plugin; +} + +nvinfer1::DimsExprs LowLatencyGemmSwigluPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 2); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + TLLM_CHECK(nbDimsA >= 2); + DimsExprs ret; + ret.nbDims = nbDimsA; + for (int ii = 0; ii < nbDimsA - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[1].d[1]->getConstantValue() / 2); + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool LowLatencyGemmSwigluPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + switch (pos) + { + case 0: + // activation + return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; + case 1: + // weights + // Weights stored in checkpoint must have fp8 type + return inOut[pos].type == nvinfer1::DataType::kFP8 && inOut[pos].format == TensorFormat::kLINEAR; + case 2: + // out + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + default: + // Never should be here + TLLM_CHECK(false); + return false; + } +} + +void LowLatencyGemmSwigluPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + int const maxN = in[1].max.d[1]; + int const minK = in[0].min.d[in[0].min.nbDims - 1]; + int const minN = in[1].min.d[1]; + + TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); + TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, maxN, maxK}; + } + mGemmId = {maxN, maxK, mType}; + + mWorkspaceMaxSize = mLowLatencyGemmSwigluRunner->getWorkspaceSize(maxM, maxN, maxK); +} + +size_t LowLatencyGemmSwigluPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return mWorkspaceMaxSize; +} + +int LowLatencyGemmSwigluPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + + // input0 activation [M,K] row-major + // input1 weights [K, N] col-major + // output0 [M,N / 2] row-major + + int64_t m64 = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m64 *= inputDesc[0].dims.d[ii]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[1]); + int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); + int const wsSize = mLowLatencyGemmSwigluRunner->getWorkspaceSize(m, n, k); + auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); + TLLM_CHECK_WITH_INFO(bestTactic, "No valid Low Latency GEMM SWIGLU tactic"); + + auto env_pdl_overlap_ratio = getFloatEnv("TRTLLM_PDL_OVERLAP_RATIO"); + auto env_prefetch_ratio = getFloatEnv("TRTLLM_PREFETCH_RATIO"); + auto valid_ratio = [](std::optional<float>& env_val, float default_val) + { + if (env_val.has_value()) + { + TLLM_CHECK_WITH_INFO(env_val.value() <= 1.0f, "Valid ratio should be less than or equal to 1.0"); + return env_val.value(); + } + return default_val; + }; + float pdl_overlap_ratio = valid_ratio(env_pdl_overlap_ratio, /*default_val=*/0.5); + float prefetch_ratio = valid_ratio(env_prefetch_ratio, /*default_val=*/-1.0); + mLowLatencyGemmSwigluRunner->gemm(const_cast<FP8Type*>(reinterpret_cast<FP8Type const*>(inputs[0])), + const_cast<FP8Type*>(reinterpret_cast<FP8Type const*>(inputs[1])), mScaleOutput, 0.0F, mScaleD0, mScaleD1, + nullptr, outputs[0], m, n, k, pdl_overlap_ratio, prefetch_ratio, *bestTactic, + reinterpret_cast<char*>(workspace), wsSize, stream); + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType LowLatencyGemmSwigluPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* LowLatencyGemmSwigluPlugin::getPluginType() const noexcept +{ + return LOW_LATENCY_GEMM_SWIGLU_PLUGIN_NAME; +} + +char const* LowLatencyGemmSwigluPlugin::getPluginVersion() const noexcept +{ + return LOW_LATENCY_GEMM_SWIGLU_PLUGIN_VERSION; +} + +int LowLatencyGemmSwigluPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int LowLatencyGemmSwigluPlugin::initialize() noexcept +{ + configGemm(); + return 0; +} + +void LowLatencyGemmSwigluPlugin::terminate() noexcept {} + +size_t LowLatencyGemmSwigluPlugin::getSerializationSize() const noexcept +{ + return sizeof(nvinfer1::DataType) + // dtype + sizeof(float) * 3 + // scales + sizeof(mDims) + mPluginProfiler->getSerializationSize(mGemmId); +} + +void LowLatencyGemmSwigluPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mScaleOutput); + write(d, mScaleD0); + write(d, mScaleD1); + write(d, mDims); + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void LowLatencyGemmSwigluPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +void LowLatencyGemmSwigluPlugin::configGemm() +{ + mPluginProfiler->profileTactics(mLowLatencyGemmSwigluRunner, mType, mDims, mGemmId); +} + +////////////////////////////////////////////////////////////////////////// + +LowLatencyGemmSwigluPluginCreator::LowLatencyGemmSwigluPluginCreator() +{ + + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("scale_output", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("scale_d0", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("scale_d1", nullptr, PluginFieldType::kFLOAT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* LowLatencyGemmSwigluPluginCreator::getPluginName() const noexcept +{ + return LOW_LATENCY_GEMM_SWIGLU_PLUGIN_NAME; +} + +char const* LowLatencyGemmSwigluPluginCreator::getPluginVersion() const noexcept +{ + return LOW_LATENCY_GEMM_SWIGLU_PLUGIN_VERSION; +} + +PluginFieldCollection const* LowLatencyGemmSwigluPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* LowLatencyGemmSwigluPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + TLLM_CHECK(fc->nbFields == 4); + nvinfer1::DataType type{}; + float scale_output{}; + float scale_d0{}; + float scale_d1{}; + for (int i = 0; i < fc->nbFields; i++) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "scale_output")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + scale_output = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + else if (!strcmp(attrName, "scale_d0")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + scale_d0 = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + else if (!strcmp(attrName, "scale_d1")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + scale_d1 = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + } + + try + { + + // + // LowLatencyGemmSwigluPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/false); + auto* obj = new LowLatencyGemmSwigluPlugin(type, scale_output, scale_d0, scale_d1, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* LowLatencyGemmSwigluPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + try + { + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/*inference=*/true); + auto* obj = new LowLatencyGemmSwigluPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h new file mode 100644 index 000000000000..3f73324e7740 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lowLatencyGemmSwigluPlugin/lowLatencyGemmSwigluPlugin.h @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "low_latency_gemm_swiglu.h" + +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <cstddef> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ +using LowLatencyGemmSwigluRunnerPtr + = std::shared_ptr<tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmSwigluRunnerInterface>; + +class LowLatencyGemmSwigluPluginProfiler + : public GemmPluginProfiler< + tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmSwigluRunnerInterface::ConfigType, + LowLatencyGemmSwigluRunnerPtr, GemmIdCore, GemmIdCoreHash> +{ + +public: + using Config + = tensorrt_llm::kernels::internal_cutlass_kernels::CutlassLowLatencyFp8GemmSwigluRunnerInterface::ConfigType; + + virtual int getMaxProfileM() const override; + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + std::vector<Config> getTactics(int m, int n, int k) const override; +}; + +class LowLatencyGemmSwigluPlugin : public BasePlugin +{ + +public: + using PluginProfilerPtr = std::shared_ptr<LowLatencyGemmSwigluPluginProfiler>; + + LowLatencyGemmSwigluPlugin() = delete; + + LowLatencyGemmSwigluPlugin(nvinfer1::DataType type, float scale_output, float scale_d0, float scale_d1, + PluginProfilerPtr const& pluginProfiler); + + LowLatencyGemmSwigluPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler); + ~LowLatencyGemmSwigluPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + void init(nvinfer1::DataType type); + void configGemm(); + +private: + std::string const mLayerName; + + LowLatencyGemmSwigluRunnerPtr mLowLatencyGemmSwigluRunner; + size_t mWorkspaceMaxSize; + + GemmDims mDims{}; + GemmIdCore mGemmId{}; + + PluginProfilerPtr mPluginProfiler; + + nvinfer1::DataType mType; + float mScaleOutput; + float mScaleD0; + float mScaleD1; +}; + +class LowLatencyGemmSwigluPluginCreator : public BaseCreator +{ +public: + LowLatencyGemmSwigluPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<LowLatencyGemmSwigluPluginProfiler> gemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/lruPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/lruPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lruPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.cpp b/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.cpp new file mode 100644 index 000000000000..9d86b8cb8acd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.cpp @@ -0,0 +1,431 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lruPlugin.h" +#include "tensorrt_llm/common/assert.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::lruPluginCreator; +using tensorrt_llm::plugins::lruPlugin; + +static char const* LRU_PLUGIN_VERSION{"1"}; +static char const* LRU_PLUGIN_NAME{"LRU"}; +PluginFieldCollection lruPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> lruPluginCreator::mPluginAttributes; + +lruPlugin::lruPlugin(int dim, int block_size, nvinfer1::DataType type, bool removePadding, bool pagedState, + bool yEnabled, bool yBiasEnabled, bool fuseGateEnabled, bool gateBiasEnabled) + : mDim(dim) + , mBlockSize(block_size) + , mType(type) + , mRemovePadding(removePadding) + , mPagedState(pagedState) + , mYEnabled(yEnabled) + , mYBiasEnabled(yBiasEnabled) + , mFuseGateEnabled(fuseGateEnabled) + , mGateBiasEnabled(gateBiasEnabled) +{ + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), + "Only support float, half, and bfloat16."); +} + +// Parameterized constructor +lruPlugin::lruPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mDim); + read(d, mBlockSize); + read(d, mType); + read(d, mRemovePadding); + read(d, mPagedState); + read(d, mYEnabled); + read(d, mYBiasEnabled); + read(d, mFuseGateEnabled); + read(d, mGateBiasEnabled); + TLLM_CHECK(d == a + length); + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), + "Only support float, half, and bfloat16."); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* lruPlugin::clone() const noexcept +{ + auto* plugin = new lruPlugin(mDim, mBlockSize, mType, mRemovePadding, mPagedState, mYEnabled, mYBiasEnabled, + mFuseGateEnabled, mGateBiasEnabled); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +// Outputs +// output_tensor: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// state: [batch_size, dim] +nvinfer1::DimsExprs lruPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + if (outputIndex == 0) + { + return inputs[getXIdx()]; + } + return inputs[getStateIdx()]; +} + +bool lruPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + if (pos == getHostRequestTypesIdx() || pos == getLastTokenIdsIdx() || (mPagedState && pos == getSlotMappingIdx())) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (mPagedState && pos == getStateIdx()) + { + return inOut[pos].type == nvinfer1::DataType::kINT64; + } + else if (pos == getStateIdx() || pos == (nbInputs + 1)) + { + // Use float for both input and output state + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else + { + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } +} + +void lruPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t lruPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +void lruPlugin::setLruParams(lruParams& params, const size_t batch, const size_t dim, const size_t block_size, + const size_t maxSeqLen, void* statePtr, void const* x, void const* gate, void const* gate_bias, void const* gate_x, + void const* gate_x_bias, void const* gate_a, void const* gate_a_bias, void const* y, void const* y_bias, + void const* A, int const* lastTokenIds, int const* slotMapping, void* out, bool removePadding) +{ + // Reset the parameters + memset(¶ms, 0, sizeof(params)); + + params.batch = batch; + params.width = dim; + params.block_size = block_size; + params.max_seqlen = maxSeqLen; + params.remove_padding = removePadding; + + // Set the pointers and strides. + params.A_ptr = const_cast<void*>(A); + params.x_ptr = const_cast<void*>(x); + params.y_ptr = const_cast<void*>(y); + params.y_bias_ptr = const_cast<void*>(y_bias); + params.gate_ptr = const_cast<void*>(gate); + params.gate_bias_ptr = const_cast<void*>(gate_bias); + params.gate_x_ptr = const_cast<void*>(gate_x); + params.gate_x_bias_ptr = const_cast<void*>(gate_x_bias); + params.gate_a_ptr = const_cast<void*>(gate_a); + params.gate_a_bias_ptr = const_cast<void*>(gate_a_bias); + params.state_ptr = statePtr; + params.out_ptr = out; + params.last_token_ids_ptr = lastTokenIds; + params.slot_mapping_ptr = slotMapping; +} + +template <typename T> +int lruPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) +{ + // inputs + // 0. x [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding + // 1. A [dim] + // 2. state [batch_size, dim] or host [1] containing only pointer for paged_state + // 3. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. + // 4. last_token_ids [batch_size] int32 + // 5. state_slot_mapping [batch_size] int32, optional for paged state + // 6. y [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding + // 7. y_bias [dim] + // 8. gate [batch_size, seq_len, 2 * dim] or [num_tokens, 2 * dim] for remove_input_padding + // 9. gate_bias [2 * dim] + // 10. gate_x [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding + // 11. gate_a [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding + // 12. gate_x_bias [2 * dim] + // 13. gate_a_bias [2 * dim] + // outputs + // 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding + // 1. state [batch_size, dim] + auto const batch_size = inputDesc[getHostRequestTypesIdx()].dims.d[0]; + int max_seq_len; + if (mRemovePadding) + { + max_seq_len = -1; + } + else + { + max_seq_len = inputDesc[getXIdx()].dims.d[1]; + } + + // only support context or generation, not for both of them + RequestType const* reqTypes = static_cast<RequestType const*>(inputs[getHostRequestTypesIdx()]); + + lruParams lru_params; + + int const* slotMapping = mPagedState ? static_cast<int const*>(inputs[getSlotMappingIdx()]) : nullptr; + void const* y = mYEnabled ? inputs[getYIdx()] : nullptr; + void const* y_bias = mYBiasEnabled ? inputs[getYBiasIdx()] : nullptr; + void const* gate = mFuseGateEnabled ? inputs[getGateIdx()] : nullptr; + void const* gate_bias = (mFuseGateEnabled && mGateBiasEnabled) ? inputs[getGateBiasIdx()] : nullptr; + void const* gate_x = mFuseGateEnabled ? nullptr : inputs[getGateXIdx()]; + void const* gate_a = mFuseGateEnabled ? nullptr : inputs[getGateAIdx()]; + void const* gate_x_bias = (!mFuseGateEnabled && mGateBiasEnabled) ? inputs[getGateXBiasIdx()] : nullptr; + void const* gate_a_bias = (!mFuseGateEnabled && mGateBiasEnabled) ? inputs[getGateABiasIdx()] : nullptr; + + void* statePtr = mPagedState ? *reinterpret_cast<void**>(const_cast<void*>(inputs[getStateIdx()])) : outputs[1]; + + setLruParams(lru_params, batch_size, mDim, mBlockSize, max_seq_len, statePtr, inputs[getXIdx()], gate, gate_bias, + gate_x, gate_x_bias, gate_a, gate_a_bias, y, y_bias, inputs[getAIdx()], + static_cast<int const*>(inputs[getLastTokenIdsIdx()]), slotMapping, outputs[0], mRemovePadding); + + if (reqTypes[0] == RequestType::kCONTEXT) + { + invokeRGLRU<T>(lru_params, stream); + } + else if (reqTypes[0] == RequestType::kGENERATION) + { + invokeRGLRUUpdate<T>(lru_params, stream); + } + sync_check_cuda_error(stream); + return 0; +} + +int lruPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + if (mType == DataType::kHALF) + { + return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kFLOAT) + { + return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16) + { + return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType lruPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + if (index == 0) + { + return inputTypes[getXIdx()]; + } + else + { + return inputTypes[getStateIdx()]; + } +} + +// IPluginV2 Methods + +char const* lruPlugin::getPluginType() const noexcept +{ + return LRU_PLUGIN_NAME; +} + +char const* lruPlugin::getPluginVersion() const noexcept +{ + return LRU_PLUGIN_VERSION; +} + +int lruPlugin::getNbOutputs() const noexcept +{ + return mPagedState ? 1 : 2; +} + +int lruPlugin::initialize() noexcept +{ + return 0; +} + +void lruPlugin::terminate() noexcept {} + +size_t lruPlugin::getSerializationSize() const noexcept +{ + return sizeof(mDim) + sizeof(mBlockSize) + sizeof(mType) + sizeof(mRemovePadding) + sizeof(mPagedState) + + sizeof(mYEnabled) + sizeof(mYBiasEnabled) + sizeof(mFuseGateEnabled) + sizeof(mGateBiasEnabled); +} + +void lruPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mDim); + write(d, mBlockSize); + write(d, mType); + write(d, mRemovePadding); + write(d, mPagedState); + write(d, mYEnabled); + write(d, mYBiasEnabled); + write(d, mFuseGateEnabled); + write(d, mGateBiasEnabled); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void lruPlugin::destroy() noexcept +{ + delete this; +} + +/////////////// + +lruPluginCreator::lruPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("dim", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("block_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("remove_input_padding", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("paged_state", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("y_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("y_bias_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("fuse_gate_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("gate_bias_enabled", nullptr, PluginFieldType::kINT8)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* lruPluginCreator::getPluginName() const noexcept +{ + return LRU_PLUGIN_NAME; +} + +char const* lruPluginCreator::getPluginVersion() const noexcept +{ + return LRU_PLUGIN_VERSION; +} + +PluginFieldCollection const* lruPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* lruPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int dim{}; + int block_size{}; + bool removePadding{}; + bool pagedState{}; + bool yEnabled{}; + bool yBiasEnabled{}; + bool fuseGateEnabled{}; + bool gateBiasEnabled{}; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "dim")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dim = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + if (!strcmp(attrName, "block_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + block_size = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "remove_input_padding")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + removePadding = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "paged_state")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + pagedState = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "y_enabled")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + yEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "y_bias_enabled")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + yBiasEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "fuse_gate_enabled")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + fuseGateEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "gate_bias_enabled")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + gateBiasEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + } + try + { + auto* obj = new lruPlugin( + dim, block_size, type, removePadding, pagedState, yEnabled, yBiasEnabled, fuseGateEnabled, gateBiasEnabled); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* lruPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call lruPlugin::destroy() + try + { + auto* obj = new lruPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.h b/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.h new file mode 100644 index 000000000000..ee4e0b989b34 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/lruPlugin/lruPlugin.h @@ -0,0 +1,239 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRT_LRU_PLUGIN_H +#define TRT_LRU_PLUGIN_H +#include "tensorrt_llm/kernels/lruKernel.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> + +namespace tensorrt_llm::plugins +{ +// batch_size = num_ctx_requests or num_gen_requests +// num_ctx_requests = number of context requests (single sequence per request). +// num_gen_requests = number of generation requests (single sequences per request). +// can not support beam search + +// inputs +// 0. x [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 1. A [dim] +// 2. state [batch_size, dim] or host [1] containing only pointer for paged_state +// 3. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. +// 4. last_token_ids [batch_size] int32 +// 5. state_slot_mapping [batch_size] int32, optional for paged state +// 6. y [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 7. y_bias [dim] +// 8. gate [batch_size, seq_len, 2 * dim] or [num_tokens, 2 * dim] for remove_input_padding +// 9. gate_bias [2 * dim] +// 10. gate_x [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 11. gate_a [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 12. gate_x_bias [2 * dim] +// 13. gate_a_bias [2 * dim] +// outputs +// 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 1. state [batch_size, dim] + +class lruPlugin : public BasePlugin +{ +public: + lruPlugin(int dim, int block_size, nvinfer1::DataType type, bool removePadding, bool pagedState, bool yEnabled, + bool yBiasEnabled, bool fuseGateEnabled, bool gateBiasEnabled); + + lruPlugin(void const* data, size_t length); + + ~lruPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + template <typename T> + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + + enum class RequestType : int32_t + { + kCONTEXT = 0, + kGENERATION = 1 + }; + +private: + using IndexType = std::int32_t; + + IndexType getXIdx() const + { + return 0; + }; + + IndexType getAIdx() const + { + return 1; + }; + + IndexType getStateIdx() const + { + return 2; + }; + + IndexType getHostRequestTypesIdx() const + { + return 3; + }; + + IndexType getLastTokenIdsIdx() const + { + return 4; + }; + + IndexType getSlotMappingIdx() const + { + if (mPagedState) + return 5; + else + return 4; + }; + + IndexType getYIdx() const + { + if (mYEnabled) + return getSlotMappingIdx() + 1; + else + return getSlotMappingIdx(); + }; + + IndexType getYBiasIdx() const + { + if (mYBiasEnabled) + return getYIdx() + 1; + else + return getYIdx(); + }; + + IndexType getGateIdx() const + { + if (mFuseGateEnabled) + return getYBiasIdx() + 1; + else + return getYBiasIdx(); + }; + + IndexType getGateBiasIdx() const + { + if (mFuseGateEnabled && mGateBiasEnabled) + return getGateIdx() + 1; + else + return getGateIdx(); + }; + + IndexType getGateXIdx() const + { + if (mFuseGateEnabled) + return getGateBiasIdx(); + else + return getGateBiasIdx() + 1; + }; + + IndexType getGateAIdx() const + { + if (mFuseGateEnabled) + return getGateXIdx(); + else + return getGateXIdx() + 1; + }; + + IndexType getGateXBiasIdx() const + { + if (!mFuseGateEnabled && mGateBiasEnabled) + return getGateAIdx() + 1; + else + return getGateAIdx(); + }; + + IndexType getGateABiasIdx() const + { + if (!mFuseGateEnabled && mGateBiasEnabled) + return getGateXBiasIdx() + 1; + else + return getGateXBiasIdx(); + }; + + static void setLruParams(tensorrt_llm::kernels::lruParams& params, + // sizes + const size_t batch, const size_t dim, const size_t block_size, const size_t maxSeqLen, + // device pointers + void* statePtr, void const* x, void const* gate, void const* gate_bias, void const* gate_x, + void const* gate_x_bias, void const* gate_a, void const* gate_a_bias, void const* y, void const* y_bias, + void const* A, int const* lastTokenIds, int const* slotMapping, void* out, bool removePadding); + +private: + int mDim; + int mBlockSize; + nvinfer1::DataType mType; + bool mRemovePadding = false; + bool mPagedState = false; + bool mYEnabled = false; + bool mYBiasEnabled = false; + bool mFuseGateEnabled = false; + bool mGateBiasEnabled = false; +}; + +class lruPluginCreator : public BaseCreator +{ +public: + lruPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins + +#endif // TRT_LRU_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.cpp b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.cpp new file mode 100644 index 000000000000..16754248b84d --- /dev/null +++ b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.cpp @@ -0,0 +1,404 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mambaConv1dPlugin.h" +#include "tensorrt_llm/common/assert.h" +#include <algorithm> + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::MambaConv1dPluginCreator; +using tensorrt_llm::plugins::MambaConv1dPlugin; + +static char const* MAMBA_CONV1D_PLUGIN_VERSION{"1"}; +static char const* MAMBA_CONV1D_PLUGIN_NAME{"MambaConv1d"}; + +PluginFieldCollection MambaConv1dPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> MambaConv1dPluginCreator::mPluginAttributes; + +MambaConv1dPlugin::MambaConv1dPlugin(int dim, int dconv, int preStride, int postStride, nvinfer1::DataType type, + bool removePadding, bool pagedState, bool applySilu) + : mDim(dim) + , mDConv(dconv) + , mPreStride(preStride) + , mPostStride(postStride) + , mType(type) + , mRemovePadding(removePadding) + , mPagedState(pagedState) + , mApplySilu(applySilu) +{ + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), + "Only support float, half, and bfloat16."); +} + +// Parameterized constructor +MambaConv1dPlugin::MambaConv1dPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mDim); + read(d, mDConv); + read(d, mPreStride); + read(d, mPostStride); + read(d, mType); + read(d, mRemovePadding); + read(d, mPagedState); + read(d, mApplySilu); + TLLM_CHECK(d == a + length); + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), + "Only support float, half, and bfloat16."); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* MambaConv1dPlugin::clone() const noexcept +{ + auto* plugin + = new MambaConv1dPlugin(mDim, mDConv, mPreStride, mPostStride, mType, mRemovePadding, mPagedState, mApplySilu); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +// Outputs +// output_tensor: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// state: [batch_size, dconv - 1, dim] +nvinfer1::DimsExprs MambaConv1dPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + if (outputIndex == 0) + { + auto ret = inputs[getInputTensorIdx()]; + ret.d[mRemovePadding ? 1 : 2] = exprBuilder.constant(mDim); + return ret; + } + return inputs[getConvStateIdx()]; +} + +bool MambaConv1dPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + if (pos == getHostRequestTypesIdx() || pos == getLastTokenIdsIdx() + || (mRemovePadding && pos == getHostContextLengthIdx()) || (mPagedState && pos == getSlotMappingIdx())) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (mPagedState && pos == getConvStateIdx()) + { + return inOut[pos].type == nvinfer1::DataType::kINT64; + } + else + { + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } +} + +void MambaConv1dPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t MambaConv1dPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +void MambaConv1dPlugin::setMambaConv1dParams(tensorrt_llm::kernels::MambaConv1dParamsBase& params, const size_t batch, + const size_t dim, const size_t maxSeqLen, const size_t dconv, const size_t preStride, const size_t postStride, + void const* inPtr, void const* stateInPtr, void* stateOutPtr, void const* convWeight, void const* convBias, + void* outPtr, int const* lastTokenIds, int const* stateSlotMapping, bool removePadding, bool applySilu) +{ + // Reset the parameters + memset(¶ms, 0, sizeof(params)); + + params.batch = batch; + params.dim = dim; + params.max_seqlen = maxSeqLen; + params.dconv = dconv; + params.pre_stride = preStride; + params.post_stride = postStride; + + params.remove_padding = removePadding; + params.apply_silu = applySilu; + + // Set the pointers and strides. + params.in_ptr = const_cast<void*>(inPtr); + params.state_in_ptr = const_cast<void*>(stateInPtr); + params.state_out_ptr = stateOutPtr; + params.weight_ptr = const_cast<void*>(convWeight); + params.bias_ptr = const_cast<void*>(convBias); + params.out_ptr = outPtr; + params.last_token_ids_ptr = lastTokenIds; + params.state_slot_mapping_ptr = stateSlotMapping; +} + +template <typename T> +int MambaConv1dPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) +{ + // inputs + // 0. input_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding + // 1. conv_state [batch_size, dconv - 1, dim] or host [1] containing only pointer for paged_state + // 2. weight [dim, 1, dconv] + // 3. bias [dim] + // 4. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. + // 5. last_token_ids [batch_size] int32 + // 6. host_context_lengths [batch_size] int32, optional for remove_input_padding + // 7. state_slot_mapping [batch_size] int32, optional + // outputs + // 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding + // 1. conv_state [batch_size, dconv - 1, dim] + auto const batchSize = inputDesc[getHostRequestTypesIdx()].dims.d[0]; + int maxSeqLen; + if (mRemovePadding) + { + int const* host_context_length = static_cast<int const*>(inputs[getHostContextLengthIdx()]); + maxSeqLen = *std::max_element(host_context_length, host_context_length + batchSize); + } + else + { + maxSeqLen = inputDesc[getInputTensorIdx()].dims.d[1]; + } + + // only support context or generation, not for both of them + RequestType const* reqTypes = static_cast<RequestType const*>(inputs[getHostRequestTypesIdx()]); + + MambaConv1dParamsBase mambaConv1dParams; + + int const* slotMapping = mPagedState ? static_cast<int const*>(inputs[getSlotMappingIdx()]) : nullptr; + void* stateInPtr = mPagedState ? *reinterpret_cast<void**>(const_cast<void*>(inputs[getConvStateIdx()])) + : const_cast<void*>(inputs[getConvStateIdx()]); + void* stateOutPtr + = mPagedState ? *reinterpret_cast<void**>(const_cast<void*>(inputs[getConvStateIdx()])) : outputs[1]; + + setMambaConv1dParams(mambaConv1dParams, batchSize, mDim, maxSeqLen, mDConv, mPreStride, mPostStride, + inputs[getInputTensorIdx()], stateInPtr, stateOutPtr, inputs[getWeightIdx()], inputs[getBiasIdx()], outputs[0], + static_cast<int const*>(inputs[getLastTokenIdsIdx()]), slotMapping, mRemovePadding, mApplySilu); + + if (reqTypes[0] == RequestType::kCONTEXT) + { + invokeMambaConv1dContext<T>(mambaConv1dParams, stream); + } + else if (reqTypes[0] == RequestType::kGENERATION) + { + invokeMambaConv1dGeneration<T>(mambaConv1dParams, stream); + } + sync_check_cuda_error(stream); + return 0; +} + +int MambaConv1dPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + if (mType == DataType::kHALF) + { + return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kFLOAT) + { + return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16) + { + return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType MambaConv1dPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + return inputTypes[getInputTensorIdx()]; +} + +// IPluginV2 Methods + +char const* MambaConv1dPlugin::getPluginType() const noexcept +{ + return MAMBA_CONV1D_PLUGIN_NAME; +} + +char const* MambaConv1dPlugin::getPluginVersion() const noexcept +{ + return MAMBA_CONV1D_PLUGIN_VERSION; +} + +int MambaConv1dPlugin::getNbOutputs() const noexcept +{ + return 2; +} + +int MambaConv1dPlugin::initialize() noexcept +{ + return 0; +} + +void MambaConv1dPlugin::terminate() noexcept {} + +size_t MambaConv1dPlugin::getSerializationSize() const noexcept +{ + return sizeof(mDim) + sizeof(mDConv) + sizeof(mPreStride) + sizeof(mPostStride) + sizeof(mType) + + sizeof(mRemovePadding) + sizeof(mPagedState) + sizeof(mApplySilu); +} + +void MambaConv1dPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mDim); + write(d, mDConv); + write(d, mPreStride); + write(d, mPostStride); + write(d, mType); + write(d, mRemovePadding); + write(d, mPagedState); + write(d, mApplySilu); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void MambaConv1dPlugin::destroy() noexcept +{ + delete this; +} + +/////////////// + +MambaConv1dPluginCreator::MambaConv1dPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("dim", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("dconv", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("pre_stride", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("post_stride", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("remove_input_padding", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("paged_state", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("apply_silu", nullptr, PluginFieldType::kINT8)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* MambaConv1dPluginCreator::getPluginName() const noexcept +{ + return MAMBA_CONV1D_PLUGIN_NAME; +} + +char const* MambaConv1dPluginCreator::getPluginVersion() const noexcept +{ + return MAMBA_CONV1D_PLUGIN_VERSION; +} + +PluginFieldCollection const* MambaConv1dPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* MambaConv1dPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int dim{}; + int dconv{}; + int pre_stride{}; + int post_stride{}; + bool removePadding{}; + bool pagedState{}; + bool applySilu{}; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "dim")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dim = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "dconv")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dconv = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "pre_stride")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + pre_stride = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "post_stride")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + post_stride = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "remove_input_padding")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + removePadding = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "paged_state")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + pagedState = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "apply_silu")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + applySilu = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + } + try + { + auto* obj + = new MambaConv1dPlugin(dim, dconv, pre_stride, post_stride, type, removePadding, pagedState, applySilu); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* MambaConv1dPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call MambaConv1dPlugin::destroy() + try + { + auto* obj = new MambaConv1dPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.h b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.h new file mode 100644 index 000000000000..d351b1cdc237 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/mambaConv1dPlugin/mambaConv1dPlugin.h @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRT_MAMBA_CONV1D_PLUGIN_H +#define TRT_MAMBA_CONV1D_PLUGIN_H +#include "tensorrt_llm/kernels/mambaConv1dKernels.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> + +namespace tensorrt_llm::plugins +{ +// batch_size = num_ctx_requests or num_gen_requests +// num_ctx_requests = number of context requests (single sequence per request). +// num_gen_requests = number of generation requests (single sequences per request). +// can not support beam search + +// inputs +// 0. input_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 1. conv_state [batch_size, dconv - 1, dim] or host [1] containing only pointer for paged_state +// 2. weight [1, dconv, dim] +// 3. bias [dim] +// 4. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. +// 5. last_token_ids [batch_size] int32 +// 6. host_context_lengths [batch_size] int32, optional for remove_input_padding +// 7. state_slot_mapping [batch_size] int32, optional +// outputs +// 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 1. conv_state [batch_size, dconv - 1, dim] + +class MambaConv1dPlugin : public BasePlugin +{ +public: + MambaConv1dPlugin(int dim, int dconv, int preStride, int postStride, nvinfer1::DataType type, bool removePadding, + bool pagedState, bool applySilu); + + MambaConv1dPlugin(void const* data, size_t length); + + ~MambaConv1dPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + template <typename T> + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + + enum class RequestType : int32_t + { + kCONTEXT = 0, + kGENERATION = 1 + }; + +private: + using IndexType = std::int32_t; + + IndexType getInputTensorIdx() const + { + return 0; + }; + + IndexType getConvStateIdx() const + { + return 1; + }; + + IndexType getWeightIdx() const + { + return 2; + }; + + IndexType getBiasIdx() const + { + return 3; + }; + + IndexType getHostRequestTypesIdx() const + { + return 4; + }; + + IndexType getLastTokenIdsIdx() const + { + return 5; + }; + + IndexType getHostContextLengthIdx() const + { + return 6; + }; + + IndexType getSlotMappingIdx() const + { + // if not remove input padding, host_context_length is not used, so the index is 6 + return mRemovePadding ? 7 : 6; + }; + + void setMambaConv1dParams(tensorrt_llm::kernels::MambaConv1dParamsBase& params, + // sizes + const size_t batch, const size_t dim, const size_t maxSeqLen, const size_t dconv, const size_t preStride, + const size_t postStride, + // device pointers + void const* inPtr, void const* stateInPtr, void* stateOutPtr, void const* convWeight, void const* convBias, + void* outPtr, int const* lastTokenIds, int const* stateSlotMapping, bool removePadding, bool applySilu); + +private: + int mDim; + int mDConv; + int mPreStride; + int mPostStride; + nvinfer1::DataType mType; + bool mRemovePadding = false; + bool mPagedState = false; + bool mApplySilu = true; +}; + +class MambaConv1dPluginCreator : public BaseCreator +{ +public: + MambaConv1dPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins + +#endif // TRT_MAMBA_CONV1D_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/mixtureOfExperts/CMakeLists.txt b/cpp/tensorrt_llm/plugins/mixtureOfExperts/CMakeLists.txt new file mode 100644 index 000000000000..7cc985b60b7a --- /dev/null +++ b/cpp/tensorrt_llm/plugins/mixtureOfExperts/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.cpp b/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.cpp new file mode 100644 index 000000000000..ccce34850730 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.cpp @@ -0,0 +1,1314 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h" +#include "tensorrt_llm/common/cudaBf16Wrapper.h" +#include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/utils/debugUtils.h" +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::plugins; +using tensorrt_llm::common::QuantMode; +using tensorrt_llm::common::nextWorkspacePtr; +using tensorrt_llm::common::calculateTotalWorkspaceSize; +using tensorrt_llm::plugins::MixtureOfExpertsPluginCreator; +using tensorrt_llm::plugins::MixtureOfExpertsPlugin; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +using LoraImpl = tensorrt_llm::kernels::LoraImpl; +using LoraParams = tensorrt_llm::kernels::LoraParams; + +static char const* MIXTURE_OF_EXPERTS_PLUGIN_VERSION{"1"}; +static char const* MIXTURE_OF_EXPERTS_PLUGIN_NAME{"MixtureOfExperts"}; +nvinfer1::PluginFieldCollection MixtureOfExpertsPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> MixtureOfExpertsPluginCreator::mPluginAttributes; + +MixtureOfExpertsPlugin::MixtureOfExpertsPlugin(bool remove_input_padding, int number_of_experts, int experts_per_token, + int expert_hidden_size, int expert_inter_size, int groupwise_quant_algo, int group_size, + ActivationType activation_type, nvinfer1::DataType type, nvinfer1::DataType weight_type, + nvinfer1::DataType output_type, QuantMode quant_mode, bool use_final_scales, bool use_bias, int tp_size, + int tp_rank, int ep_size, int ep_rank, bool force_determinism, int side_stream_id, + MixtureOfExpertsPluginProfilerPtr gemm_profiler_ptr, bool use_lora, nvinfer1::DataType lora_type, + LoraPluginProfilerPtr lora_profiler, int max_low_rank) + : mNumExperts(number_of_experts) + , mExpertsPerToken(experts_per_token) + , mExpertHiddenSize(expert_hidden_size) + , mExpertInterSize(expert_inter_size) + , mGroupwiseQuantAlgo(groupwise_quant_algo) + , mGroupSize(group_size) + , mActivationType(activation_type) + , mType(type) + , mWeightType(weight_type) + , mOutputType(output_type) + , mQuantMode(quant_mode) + , mUseFinalScales(use_final_scales) + , mUseBias(use_bias) + , mParallelismConfig(MOEParallelismConfig{tp_size, tp_rank, ep_size, ep_rank}) + , mUseDeterministicKernels(force_determinism) + , mSideStreamId(side_stream_id) + , mGemmProfiler(std::move(gemm_profiler_ptr)) + , mUseLora(use_lora) + , mLoraType(lora_type) + , mMaxLowRank(max_low_rank) + , mRemoveInputPadding(remove_input_padding) + , mLoraProfiler(std::move(lora_profiler)) +{ + init(); +} + +tensorrt_llm::plugins::MixtureOfExpertsPlugin::MixtureOfExpertsPlugin(MixtureOfExpertsPlugin const& other) + : mMOERunner() + , mNumExperts(other.mNumExperts) + , mExpertsPerToken(other.mExpertsPerToken) + , mExpertHiddenSize(other.mExpertHiddenSize) + , mExpertInterSize(other.mExpertInterSize) + , mGroupwiseQuantAlgo(other.mGroupwiseQuantAlgo) + , mGroupSize(other.mGroupSize) + , mActivationType(other.mActivationType) + , mType(other.mType) + , mWeightType(other.mWeightType) + , mOutputType(other.mOutputType) + , mQuantMode(other.mQuantMode) + , mUseFinalScales(other.mUseFinalScales) + , mUseBias(other.mUseBias) + , mParallelismConfig(other.mParallelismConfig) + , mDims(other.mDims) + , mUseDeterministicKernels(other.mUseDeterministicKernels) + , mSideStreamId(other.mSideStreamId) + , mGemmId1(other.mGemmId1) + , mGemmId2(other.mGemmId2) + , mGemmProfiler(other.mGemmProfiler) + , mUseLora(other.mUseLora) + , mLoraType(other.mLoraType) + , mMaxLowRank(other.mMaxLowRank) + , mRemoveInputPadding(other.mRemoveInputPadding) + , mLoraImpl1(other.mLoraImpl1) + , mLoraImpl2(other.mLoraImpl2) + , mLoraGemmId1(other.mLoraGemmId1) + , mLoraGemmId2(other.mLoraGemmId2) + , mLoraProfiler(other.mLoraProfiler) + , mLayerName(other.mLayerName) + , mNamespace(other.mNamespace) +{ + init(); +} + +size_t MixtureOfExpertsPlugin::getSerializationSize() const noexcept +{ + size_t size = sizeof(mRemoveInputPadding) + sizeof(mNumExperts) + sizeof(mExpertsPerToken) + + sizeof(mExpertHiddenSize) + sizeof(mExpertInterSize) + sizeof(mGroupwiseQuantAlgo) + sizeof(mGroupSize) + + sizeof(mActivationType) + sizeof(mType) + sizeof(mWeightType) + sizeof(mOutputType) + + sizeof(QuantMode::BaseType) + sizeof(mUseFinalScales) + sizeof(mUseBias) + sizeof(mParallelismConfig) + + sizeof(mDims) + sizeof(mUseDeterministicKernels) + sizeof(mSideStreamId) + + mGemmProfiler->getSerializationSize(mGemmId1) + mGemmProfiler->getSerializationSize(mGemmId2) + + sizeof(mUseLora) + sizeof(mLoraType) + sizeof(mMaxLowRank); + + if (hasLora()) + { + size += mLoraProfiler->getSerializationSize(mLoraGemmId1); + size += mLoraProfiler->getSerializationSize(mLoraGemmId2); + } + + return size; +} + +MixtureOfExpertsPlugin::MixtureOfExpertsPlugin(void const* data, size_t length, + MixtureOfExpertsPluginProfilerPtr gemm_profiler_ptr, LoraPluginProfilerPtr lora_profiler) + : mGemmProfiler(gemm_profiler_ptr) + , mLoraProfiler(lora_profiler) +{ + char const* d = reinterpret_cast<char const*>(data); + char const* a = d; + read(d, mRemoveInputPadding); + read(d, mNumExperts); + read(d, mExpertsPerToken); + read(d, mExpertHiddenSize); + read(d, mExpertInterSize); + read(d, mGroupwiseQuantAlgo); + read(d, mGroupSize); + read(d, mActivationType); + read(d, mType); + read(d, mWeightType); + read(d, mOutputType); + QuantMode::BaseType quant_mode; + read(d, quant_mode); + mQuantMode = QuantMode{quant_mode}; + read(d, mUseFinalScales); + read(d, mUseBias); + read(d, mParallelismConfig); + read(d, mDims); + read(d, mUseDeterministicKernels); + read(d, mSideStreamId); + read(d, mUseLora); + read(d, mLoraType); + read(d, mMaxLowRank); + + // Call init before deserialising the profiler to initialize mGemmId + init(); + mGemmProfiler->deserialize(d, mDims, mGemmId1); + mGemmProfiler->deserialize(d, mDims, mGemmId2); + + if (hasLora()) + { + mLoraProfiler->deserialize(d, mDims, mLoraGemmId1); + mLoraProfiler->deserialize(d, mDims, mLoraGemmId2); + } + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +void MixtureOfExpertsPlugin::serialize(void* buffer) const noexcept +{ + char* d = static_cast<char*>(buffer); + char* a = d; + + write(d, mRemoveInputPadding); + write(d, mNumExperts); + write(d, mExpertsPerToken); + write(d, mExpertHiddenSize); + write(d, mExpertInterSize); + write(d, mGroupwiseQuantAlgo); + write(d, mGroupSize); + write(d, mActivationType); + write(d, mType); + write(d, mWeightType); + write(d, mOutputType); + write(d, mQuantMode.value()); + write(d, mUseFinalScales); + write(d, mUseBias); + write(d, mParallelismConfig); + write(d, mDims); + write(d, mUseDeterministicKernels); + write(d, mSideStreamId); + write(d, mUseLora); + write(d, mLoraType); + write(d, mMaxLowRank); + + mGemmProfiler->serialize(d, mGemmId1); + mGemmProfiler->serialize(d, mGemmId2); + + if (hasLora()) + { + mLoraProfiler->serialize(d, mLoraGemmId1); + mLoraProfiler->serialize(d, mLoraGemmId2); + } + + TLLM_CHECK(d == a + getSerializationSize()); +} + +template <typename Type, bool NeedQuant = false> +std::unique_ptr<kernels::CutlassMoeFCRunnerInterface> switch_output_type(nvinfer1::DataType output_type) +{ + switch (output_type) + { + case nvinfer1::DataType::kFP4: + case nvinfer1::DataType::kFP8: + // TODO We need an atomic FP8 reduction for the finalize fusions + TLLM_THROW("Outputting %d directly is not currently supported", static_cast<int>(output_type)); + // return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type>>(); + case nvinfer1::DataType::kHALF: + if constexpr (NeedQuant) + { + return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type, half, half>>(); + } + else + { + return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type, half, Type>>(); + } +#ifdef ENABLE_BF16 + case nvinfer1::DataType::kBF16: + if constexpr (NeedQuant) + { + return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type, __nv_bfloat16, __nv_bfloat16>>(); + } + else + { + return std::make_unique<kernels::CutlassMoeFCRunner<Type, Type, __nv_bfloat16, Type>>(); + } +#endif + default: TLLM_THROW("Invalid output type %d", static_cast<int>(output_type)); + } +}; + +void MixtureOfExpertsPlugin::init() +{ + TLLM_CHECK_WITH_INFO(mType == DataType::kFP8 || mType == DataType::kFP4 || mOutputType == mType, + "MOE plugin only supports a different output type for FP4/FP8"); + TLLM_CHECK_WITH_INFO(mType != DataType::kFP8 || tensorrt_llm::common::getSMVersion() >= 89, + "MoE FP8 is not supported for architectures less than SM89"); + TLLM_CHECK_WITH_INFO(mType != DataType::kFP4 || (tensorrt_llm::common::getSMVersion() >= 100), + "MoE FP4 is only supported on architecture SM100 or later"); + + TLLM_CHECK_WITH_INFO(!hasLora() || mLoraType == mOutputType, "The LoraType need to keep same with moe OutputType."); + + if (mWeightType == nvinfer1::DataType::kINT8 && mQuantMode.hasInt4Weights()) + { + mWeightType = DataType::kINT4; + } + + if (mType == DataType::kHALF && mWeightType == DataType::kHALF) + { + mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<half, half>>(); + } + else if (mType == DataType::kFLOAT && mWeightType == DataType::kFLOAT) + { + mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<float, float>>(); + } + else if (mType == DataType::kHALF && mWeightType == DataType::kINT8) + { + mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<half, uint8_t>>(); + } + else if (mType == DataType::kHALF && mWeightType == DataType::kINT4) + { + mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<half, cutlass::uint4b_t>>(); + } +#ifdef ENABLE_FP8 + else if (mType == DataType::kFP8 && mWeightType == DataType::kINT4 && mOutputType == DataType::kHALF) + { + mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<__nv_fp8_e4m3, cutlass::uint4b_t, half, half>>(); + } +#endif +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16 && mWeightType == DataType::kBF16) + { + mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<__nv_bfloat16, __nv_bfloat16>>(); + } + else if (mType == DataType::kBF16 && mWeightType == DataType::kINT8) + { + mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<__nv_bfloat16, uint8_t>>(); + } + else if (mType == DataType::kBF16 && mWeightType == DataType::kINT4) + { + mMOERunner = std::make_unique<kernels::CutlassMoeFCRunner<__nv_bfloat16, cutlass::uint4b_t>>(); + } +#ifdef ENABLE_FP8 + else if (mType == DataType::kFP8 && mWeightType == DataType::kINT4 && mOutputType == DataType::kBF16) + { + mMOERunner = std::make_unique< + kernels::CutlassMoeFCRunner<__nv_fp8_e4m3, cutlass::uint4b_t, __nv_bfloat16, __nv_bfloat16>>(); + } +#endif +#endif + +#ifdef ENABLE_FP8 + if (mType == DataType::kFP8 && mWeightType == DataType::kFP8) + { + mMOERunner = switch_output_type<__nv_fp8_e4m3>(mOutputType); + } +#endif +#ifdef ENABLE_FP4 + if (mType == DataType::kFP4 && mWeightType == DataType::kFP4) + { + mMOERunner = switch_output_type<__nv_fp4_e2m1, true>(mOutputType); + } +#endif + + if (!mMOERunner) + { + TLLM_THROW( + "Could not construct the mixture of experts plugin with the requested input combination Activation: %d " + "Weight: %d Output: %d", + static_cast<int>(mType), static_cast<int>(mWeightType), static_cast<int>(mOutputType)); + } + + // Finalize fusion should be disabled if Lora is used. + mMOERunner->use_fused_finalize_ + = (mExpertsPerToken < 3 || !mUseDeterministicKernels) && !getEnvMOEDisableFinalizeFusion() && !hasLora(); + + mGemmId1 = GemmIDMoe{1, mNumExperts, mExpertsPerToken, mParallelismConfig, mExpertHiddenSize, mExpertInterSize, + mGroupSize, mActivationType, mType, mWeightType, mQuantMode, !mMOERunner->use_fused_finalize_}; + mGemmId2 = GemmIDMoe{2, mNumExperts, mExpertsPerToken, mParallelismConfig, mExpertHiddenSize, mExpertInterSize, + mGroupSize, mActivationType, mType, mWeightType, mQuantMode, !mMOERunner->use_fused_finalize_}; + mGemmProfiler->setMaxProfileM(16384 * mNumExperts / mExpertsPerToken); + + if (hasLora()) + { + auto cublasHandle = getCublasHandle(); + auto cublasLtHandle = getCublasLtHandle(); + auto cublasWrapper = std::make_shared<CublasMMWrapper>(cublasHandle, cublasLtHandle, nullptr, nullptr); + mLoraGemmId1 = GemmIdCublas(mExpertInterSize, mExpertHiddenSize, mLoraType, false, true, mLoraType); + mLoraGemmId2 = GemmIdCublas(mExpertHiddenSize, mExpertInterSize, mLoraType, false, true, mLoraType); + std::vector<int> loraOutSizes1 = {static_cast<int>(mExpertInterSize)}; + mLoraImpl1 = std::make_shared<LoraImpl>( + mExpertHiddenSize, loraOutSizes1, false, true, 1, mLoraType, mMaxLowRank, cublasWrapper); + std::vector<int> loraOutSizes2 = {static_cast<int>(mExpertHiddenSize)}; + mLoraImpl2 = std::make_shared<LoraImpl>( + mExpertInterSize, loraOutSizes2, false, true, 1, mLoraType, mMaxLowRank, cublasWrapper); + + TLLM_CUDA_CHECK(cudaEventCreate(&mMemcpyEvent)); + } + mSideStreamPtr = nullptr; + mDebugStallMain = tensorrt_llm::runtime::utils::stallStream("TLLM_DEBUG_MOE_STALL_MAIN"); + mDebugStallSide = tensorrt_llm::runtime::utils::stallStream("TLLM_DEBUG_MOE_STALL_SIDE"); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* MixtureOfExpertsPlugin::clone() const noexcept +{ + auto* plugin = new MixtureOfExpertsPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs MixtureOfExpertsPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + assert(outputIndex == getOutputTensorIndex() || outputIndex == getOutputDummyTensorIndex()); + return inputs[getInputTensorIndex()]; +} + +bool MixtureOfExpertsPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + TLLM_CHECK(0 <= pos && pos < getNbInputs() + getNbOutputs()); + TLLM_CHECK_WITH_INFO( + nbInputs == getNbInputs(), "Required input to plugin is missing. Expected %d Got %d", getNbInputs(), nbInputs); + TLLM_CHECK_WITH_INFO(nbOutputs == getNbOutputs(), "Required output to plugin is missing. Expected %d Got %d", + getNbOutputs(), nbOutputs); + + if (inOut[pos].format != TensorFormat::kLINEAR) + { + return false; + } + + if (pos == getExpertWeights1Index() || pos == getExpertWeights2Index()) + { + if (mGroupwiseQuantAlgo == 0) + { + auto normalized_weight_type + = mWeightType == nvinfer1::DataType::kINT4 ? nvinfer1::DataType::kINT8 : mWeightType; + return inOut[pos].type == normalized_weight_type; + } + else + { + return inOut[pos].type == mOutputType; + } + } + else if (pos == getTokenSelectedExpertsIndex()) + { + return inOut[pos].type == DataType::kINT32; + } + else if (pos == getTokenFinalScalesIndex()) + { + return inOut[pos].type == DataType::kFLOAT; + } + else if (pos == getExpertBias1Index() || pos == getExpertBias2Index()) + { + return inOut[pos].type == mOutputType; + } + else if (pos == nbInputs + getOutputTensorIndex()) + { + return inOut[pos].type == mOutputType; + } + else if (useSideStream() && pos == nbInputs + getOutputDummyTensorIndex()) + { + return inOut[pos].type == inOut[getInputDummyTensorIndex()].type; + } + else if (useSideStream() && pos == getInputDummyTensorIndex()) + { + return true; + } + else if (hasExpertFp8QuantScales() && getExpertFP8Dequant1Index() <= pos && pos <= getExpertFP8QuantFinalIndex()) + { + return inOut[pos].type == DataType::kFLOAT; + } + else if (hasExpertIntQuantScales() && getExpertIntQuantScale1Index() <= pos + && pos <= getExpertIntQuantScale2Index()) + { + return inOut[pos].type == mOutputType; + } + else if (hasFP4QuantScales() && getFP4GlobalActSF1Index() <= pos && pos <= getFP4GlobalSF2Index()) + { + if (pos == getFP4WeightSF1Index() || pos == getFP4WeightSF2Index()) + return inOut[pos].type == nvinfer1::DataType::kFP8; + else + return inOut[pos].type == nvinfer1::DataType::kFLOAT; + } + else if (hasLora() && hasExpertFp8QuantScales() && pos == getInputFP8DequantIndex()) + { + return inOut[pos].type == nvinfer1::DataType::kFLOAT; + } + else if (hasExpertWeightQuantZeros() && getExpertIntQuantZeros1Index() <= pos + && pos <= getExpertIntQuantZeros2Index()) + { + return inOut[pos].type == mOutputType; + } + else if (hasExpertPrequantScales() && getExpertPrequantScales1Index() <= pos + && pos <= getExpertPrequantScales2Index()) + { + return inOut[pos].type == mOutputType; + } + else if (hasGroupwiseFp8Alpha() && getExpertFp8Alpha1Index() <= pos && pos <= getExpertFp8Alpha2Index()) + { + return inOut[pos].type == DataType::kFLOAT; + } + else if (hasLora() && pos == getHostRequestTypeIndex()) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (hasLora() && (pos == getLoraFC1RanksIndex() || pos == getLoraFC2RanksIndex())) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (hasGatedLoraWeightsAndRanks() && pos == getLoraGatedRanksIndex()) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (hasLora() && (pos == getLoraFC1WeightPtrsIndex() || pos == getLoraFC2WeightPtrsIndex())) + { + return inOut[pos].type == nvinfer1::DataType::kINT64; + } + else if (hasGatedLoraWeightsAndRanks() && pos == getLoraGatedWeightPtrsIndex()) + { + return inOut[pos].type == nvinfer1::DataType::kINT64; + } + else if (hasLora() && mRemoveInputPadding && pos == getHostContextLengthIndex()) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if ((hasFP4QuantScales() || hasGroupwiseFp8Alpha()) && pos == getInputTensorIndex()) + { + return inOut[pos].type == mOutputType; + } + else + { + return inOut[pos].type == mType; + } + + return false; +} + +void MixtureOfExpertsPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + TLLM_CHECK_WITH_INFO( + nbInputs == getNbInputs(), "Required input to plugin is missing. Expected %d Got %d", getNbInputs(), nbInputs); + TLLM_CHECK_WITH_INFO(nbOutputs == getNbOutputs(), "Required output to plugin is missing. Expected %d Got %d", + getNbOutputs(), nbOutputs); + + auto in_tensor = in[getInputTensorIndex()]; + + auto const minM + = std::accumulate(in_tensor.min.d, in_tensor.min.d + in_tensor.min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM + = std::accumulate(in_tensor.max.d, in_tensor.max.d + in_tensor.max.nbDims - 1, 1, std::multiplies<int>()); + + auto weights_1 = in[getExpertWeights1Index()]; + auto weights_2 = in[getExpertWeights2Index()]; + int inner_dim_idx = getGemmShapeInnerDimIndex(); + int const maxK = weights_1.max.d[inner_dim_idx]; + int const maxN = weights_2.max.d[inner_dim_idx]; + int const minK = weights_1.min.d[inner_dim_idx]; + int const minN = weights_2.min.d[inner_dim_idx]; + + TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); + TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); + TLLM_CHECK_WITH_INFO(maxK == mExpertHiddenSize && maxN == mExpertInterSize, + "Configured tensor sizes %dx%d does not match constructor param size %ldx%ld", maxK, maxN, mExpertHiddenSize, + mExpertInterSize); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, maxN, maxK}; + } + + mGemmId1 = GemmIDMoe{1, mNumExperts, mExpertsPerToken, mParallelismConfig, mExpertHiddenSize, mExpertInterSize, + mGroupSize, mActivationType, mType, mWeightType, mQuantMode, !mMOERunner->use_fused_finalize_}; + mGemmId2 = GemmIDMoe{2, mNumExperts, mExpertsPerToken, mParallelismConfig, mExpertHiddenSize, mExpertInterSize, + mGroupSize, mActivationType, mType, mWeightType, mQuantMode, !mMOERunner->use_fused_finalize_}; + + if (hasLora()) + { + auto const N = utils::computeNDimension(true, in[getHostRequestTypeIndex()].max); + mLoraGemmId1 = GemmIdCublas(N, mExpertHiddenSize, mLoraType, false, true, mLoraType); + mLoraGemmId2 = GemmIdCublas(N, mExpertInterSize, mLoraType, false, true, mLoraType); + } +} + +auto MixtureOfExpertsPlugin::setupWorkspace(void* base_ptr, int64_t num_tokens, int num_reqs) const -> WorkspaceInfo +{ + size_t moe_workspace_size + = mMOERunner->getWorkspaceSize(num_tokens, mExpertHiddenSize, mExpertInterSize, mNumExperts, mExpertsPerToken, + mActivationType, mParallelismConfig, hasLora(), /*use_deepseek_fp8_block_scale=*/false, + /*min_latency_mode=*/false, hasExpertPrequantScales()); + + // Permutation map + size_t src_to_dest_map_size = mExpertsPerToken * num_tokens * sizeof(int); + + size_t lora_workspace_size = 0; + if (hasLora()) + { + int64_t num_reqs_lora = std::min(num_tokens * mExpertsPerToken, static_cast<int64_t>(num_reqs * mNumExperts)); + lora_workspace_size + = std::max(mLoraImpl1->getWorkspaceSize(num_tokens * mExpertsPerToken, num_reqs_lora, mLoraType), + mLoraImpl2->getWorkspaceSize(num_tokens * mExpertsPerToken, num_reqs_lora, mLoraType)); + } + + std::vector<size_t> workspaces{ + moe_workspace_size, + src_to_dest_map_size, + lora_workspace_size, + }; + + WorkspaceInfo info{}; + info.size = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + + if (base_ptr) + { + info.workspace = base_ptr; + info.src_to_dest_map = nextWorkspacePtr((int8_t*) info.workspace, moe_workspace_size); + info.lora_workspace = nextWorkspacePtr((int8_t*) info.src_to_dest_map, src_to_dest_map_size); + } + + return info; +} + +int64_t MixtureOfExpertsPlugin::getNumTokens(nvinfer1::PluginTensorDesc const* input_tensors) const +{ + int ndim = input_tensors[getInputTensorIndex()].dims.nbDims; + TLLM_CHECK_WITH_INFO( + 3 == ndim || 2 == ndim, "hidden_state dimension should be either 2 [b*s, hidden], or 3 [b, s, hidden]"); + int64_t num_tokens = input_tensors[getInputTensorIndex()].dims.d[0]; + if (ndim == 3) + { + num_tokens *= input_tensors[getInputTensorIndex()].dims.d[1]; + } + return num_tokens; +} + +size_t MixtureOfExpertsPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + TLLM_CHECK_WITH_INFO( + nbInputs == getNbInputs(), "Required input to plugin is missing. Expected %d Got %d", getNbInputs(), nbInputs); + TLLM_CHECK_WITH_INFO(nbOutputs == getNbOutputs(), "Required output to plugin is missing. Expected %d Got %d", + getNbOutputs(), nbOutputs); + + if (useSideStream()) + { + return 0; + } + int const num_tokens = getNumTokens(inputs); + int const num_lora_reqs = getNumLoraRequests(inputs); + return setupWorkspace(nullptr, num_tokens, num_lora_reqs).size; +} + +MOEParallelismConfig MixtureOfExpertsPlugin::getParallelismConfig() const +{ + return mParallelismConfig; +} + +QuantParams tensorrt_llm::plugins::MixtureOfExpertsPlugin::getQuantParams(nvinfer1::PluginTensorDesc const* inputDesc, + void const* const* inputs, int scale_1_idx, int scale_2_idx, int scale_3_idx, int scale_4_idx, int scale_5_idx, + int scale_6_idx, int scale_7_idx, int scale_8_idx) const +{ + void const* scale_1 = scale_1_idx >= 0 ? inputs[scale_1_idx] : nullptr; + void const* scale_2 = scale_2_idx >= 0 ? inputs[scale_2_idx] : nullptr; + void const* scale_3 = scale_3_idx >= 0 ? inputs[scale_3_idx] : nullptr; + void const* scale_4 = scale_4_idx >= 0 ? inputs[scale_4_idx] : nullptr; + void const* scale_5 = scale_5_idx >= 0 ? inputs[scale_5_idx] : nullptr; + void const* scale_6 = scale_6_idx >= 0 ? inputs[scale_6_idx] : nullptr; + void const* scale_7 = scale_7_idx >= 0 ? inputs[scale_7_idx] : nullptr; + void const* scale_8 = scale_8_idx >= 0 ? inputs[scale_8_idx] : nullptr; + nvinfer1::PluginTensorDesc const* desc_1 = scale_1_idx >= 0 ? &inputDesc[scale_1_idx] : nullptr; + nvinfer1::PluginTensorDesc const* desc_2 = scale_2_idx >= 0 ? &inputDesc[scale_2_idx] : nullptr; + nvinfer1::PluginTensorDesc const* desc_3 = scale_3_idx >= 0 ? &inputDesc[scale_3_idx] : nullptr; + nvinfer1::PluginTensorDesc const* desc_4 = scale_4_idx >= 0 ? &inputDesc[scale_4_idx] : nullptr; + nvinfer1::PluginTensorDesc const* desc_5 = scale_5_idx >= 0 ? &inputDesc[scale_5_idx] : nullptr; + nvinfer1::PluginTensorDesc const* desc_6 = scale_6_idx >= 0 ? &inputDesc[scale_6_idx] : nullptr; + auto const gated_inter_size = isGatedActivation(mActivationType) ? mExpertInterSize * 2 : mExpertInterSize; + auto const experts_per_node = mNumExperts / mParallelismConfig.ep_size; + if (hasExpertIntQuantScales()) + { + TLLM_CHECK(scale_1 && scale_2); + if (!hasGroupwiseIntQuantScales()) + { + TLLM_CHECK(!scale_3 && !scale_4 && !scale_5 && !scale_6); + TLLM_CHECK(desc_1->dims.nbDims == 2); + TLLM_CHECK(desc_2->dims.nbDims == 2); + TLLM_CHECK_WITH_INFO( + desc_1->dims.d[0] == experts_per_node, "Incorrect number of experts in int quant scale"); + TLLM_CHECK(desc_1->dims.d[1] == gated_inter_size); + TLLM_CHECK_WITH_INFO( + desc_2->dims.d[0] == experts_per_node, "Incorrect number of experts in int quant scale"); + TLLM_CHECK(desc_2->dims.d[1] == mExpertHiddenSize); + return QuantParams::Int(scale_1, scale_2); + } + else + { + TLLM_CHECK(desc_1->dims.nbDims == 3); + TLLM_CHECK(desc_2->dims.nbDims == 3); + TLLM_CHECK((scale_3 && scale_4) || !hasExpertPrequantScales()); + TLLM_CHECK((scale_5 && scale_6) || !hasExpertWeightQuantZeros()); + TLLM_CHECK((scale_7 && scale_8) || !hasGroupwiseFp8Alpha()); + return QuantParams::GroupWise(mGroupSize, scale_1, scale_2, scale_3, scale_4, scale_5, scale_6, + static_cast<float const*>(scale_7), static_cast<float const*>(scale_8)); + } + } + else if (hasExpertFp8QuantScales()) + { + TLLM_CHECK(scale_1 && scale_2 && scale_3); + TLLM_CHECK(scale_4 || !hasExpertFp8FinalQuantScales()); + TLLM_CHECK((scale_5 != nullptr) == hasLora()); + TLLM_CHECK(!scale_6); + TLLM_CHECK(desc_1->dims.nbDims == 2); + TLLM_CHECK(desc_2->dims.nbDims == 1); + TLLM_CHECK(desc_3->dims.nbDims == 2); + TLLM_CHECK_WITH_INFO( + desc_1->dims.d[0] == experts_per_node && desc_1->dims.d[1] == 1, "Incorrect shape for weight FP8 scale"); + TLLM_CHECK(desc_2->dims.d[0] == 1); + TLLM_CHECK_WITH_INFO( + desc_3->dims.d[0] == experts_per_node && desc_3->dims.d[1] == 1, "Incorrect shape for weight FP8 scale"); + return QuantParams::FP8(static_cast<float const*>(scale_1), static_cast<float const*>(scale_2), + static_cast<float const*>(scale_3), static_cast<float const*>(scale_4), static_cast<float const*>(scale_5)); + } + else if (hasFP4QuantScales()) + { + TLLM_CHECK(scale_1 && scale_2 && scale_3 && scale_4 && scale_5 && scale_6); + TLLM_CHECK(desc_1->dims.nbDims == 1); + TLLM_CHECK(desc_2->dims.nbDims == 3); + TLLM_CHECK(desc_3->dims.nbDims == 1); + TLLM_CHECK(desc_4->dims.nbDims == 1); + TLLM_CHECK(desc_5->dims.nbDims == 3); + TLLM_CHECK(desc_6->dims.nbDims == 1); + TLLM_CHECK(desc_1->dims.d[0] == 1); + TLLM_CHECK_WITH_INFO(desc_2->dims.d[0] == experts_per_node && desc_2->dims.d[1] == gated_inter_size + && desc_2->dims.d[2] + == mExpertHiddenSize / TmaWarpSpecializedGroupedGemmInput::NVFP4BlockScaleVectorSize, + "Incorrect shape for FP4 scale"); + TLLM_CHECK_WITH_INFO(desc_3->dims.d[0] == experts_per_node, "Incorrect shape for FP4 scale"); + TLLM_CHECK(desc_4->dims.d[0] == 1); + TLLM_CHECK_WITH_INFO(desc_5->dims.d[0] == experts_per_node && desc_5->dims.d[1] == mExpertHiddenSize + && desc_5->dims.d[2] + == mExpertInterSize / TmaWarpSpecializedGroupedGemmInput::NVFP4BlockScaleVectorSize, + "Incorrect shape for FP4 scale"); + TLLM_CHECK_WITH_INFO(desc_6->dims.d[0] == experts_per_node, "Incorrect shape for FP4 scale"); + return QuantParams::FP4(static_cast<float const*>(scale_1), + static_cast<TmaWarpSpecializedGroupedGemmInput::ElementSF const*>(scale_2), + static_cast<float const*>(scale_3), static_cast<float const*>(scale_4), + static_cast<TmaWarpSpecializedGroupedGemmInput::ElementSF const*>(scale_5), + static_cast<float const*>(scale_6)); + } + return {}; +} + +int MixtureOfExpertsPlugin::getNumLoraRequests(nvinfer1::PluginTensorDesc const* input_tensors) const +{ + if (!hasLora()) + return 0; + int num_reqs = input_tensors[getLoraFC1RanksIndex()].dims.d[0]; + return num_reqs; +} + +LoraParams MixtureOfExpertsPlugin::getLoraParams( + nvinfer1::PluginTensorDesc const* inputDesc, void const* const* inputs, void* workspace) +{ + TLLM_CHECK(hasLora()); + + int const num_reqs = getNumLoraRequests(inputDesc); + int64_t const num_tokens = getNumTokens(inputDesc); + bool is_gated_actiation = isGatedActivation(mActivationType); + + mLoraExpandFC1WeightPtrs.clear(); + mLoraExpandFC2WeightPtrs.clear(); + mLoraExpandFC1Ranks.clear(); + mLoraExpandFC2Ranks.clear(); + + mLoraExpandFC1WeightPtrs.reserve(num_tokens * 2); + mLoraExpandFC2WeightPtrs.reserve(num_tokens * 2); + mLoraExpandFC1Ranks.reserve(num_tokens); + mLoraExpandFC2Ranks.reserve(num_tokens); + + if (is_gated_actiation) + { + mLoraExpandGatedWeightPtrs.clear(); + mLoraExpandGatedRanks.clear(); + mLoraExpandGatedWeightPtrs.reserve(num_tokens * 2); + mLoraExpandGatedRanks.reserve(num_tokens); + } + + int const seq_len = mRemoveInputPadding ? 0 : inputDesc[getInputTensorIndex()].dims.d[1]; + int32_t const* req_types = static_cast<int32_t const*>(inputs[getHostRequestTypeIndex()]); + int32_t const* host_context_lens + = mRemoveInputPadding ? static_cast<int32_t const*>(inputs[getHostContextLengthIndex()]) : nullptr; + + auto const fc1_lora_weight_ptrs = static_cast<void const* const*>(inputs[getLoraFC1WeightPtrsIndex()]); + auto const fc1_lora_ranks = static_cast<int32_t const*>(inputs[getLoraFC1RanksIndex()]); + + auto const fc2_lora_weight_ptrs = static_cast<void const* const*>(inputs[getLoraFC2WeightPtrsIndex()]); + auto const fc2_lora_ranks = static_cast<int32_t const*>(inputs[getLoraFC2RanksIndex()]); + + auto const gated_lora_weight_ptrs + = is_gated_actiation ? static_cast<void const* const*>(inputs[getLoraGatedWeightPtrsIndex()]) : nullptr; + auto const gated_lora_ranks + = is_gated_actiation ? static_cast<int32_t const*>(inputs[getLoraGatedRanksIndex()]) : nullptr; + + int idx = 0; + for (int req_id = 0; req_id < num_reqs; req_id++) + { + RequestType const reqType = static_cast<RequestType const>(req_types[req_id]); + if (reqType == RequestType::kGENERATION) + { + // lora_weight_ptrs has 3 pointers for each module: A,B, and an optional DoRA magnitude + // the current LoRA implementation does not apply DoRA scaling, so the magnitude is ignored + mLoraExpandFC1WeightPtrs.push_back(fc1_lora_weight_ptrs[req_id * 3]); + mLoraExpandFC1WeightPtrs.push_back(fc1_lora_weight_ptrs[req_id * 3 + 1]); + mLoraExpandFC1Ranks.push_back(fc1_lora_ranks[req_id]); + + mLoraExpandFC2WeightPtrs.push_back(fc2_lora_weight_ptrs[req_id * 3]); + mLoraExpandFC2WeightPtrs.push_back(fc2_lora_weight_ptrs[req_id * 3 + 1]); + mLoraExpandFC2Ranks.push_back(fc2_lora_ranks[req_id]); + + if (is_gated_actiation) + { + mLoraExpandGatedWeightPtrs.push_back(gated_lora_weight_ptrs[req_id * 3]); + mLoraExpandGatedWeightPtrs.push_back(gated_lora_weight_ptrs[req_id * 3 + 1]); + mLoraExpandGatedRanks.push_back(gated_lora_ranks[req_id]); + } + + idx += 1; + } + else + { + int context_len = (mRemoveInputPadding ? host_context_lens[req_id] : seq_len); + + for (int context_id = 0; context_id < context_len; context_id++) + { + mLoraExpandFC1WeightPtrs.push_back(fc1_lora_weight_ptrs[req_id * 3]); + mLoraExpandFC1WeightPtrs.push_back(fc1_lora_weight_ptrs[req_id * 3 + 1]); + mLoraExpandFC1Ranks.push_back(fc1_lora_ranks[req_id]); + + mLoraExpandFC2WeightPtrs.push_back(fc2_lora_weight_ptrs[req_id * 3]); + mLoraExpandFC2WeightPtrs.push_back(fc2_lora_weight_ptrs[req_id * 3 + 1]); + mLoraExpandFC2Ranks.push_back(fc2_lora_ranks[req_id]); + + if (is_gated_actiation) + { + mLoraExpandGatedWeightPtrs.push_back(gated_lora_weight_ptrs[req_id * 3]); + mLoraExpandGatedWeightPtrs.push_back(gated_lora_weight_ptrs[req_id * 3 + 1]); + mLoraExpandGatedRanks.push_back(gated_lora_ranks[req_id]); + } + } + idx += context_len; + } + } + + TLLM_CHECK_WITH_INFO(idx == num_tokens, fmtstr("idx %d num_tokens %ld", idx, num_tokens)); + + return LoraParams(num_reqs, mLoraExpandFC1Ranks.data(), mLoraExpandFC1WeightPtrs.data(), mLoraExpandFC2Ranks.data(), + mLoraExpandFC2WeightPtrs.data(), mLoraImpl1, mLoraImpl2, workspace, &mMemcpyEvent, mLoraExpandGatedRanks.data(), + mLoraExpandGatedWeightPtrs.data()); +} + +int MixtureOfExpertsPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace_ptr, + cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + + int64_t const num_tokens = getNumTokens(inputDesc); + int64_t const num_reqs = getNumLoraRequests(inputDesc); + + if (useSideStream()) + { + // Prepare the side stream + if (!mSideStreamPtr) + { + auto const resource_name = nvinfer1::pluginInternal::SideStream::getResourceKey(mSideStreamId); + nvinfer1::pluginInternal::SideStream side_stream{}; + mSideStreamPtr = reinterpret_cast<nvinfer1::pluginInternal::SideStream*>( + getPluginRegistry()->acquirePluginResource(resource_name.c_str(), &side_stream)); + } + // Debug the code with the main stream stalled (only executed when the environment variable + // TLLM_DEBUG_MOE_STALL_MAIN is set and has a positive value) + mSideStreamPtr->stallMainStream("TLLM_DEBUG_MOE_STALL_MAIN", stream, mDebugStallMain); + // The side stream waits for the inputs managed by the main stream to be ready + mSideStreamPtr->waitMainStreamOnSideStream(stream); + // Provide data dependency for the shared experts running after this plugin by copying inputs on the main stream + size_t count = 1; + for (int i = 0; i < inputDesc[getInputDummyTensorIndex()].dims.nbDims; ++i) + { + count *= inputDesc[getInputDummyTensorIndex()].dims.d[i]; + } + count *= tensorrt_llm::runtime::BufferDataType(inputDesc[getInputDummyTensorIndex()].type).getSize(); + TLLM_CUDA_CHECK(cudaMemcpyAsync(outputs[getOutputDummyTensorIndex()], inputs[getInputDummyTensorIndex()], count, + cudaMemcpyDeviceToDevice, stream)); + // Switch from the main stream to the side stream + stream = mSideStreamPtr->getStream(); + // The workspace is managed by the side stream (otherwise, the lifetime of workspace may be incorrect) + auto const workspace_size = setupWorkspace(nullptr, num_tokens, num_reqs).size; + workspace_ptr = mSideStreamPtr->getWorkspacePtr(workspace_size); + } + auto workspace = setupWorkspace(workspace_ptr, num_tokens, num_reqs); + + auto w1_desc = inputDesc[getExpertWeights1Index()]; + auto w2_desc = inputDesc[getExpertWeights2Index()]; + TLLM_CHECK(w1_desc.dims.nbDims == 3); + auto const experts_per_node = mNumExperts / mParallelismConfig.ep_size; + TLLM_CHECK(w1_desc.dims.d[0] == experts_per_node); + TLLM_CHECK(w2_desc.dims.nbDims == 3); + TLLM_CHECK(w2_desc.dims.d[0] == experts_per_node); + + auto [inner_packed_elements, outer_packed_elements] = getWeightPackedElements(); + int inner_dim_idx = getGemmShapeInnerDimIndex(); + int outer_dim_idx = getGemmShapeOuterDimIndex(); + TLLM_CHECK(w1_desc.dims.d[inner_dim_idx] * inner_packed_elements == mExpertHiddenSize); + if (isGatedActivation(mActivationType)) + { + TLLM_CHECK(w1_desc.dims.d[outer_dim_idx] * outer_packed_elements == mExpertInterSize * 2); + } + else + { + TLLM_CHECK(w1_desc.dims.d[outer_dim_idx] * outer_packed_elements == mExpertInterSize); + } + + TLLM_CHECK(w2_desc.dims.d[inner_dim_idx] * inner_packed_elements == mExpertInterSize); + TLLM_CHECK(w2_desc.dims.d[outer_dim_idx] * outer_packed_elements == mExpertHiddenSize); + + QuantParams quant_params{}; + if (hasExpertIntQuantScales()) + { + if (mGroupSize > 0) + { + quant_params = getQuantParams(inputDesc, inputs, getExpertIntQuantScale1Index(), + getExpertIntQuantScale2Index(), hasExpertPrequantScales() ? getExpertPrequantScales1Index() : -1, + hasExpertPrequantScales() ? getExpertPrequantScales2Index() : -1, + hasExpertWeightQuantZeros() ? getExpertIntQuantZeros1Index() : -1, + hasExpertWeightQuantZeros() ? getExpertIntQuantZeros2Index() : -1, + hasGroupwiseFp8Alpha() ? getExpertFp8Alpha1Index() : -1, + hasGroupwiseFp8Alpha() ? getExpertFp8Alpha2Index() : -1); + } + else + { + quant_params + = getQuantParams(inputDesc, inputs, getExpertIntQuantScale1Index(), getExpertIntQuantScale2Index()); + } + } + else if (hasExpertFp8QuantScales()) + { + quant_params = getQuantParams(inputDesc, inputs, // + getExpertFP8Dequant1Index(), // + getExpertFP8Quant2Index(), // + getExpertFP8Dequant2Index(), // + hasExpertFp8FinalQuantScales() ? getExpertFP8QuantFinalIndex() : -1, + hasLora() ? getInputFP8DequantIndex() : -1); + } + else if (hasFP4QuantScales()) + { + quant_params = getQuantParams(inputDesc, inputs, // + getFP4GlobalActSF1Index(), // + getFP4WeightSF1Index(), // + getFP4GlobalSF1Index(), // + getFP4GlobalActSF2Index(), // + getFP4WeightSF2Index(), // + getFP4GlobalSF2Index() // + ); + } + + LoraParams lora_params{}; + + if (hasLora()) + { + lora_params = getLoraParams(inputDesc, inputs, workspace.lora_workspace); + auto lora_gemm1 = mLoraProfiler->getBestConfig(num_tokens, mLoraGemmId1); + auto lora_gemm2 = mLoraProfiler->getBestConfig(num_tokens, mLoraGemmId2); + + mLoraImpl1->setBestTactic(lora_gemm1); + mLoraImpl2->setBestTactic(lora_gemm2); + } + + std::optional<tensorrt_llm::cutlass_extensions::CutlassGemmConfig> gemm1; + std::optional<tensorrt_llm::cutlass_extensions::CutlassGemmConfig> gemm2; + if (common::getEnvForceDeterministicMOE()) + { + gemm1 = mMOERunner->getTactics(MoeGemmId::GEMM_1)[0]; + gemm2 = mMOERunner->getTactics(MoeGemmId::GEMM_2)[0]; + } + else + { + gemm1 = mGemmProfiler->getBestConfig(num_tokens, mGemmId1); + gemm2 = mGemmProfiler->getBestConfig(num_tokens, mGemmId2); + } + + MoeMinLatencyParams min_latency_params{}; + mMOERunner->setTactic(gemm1, gemm2); +#ifdef USING_OSS_CUTLASS_MOE_GEMM + mMOERunner->runMoe(inputs[getInputTensorIndex()], nullptr, true, + static_cast<int const*>(inputs[getTokenSelectedExpertsIndex()]), + hasFinalScales() ? static_cast<float const*>(inputs[getTokenFinalScalesIndex()]) : nullptr, + inputs[getExpertWeights1Index()], hasBias() ? inputs[getExpertBias1Index()] : nullptr, + ActivationParams(mActivationType), inputs[getExpertWeights2Index()], + hasBias() ? inputs[getExpertBias2Index()] : nullptr, quant_params, num_tokens, num_tokens, mExpertHiddenSize, + mExpertHiddenSize /*TRT does not support padding, safe to assume padded/unpadded hidden sizes are the same*/, + mExpertInterSize, mNumExperts, mExpertsPerToken, static_cast<char*>(workspace.workspace), + // Outputs + outputs[getOutputTensorIndex()], static_cast<int*>(workspace.src_to_dest_map), mParallelismConfig, + /*enable_alltoall=*/false, hasLora(), lora_params, /*use_deepseek_fp8_block_scale=*/false, + /*min_latency_mode=*/false, min_latency_params, stream); +#else + mMOERunner->runMoe(inputs[getInputTensorIndex()], nullptr, true, + static_cast<int const*>(inputs[getTokenSelectedExpertsIndex()]), + hasFinalScales() ? static_cast<float const*>(inputs[getTokenFinalScalesIndex()]) : nullptr, + inputs[getExpertWeights1Index()], hasBias() ? inputs[getExpertBias1Index()] : nullptr, + ActivationParams(mActivationType), inputs[getExpertWeights2Index()], + hasBias() ? inputs[getExpertBias2Index()] : nullptr, quant_params, num_tokens, num_tokens, mExpertHiddenSize, + mExpertInterSize, mNumExperts, mExpertsPerToken, static_cast<char*>(workspace.workspace), + // Outputs + outputs[getOutputTensorIndex()], static_cast<int*>(workspace.src_to_dest_map), mParallelismConfig, hasLora(), + lora_params, /*use_deepseek_fp8_block_scale=*/false, + /*min_latency_mode=*/false, min_latency_params, stream); +#endif + + if (useSideStream()) + { + // Debug the code with the side stream stalled (only executed when the environment variable + // TLLM_DEBUG_MOE_STALL_SIDE is set and has a positive value) + mSideStreamPtr->stallSideStream("TLLM_DEBUG_MOE_STALL_SIDE", mDebugStallSide); + } + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType MixtureOfExpertsPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == getOutputTensorIndex() || index == getOutputDummyTensorIndex()); + if (useSideStream() && index == getOutputDummyTensorIndex()) + { + return inputTypes[getInputDummyTensorIndex()]; + } + return mOutputType; +} + +// IPluginV2 Methods +char const* MixtureOfExpertsPlugin::getPluginType() const noexcept +{ + return MIXTURE_OF_EXPERTS_PLUGIN_NAME; +} + +char const* MixtureOfExpertsPlugin::getPluginVersion() const noexcept +{ + return MIXTURE_OF_EXPERTS_PLUGIN_VERSION; +} + +int MixtureOfExpertsPlugin::initialize() noexcept +{ + mGemmProfiler->setGemmToProfile(kernels::GemmProfilerBackend::GemmToProfile::GEMM_1); + mGemmProfiler->profileTactics(this, mType, mDims, mGemmId1); + mGemmProfiler->setGemmToProfile(kernels::GemmProfilerBackend::GemmToProfile::GEMM_2); + mGemmProfiler->profileTactics(this, mType, mDims, mGemmId2); + + if (hasLora()) + { + mLoraImpl1->setGemmConfig(); + mLoraImpl2->setGemmConfig(); + + mLoraProfiler->profileTactics(mLoraImpl1->getCublasWrapper(), mType, mDims, mLoraGemmId1); + mLoraProfiler->profileTactics(mLoraImpl2->getCublasWrapper(), mType, mDims, mLoraGemmId2); + } + return 0; +} + +void MixtureOfExpertsPlugin::terminate() noexcept +{ + if (mSideStreamPtr) + { + auto const resource_name = nvinfer1::pluginInternal::SideStream::getResourceKey(mSideStreamId); + getPluginRegistry()->releasePluginResource(resource_name.c_str()); + mSideStreamPtr = nullptr; + } +} + +void MixtureOfExpertsPlugin::destroy() noexcept +{ + if (hasLora()) + { + TLLM_CUDA_CHECK(cudaEventDestroy(mMemcpyEvent)); + } + // This gets called when the network containing plugin is destroyed + delete this; +} + +void MixtureOfExpertsPlugin::setPluginNamespace(char const* libNamespace) noexcept +{ + mNamespace = libNamespace; +} + +char const* MixtureOfExpertsPlugin::getPluginNamespace() const noexcept +{ + return mNamespace.c_str(); +} + +/////////////// + +char const* MixtureOfExpertsPluginCreator::getPluginName() const noexcept +{ + return MIXTURE_OF_EXPERTS_PLUGIN_NAME; +} + +char const* MixtureOfExpertsPluginCreator::getPluginVersion() const noexcept +{ + return MIXTURE_OF_EXPERTS_PLUGIN_VERSION; +} + +nvinfer1::PluginFieldCollection const* MixtureOfExpertsPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +MixtureOfExpertsPluginCreator::MixtureOfExpertsPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(nvinfer1::PluginField("remove_input_padding", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("number_of_experts", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("experts_per_token", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("expert_hidden_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("expert_inter_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("groupwise_quant_algo", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("group_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("activation_type", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("weight_type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("quant_mode", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("use_final_scales", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("use_bias", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("tp_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("tp_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("ep_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("ep_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("side_stream_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("use_lora", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("lora_type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(nvinfer1::PluginField("max_low_rank", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +IPluginV2* MixtureOfExpertsPluginCreator::createPlugin( + char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept +{ + nvinfer1::PluginField const* fields = fc->fields; + int mRemoveInputPadding{}; + int mNumExperts{}; + int mExpertsPerToken{}; + int mExpertHiddenSize{}; + int mExpertInterSize{}; + int mGroupwiseQuantAlgo{}; + int mGroupSize{}; + int mActivationType{}; + int mType{}; + int mWeightType{}; + int mOutputType{INT_MAX}; + int mQuantMode{}; + int mUseFinalScales{1}; // Default to true + int mUseBias{0}; + int mTPSize{}; + int mTPRank{}; + int mEPSize{}; + int mEPRank{}; + int mRequiresDeterminism{0}; + int mSideStreamId{0}; + int mUseLora{}; + int mLoraType{INT_MAX}; + int mMaxLowRank{0}; + + // Read configurations from each fields + struct MapPair + { + char const* key; + int& field; + bool optional = false; + bool set = false; + }; + + std::array input_map{ + MapPair{"remove_input_padding", std::ref(mRemoveInputPadding)}, + MapPair{"number_of_experts", std::ref(mNumExperts)}, + MapPair{"experts_per_token", std::ref(mExpertsPerToken)}, + MapPair{"expert_hidden_size", std::ref(mExpertHiddenSize)}, + MapPair{"expert_inter_size", std::ref(mExpertInterSize)}, + MapPair{"groupwise_quant_algo", std::ref(mGroupwiseQuantAlgo)}, + MapPair{"group_size", std::ref(mGroupSize)}, + MapPair{"activation_type", std::ref(mActivationType)}, + MapPair{"type_id", std::ref(mType)}, + MapPair{"weight_type_id", std::ref(mWeightType)}, + MapPair{"quant_mode", std::ref(mQuantMode)}, + MapPair{"tp_size", std::ref(mTPSize)}, + MapPair{"tp_rank", std::ref(mTPRank)}, + MapPair{"ep_size", std::ref(mEPSize)}, + MapPair{"ep_rank", std::ref(mEPRank)}, + MapPair{"use_lora", std::ref(mUseLora)}, + MapPair{"use_final_scales", std::ref(mUseFinalScales)}, + + // Optional + MapPair{"use_bias", std::ref(mUseBias), true}, + MapPair{"output_type_id", std::ref(mOutputType), true}, + MapPair{"force_determinism", std::ref(mRequiresDeterminism), true}, + MapPair{"side_stream_id", std::ref(mSideStreamId), true}, + MapPair{"lora_type_id", std::ref(mLoraType), true}, + MapPair{"max_low_rank", std::ref(mMaxLowRank), true}, + }; + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + for (auto& item : input_map) + { + if (!strcmp(item.key, attrName)) + { + TLLM_CHECK(fields[i].type == nvinfer1::PluginFieldType::kINT32); + TLLM_CHECK_WITH_INFO(!item.set, "Parameter %s was set twice", item.key); + item.field = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + item.set = true; + } + } + } + + for (auto& item : input_map) + { + TLLM_CHECK_WITH_INFO(item.set || item.optional, "Parameter %s is required but not set", item.key); + } + + // Output type is optional, if not set it to the same as mType + if (mOutputType == INT_MAX) + { + mOutputType = mType; + } + + if (mUseLora) + { + TLLM_CHECK_WITH_INFO(mLoraType != INT_MAX && mMaxLowRank != 0, + "MoE fuse lora, lora_type_id and max_low_rank are required but not set"); + } + + try + { + auto gemmProfiler = moePluginProfiler.createGemmPluginProfiler(/* inference */ false); + auto loraProfiler = loraPluginProfileManager.createGemmPluginProfiler(/* inference */ false, /* skip */ true); + auto* obj = new MixtureOfExpertsPlugin( + // Constructor parameters + mRemoveInputPadding, mNumExperts, mExpertsPerToken, mExpertHiddenSize, mExpertInterSize, + mGroupwiseQuantAlgo, mGroupSize, static_cast<ActivationType>(mActivationType), + static_cast<nvinfer1::DataType>(mType), static_cast<nvinfer1::DataType>(mWeightType), + static_cast<nvinfer1::DataType>(mOutputType), QuantMode(mQuantMode), mUseFinalScales != 0, mUseBias != 0, + mTPSize, mTPRank, mEPSize, mEPRank, mRequiresDeterminism != 0, mSideStreamId, gemmProfiler, mUseLora != 0, + static_cast<nvinfer1::DataType>(mLoraType), loraProfiler, mMaxLowRank); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* MixtureOfExpertsPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call MixtureOfExpertsPlugin::destroy() + try + { + auto gemmProfiler = moePluginProfiler.createGemmPluginProfiler(/* inference */ true); + auto loraProfiler = loraPluginProfileManager.createGemmPluginProfiler(/* inference */ false, /* skip */ true); + + auto* obj = new MixtureOfExpertsPlugin( + // Constructor parameters + serialData, serialLength, gemmProfiler, loraProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +void MixtureOfExpertsPluginCreator::setPluginNamespace(char const* libNamespace) noexcept +{ + mNamespace = libNamespace; +} + +char const* MixtureOfExpertsPluginCreator::getPluginNamespace() const noexcept +{ + return mNamespace.c_str(); +} + +void MixtureOfExpertsGemmProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + checkInit(); + size_t bytes = backend.getWorkspaceSize(maxM); + this->setTmpWorkspaceSizeInBytes(bytes); +} + +void MixtureOfExpertsGemmProfiler::runTactic(int m, int n, int k, MixtureOfExpertsGemmProfiler::Config const& tactic, + char* workspace_ptr_char, cudaStream_t const& stream) +{ + checkInit(); + backend.runProfiler(m, tactic, workspace_ptr_char, /*expert_weights*/ nullptr, stream); +} + +auto MixtureOfExpertsGemmProfiler::getTactics(int m, int n, int k) const -> std::vector<Config> +{ + assert(mRunner); + return mRunner->mMOERunner->getTactics(backend.mGemmToProfile); +} + +void MixtureOfExpertsGemmProfiler::initTmpData( + int m, int n, int k, char* workspace, size_t ws_size, cudaStream_t stream) +{ + checkInit(); + backend.prepare(m, workspace, /*expert_weights*/ nullptr, stream); +} + +void MixtureOfExpertsGemmProfiler::checkInit() +{ + assert(mRunner); + if (init_backend) + { + return; + } + init_backend = true; + auto& plugin = *mRunner; +#ifdef USING_OSS_CUTLASS_MOE_GEMM + backend.init(*plugin.mMOERunner, backend.mGemmToProfile, plugin.mType, plugin.mWeightType, plugin.mOutputType, + plugin.mNumExperts, plugin.mExpertsPerToken, plugin.mExpertHiddenSize, + plugin.mExpertHiddenSize /*TRT backend does not support unpadded hidden size*/, plugin.mExpertInterSize, + plugin.mGroupSize, plugin.mActivationType, plugin.hasBias(), plugin.hasLora(), /*min_latency_mode=*/false, + /*need_weights=*/true, plugin.getParallelismConfig(), /*enable_alltoall=*/false); +#else + backend.init(*plugin.mMOERunner, backend.mGemmToProfile, plugin.mType, plugin.mWeightType, plugin.mOutputType, + plugin.mNumExperts, plugin.mExpertsPerToken, plugin.mExpertHiddenSize, plugin.mExpertInterSize, + plugin.mGroupSize, plugin.mActivationType, plugin.hasBias(), plugin.hasLora(), /*min_latency_mode=*/false, + /*need_weights=*/true, plugin.getParallelismConfig()); +#endif +} diff --git a/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h b/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h new file mode 100644 index 000000000000..feb1f10cdc70 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/mixtureOfExperts/mixtureOfExpertsPlugin.h @@ -0,0 +1,637 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TRT_MIXTURE_OF_EXPERTS_PLUGIN_H +#define TRT_MIXTURE_OF_EXPERTS_PLUGIN_H + +#include "NvInferPlugin.h" +#include "tensorrt_llm/kernels/cutlass_kernels/include/cutlass_kernel_selector.h" +#if defined(USING_OSS_CUTLASS_MOE_GEMM) +#include "tensorrt_llm/kernels/cutlass_kernels/include/moe_kernels.h" +#else +#include "moe_kernels.h" +#endif +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/kernels/lora/lora.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/plugins/cudaStreamPlugin/cudaStreamPlugin.h" +#include "tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h" +#include "tensorrt_llm/runtime/cudaStream.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ +namespace kernels = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE; +using MoeMinLatencyParams = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE::MoeMinLatencyParams; +using MOEParallelismConfig = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE::MOEParallelismConfig; +using QuantParams = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE::QuantParams; +using MoeGemmId = CUTLASS_MOE_GEMM_NAMESPACE::MoeGemmId; +using ActivationType = CUTLASS_MOE_GEMM_NAMESPACE::ActivationType; +using ActivationParams = CUTLASS_MOE_GEMM_KERNELS_NAMESPACE::ActivationParams; +using TmaWarpSpecializedGroupedGemmInput = CUTLASS_MOE_GEMM_NAMESPACE::TmaWarpSpecializedGroupedGemmInput; +using CUTLASS_MOE_GEMM_NAMESPACE::isGatedActivation; + +class MixtureOfExpertsGemmProfiler; +using MixtureOfExpertsPluginProfilerPtr = std::shared_ptr<MixtureOfExpertsGemmProfiler>; +using GroupwiseQuantAlgo = tensorrt_llm::common::GroupwiseQuantAlgo; + +struct GemmIDMoe +{ + int gemm_idx; + int num_experts{}; + int experts_per_token{}; + kernels::MOEParallelismConfig parallelism_config{}; + int64_t hidden{}; + int64_t inter{}; + int64_t group_size{}; + ActivationType actfn{}; + nvinfer1::DataType dtype{}; + nvinfer1::DataType wdtype{}; + tensorrt_llm::common::QuantMode quant_mode; + bool determinism_mode = false; + + bool operator==(GemmIDMoe const& id) const + { + return id.gemm_idx == gemm_idx && id.num_experts == num_experts && id.experts_per_token == experts_per_token + && id.parallelism_config == parallelism_config && id.hidden == hidden && id.inter == inter + && id.group_size == group_size && id.actfn == actfn && id.dtype == dtype && id.wdtype == wdtype + && id.quant_mode == quant_mode && id.determinism_mode == determinism_mode; + } + + friend std::ostream& operator<<(std::ostream& out, GemmIDMoe const& id) + { + out << "gemm idx, experts, experts_per_token, parallelism_config, hidden, inter, group_size, actfn, dtype, " + "weight " + "type, parallelism mode, determinism mode=" + + << id.gemm_idx << "," << id.num_experts << "," << id.experts_per_token << "," << id.parallelism_config + << "," << id.hidden << "," << id.inter << "," << id.group_size << "," << static_cast<int>(id.actfn) << "," + << static_cast<int>(id.dtype) << "," << static_cast<int>(id.wdtype) << "," << id.quant_mode.value() << "," + << id.determinism_mode; + return out; + } +}; + +// Hash of GemmIDMoe +struct GemmIDMoeHash +{ + std::size_t operator()(GemmIDMoe const& id) const + { + size_t hash = std::hash<int>{}(id.gemm_idx); + hash ^= std::hash<int>{}(id.num_experts); + hash ^= std::hash<int>{}(id.experts_per_token); + hash ^= std::hash<int>{}(id.parallelism_config.tp_size); + hash ^= std::hash<int>{}(id.parallelism_config.ep_size); + hash ^= std::hash<int>{}(id.parallelism_config.tp_rank); + hash ^= std::hash<int>{}(id.parallelism_config.ep_rank); + hash ^= std::hash<int>{}(id.hidden); + hash ^= std::hash<int>{}(id.inter); + hash ^= std::hash<int>{}(id.group_size); + hash ^= std::hash<int>{}(static_cast<int>(id.actfn)); + hash ^= std::hash<int>{}(static_cast<int>(id.dtype)); + hash ^= std::hash<int>{}(static_cast<int>(id.wdtype)); + hash ^= std::hash<int>{}(static_cast<int>(id.quant_mode.value())); + return hash; + } +}; + +class MixtureOfExpertsPlugin : public nvinfer1::IPluginV2DynamicExt +{ +public: + using LoraPluginProfilerPtr = std::shared_ptr<CublasLtGemmPluginProfiler>; + using LoraImplPtr = std::shared_ptr<tensorrt_llm::kernels::LoraImpl>; + MixtureOfExpertsPlugin() = delete; + MixtureOfExpertsPlugin(bool remove_input_padding, int number_of_experts, int experts_per_token, + int expert_hidden_size, int expert_inter_size, int groupwise_quant_algo, int group_size, + ActivationType activation_type, nvinfer1::DataType type, nvinfer1::DataType weight_type, + nvinfer1::DataType output_type, tensorrt_llm::common::QuantMode quant_mode, bool use_final_scales, + bool use_bias, int tp_size, int tp_rank, int ep_size, int ep_rank, bool force_determinism, int side_stream_id, + MixtureOfExpertsPluginProfilerPtr gemm_profiler_ptr, bool use_lora, nvinfer1::DataType lora_type, + LoraPluginProfilerPtr lora_profiler, int max_low_rank); + MixtureOfExpertsPlugin(void const* data, size_t length, MixtureOfExpertsPluginProfilerPtr gemm_profiler_ptr, + LoraPluginProfilerPtr lora_profiler); + MixtureOfExpertsPlugin(MixtureOfExpertsPlugin const&); + + void init(); + + ~MixtureOfExpertsPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + + int getNbOutputs() const noexcept override + { + return 1 + useSideStream(); + } + + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + void setPluginNamespace(char const* pluginNamespace) noexcept override; + char const* getPluginNamespace() const noexcept override; + +private: + friend class MixtureOfExpertsGemmProfiler; + std::unique_ptr<kernels::CutlassMoeFCRunnerInterface> mMOERunner{}; + int mNumExperts{}; + int mExpertsPerToken{}; + int64_t mExpertHiddenSize{}; + int64_t mExpertInterSize{}; + int64_t mGroupwiseQuantAlgo{}; + int64_t mGroupSize{}; + ActivationType mActivationType; + nvinfer1::DataType mType{}; + nvinfer1::DataType mWeightType{}; + nvinfer1::DataType mOutputType{}; + tensorrt_llm::common::QuantMode mQuantMode; + bool mUseFinalScales{}; + bool mUseBias{}; + MOEParallelismConfig mParallelismConfig{}; + + GemmDims mDims{}; + bool mUseDeterministicKernels = false; + int mSideStreamId = 0; + + int mDebugStallMain = 0; + int mDebugStallSide = 0; + + GemmIDMoe mGemmId1{}; + GemmIDMoe mGemmId2{}; + + MixtureOfExpertsPluginProfilerPtr mGemmProfiler; + + // lora related + bool mUseLora{}; + nvinfer1::DataType mLoraType{}; + int mMaxLowRank{}; + bool mRemoveInputPadding{}; + + LoraImplPtr mLoraImpl1; + LoraImplPtr mLoraImpl2; + + GemmIdCublas mLoraGemmId1{}; + GemmIdCublas mLoraGemmId2{}; + LoraPluginProfilerPtr mLoraProfiler; + + std::vector<void const*> mLoraExpandFC1WeightPtrs{}; + std::vector<void const*> mLoraExpandFC2WeightPtrs{}; + std::vector<void const*> mLoraExpandGatedWeightPtrs{}; + std::vector<int32_t> mLoraExpandFC1Ranks{}; + std::vector<int32_t> mLoraExpandFC2Ranks{}; + std::vector<int32_t> mLoraExpandGatedRanks{}; + + cudaEvent_t mMemcpyEvent; + nvinfer1::pluginInternal::SideStream* mSideStreamPtr; + + // The below are not serialised + std::string const mLayerName{}; + std::string mNamespace{}; + + struct WorkspaceInfo + { + void* workspace{}; + void* src_to_dest_map{}; + void* lora_workspace{}; + size_t size{}; + }; + + int64_t getNumTokens(nvinfer1::PluginTensorDesc const* input_tensor) const; + WorkspaceInfo setupWorkspace(void* base_ptr, int64_t num_tokens, int num_reqs = 0) const; + + MOEParallelismConfig getParallelismConfig() const; + QuantParams getQuantParams(nvinfer1::PluginTensorDesc const* inputDesc, void const* const* inputs, + int scale_1_idx = -1, int scale_2_idx = -1, int scale_3_idx = -1, int scale_4_idx = -1, int scale_5_idx = -1, + int scale_6_idx = -1, int scale_7_idx = -1, int scale_8_idx = -1) const; + + int getNumLoraRequests(nvinfer1::PluginTensorDesc const* input_tensor) const; + tensorrt_llm::kernels::LoraParams getLoraParams( + nvinfer1::PluginTensorDesc const* inputDesc, void const* const* inputs, void* workspace); + + enum class RequestType : int32_t + { + kCONTEXT = 0, + kGENERATION = 1 + }; + + using IndexType = std::int32_t; + + // Inputs + constexpr static IndexType getInputTensorIndex() + { + return 0; + } + + constexpr static IndexType getExpertWeights1Index() + { + return getInputTensorIndex() + 1; + } + + constexpr static IndexType getExpertWeights2Index() + { + return getExpertWeights1Index() + 1; + } + + constexpr static IndexType getTokenSelectedExpertsIndex() + { + return getExpertWeights2Index() + 1; + } + + // Conditional inputs, we only allocate a new index if actually used + bool hasBias() const + { + return mUseBias; + } + + bool hasFinalScales() const + { + return mUseFinalScales; + } + + bool hasExpertIntQuantScales() const + { + return mQuantMode.hasInt4Weights() || mQuantMode.hasInt8Weights(); + } + + bool hasExpertFp8QuantScales() const + { + return mQuantMode.hasFp8Qdq(); + } + + bool hasExpertFp8FinalQuantScales() const + { + return hasExpertFp8QuantScales() && mOutputType == nvinfer1::DataType::kFP8; + } + + bool hasFP4QuantScales() const + { + return mQuantMode.hasNvfp4(); + } + + bool hasGroupwiseIntQuantScales() const + { + return mGroupwiseQuantAlgo > 0; + } + + bool hasExpertWeightQuantZeros() const + { + return mGroupwiseQuantAlgo & GroupwiseQuantAlgo::ZERO; + } + + bool hasExpertPrequantScales() const + { + return mGroupwiseQuantAlgo & GroupwiseQuantAlgo::PRE_QUANT_SCALE; + } + + bool hasGroupwiseFp8Alpha() const + { + return mGroupwiseQuantAlgo & GroupwiseQuantAlgo::FP8_ALPHA; + } + + bool useSideStream() const + { + return mSideStreamId > 0; + } + + bool hasLora() const + { + return mUseLora; + } + + bool hasGatedLoraWeightsAndRanks() const + { + return mUseLora && isGatedActivation(mActivationType); + } + + IndexType getTokenFinalScalesIndex() const + { + return getTokenSelectedExpertsIndex() + hasFinalScales(); + } + + IndexType getExpertBias1Index() const + { + return getTokenFinalScalesIndex() + hasBias(); + } + + IndexType getExpertBias2Index() const + { + return getExpertBias1Index() + hasBias(); + } + + /* + * Weight-Only int quant scales + */ + IndexType getExpertIntQuantScale1Index() const + { + return getExpertBias2Index() + hasExpertIntQuantScales(); + } + + IndexType getExpertIntQuantScale2Index() const + { + return getExpertIntQuantScale1Index() + hasExpertIntQuantScales(); + } + + /* + * FP8 Quant Scales + */ + IndexType getExpertFP8Dequant1Index() const + { + return getExpertIntQuantScale2Index() + hasExpertFp8QuantScales(); + } + + IndexType getExpertFP8Quant2Index() const + { + return getExpertFP8Dequant1Index() + hasExpertFp8QuantScales(); + } + + IndexType getExpertFP8Dequant2Index() const + { + return getExpertFP8Quant2Index() + hasExpertFp8QuantScales(); + } + + IndexType getExpertFP8QuantFinalIndex() const + { + return getExpertFP8Dequant2Index() + hasExpertFp8FinalQuantScales(); + } + + IndexType getInputFP8DequantIndex() const + { + return getExpertFP8QuantFinalIndex() + (hasExpertFp8QuantScales() && hasLora()); + } + + /* + * FP4 Quant Scales + */ + IndexType getFP4GlobalActSF1Index() const + { + return getInputFP8DequantIndex() + hasFP4QuantScales(); + } + + IndexType getFP4WeightSF1Index() const + { + return getFP4GlobalActSF1Index() + hasFP4QuantScales(); + } + + IndexType getFP4GlobalSF1Index() const + { + return getFP4WeightSF1Index() + hasFP4QuantScales(); + } + + IndexType getFP4GlobalActSF2Index() const + { + return getFP4GlobalSF1Index() + hasFP4QuantScales(); + } + + IndexType getFP4WeightSF2Index() const + { + return getFP4GlobalActSF2Index() + hasFP4QuantScales(); + } + + IndexType getFP4GlobalSF2Index() const + { + return getFP4WeightSF2Index() + hasFP4QuantScales(); + } + + /* + * Groupwise Params + */ + IndexType getExpertPrequantScales1Index() const + { + return getFP4GlobalSF2Index() + hasExpertPrequantScales(); + } + + IndexType getExpertPrequantScales2Index() const + { + return getExpertPrequantScales1Index() + hasExpertPrequantScales(); + } + + IndexType getExpertIntQuantZeros1Index() const + { + return getExpertPrequantScales2Index() + hasExpertWeightQuantZeros(); + } + + IndexType getExpertIntQuantZeros2Index() const + { + return getExpertIntQuantZeros1Index() + hasExpertWeightQuantZeros(); + } + + IndexType getExpertFp8Alpha1Index() const + { + return getExpertIntQuantZeros2Index() + hasGroupwiseFp8Alpha(); + } + + IndexType getExpertFp8Alpha2Index() const + { + return getExpertFp8Alpha1Index() + hasGroupwiseFp8Alpha(); + } + + /* + * LoRA params + */ + IndexType getLoraFC1WeightPtrsIndex() const + { + return getExpertFp8Alpha2Index() + hasLora(); + } + + IndexType getLoraFC1RanksIndex() const + { + return getLoraFC1WeightPtrsIndex() + hasLora(); + } + + IndexType getLoraFC2WeightPtrsIndex() const + { + return getLoraFC1RanksIndex() + hasLora(); + } + + IndexType getLoraFC2RanksIndex() const + { + return getLoraFC2WeightPtrsIndex() + hasLora(); + } + + IndexType getLoraGatedWeightPtrsIndex() const + { + return getLoraFC2RanksIndex() + hasGatedLoraWeightsAndRanks(); + } + + IndexType getLoraGatedRanksIndex() const + { + return getLoraGatedWeightPtrsIndex() + hasGatedLoraWeightsAndRanks(); + } + + IndexType getHostRequestTypeIndex() const + { + return getLoraGatedRanksIndex() + hasLora(); + } + + IndexType getHostContextLengthIndex() const + { + return getHostRequestTypeIndex() + (mRemoveInputPadding && hasLora()); + } + + IndexType getInputDummyTensorIndex() const + { + return getHostContextLengthIndex() + useSideStream(); + } + + IndexType getNbInputs() const + { + return getInputDummyTensorIndex() + 1; + } + + // Outputs + constexpr static IndexType getOutputTensorIndex() + { + return 0; + } + + IndexType getOutputDummyTensorIndex() const + { + return getOutputTensorIndex() + useSideStream(); + } + + /** + * Get the index of the expert shape tuple that represents the inner dimension + */ + int getGemmShapeInnerDimIndex() const + { + // In weight only mode the shape is transposed + return hasExpertIntQuantScales() ? 1 : 2; + } + + /** + * Get the index of the expert shape tuple that represents the outer dimension + */ + int getGemmShapeOuterDimIndex() const + { + // In weight only mode the shape is transposed + return hasExpertIntQuantScales() ? 2 : 1; + } + + /** + * Get quantization dimension scaling factor + */ + std::pair<int, int> getWeightPackedElements() const + { + if (mGroupwiseQuantAlgo == 0) + { + return {1, mQuantMode.hasInt4Weights() ? 2 : 1}; + } + else + { + return {1, 4}; + } + } +}; + +class MixtureOfExpertsGemmProfiler + : public tensorrt_llm::plugins::GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + MixtureOfExpertsPlugin*, GemmIDMoe, GemmIDMoeHash> +{ +public: + MixtureOfExpertsGemmProfiler() + { + // NOTE: Do not access mPlugin here, since we are called from the constructor before all fields are init + } + + void setGemmToProfile(kernels::GemmProfilerBackend::GemmToProfile gemm_to_profile) + { + // Just set the backend directly. This will just be reused in checkInit(). + backend.mGemmToProfile = gemm_to_profile; + // We need to set the backend to reinitialise itself with the new GEMM + init_backend = false; + } + + void setMaxProfileM(int maxProfileM) + { + mMaxProfileM = maxProfileM; + } + + virtual int getMaxProfileM() const override + { + return mMaxProfileM; + } + +protected: + using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + std::vector<Config> getTactics(int m, int n, int k) const override; + void initTmpData(int maxM, int n, int k, char* workspace, size_t size, cudaStream_t stream) override; + + void checkInit(); + + bool init_backend = false; + kernels::GemmProfilerBackend backend{}; + +private: + int mMaxProfileM = 0; +}; + +class MixtureOfExpertsPluginCreator : public nvinfer1::IPluginCreator +{ +public: + MixtureOfExpertsPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + + void setPluginNamespace(char const* pluginNamespace) noexcept override; + + char const* getPluginNamespace() const noexcept override; + +private: + GemmPluginProfilerManager<MixtureOfExpertsGemmProfiler> moePluginProfiler; + GemmPluginProfilerManager<CublasLtGemmPluginProfiler> loraPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; + std::string mNamespace; +}; + +} // namespace tensorrt_llm::plugins + +#endif // TRT_MIXTURE_OF_EXPERTS_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/ncclPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.cpp new file mode 100644 index 000000000000..4825dd51bbab --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.cpp @@ -0,0 +1,253 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "allgatherPlugin.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include <nccl.h> + +using namespace nvinfer1; +using tensorrt_llm::plugins::AllgatherPluginCreator; +using tensorrt_llm::plugins::AllgatherPlugin; + +static char const* ALLGATHER_PLUGIN_VERSION{"1"}; +static char const* ALLGATHER_PLUGIN_NAME{"AllGather"}; +PluginFieldCollection AllgatherPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> AllgatherPluginCreator::mPluginAttributes; + +AllgatherPlugin::AllgatherPlugin(std::set<int> group, nvinfer1::DataType type) + : mGroup(std::move(group)) + , mType(type) +{ +} + +// Parameterized constructor +AllgatherPlugin::AllgatherPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mType); + mGroup.clear(); + int groupItem = 0; + while (d != a + length) + { + read(d, groupItem); + mGroup.insert(groupItem); + } + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* AllgatherPlugin::clone() const noexcept +{ + auto* plugin = new AllgatherPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs AllgatherPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + auto ret = inputs[0]; + auto groupSize = exprBuilder.constant(mGroup.size()); + ret.d[0] = exprBuilder.operation(DimensionOperation::kPROD, *ret.d[0], *groupSize); + return ret; +} + +bool AllgatherPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); +} + +void AllgatherPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t AllgatherPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int AllgatherPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + size_t size = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) + { + size *= inputDesc[0].dims.d[i]; + } + + TLLM_CHECK_WITH_INFO(mNcclComm.get() != nullptr, "mNcclComm should be initialized before used"); + NCCLCHECK(ncclAllGather(inputs[0], outputs[0], size, (*getDtypeMap())[inputDesc[0].type], *mNcclComm, stream)); + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType AllgatherPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + assert(index == 0); + return inputTypes[0]; +} + +// IPluginV2 Methods + +char const* AllgatherPlugin::getPluginType() const noexcept +{ + return ALLGATHER_PLUGIN_NAME; +} + +char const* AllgatherPlugin::getPluginVersion() const noexcept +{ + return ALLGATHER_PLUGIN_VERSION; +} + +int AllgatherPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int AllgatherPlugin::initialize() noexcept +{ + if (isBuilding()) + { + return 0; + } + TLLM_LOG_TRACE("%s start for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); + mNcclComm = getComm(mGroup); + TLLM_LOG_TRACE("%s stop for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); + return 0; +} + +void AllgatherPlugin::terminate() noexcept {} + +size_t AllgatherPlugin::getSerializationSize() const noexcept +{ + return sizeof(int) * mGroup.size() + sizeof(mType); +} + +void AllgatherPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + for (auto it = mGroup.begin(); it != mGroup.end(); ++it) + { + write(d, *it); + } + TLLM_CHECK(d == a + getSerializationSize()); +} + +void AllgatherPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +AllgatherPluginCreator::AllgatherPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("group", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* AllgatherPluginCreator::getPluginName() const noexcept +{ + return ALLGATHER_PLUGIN_NAME; +} + +char const* AllgatherPluginCreator::getPluginVersion() const noexcept +{ + return ALLGATHER_PLUGIN_VERSION; +} + +PluginFieldCollection const* AllgatherPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* AllgatherPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + std::set<int> group; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "group")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + auto const* r = static_cast<int const*>(fields[i].data); + for (int j = 0; j < fields[i].length; ++j) + { + group.insert(*r); + ++r; + } + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + + try + { + auto* obj = new AllgatherPlugin(group, type); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* AllgatherPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call AllgatherPlugin::destroy() + try + { + auto* obj = new AllgatherPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.h new file mode 100644 index 000000000000..3d7810e6bd49 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/allgatherPlugin.h @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class AllgatherPlugin : public BasePlugin +{ +public: + AllgatherPlugin(std::set<int> group, nvinfer1::DataType type); + + AllgatherPlugin(void const* data, size_t length); + + ~AllgatherPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + const std::string mLayerName; + std::set<int> mGroup; + nvinfer1::DataType mType; + std::shared_ptr<ncclComm_t> mNcclComm; +}; + +class AllgatherPluginCreator : public BaseCreator +{ +public: + AllgatherPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.cpp new file mode 100644 index 000000000000..24d9aff418f7 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.cpp @@ -0,0 +1,986 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "allreducePlugin.h" + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/customAllReduceUtils.h" +#include "tensorrt_llm/common/dataType.h" +#include "tensorrt_llm/common/nvmlWrapper.h" +#include "tensorrt_llm/kernels/customAllReduceKernels.h" +#include "tensorrt_llm/kernels/userbuffers/ub_interface.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include <nccl.h> + +#include <unordered_set> + +using namespace nvinfer1; +using tensorrt_llm::plugins::AllreducePluginCreator; +using tensorrt_llm::plugins::AllreducePlugin; +using tensorrt_llm::kernels::AllReduceFusionOp; +using tensorrt_llm::kernels::AllReduceStrategyType; +using tensorrt_llm::kernels::AllReduceStrategyConfig; +using tensorrt_llm::mpi::MpiTag; + +static char const* ALLREDUCE_PLUGIN_VERSION{"1"}; +static char const* ALLREDUCE_PLUGIN_NAME{"AllReduce"}; +PluginFieldCollection AllreducePluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> AllreducePluginCreator::mPluginAttributes; + +AllreducePlugin::AllreducePlugin(std::set<int> group, nvinfer1::DataType type, AllReduceStrategyType strategy, + AllReduceStrategyConfig config, AllReduceFusionOp op, int32_t counter, float eps, int8_t affine, int8_t bias, + int8_t scale) + : mGroup(std::move(group)) + , mType(type) + , mStrategy(strategy) + , mConfig(config) + , mOp(op) + , mEps(eps) + , mAffine(affine) + , mBias(bias) + , mScale(scale) +{ + check(); +} + +// Parameterized constructor +AllreducePlugin::AllreducePlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mType); + read(d, mStrategy); + read(d, mConfig); + read(d, mOp); + read(d, mEps); + read(d, mAffine); + read(d, mBias); + read(d, mScale); + mGroup.clear(); + int groupItem = 0; + while (d != a + length) + { + read(d, groupItem); + mGroup.insert(groupItem); + } + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); + check(); +} + +void AllreducePlugin::check() noexcept +{ + if (mStrategy != AllReduceStrategyType::UB) + { + TLLM_CHECK(mOp != AllReduceFusionOp::LAST_PROCESS_FOR_UB); + } +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* AllreducePlugin::clone() const noexcept +{ + auto* plugin = new AllreducePlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs AllreducePlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4 && mStrategy == AllReduceStrategyType::UB && mScale) + { + if (outputIndex == 0) + { + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int di = 0; di < ret.nbDims; ++di) + { + ret.d[di] = inputs[0].d[di]; + } + return ret; + } + else if (outputIndex == 2) + { + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int di = 0; di < ret.nbDims; ++di) + { + ret.d[di] = inputs[0].d[di]; + } + auto dimM = exprBuilder.operation( + DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 2], *exprBuilder.constant(128)); + ret.d[ret.nbDims - 2] = exprBuilder.operation(DimensionOperation::kPROD, *dimM, *exprBuilder.constant(128)); + ret.d[ret.nbDims - 1] = exprBuilder.operation( + DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 1], *exprBuilder.constant(16)); + return ret; + } + } + return inputs[0]; +} + +bool AllreducePlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + int base_inputs = 0; + switch (mStrategy) + { + case AllReduceStrategyType::NCCL: + case AllReduceStrategyType::UB: + case AllReduceStrategyType::NCCL_SYMMETRIC: base_inputs = 1; break; + default: base_inputs = 2; break; + } + int fusion_op_extra_inputs = 0; + int scale_idx = 0; + if (mOp != AllReduceFusionOp::NONE) + { + ++fusion_op_extra_inputs; + if (mAffine) + { + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_PREPOST_NORM) + ++fusion_op_extra_inputs; + ++fusion_op_extra_inputs; + } + if (mBias) + { + ++fusion_op_extra_inputs; + } + if (mScale) + { + scale_idx = base_inputs + fusion_op_extra_inputs; + ++fusion_op_extra_inputs; + } + } + + TLLM_CHECK(nbInputs == (base_inputs + fusion_op_extra_inputs)); + + if (pos == 1) + { + switch (mStrategy) + { + case AllReduceStrategyType::NCCL: + case AllReduceStrategyType::UB: + case AllReduceStrategyType::NCCL_SYMMETRIC: break; + default: return (inOut[pos].type == nvinfer1::DataType::kINT64) && (inOut[pos].format == TensorFormat::kLINEAR); + } + } + if (mStrategy == AllReduceStrategyType::UB) + { + if (mScale && pos == scale_idx) + { + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4) + { + if (pos == nbInputs) + { + return (inOut[pos].type == nvinfer1::DataType::kFP4) && (inOut[pos].format == TensorFormat::kLINEAR); + } + if (pos == (nbInputs + 2)) + { + return (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); + } + } + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_FP8) + { + if (pos == nbInputs) + { + return (inOut[pos].type == nvinfer1::DataType::kFP8) && (inOut[pos].format == TensorFormat::kLINEAR); + } + } + } + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); +} + +void AllreducePlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t AllreducePlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +AllReduceStrategyType AllreducePlugin::selectImplementation( + size_t messageSize, int worldSize, nvinfer1::DataType type) noexcept +{ + bool const isAuto = (mStrategy == AllReduceStrategyType::AUTO); + + bool const forceDeterministic = common::getEnvForceDeterministicAllReduce(); + if (!mIsP2PSupported) + { + if (!isAuto) + { + TLLM_LOG_INFO("Since Peer to Peer not supported, fallback to AllReduceStrategy: NCCL_SYMMETRIC"); + } + else if (forceDeterministic) + { + TLLM_LOG_WARNING( + "Since Peer to Peer not supported, fallback to AllReduceStrategy: NCCL_SYMMETRIC. NCCL_SYMMETRIC might " + "produce " + "non-deterministic results."); + } + return AllReduceStrategyType::NCCL_SYMMETRIC; + } + + if (isAuto && !mIsNVLINKSupported && !forceDeterministic) + { + return AllReduceStrategyType::NCCL_SYMMETRIC; + } + + auto const maxWorkspaceSize = utils::customAllReduceUtils::getMaxRequiredWorkspaceSize(worldSize); + + AllReduceStrategyType strat = AllReduceStrategyType::NCCL_SYMMETRIC; + auto const messageSizeBytes = messageSize * common::getDTypeSize(type); + + if (messageSizeBytes <= maxWorkspaceSize) + { + // In some instances, the two-shot strategy has exhibited significant performance issues. + // As a temporary measure, we have disabled the two-shot strategy. + // TODO: remove this WAR after https://nvbugspro.nvidia.com/bug/4718747 is fixed. + if (!isAuto) + { + strat = mStrategy; + } + else if (forceDeterministic) + { + strat = AllReduceStrategyType::ONESHOT; + } + else if (worldSize <= 2) + { + strat = AllReduceStrategyType::ONESHOT; + } + else if (worldSize <= 4) + { + if (messageSizeBytes < 1 * 1000 * 1000) + { + strat = AllReduceStrategyType::ONESHOT; + } + else + { + strat = AllReduceStrategyType::NCCL_SYMMETRIC; + } + } + else + { + if (messageSizeBytes < 500 * 1000) + { + strat = AllReduceStrategyType::ONESHOT; + } + else + { + strat = AllReduceStrategyType::NCCL_SYMMETRIC; + } + } + + if (!kernels::configurationSupported(strat, messageSize, worldSize, type)) + { + if (!isAuto) + { + TLLM_LOG_WARNING("Since not aligned, fallback to AllReduceStrategy: NCCL_SYMMETRIC"); + } + else if (forceDeterministic) + { + TLLM_LOG_WARNING( + "Since not aligned, fallback to AllReduceStrategy: NCCL_SYMMETRIC. NCCL_SYMMETRIC might produce " + "non-deterministic results."); + } + strat = AllReduceStrategyType::NCCL_SYMMETRIC; + } + } + else + { + if (!isAuto) + { + TLLM_LOG_WARNING("Since messageSize > maxWorkspace, fallback to AllReduceStrategy: NCCL_SYMMETRIC"); + } + else if (forceDeterministic) + { + TLLM_LOG_WARNING( + "Since messageSize > maxWorkspace, fallback to AllReduceStrategy: NCCL_SYMMETRIC. NCCL_SYMMETRIC might " + "produce " + "non-deterministic results."); + } + strat = AllReduceStrategyType::NCCL_SYMMETRIC; + } + + return strat; +} + +int AllreducePlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + size_t size = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) + { + size *= inputDesc[0].dims.d[i]; + } + + kernels::AllReduceStrategyType runtimeStrategy; + + static char* forceNcclAllReduceStrategyChar = std::getenv("FORCE_NCCL_ALL_REDUCE_STRATEGY"); + bool forceNcclAllReduceStrategy = (forceNcclAllReduceStrategyChar != nullptr); + if (forceNcclAllReduceStrategy || mStrategy == AllReduceStrategyType::NCCL) + { + runtimeStrategy = AllReduceStrategyType::NCCL; + } + else if (mStrategy == AllReduceStrategyType::NCCL_SYMMETRIC) + { + runtimeStrategy = AllReduceStrategyType::NCCL_SYMMETRIC; + } + else if (mStrategy == AllReduceStrategyType::UB) + { + runtimeStrategy = AllReduceStrategyType::UB; + } + else + { + runtimeStrategy = selectImplementation(size, mGroup.size(), mType); + } + + // Log runtime strategy + auto const rank = COMM_SESSION.getRank(); + switch (runtimeStrategy) + { + case AllReduceStrategyType::NCCL: + { + TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: NCCL", rank); + break; + } + case AllReduceStrategyType::NCCL_SYMMETRIC: + { + TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: NCCL_SYMMETRIC", rank); + break; + } + case AllReduceStrategyType::ONESHOT: + { + TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: ONESHOT", rank); + break; + } + case AllReduceStrategyType::TWOSHOT: + { + TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: TWOSHOT", rank); + break; + } + case AllReduceStrategyType::UB: + { + TLLM_LOG_DEBUG("AllReducePlugin strategy for rank %d: UB", rank); + break; + } + default: break; + } + + if (runtimeStrategy == AllReduceStrategyType::NCCL || runtimeStrategy == AllReduceStrategyType::NCCL_SYMMETRIC) + { + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM || mOp == AllReduceFusionOp::RESIDUAL_RMS_PREPOST_NORM) + { + NCCLCHECK(ncclAllReduce(inputs[0], outputs[1], size, (*getDtypeMap())[mType], ncclSum, *mNcclComm, stream)); + tensorrt_llm::kernels::AllReduceParams params; + int fusion_ptr_idx = 0; + if (mStrategy == AllReduceStrategyType::NCCL || mStrategy == AllReduceStrategyType::NCCL_SYMMETRIC) + { + fusion_ptr_idx = 1; + } + else + { + fusion_ptr_idx = 2; + } + params.fusion_params.bias_buffer = mBias ? inputs[fusion_ptr_idx++] : nullptr; + params.fusion_params.residual_buffer = inputs[fusion_ptr_idx++]; + params.fusion_params.weight_buffer = mAffine ? inputs[fusion_ptr_idx++] : nullptr; + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_PREPOST_NORM) + { + params.fusion_params.weight_buffer_pre_residual_norm = mAffine ? inputs[fusion_ptr_idx++] : nullptr; + } + params.local_output_buffer_ptr = outputs[0]; + params.elts_total = size; + params.fusion_params.hidden_size = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; + params.fusion_params.eps = mEps; + params.fusion_params.intermediate_buffer = outputs[1]; + TLLM_LOG_DEBUG("residualRmsNorm called"); + tensorrt_llm::kernels::residualRmsNorm(params, mType, stream, mOp); + } + else + { + NCCLCHECK(ncclAllReduce(inputs[0], outputs[0], size, (*getDtypeMap())[mType], ncclSum, *mNcclComm, stream)); + } + } + else if (runtimeStrategy == AllReduceStrategyType::UB) + { + TLLM_CHECK(!mBias); + + size_t dtype_size = tensorrt_llm::common::getDTypeSize(mType); + int hidden_size = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; + + TLLM_CHECK_WITH_INFO(tensorrt_llm::runtime::ub::ub_is_initialized(), "UserBuffer has not been initialized!"); + auto ub_buffer0 = tensorrt_llm::runtime::ub::ub_get(0); + auto ub_buffer1 = tensorrt_llm::runtime::ub::ub_get(1); + TLLM_CHECK(inputs[0] == ub_buffer0.addr); + auto ub_comm = tensorrt_llm::runtime::ub::ub_comm(); + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_FP8) + { + TLLM_CHECK(mAffine); + TLLM_CHECK(mScale); + TLLM_CHECK(outputs[0] == ub_buffer1.addr); + void* residual = const_cast<void*>(inputs[1]); + void* gamma = const_cast<void*>(inputs[2]); + float* scale = const_cast<float*>(reinterpret_cast<float const*>(inputs[3])); + tensorrt_llm::kernels::ub::allreduce2_userbuff_inplace_rmsnorm_quant_launcher(ub_buffer0.handle, 0, + ub_buffer1.handle, 0, size, hidden_size, nullptr, gamma, mEps, scale, residual, outputs[1], mType, + ub_comm, stream); + } + else if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4) + { + auto ub_buffer2 = tensorrt_llm::runtime::ub::ub_get(2); + TLLM_CHECK(mAffine); + TLLM_CHECK(mScale); + TLLM_CHECK(outputs[0] == ub_buffer1.addr); + TLLM_CHECK(outputs[2] == ub_buffer2.addr); + void* residual = const_cast<void*>(inputs[1]); + void* gamma = const_cast<void*>(inputs[2]); + float* scale = const_cast<float*>(reinterpret_cast<float const*>(inputs[3])); + tensorrt_llm::kernels::ub::allreduce2_userbuff_inplace_rmsnorm_quant_fp4_launcher(ub_buffer0.handle, 0, + ub_buffer1.handle, 0, ub_buffer2.handle, 0, size, hidden_size, nullptr, gamma, mEps, scale, residual, + outputs[1], mType, ub_comm, stream); + } + else if (mOp == AllReduceFusionOp::LAST_PROCESS_FOR_UB) + { + TLLM_CHECK(outputs[1] == ub_buffer1.addr); + void* residual = const_cast<void*>(inputs[1]); + tensorrt_llm::kernels::ub::allreduce2_userbuff_inplace_launcher( + ub_buffer0.handle, 0, size, mType, ub_comm, stream); + tensorrt_llm::kernels::ub::allgather2_userbuff_residual_launcher( + ub_buffer1.handle, 0, size, hidden_size, residual, mType, ub_comm, stream); + TLLM_CUDA_CHECK( + cudaMemcpyAsync(outputs[0], ub_buffer0.addr, size * dtype_size, cudaMemcpyDeviceToDevice, stream)); + } + else if (mOp == AllReduceFusionOp::NONE) + { + tensorrt_llm::kernels::ub::allreduce2_userbuff_inplace_launcher( + ub_buffer0.handle, 0, size, mType, ub_comm, stream); + TLLM_CUDA_CHECK( + cudaMemcpyAsync(outputs[0], ub_buffer0.addr, size * dtype_size, cudaMemcpyDeviceToDevice, stream)); + } + else + { + TLLM_CHECK_WITH_INFO(false, "Unsupported UB allreduce fusion op"); + } + } + else + { + auto const tpSize = mGroup.size(); + int tpRank = 0; + for (auto const& currentRank : mGroup) + { + if (rank == currentRank) + break; + ++tpRank; + } + + int token_num = size / inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; + int hidden_size = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; + auto params = tensorrt_llm::kernels::AllReduceParams::deserialize( + reinterpret_cast<int64_t*>(const_cast<void*>(inputs[1])), tpSize, tpRank, mType, token_num, hidden_size, + mOp); + + params.local_output_buffer_ptr = outputs[0]; + params.local_input_buffer_ptr = inputs[0]; + params.elts_total = size; + + int fusion_ptr_idx = 2; + params.fusion_params.bias_buffer = mBias ? inputs[fusion_ptr_idx++] : nullptr; + params.fusion_params.residual_buffer = inputs[fusion_ptr_idx++]; + params.fusion_params.weight_buffer = mAffine ? inputs[fusion_ptr_idx++] : nullptr; + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_PREPOST_NORM) + params.fusion_params.weight_buffer_pre_residual_norm = mAffine ? inputs[fusion_ptr_idx++] : nullptr; + params.fusion_params.hidden_size = hidden_size; + params.fusion_params.eps = mEps; + params.fusion_params.intermediate_buffer = outputs[1]; + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM) + { + for (size_t i = 0; i < tpSize; ++i) + { + params.fusion_params.lamport_peer_comm_buffer_ptrs[i] + = reinterpret_cast<void**>(const_cast<void*>(inputs[1]))[tpSize * 4 + i]; + params.fusion_params.lamport_peer_comm_buffer_ptrs[i + tensorrt_llm::kernels::MAX_RANKS_PER_NODE] + = reinterpret_cast<void**>(const_cast<void*>(inputs[1]))[tpSize * 5 + i]; + params.fusion_params.lamport_peer_comm_buffer_ptrs[i + tensorrt_llm::kernels::MAX_RANKS_PER_NODE * 2] + = reinterpret_cast<void**>(const_cast<void*>(inputs[1]))[tpSize * 6 + i]; + } + } + TLLM_LOG_DEBUG("customAllReduce called"); + tensorrt_llm::kernels::customAllReduce(params, mType, runtimeStrategy, mConfig, mOp, stream); + } + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType AllreducePlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index < getNbOutputs()); + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4) + { + if (index == 0) + { + return nvinfer1::DataType::kFP4; + } + else if (index == 2) + { + return nvinfer1::DataType::kFP8; + } + } + if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_FP8) + { + if (index == 0) + { + return nvinfer1::DataType::kFP8; + } + } + return inputTypes[0]; +} + +// IPluginV2 Methods + +char const* AllreducePlugin::getPluginType() const noexcept +{ + return ALLREDUCE_PLUGIN_NAME; +} + +char const* AllreducePlugin::getPluginVersion() const noexcept +{ + return ALLREDUCE_PLUGIN_VERSION; +} + +int AllreducePlugin::getNbOutputs() const noexcept +{ + if (mOp == AllReduceFusionOp::NONE) + { + return 1; + } + else if (mOp == AllReduceFusionOp::RESIDUAL_RMS_NORM_QUANT_NVFP4) + { + return 3; + } + else + { + return 2; + } +} + +bool AllreducePlugin::isCustomAllReduceSupported(int ranks_per_node) const noexcept +{ + constexpr bool isCudaVersionSupported = +#if defined(CUDART_VERSION) && CUDART_VERSION >= 11020 + true; +#else + false; +#endif + + return isCudaVersionSupported && (ranks_per_node % 2 == 0) + && (static_cast<size_t>(ranks_per_node) <= kernels::MAX_RANKS_PER_NODE) && (ranks_per_node > 0); +} + +using tensorrt_llm::common::NvmlManager; +using tensorrt_llm::common::NVMLWrapper; + +std::set<int> getLocalGroup(std::set<int> const& group) +{ + auto const myRank = COMM_SESSION.getRank(); + auto const myLocalRank = LOCAL_COMM_SESSION.getRank(); + auto const localSize = LOCAL_COMM_SESSION.getSize(); + + std::vector<int32_t> ranks(localSize, 0); + std::vector<int32_t> localRanks(localSize, 0); + if (group.size() >= static_cast<size_t>(localSize)) + { + LOCAL_COMM_SESSION.allgather(&myRank, ranks.data(), 1, tensorrt_llm::mpi::MpiType::kINT32); + LOCAL_COMM_SESSION.allgather(&myLocalRank, localRanks.data(), 1, tensorrt_llm::mpi::MpiType::kINT32); + } + else + { + if (myRank == *group.begin()) + { + ranks.clear(); + int rank; + ranks.push_back(myRank); + for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it) + { + COMM_SESSION.recvValue(rank, *it, MpiTag::kDefault); + ranks.push_back(rank); + } + for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it) + { + COMM_SESSION.send(ranks.data(), localSize, tensorrt_llm::mpi::MpiType::kINT32, *it, MpiTag::kDefault); + } + + localRanks.clear(); + localRanks.push_back(myLocalRank); + for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it) + { + COMM_SESSION.recvValue(rank, *it, MpiTag::kDefault); + localRanks.push_back(rank); + } + for (auto it = std::next(std::begin(group), 1); it != group.end(); ++it) + { + COMM_SESSION.send( + localRanks.data(), localSize, tensorrt_llm::mpi::MpiType::kINT32, *it, MpiTag::kDefault); + } + } + else + { + COMM_SESSION.sendValue(myRank, *group.begin(), MpiTag::kDefault); + COMM_SESSION.recv( + ranks.data(), localSize, tensorrt_llm::mpi::MpiType::kINT32, *group.begin(), MpiTag::kDefault); + + COMM_SESSION.sendValue(myLocalRank, *group.begin(), MpiTag::kDefault); + COMM_SESSION.recv( + localRanks.data(), localSize, tensorrt_llm::mpi::MpiType::kINT32, *group.begin(), MpiTag::kDefault); + } + } + + std::set<int> localGroup; + for (size_t i = 0; i < ranks.size(); ++i) + { + auto rank = ranks[i]; + if (group.find(rank) != group.end()) + { + localGroup.insert(localRanks[i]); + } + } + return localGroup; +} + +void AllreducePlugin::initGroupTopology() noexcept +{ + static std::map<std::set<int>, std::tuple<bool, bool>> cache; + if (cache.find(mGroup) != cache.end()) + { + auto [isNVLINKSupported, isP2PSupported] = cache[mGroup]; + mIsNVLINKSupported = isNVLINKSupported; + mIsP2PSupported = isP2PSupported; + return; + } + setGroupTopology(); + cache[mGroup] = {mIsNVLINKSupported, mIsP2PSupported}; +} + +void AllreducePlugin::setGroupTopology() noexcept +{ + auto const rank = COMM_SESSION.getRank(); + TLLM_LOG_INFO("Detecting local TP group for rank %d", rank); + std::set<int> localGroup = getLocalGroup(mGroup); + if (mGroup.size() != localGroup.size()) + { + mIsP2PSupported = false; + mIsNVLINKSupported = false; + TLLM_LOG_INFO("Found inter-node TP group for rank %d", rank); + return; + } + TLLM_LOG_INFO("TP group is intra-node for rank %d", rank); + + NvmlManager nvmlManager; + auto const& nvml = nvmlManager.sharedWrapper(); + std::unordered_set<int> visitedDevice; + mIsP2PSupported = true; + mIsNVLINKSupported = true; + + // Use cudaDeviceCanAccessPeer to determine whether p2p is supported, + // and use nvml to determine whether there are nvlink links between ranks. + for (int firstDeviceId : localGroup) + { + for (int secondDeviceId : localGroup) + { + if (firstDeviceId == secondDeviceId || visitedDevice.find(secondDeviceId) != visitedDevice.end()) + { + continue; + } + + int canAccessPeer = 0; + TLLM_CUDA_CHECK(cudaDeviceCanAccessPeer(&canAccessPeer, firstDeviceId, secondDeviceId)); + + if (!canAccessPeer) + { + mIsP2PSupported = false; + mIsNVLINKSupported = false; + + return; + } + + nvmlDevice_t firstDevice; + NVML_CHECK(nvml->nvmlDeviceGetHandleByIndex(firstDeviceId, &firstDevice)); + + bool isNVLINK = false; + + for (unsigned int link = 0; link < NVML_NVLINK_MAX_LINKS; link++) + { + nvmlPciInfo_t remotePciInfo; + if (nvml->nvmlDeviceGetNvLinkRemotePciInfo(firstDevice, link, &remotePciInfo) != NVML_SUCCESS) + { + continue; + } + + nvmlDevice_t remoteDevice; + auto const result = nvml->nvmlDeviceGetHandleByPciBusId(remotePciInfo.busId, &remoteDevice); + + if (result == NVML_SUCCESS) + { + // Two GPUs are connected directly through nvlink + unsigned int remoteDeviceId; + NVML_CHECK(nvml->nvmlDeviceGetIndex(remoteDevice, &remoteDeviceId)); + + if (remoteDeviceId == static_cast<unsigned int>(secondDeviceId)) + { + isNVLINK = true; + } + } + else if (result == NVML_ERROR_NOT_FOUND) + { + // Maybe Two GPUs are connected via nvswitch, + // now remotePciInfo represents the pci information of nvswitch, + // determine whether nvlink is supported by whether two GPUs are connected to the same nvswitch. + nvmlDevice_t secondDevice; + NVML_CHECK(nvml->nvmlDeviceGetHandleByIndex(secondDeviceId, &secondDevice)); + + for (unsigned int secondLink = 0; secondLink < NVML_NVLINK_MAX_LINKS; secondLink++) + { + nvmlPciInfo_t secondRemotePciInfo; + if (nvml->nvmlDeviceGetNvLinkRemotePciInfo(secondDevice, secondLink, &secondRemotePciInfo) + != NVML_SUCCESS) + { + continue; + } + + if (strcmp(remotePciInfo.busId, secondRemotePciInfo.busId) == 0) + { + isNVLINK = true; + break; + } + } + } + else + { + NVML_CHECK(result); + } + + if (isNVLINK) + { + break; + } + } + + mIsNVLINKSupported &= isNVLINK; + } + visitedDevice.insert(firstDeviceId); + } +} + +int AllreducePlugin::initialize() noexcept +{ + if (isBuilding()) + { + return 0; + } + + TLLM_LOG_TRACE("%s start for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); + mNcclComm = getComm(mGroup); + if (mStrategy != AllReduceStrategyType::NCCL) + { + initGroupTopology(); + } + + TLLM_LOG_TRACE("%s stop for rank %d", __PRETTY_FUNCTION__, COMM_SESSION.getRank()); + return 0; +} + +void AllreducePlugin::terminate() noexcept {} + +size_t AllreducePlugin::getSerializationSize() const noexcept +{ + return sizeof(int) * mGroup.size() + sizeof(mType) + sizeof(mStrategy) + sizeof(mConfig) + sizeof(mOp) + + sizeof(mEps) + sizeof(mAffine) + sizeof(mBias) + sizeof(mScale); +} + +void AllreducePlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mStrategy); + write(d, mConfig); + write(d, mOp); + write(d, mEps); + write(d, mAffine); + write(d, mBias); + write(d, mScale); + for (auto it = mGroup.begin(); it != mGroup.end(); ++it) + { + write(d, *it); + } + TLLM_CHECK(d == a + getSerializationSize()); +} + +void AllreducePlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +AllreducePluginCreator::AllreducePluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("group", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("strategy", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("config", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("fusion_op", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("counter", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("eps", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("affine", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("bias", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("scale", nullptr, PluginFieldType::kINT8)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* AllreducePluginCreator::getPluginName() const noexcept +{ + return ALLREDUCE_PLUGIN_NAME; +} + +char const* AllreducePluginCreator::getPluginVersion() const noexcept +{ + return ALLREDUCE_PLUGIN_VERSION; +} + +PluginFieldCollection const* AllreducePluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* AllreducePluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + std::set<int> group; + nvinfer1::DataType type{}; + AllReduceStrategyType strategy{}; + AllReduceStrategyConfig config{}; + AllReduceFusionOp fusion_op{}; + int32_t counter{}; + float eps{}; + int8_t affine{}; + int8_t bias{}; + int8_t scale{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "group")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + auto const* r = static_cast<int const*>(fields[i].data); + for (int j = 0; j < fields[i].length; ++j) + { + group.insert(*r); + ++r; + } + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "strategy")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + strategy = static_cast<AllReduceStrategyType>(*static_cast<int8_t const*>(fields[i].data)); + } + else if (!strcmp(attrName, "config")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + config = static_cast<AllReduceStrategyConfig>(*static_cast<int8_t const*>(fields[i].data)); + } + else if (!strcmp(attrName, "fusion_op")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + fusion_op = static_cast<AllReduceFusionOp>(*static_cast<int8_t const*>(fields[i].data)); + } + else if (!strcmp(attrName, "counter")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + counter = *static_cast<int32_t const*>(fields[i].data); + } + else if (!strcmp(attrName, "eps")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + eps = *static_cast<float const*>(fields[i].data); + } + else if (!strcmp(attrName, "affine")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + affine = *static_cast<int8_t const*>(fields[i].data); + } + else if (!strcmp(attrName, "bias")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + bias = *static_cast<int8_t const*>(fields[i].data); + } + else if (!strcmp(attrName, "scale")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + scale = *static_cast<int8_t const*>(fields[i].data); + } + } + try + { + auto* obj = new AllreducePlugin(group, type, strategy, config, fusion_op, counter, eps, affine, bias, scale); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* AllreducePluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call AllreducePlugin::destroy() + try + { + auto* obj = new AllreducePlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.h new file mode 100644 index 000000000000..881fbf3b89a5 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.h @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/kernels/customAllReduceKernels.h" +#include "tensorrt_llm/plugins/common/plugin.h" + +#include <cassert> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ +namespace tk = ::tensorrt_llm::kernels; + +class AllreducePlugin : public BasePlugin +{ +public: + AllreducePlugin(std::set<int> group, nvinfer1::DataType type, tk::AllReduceStrategyType strategy, + tk::AllReduceStrategyConfig config, tk::AllReduceFusionOp op, int32_t counter, float eps, int8_t affine, + int8_t bias, int8_t scale); + + AllreducePlugin(void const* data, size_t length); + + ~AllreducePlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + bool isCustomAllReduceSupported(int ranks_per_node) const noexcept; + void initGroupTopology() noexcept; + void setGroupTopology() noexcept; + tk::AllReduceStrategyType selectImplementation(size_t messageSize, int worldSize, nvinfer1::DataType type) noexcept; + void check() noexcept; + +private: + std::string const mLayerName; + std::set<int> mGroup; + bool mIsNVLINKSupported; + bool mIsP2PSupported; + nvinfer1::DataType mType; + tk::AllReduceStrategyType mStrategy; + tk::AllReduceStrategyConfig mConfig; + tk::AllReduceFusionOp mOp; + float mEps; + std::shared_ptr<ncclComm_t> mNcclComm; + int8_t mAffine; + int8_t mBias; + int8_t mScale; +}; + +class AllreducePluginCreator : public BaseCreator +{ +public: + AllreducePluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.cpp new file mode 100644 index 000000000000..089ed31175b2 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.cpp @@ -0,0 +1,252 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "recvPlugin.h" + +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include <nccl.h> + +using namespace nvinfer1; +using tensorrt_llm::plugins::RecvPluginCreator; +using tensorrt_llm::plugins::RecvPlugin; +using tensorrt_llm::mpi::MpiTag; + +static char const* RECV_PLUGIN_VERSION{"1"}; +static char const* RECV_PLUGIN_NAME{"Recv"}; +PluginFieldCollection RecvPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> RecvPluginCreator::mPluginAttributes; + +RecvPlugin::RecvPlugin(int srcRank, nvinfer1::DataType type) + : mSrcRank(srcRank) + , mType(type) +{ +} + +// Parameterized constructor +RecvPlugin::RecvPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mType); + read(d, mSrcRank); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* RecvPlugin::clone() const noexcept +{ + auto* plugin = new RecvPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs RecvPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + return inputs[0]; +} + +bool RecvPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); +} + +void RecvPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t RecvPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int RecvPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + size_t size = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) + { + size *= inputDesc[0].dims.d[i]; + } + TLLM_LOG_DEBUG("start ncclRecv with size %d", size); + NCCLCHECK(ncclRecv(outputs[0], size, (*getDtypeMap())[inputDesc[0].type], 0, mComm, stream)); + TLLM_LOG_DEBUG("end ncclRecv with size %d", size); + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType RecvPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + assert(index == 0); + return inputTypes[0]; +} + +// IPluginV2 Methods + +char const* RecvPlugin::getPluginType() const noexcept +{ + return RECV_PLUGIN_NAME; +} + +char const* RecvPlugin::getPluginVersion() const noexcept +{ + return RECV_PLUGIN_VERSION; +} + +int RecvPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int RecvPlugin::initialize() noexcept +{ + if (isBuilding()) + { + return 0; + } + ncclUniqueId id; + COMM_SESSION.recvValue(id, mSrcRank, MpiTag::kDefault); +// Need static connection initialization for accurate KV cache size estimation +#if defined(_WIN32) + if (getenv("NCCL_RUNTIME_CONNECT") == nullptr) + _putenv_s("NCCL_RUNTIME_CONNECT", "0"); +#else + setenv("NCCL_RUNTIME_CONNECT", "0", 0); +#endif // _WIN32 + NCCLCHECK(ncclCommInitRank(&mComm, 2, id, 1)); + return 0; +} + +void RecvPlugin::terminate() noexcept +{ + if (isBuilding()) + { + return; + } + NCCLCHECK(ncclCommDestroy(mComm)); +} + +size_t RecvPlugin::getSerializationSize() const noexcept +{ + return sizeof(mSrcRank) + sizeof(mType); +} + +void RecvPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mSrcRank); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void RecvPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +RecvPluginCreator::RecvPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("src_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* RecvPluginCreator::getPluginName() const noexcept +{ + return RECV_PLUGIN_NAME; +} + +char const* RecvPluginCreator::getPluginVersion() const noexcept +{ + return RECV_PLUGIN_VERSION; +} + +PluginFieldCollection const* RecvPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* RecvPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int srcRank{}; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "src_rank")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + srcRank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + + try + { + auto* obj = new RecvPlugin(srcRank, type); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* RecvPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call RecvPlugin::destroy() + try + { + auto* obj = new RecvPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.h new file mode 100644 index 000000000000..5c8eedfb5218 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/recvPlugin.h @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class RecvPlugin : public BasePlugin +{ +public: + RecvPlugin(int srcRank, nvinfer1::DataType type); + + RecvPlugin(void const* data, size_t length); + + ~RecvPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + ncclComm_t mComm; // TODO: Remove this + int mSrcRank; + nvinfer1::DataType mType; +}; + +class RecvPluginCreator : public BaseCreator +{ +public: + RecvPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.cpp new file mode 100644 index 000000000000..fe17c44fc418 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.cpp @@ -0,0 +1,252 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "reduceScatterPlugin.h" + +#include <cassert> +#include <nccl.h> + +using namespace nvinfer1; +using tensorrt_llm::plugins::ReduceScatterPluginCreator; +using tensorrt_llm::plugins::ReduceScatterPlugin; + +static char const* REDUCE_SCATTER_PLUGIN_VERSION{"1"}; +static char const* REDUCE_SCATTER_PLUGIN_NAME{"ReduceScatter"}; +PluginFieldCollection ReduceScatterPluginCreator::mFC{}; +std::vector<PluginField> ReduceScatterPluginCreator::mPluginAttributes; + +ReduceScatterPlugin::ReduceScatterPlugin(std::set<int> group, nvinfer1::DataType type) + : mGroup(std::move(group)) + , mType(type) +{ +} + +// Parameterized constructor +ReduceScatterPlugin::ReduceScatterPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mType); + mGroup.clear(); + int groupItem = 0; + while (d != a + length) + { + read(d, groupItem); + mGroup.insert(groupItem); + } + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* ReduceScatterPlugin::clone() const noexcept +{ + auto* plugin = new ReduceScatterPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs ReduceScatterPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + auto output = inputs[0]; + output.d[0] + = exprBuilder.operation(DimensionOperation::kFLOOR_DIV, *output.d[0], *exprBuilder.constant(mGroup.size())); + return output; +} + +bool ReduceScatterPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); +} + +void ReduceScatterPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t ReduceScatterPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int ReduceScatterPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + size_t size = 1; + for (int i = 0; i < outputDesc[0].dims.nbDims; ++i) + { + size *= outputDesc[0].dims.d[i]; + } + + TLLM_CHECK_WITH_INFO(mNcclComm.get() != nullptr, "mNcclComm should be initialized before used"); + NCCLCHECK(ncclReduceScatter( + inputs[0], outputs[0], size, (*getDtypeMap())[inputDesc[0].type], ncclSum, *mNcclComm, stream)); + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType ReduceScatterPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + assert(index == 0); + return inputTypes[0]; +} + +// IPluginV2 Methods + +char const* ReduceScatterPlugin::getPluginType() const noexcept +{ + return REDUCE_SCATTER_PLUGIN_NAME; +} + +char const* ReduceScatterPlugin::getPluginVersion() const noexcept +{ + return REDUCE_SCATTER_PLUGIN_VERSION; +} + +int ReduceScatterPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int ReduceScatterPlugin::initialize() noexcept +{ + if (isBuilding()) + { + return 0; + } + mNcclComm = getComm(mGroup); + return 0; +} + +void ReduceScatterPlugin::terminate() noexcept {} + +size_t ReduceScatterPlugin::getSerializationSize() const noexcept +{ + return sizeof(int) * mGroup.size() + sizeof(mType); +} + +void ReduceScatterPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + for (auto it = mGroup.begin(); it != mGroup.end(); ++it) + { + write(d, *it); + } + TLLM_CHECK(d == a + getSerializationSize()); +} + +void ReduceScatterPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +ReduceScatterPluginCreator::ReduceScatterPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("group", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* ReduceScatterPluginCreator::getPluginName() const noexcept +{ + return REDUCE_SCATTER_PLUGIN_NAME; +} + +char const* ReduceScatterPluginCreator::getPluginVersion() const noexcept +{ + return REDUCE_SCATTER_PLUGIN_VERSION; +} + +PluginFieldCollection const* ReduceScatterPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* ReduceScatterPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + std::set<int> group; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "group")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + auto const* r = static_cast<int const*>(fields[i].data); + for (int j = 0; j < fields[i].length; ++j) + { + group.insert(*r); + ++r; + } + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + + try + { + auto* obj = new ReduceScatterPlugin(group, type); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* ReduceScatterPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call ReduceScatterPlugin::destroy() + try + { + auto* obj = new ReduceScatterPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.h new file mode 100644 index 000000000000..c630b57a2b98 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/reduceScatterPlugin.h @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class ReduceScatterPlugin : public BasePlugin +{ +public: + ReduceScatterPlugin(std::set<int> group, nvinfer1::DataType type); + + ReduceScatterPlugin(void const* data, size_t length); + + ~ReduceScatterPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + const std::string mLayerName; + std::set<int> mGroup; + nvinfer1::DataType mType; + std::shared_ptr<ncclComm_t> mNcclComm; +}; + +class ReduceScatterPluginCreator : public BaseCreator +{ +public: + ReduceScatterPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.cpp b/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.cpp new file mode 100644 index 000000000000..81d66aa8211e --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.cpp @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "sendPlugin.h" + +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include <cassert> +#include <nccl.h> + +using namespace nvinfer1; +using tensorrt_llm::plugins::SendPluginCreator; +using tensorrt_llm::plugins::SendPlugin; +using tensorrt_llm::mpi::MpiTag; + +static char const* SEND_PLUGIN_VERSION{"1"}; +static char const* SEND_PLUGIN_NAME{"Send"}; +PluginFieldCollection SendPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> SendPluginCreator::mPluginAttributes; + +SendPlugin::SendPlugin(int tgtRank, nvinfer1::DataType type) + : mTgtRank(tgtRank) + , mType(type) +{ +} + +// Parameterized constructor +SendPlugin::SendPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mType); + read(d, mTgtRank); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* SendPlugin::clone() const noexcept +{ + auto* plugin = new SendPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs SendPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + return inputs[0]; +} + +bool SendPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); +} + +void SendPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t SendPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int SendPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + size_t size = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims; ++i) + { + size *= inputDesc[0].dims.d[i]; + } + + TLLM_LOG_DEBUG("start ncclSend with size %d", size); + NCCLCHECK(ncclSend(inputs[0], size, (*getDtypeMap())[inputDesc[0].type], 1, mComm, stream)); + TLLM_LOG_DEBUG("end ncclSend with size %d", size); + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType SendPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + assert(index == 0); + return inputTypes[0]; +} + +// IPluginV2 Methods + +char const* SendPlugin::getPluginType() const noexcept +{ + return SEND_PLUGIN_NAME; +} + +char const* SendPlugin::getPluginVersion() const noexcept +{ + return SEND_PLUGIN_VERSION; +} + +int SendPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int SendPlugin::initialize() noexcept +{ + if (isBuilding()) + { + return 0; + } + + ncclUniqueId id; + ncclGetUniqueId(&id); + COMM_SESSION.sendValue(id, mTgtRank, MpiTag::kDefault); +// Need static connection initialization for accurate KV cache size estimation +#if defined(_WIN32) + if (getenv("NCCL_RUNTIME_CONNECT") == nullptr) + _putenv_s("NCCL_RUNTIME_CONNECT", "0"); +#else + setenv("NCCL_RUNTIME_CONNECT", "0", 0); +#endif // _WIN32 + NCCLCHECK(ncclCommInitRank(&mComm, 2, id, 0)); + return 0; +} + +void SendPlugin::terminate() noexcept +{ + if (isBuilding()) + { + return; + } + NCCLCHECK(ncclCommDestroy(mComm)); +} + +size_t SendPlugin::getSerializationSize() const noexcept +{ + return sizeof(mTgtRank) + sizeof(mType); +} + +void SendPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mTgtRank); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void SendPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +SendPluginCreator::SendPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("tgt_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* SendPluginCreator::getPluginName() const noexcept +{ + return SEND_PLUGIN_NAME; +} + +char const* SendPluginCreator::getPluginVersion() const noexcept +{ + return SEND_PLUGIN_VERSION; +} + +PluginFieldCollection const* SendPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* SendPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int tgtRank{}; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "tgt_rank")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + tgtRank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + + try + { + auto* obj = new SendPlugin(tgtRank, type); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* SendPluginCreator::deserializePlugin(char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call SendPlugin::destroy() + try + { + auto* obj = new SendPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.h b/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.h new file mode 100644 index 000000000000..0d36b0ebff28 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/ncclPlugin/sendPlugin.h @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class SendPlugin : public BasePlugin +{ +public: + SendPlugin(int tgtRank, nvinfer1::DataType type); + + SendPlugin(void const* data, size_t length); + + ~SendPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + ncclComm_t mComm; // TODO: Remove this + int mTgtRank; + nvinfer1::DataType mType; +}; + +class SendPluginCreator : public BaseCreator +{ +public: + SendPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/CMakeLists.txt new file mode 100755 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.cpp b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.cpp new file mode 100644 index 000000000000..166f1cc32cbe --- /dev/null +++ b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.cpp @@ -0,0 +1,416 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qserveGemmPlugin.h" +#include "tensorrt_llm/kernels/qserveGemm.h" +#include <cassert> +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::QServeGemmPluginCreator; +using tensorrt_llm::plugins::QServeGemmPlugin; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; +using namespace tensorrt_llm::kernels::qserve; + +static char const* QSERVE_GEMM_PLUGIN_VERSION{"1"}; +static char const* QSERVE_GEMM_PLUGIN_NAME{"QServeGemm"}; + +PluginFieldCollection QServeGemmPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> QServeGemmPluginCreator::mPluginAttributes; + +namespace tensorrt_llm::plugins +{ + +QServeGemmPlugin::QServeGemmPlugin( + // QuantMode quantMode, + nvinfer1::DataType dtype, int groupSize) +{ + init(dtype, groupSize); +} + +QServeGemmPlugin::QServeGemmPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + + nvinfer1::DataType type; + unsigned int quantMode; + int groupSize; + + read(d, quantMode); + read(d, type); + read(d, groupSize); + + read(d, mDims); + + // mQuantMode = QuantMode(quantMode); + + init(type, groupSize); + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +void QServeGemmPlugin::init(nvinfer1::DataType dtype, int groupSize) +{ + if (groupSize <= 0) + groupSize = -1; // Per-channel + mGroupSize = groupSize; + mType = dtype; + mRunner = std::make_shared<QServeGemmRunner>(); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* QServeGemmPlugin::clone() const noexcept +{ + auto* plugin = new QServeGemmPlugin(*this); + return plugin; +} + +nvinfer1::DimsExprs QServeGemmPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 6); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + TLLM_CHECK(nbDimsA >= 2); + DimsExprs ret; + ret.nbDims = nbDimsA; + for (int ii = 0; ii < nbDimsA - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + ret.d[nbDimsA - 1] = inputs[1].d[0]; + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool QServeGemmPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + if (mGroupSize != -1) + { // Per-group + switch (pos) + { + case 0: + // activation + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + case 1: + // uint4 weights packed in int8 + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + case 2: + // int8 weight s2_zeros + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + case 3: + // int8 weight s2_scales + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + case 4: + // fp16 weight s1_scales + return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; + case 5: + // fp16 activation scales + return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; + case 6: + // fp16 output activation + return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; + default: return false; + } + } + + else + { // Per-channel + switch (pos) + { + case 0: + // activation + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + case 1: + // uint4 weights packed in int8 + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + case 2: + // fp16 s1_scales + return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; + case 3: + // fp16 s1_szeros + return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; + case 4: + // fp16 act_sums + return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; + case 5: + // fp16 act_scales + return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; + case 6: + // fp16 output activation + return inOut[pos].type == nvinfer1::DataType::kHALF && inOut[pos].format == TensorFormat::kLINEAR; + default: return false; + } + } +} + +void QServeGemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + int const maxN = in[1].max.d[0]; + int const minK = in[0].min.d[in[0].min.nbDims - 1]; + int const minN = in[1].min.d[0]; + + TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); + TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, maxN, maxK}; + } + m_workspaceMaxSize = mRunner->getWorkspaceSize(maxM, maxN, maxK); +} + +size_t QServeGemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return m_workspaceMaxSize; +} + +int QServeGemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept +{ + // inputs + + // Per group: + // activation [M, K] int8_t Quantized sint8 activations + // weights [N, K/2] int8_t Quantized uint4 weights (packed as int8_t) + // s2_zeros [K/group_size, N] int8_t Level-2 sint8 scaled zeros of weights + // s2_scales [K/group_size, N] int8_t Level-2 sint8 scales of weights + // s1_scales [N] half Level-1 fp16 scales of weights + // act_scales [M] half Scales of activations + + // Per channel: + // activation [M, K] int8_t Quantized sint8 activations + // weights [N, K/2] int8_t Quantized uint4 weights (packed as int8_t) + // s1_scales [N] half Level-1 scales of weights + // s1_szeros [N] half Level-1 scaled zeros of weights + // act_sums [M] half Per-token sums of activations + // act_scales [M] half Scales of activations + + // outputs + // mat [M(*), N] half + + int64_t m64 = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m64 *= inputDesc[0].dims.d[ii]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); + int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); + + // TODO: Implement optimized kernels if (m <= 4) + + if (mGroupSize != -1) + { + ParamsPerGroup params = {reinterpret_cast<int8_t const*>(inputs[0]), // A + reinterpret_cast<int8_t const*>(inputs[1]), // B + reinterpret_cast<int8_t const*>(inputs[2]), // s2_zeros + reinterpret_cast<int8_t const*>(inputs[3]), // s2_scales + reinterpret_cast<half const*>(inputs[4]), // s1_scales + reinterpret_cast<half const*>(inputs[5]), // act_scales + reinterpret_cast<half*>(outputs[0]), // C + m, n, k}; + mRunner->gemmPerGroup(params, stream); + } + else + { + ParamsPerChannel params = {reinterpret_cast<int8_t const*>(inputs[0]), // A + reinterpret_cast<int8_t const*>(inputs[1]), // B + reinterpret_cast<half const*>(inputs[2]), // s1_scales + reinterpret_cast<half const*>(inputs[3]), // s1_szeros + reinterpret_cast<half const*>(inputs[4]), // act_sums + reinterpret_cast<half const*>(inputs[5]), // act_scales + reinterpret_cast<half*>(outputs[0]), // C + m, n, k}; + mRunner->gemmPerChannel(params, stream); + } + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType QServeGemmPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* QServeGemmPlugin::getPluginType() const noexcept +{ + return QSERVE_GEMM_PLUGIN_NAME; +} + +char const* QServeGemmPlugin::getPluginVersion() const noexcept +{ + return QSERVE_GEMM_PLUGIN_VERSION; +} + +int QServeGemmPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int QServeGemmPlugin::initialize() noexcept +{ + configGemm(); + return 0; +} + +void QServeGemmPlugin::terminate() noexcept {} + +size_t QServeGemmPlugin::getSerializationSize() const noexcept +{ + return sizeof(mQuantMode) + // QuantMode + sizeof(mType) + // dtype + sizeof(mGroupSize) + // GroupSize + sizeof(mDims); // Dimensions +} + +void QServeGemmPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mQuantMode.value()); + write(d, mType); + write(d, mGroupSize); + write(d, mDims); + + TLLM_CHECK(d == a + getSerializationSize()); +} + +void QServeGemmPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +void QServeGemmPlugin::configGemm() {} + +/////////////// + +QServeGemmPluginCreator::QServeGemmPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.push_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.push_back(PluginField("group_size", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* QServeGemmPluginCreator::getPluginName() const noexcept +{ + return QSERVE_GEMM_PLUGIN_NAME; +} + +char const* QServeGemmPluginCreator::getPluginVersion() const noexcept +{ + return QSERVE_GEMM_PLUGIN_VERSION; +} + +PluginFieldCollection const* QServeGemmPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* QServeGemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + // We do not use any fields for now. + + PluginField const* fields = fc->fields; + + // bool perTokenScaling, perChannelScaling; + DataType dtype{}; + int group_size = -1; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dtype = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + // Only supports fp16 for now. + assert(dtype == nvinfer1::DataType::kHALF); + } + else if (!strcmp(attrName, "group_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + group_size = *static_cast<int const*>(fields[i].data); + // Currently only support per-channel or g128. + assert(group_size == -1 || group_size == 128); + } + } + try + { + // QServeGemmPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + // auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); + // QuantMode quantMode = QuantMode::fromQuantAlgo("W4A8_QSERVE"); + auto* obj = new QServeGemmPlugin(dtype, group_size); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* QServeGemmPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call QServeGemmPlugin::destroy() + try + { + // Create plugin profiler with private tactics map which is read from the serialized engine + // auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); + auto* obj = new QServeGemmPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.h b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.h new file mode 100644 index 000000000000..086460863c4f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/qserveGemmPlugin/qserveGemmPlugin.h @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <memory> +#include <string> +#include <tensorrt_llm/kernels/qserveGemm.h> + +namespace tensorrt_llm::plugins +{ + +using QServeGemmRunnerPtr = std::shared_ptr<tensorrt_llm::kernels::qserve::QServeGemmRunner>; + +class QServeGemmPlugin : public BasePlugin +{ +public: + // using PluginProfilerPtr = std::shared_ptr<QServeGemmPluginProfiler>; + + QServeGemmPlugin(void const* data, size_t length); + + QServeGemmPlugin(nvinfer1::DataType dtype, int groupSize); + + ~QServeGemmPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + void init(nvinfer1::DataType dtype, int groupSize); + + void configGemm(); + + std::string const mLayerName; + + QServeGemmRunnerPtr mRunner; + + tensorrt_llm::common::QuantMode mQuantMode; // Not used for now + GemmDims mDims{}; + + size_t m_workspaceMaxSize; + + // Only supports fp16 output for now. + nvinfer1::DataType mType; + + int mGroupSize; +}; + +class QServeGemmPluginCreator : public BaseCreator +{ +public: + QServeGemmPluginCreator(); + + QServeGemmPluginCreator(void const* data, size_t length); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/CMakeLists.txt new file mode 100755 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.cpp b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.cpp new file mode 100644 index 000000000000..23d0b80390e3 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.cpp @@ -0,0 +1,353 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "quantizePerTokenPlugin.h" +#include "tensorrt_llm/kernels/quantization.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels; +using tensorrt_llm::plugins::QuantizePerTokenPluginCreator; +using tensorrt_llm::plugins::QuantizePerTokenPlugin; + +static char const* QUANTIZE_PER_TOKEN_PLUGIN_VERSION{"1"}; +static char const* QUANTIZE_PER_TOKEN_PLUGIN_NAME{"QuantizePerToken"}; +PluginFieldCollection QuantizePerTokenPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> QuantizePerTokenPluginCreator::mPluginAttributes; + +QuantizePerTokenPlugin::QuantizePerTokenPlugin( + nvinfer1::DataType outputType, QuantMode quantMode, bool clampValEnabled, bool sumPerToken) + : mOutputType{outputType} + , mQuantMode{quantMode} + , mClampValEnabled{clampValEnabled} + , mSumPerToken{sumPerToken} +{ + TLLM_CHECK_WITH_INFO(mOutputType == nvinfer1::DataType::kINT8 || mOutputType == nvinfer1::DataType::kFP8, + "Only int8 or fp8 output type is allowed."); + // Check if the quant mode is valid. + TLLM_CHECK_WITH_INFO(mQuantMode.hasPerTokenScaling(), "The quant mode is not valid."); +} + +// Parameterized constructor +QuantizePerTokenPlugin::QuantizePerTokenPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mOutputType); + read(d, mQuantMode); + read(d, mClampValEnabled); + read(d, mSumPerToken); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* QuantizePerTokenPlugin::clone() const noexcept +{ + auto* plugin = new QuantizePerTokenPlugin(mOutputType, mQuantMode, mClampValEnabled, mSumPerToken); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs QuantizePerTokenPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs <= 2); + TLLM_CHECK(outputIndex <= 2); + if (outputIndex == 2) + { + // Per token sums. + TLLM_CHECK(mSumPerToken); + } + + if (outputIndex == 0) + { + // Quantized input + return inputs[0]; + } + + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int ii = 0; ii < ret.nbDims - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + ret.d[ret.nbDims - 1] = exprBuilder.constant(1); + // [M(*), 1] dynamic per token scales or sums + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool QuantizePerTokenPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + if (pos == 0) + { + // activation + return (inOut[pos].type == nvinfer1::DataType::kFLOAT || inOut[pos].type == nvinfer1::DataType::kHALF +#ifdef ENABLE_BF16 + || inOut[pos].type == nvinfer1::DataType::kBF16 +#endif + ) + && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (pos == 1 && mClampValEnabled) + { + // clamp_max_v + return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (pos == 1 + int(mClampValEnabled)) + { + // quantized activation + return inOut[pos].type == mOutputType && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (pos == 2 + int(mClampValEnabled)) + { + // scales + return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (pos == 3 + int(mClampValEnabled)) + { + TLLM_CHECK(mSumPerToken); + // per-token sums + return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + } + + // Never should be here + assert(false); + return false; +} + +void QuantizePerTokenPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t QuantizePerTokenPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +template <typename T, typename QuantT> +void QuantizePerTokenPlugin::dispatchDataType(void* output, void const* input, void const* clampValPtr, void* scalePtr, + void* sumPtr, int dim0, int dim1, cudaStream_t stream) noexcept +{ + // inputs + // activation [dim0(*), dim1] + // clamp_value [2], contains min val, and max val (optional) + // outputs + // quant [dim0(*), dim1] + // scale_tokens [dim0(*), 1] + + invokePerTokenQuantization(reinterpret_cast<QuantT*>(output), reinterpret_cast<T const*>(input), dim0, dim1, + reinterpret_cast<float const*>(clampValPtr), reinterpret_cast<float*>(scalePtr), + reinterpret_cast<float*>(sumPtr), mQuantMode, stream); +} + +int QuantizePerTokenPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // activation [M(*), K] + // clamp_value [2], contains min val, and max val (optional) + // outputs + // quant [M(*), K] Quantized activations. + // scale_tokens [M(*), 1] Per-token scales. + // token_sums [M(*), 1] (Optional) Per-token sums of all the channels (before quantization). + + int64_t m = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m *= inputDesc[0].dims.d[ii]; + } + int64_t const k = inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]; + + void const* clampValPtr = mClampValEnabled ? inputs[1] : nullptr; + void* sumPtr = mSumPerToken ? outputs[2] : nullptr; + + if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kINT8) + { + dispatchDataType<float, int8_t>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kFP8) + { + dispatchDataType<float, __nv_fp8_e4m3>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); + } +#endif // ENABLE_FP8 + else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kINT8) + { + dispatchDataType<half, int8_t>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kFP8) + { + dispatchDataType<half, __nv_fp8_e4m3>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); + } +#endif // ENABLE_FP8 +#ifdef ENABLE_BF16 + else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kINT8) + { + dispatchDataType<__nv_bfloat16, int8_t>(outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kFP8) + { + dispatchDataType<__nv_bfloat16, __nv_fp8_e4m3>( + outputs[0], inputs[0], clampValPtr, outputs[1], sumPtr, m, k, stream); + } +#endif // ENABLE_FP8 +#endif // ENABLE_BF16 + sync_check_cuda_error(stream); + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType QuantizePerTokenPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(nbInputs >= 1); + TLLM_CHECK(index <= 2); + if (index == 2) + { + // Per token sums. + TLLM_CHECK(mSumPerToken); + } + return index == 0 ? mOutputType : nvinfer1::DataType::kFLOAT; +} + +// IPluginV2 Methods + +char const* QuantizePerTokenPlugin::getPluginType() const noexcept +{ + return QUANTIZE_PER_TOKEN_PLUGIN_NAME; +} + +char const* QuantizePerTokenPlugin::getPluginVersion() const noexcept +{ + return QUANTIZE_PER_TOKEN_PLUGIN_VERSION; +} + +int QuantizePerTokenPlugin::getNbOutputs() const noexcept +{ + return 2 + static_cast<int>(mSumPerToken); +} + +int QuantizePerTokenPlugin::initialize() noexcept +{ + return 0; +} + +void QuantizePerTokenPlugin::terminate() noexcept {} + +size_t QuantizePerTokenPlugin::getSerializationSize() const noexcept +{ + return sizeof(mOutputType) + sizeof(mQuantMode) + sizeof(mClampValEnabled) + sizeof(mSumPerToken); +} + +void QuantizePerTokenPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mOutputType); + write(d, mQuantMode); + write(d, mClampValEnabled); + write(d, mSumPerToken); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void QuantizePerTokenPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +QuantizePerTokenPluginCreator::QuantizePerTokenPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("quant_mode", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("clamp_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("sum_per_token", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* QuantizePerTokenPluginCreator::getPluginName() const noexcept +{ + return QUANTIZE_PER_TOKEN_PLUGIN_NAME; +} + +char const* QuantizePerTokenPluginCreator::getPluginVersion() const noexcept +{ + return QUANTIZE_PER_TOKEN_PLUGIN_VERSION; +} + +PluginFieldCollection const* QuantizePerTokenPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* QuantizePerTokenPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginFieldParser p{fc->nbFields, fc->fields}; + try + { + auto* obj = new QuantizePerTokenPlugin(static_cast<nvinfer1::DataType>(p.getScalar<int32_t>("type_id").value()), + QuantMode(p.getScalar<int32_t>("quant_mode").value()), + static_cast<bool>(p.getScalar<int8_t>("clamp_enabled").value()), + static_cast<bool>(p.getScalar<int32_t>("sum_per_token").value())); + + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* QuantizePerTokenPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call QuantizePerTokenPlugin::destroy() + try + { + auto* obj = new QuantizePerTokenPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.h b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.h new file mode 100644 index 000000000000..47b218acfd28 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizePerTokenPlugin/quantizePerTokenPlugin.h @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class QuantizePerTokenPlugin : public BasePlugin +{ +public: + QuantizePerTokenPlugin(nvinfer1::DataType outputType, tensorrt_llm::common::QuantMode quantMode, + bool clampValEnabled, bool sumPerToken); + + QuantizePerTokenPlugin(void const* data, size_t length); + + ~QuantizePerTokenPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + template <typename T, typename QuantT> + void dispatchDataType(void* output, void const* input, void const* clampValPtr, void* scalePtr, void* sumPtr, + int dim0, int dim1, cudaStream_t stream) noexcept; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + std::string const mLayerName; + // The quantized output data type. + nvinfer1::DataType mOutputType; + // The quantization mode. + tensorrt_llm::common::QuantMode mQuantMode; + // Do we clamp the input tensor ? + bool mClampValEnabled; + // Do we output the per-token sum? + bool mSumPerToken; +}; + +class QuantizePerTokenPluginCreator : public BaseCreator +{ +public: + QuantizePerTokenPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/CMakeLists.txt new file mode 100755 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.cpp b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.cpp new file mode 100644 index 000000000000..cacb32b809bf --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.cpp @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "quantizeTensorPlugin.h" +#include "tensorrt_llm/kernels/quantization.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using tensorrt_llm::plugins::QuantizeTensorPluginCreator; +using tensorrt_llm::plugins::QuantizeTensorPlugin; + +static char const* QUANTIZE_TENSOR_PLUGIN_VERSION{"1"}; +static char const* QUANTIZE_TENSOR_PLUGIN_NAME{"QuantizeTensor"}; +PluginFieldCollection QuantizeTensorPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> QuantizeTensorPluginCreator::mPluginAttributes; + +QuantizeTensorPlugin::QuantizeTensorPlugin() {} + +// Parameterized constructor +QuantizeTensorPlugin::QuantizeTensorPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* QuantizeTensorPlugin::clone() const noexcept +{ + return new QuantizeTensorPlugin(*this); +} + +nvinfer1::DimsExprs QuantizeTensorPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 2); + TLLM_CHECK(outputIndex < 1); + // Quantized input + return inputs[0]; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool QuantizeTensorPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + switch (pos) + { + case 0: + // activation + return (inOut[pos].type == nvinfer1::DataType::kFLOAT || inOut[pos].type == nvinfer1::DataType::kHALF +#ifdef ENABLE_BF16 + || inOut[pos].type == nvinfer1::DataType::kBF16 +#endif + ) + && inOut[pos].format == TensorFormat::kLINEAR; + case 1: + // scales + return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + case 2: + // quantized activation + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + default: + // Never should be here + TLLM_CHECK(false); + return false; + } +} + +void QuantizeTensorPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t QuantizeTensorPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int QuantizeTensorPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // activation [M(*), K] + // scale [1, 1] + // outputs + // quant [M(*), K] + + int64_t numElts = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims; ++ii) + { + numElts *= inputDesc[0].dims.d[ii]; + } + + if (inputDesc[0].type == DataType::kFLOAT) + { + invokeQuantization<float>(reinterpret_cast<int8_t*>(outputs[0]), reinterpret_cast<float const*>(inputs[0]), + numElts, reinterpret_cast<float const*>(inputs[1]), stream, mProp.maxGridSize[0]); + } + else if (inputDesc[0].type == DataType::kHALF) + { + invokeQuantization<half>(reinterpret_cast<int8_t*>(outputs[0]), reinterpret_cast<half const*>(inputs[0]), + numElts, reinterpret_cast<float const*>(inputs[1]), stream, mProp.maxGridSize[0]); + } +#ifdef ENABLE_BF16 + else if (inputDesc[0].type == DataType::kBF16) + { + invokeQuantization<__nv_bfloat16>(reinterpret_cast<int8_t*>(outputs[0]), + reinterpret_cast<__nv_bfloat16 const*>(inputs[0]), numElts, reinterpret_cast<float const*>(inputs[1]), + stream, mProp.maxGridSize[0]); + } +#endif + sync_check_cuda_error(stream); + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType QuantizeTensorPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(nbInputs == 2); + TLLM_CHECK(index == 0); + return nvinfer1::DataType::kINT8; +} + +// IPluginV2 Methods + +char const* QuantizeTensorPlugin::getPluginType() const noexcept +{ + return QUANTIZE_TENSOR_PLUGIN_NAME; +} + +char const* QuantizeTensorPlugin::getPluginVersion() const noexcept +{ + return QUANTIZE_TENSOR_PLUGIN_VERSION; +} + +int QuantizeTensorPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int QuantizeTensorPlugin::initialize() noexcept +{ + int deviceId = 0; + tensorrt_llm::common::check_cuda_error(cudaGetDevice(&deviceId)); + tensorrt_llm::common::check_cuda_error(cudaGetDeviceProperties(&mProp, deviceId)); + return 0; +} + +void QuantizeTensorPlugin::terminate() noexcept {} + +size_t QuantizeTensorPlugin::getSerializationSize() const noexcept +{ + return 0; +} + +void QuantizeTensorPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + TLLM_CHECK(d == a + getSerializationSize()); +} + +void QuantizeTensorPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +QuantizeTensorPluginCreator::QuantizeTensorPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* QuantizeTensorPluginCreator::getPluginName() const noexcept +{ + return QUANTIZE_TENSOR_PLUGIN_NAME; +} + +char const* QuantizeTensorPluginCreator::getPluginVersion() const noexcept +{ + return QUANTIZE_TENSOR_PLUGIN_VERSION; +} + +PluginFieldCollection const* QuantizeTensorPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* QuantizeTensorPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + try + { + auto* obj = new QuantizeTensorPlugin(); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* QuantizeTensorPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call QuantizeTensorPlugin::destroy() + try + { + auto* obj = new QuantizeTensorPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.h b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.h new file mode 100644 index 000000000000..6f1ce864ec35 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizeTensorPlugin/quantizeTensorPlugin.h @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class QuantizeTensorPlugin : public BasePlugin +{ +public: + QuantizeTensorPlugin(); + + QuantizeTensorPlugin(void const* data, size_t length); + + ~QuantizeTensorPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + const std::string mLayerName; + cudaDeviceProp mProp; +}; + +class QuantizeTensorPluginCreator : public BaseCreator +{ +public: + QuantizeTensorPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.cpp b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.cpp new file mode 100644 index 000000000000..b5eaffeeda2a --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.cpp @@ -0,0 +1,301 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "quantizeToFP4Plugin.h" +#include "pluginUtils.h" +#include "tensorrt_llm/kernels/quantization.h" +#include <NvInferRuntimeBase.h> + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::QuantizeToFP4PluginCreator; +using tensorrt_llm::plugins::QuantizeToFP4Plugin; + +constexpr nvinfer1::DataType FP4_DTYPE = nvinfer1::DataType::kFP4; +constexpr nvinfer1::DataType FP8_DTYPE = nvinfer1::DataType::kFP8; + +static char const* QUANT_FP4_PLUGIN_VERSION{"1"}; +static char const* QUANT_FP4_PLUGIN_NAME{"QuantizeToFP4"}; +PluginFieldCollection QuantizeToFP4PluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> QuantizeToFP4PluginCreator::mPluginAttributes; + +QuantizeToFP4Plugin::QuantizeToFP4Plugin(){}; + +// Parameterized constructor +QuantizeToFP4Plugin::QuantizeToFP4Plugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* QuantizeToFP4Plugin::clone() const noexcept +{ + auto* plugin = new QuantizeToFP4Plugin(); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs QuantizeToFP4Plugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + // Quantized output in FP4 datatype. + if (outputIndex == 0) + { + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int di = 0; di < ret.nbDims; ++di) + { + ret.d[di] = inputs[0].d[di]; + } + // // Div up by 16 as the storage type has 16 FP4 values per element. + // ret.d[ret.nbDims - 1] + // = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 1], + // *exprBuilder.constant(16)); + return ret; + } + // Scaling Factors in FP8. + else if (outputIndex == 1) + { + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int di = 0; di < ret.nbDims; ++di) + { + ret.d[di] = inputs[0].d[di]; + } + // Sequence dimension or token dimension. + // Pad to multiple of 128. + auto dimM + = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 2], *exprBuilder.constant(128)); + ret.d[ret.nbDims - 2] = exprBuilder.operation(DimensionOperation::kPROD, *dimM, *exprBuilder.constant(128)); + // Hidden size dimension. + // Div (rounding up) by 16 since 16 elements share one SF and SF padded to k%4==0. + ret.d[ret.nbDims - 1] + = exprBuilder.operation(DimensionOperation::kCEIL_DIV, *ret.d[ret.nbDims - 1], *exprBuilder.constant(16)); + return ret; + } + return DimsExprs{}; +} + +bool QuantizeToFP4Plugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + // half input + float global_sf + fp4 output (e2m1) + fp8 SF output. + int const totalPoses = 2 + 2; + TLLM_CHECK(0 <= pos && pos < totalPoses); + TLLM_CHECK(nbInputs == 2); + switch (pos) + { + case 0: + return (inOut[pos].type == nvinfer1::DataType::kHALF || inOut[pos].type == nvinfer1::DataType::kBF16 + || inOut[pos].type == nvinfer1::DataType::kFP8) + && (inOut[pos].format == TensorFormat::kLINEAR); + case 1: return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + case 2: return (inOut[pos].type == FP4_DTYPE) && (inOut[pos].format == TensorFormat::kLINEAR); + case 3: return (inOut[pos].type == FP8_DTYPE) && (inOut[pos].format == TensorFormat::kLINEAR); + default: break; + } + return false; +} + +void QuantizeToFP4Plugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t QuantizeToFP4Plugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +int QuantizeToFP4Plugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // input [M(*), N] half data type + // SF scale [1] float data type + // used to scale SF from input range to fp8 range (448.f / (MaxVal of input / 6.f)) + // outputs + // output [M(*), N] fp4 storage (E2M1) + // SF output [M, N / 16] fp8 storage (UE4M3) + + int64_t m64 = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims - 1; ++i) + { + m64 *= inputDesc[0].dims.d[i]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); + + TLLM_CHECK_WITH_INFO(n % 16 == 0, "the N dimension must be multiple of 16."); + + float const* SFScale = static_cast<float const*>(inputs[1]); + int64_t* output = reinterpret_cast<int64_t*>(outputs[0]); + int32_t* SFoutput = reinterpret_cast<int32_t*>(outputs[1]); + + DataType inputDtype = inputDesc[0].type; + + switch (inputDtype) + { + case DataType::kHALF: + { + auto input = reinterpret_cast<half const*>(inputs[0]); + invokeFP4Quantization(1, m, n, input, SFScale, output, SFoutput, false, QuantizationSFLayout::SWIZZLED, + mMultiProcessorCount, stream); + break; + } + + case DataType::kBF16: + { + auto input = reinterpret_cast<__nv_bfloat16 const*>(inputs[0]); + invokeFP4Quantization(1, m, n, input, SFScale, output, SFoutput, false, QuantizationSFLayout::SWIZZLED, + mMultiProcessorCount, stream); + break; + } + + case DataType::kFP8: + { + auto input = reinterpret_cast<__nv_fp8_e4m3 const*>(inputs[0]); + invokeFP4Quantization(1, m, n, input, SFScale, output, SFoutput, false, QuantizationSFLayout::SWIZZLED, + mMultiProcessorCount, stream); + break; + } + + default: TLLM_LOG_ERROR("only half, bfloat16 and fp8 data type are supported."); break; + } + + // Use UE4M3 scales by default. + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType QuantizeToFP4Plugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + if (index == 0) + { + // Output 0 quantized output. + return FP4_DTYPE; + } + // Output 1 SF (scaling factors). + return FP8_DTYPE; +} + +// IPluginV2 Methods + +char const* QuantizeToFP4Plugin::getPluginType() const noexcept +{ + return QUANT_FP4_PLUGIN_NAME; +} + +char const* QuantizeToFP4Plugin::getPluginVersion() const noexcept +{ + return QUANT_FP4_PLUGIN_VERSION; +} + +int QuantizeToFP4Plugin::getNbOutputs() const noexcept +{ + return 2; +} + +int QuantizeToFP4Plugin::initialize() noexcept +{ + return 0; +} + +void QuantizeToFP4Plugin::terminate() noexcept {} + +size_t QuantizeToFP4Plugin::getSerializationSize() const noexcept +{ + return 0; +} + +void QuantizeToFP4Plugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + TLLM_CHECK(d == a + getSerializationSize()); +} + +void QuantizeToFP4Plugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +QuantizeToFP4PluginCreator::QuantizeToFP4PluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* QuantizeToFP4PluginCreator::getPluginName() const noexcept +{ + return QUANT_FP4_PLUGIN_NAME; +} + +char const* QuantizeToFP4PluginCreator::getPluginVersion() const noexcept +{ + return QUANT_FP4_PLUGIN_VERSION; +} + +PluginFieldCollection const* QuantizeToFP4PluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* QuantizeToFP4PluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + try + { + auto* obj = new QuantizeToFP4Plugin(); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* QuantizeToFP4PluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call QuantizeToFP4Plugin::destroy() + try + { + auto* obj = new QuantizeToFP4Plugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.h b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.h new file mode 100644 index 000000000000..b584837a447a --- /dev/null +++ b/cpp/tensorrt_llm/plugins/quantizeToFP4Plugin/quantizeToFP4Plugin.h @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class QuantizeToFP4Plugin : public BasePlugin +{ +public: + QuantizeToFP4Plugin(); + + QuantizeToFP4Plugin(void const* data, size_t length); + + ~QuantizeToFP4Plugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + const std::string mLayerName; + int const mMultiProcessorCount = tensorrt_llm::common::getMultiProcessorCount(); +}; + +class QuantizeToFP4PluginCreator : public BaseCreator +{ +public: + QuantizeToFP4PluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/CMakeLists.txt new file mode 100755 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.cpp b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.cpp new file mode 100644 index 000000000000..16d0bf2dc356 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.cpp @@ -0,0 +1,452 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "rmsnormQuantizationPlugin.h" +#include "pluginUtils.h" +#include "tensorrt_llm/kernels/rmsnormKernels.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::RmsnormQuantizationPluginCreator; +using tensorrt_llm::plugins::RmsnormQuantizationPlugin; + +static char const* RMSNORM_QUANTIZATION_PLUGIN_VERSION{"1"}; +static char const* RMSNORM_QUANTIZATION_PLUGIN_NAME{"RmsnormQuantization"}; +PluginFieldCollection RmsnormQuantizationPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> RmsnormQuantizationPluginCreator::mPluginAttributes; + +RmsnormQuantizationPlugin::RmsnormQuantizationPlugin(float eps, bool dynamicActivationScaling, bool sumPerToken, + bool clampValEnabled, QuantMode quantMode, nvinfer1::DataType type, nvinfer1::DataType outputType) + : mEps(eps) + , mDynActScaling(dynamicActivationScaling) + , mType(type) + , mOutputType{outputType} + , mClampValEnabled{clampValEnabled} + , mQuantMode{quantMode} + , mSumPerToken(sumPerToken) +{ + TLLM_CHECK_WITH_INFO(mOutputType == nvinfer1::DataType::kINT8 || mOutputType == nvinfer1::DataType::kFP8, + "Only int8 or fp8 output type is allowed."); + // Check if the quant mode is valid. + TLLM_CHECK_WITH_INFO(mQuantMode.hasPerTokenScaling(), "The quant mode is not valid."); +} + +// Parameterized constructor +RmsnormQuantizationPlugin::RmsnormQuantizationPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mEps); + read(d, mDynActScaling); + read(d, mSumPerToken); + read(d, mClampValEnabled); + read(d, mQuantMode); + read(d, mType); + read(d, mOutputType); + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* RmsnormQuantizationPlugin::clone() const noexcept +{ + auto* plugin = new RmsnormQuantizationPlugin( + mEps, mDynActScaling, mSumPerToken, mClampValEnabled, mQuantMode, mType, mOutputType); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs RmsnormQuantizationPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + if (outputIndex == 0) + { + // Quantized output + return inputs[outputIndex]; + } + + // Dynamic scaling or per-token sum if enabled. + try + { + if (outputIndex == 1) + { + TLLM_CHECK(mDynActScaling); + } + else if (outputIndex == 2) + { + TLLM_CHECK(mSumPerToken); + } + else + { + TLLM_CHECK(false); + } + + DimsExprs ret; + ret.nbDims = inputs[0].nbDims; + for (int di = 0; di < ret.nbDims - 1; ++di) + { + ret.d[di] = inputs[0].d[di]; + } + ret.d[ret.nbDims - 1] = exprBuilder.constant(1); + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool RmsnormQuantizationPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + int const totalPoses + = 6 + static_cast<int>(mClampValEnabled) + static_cast<int>(mDynActScaling) + static_cast<int>(mSumPerToken); + TLLM_CHECK(0 <= pos && pos < totalPoses); + TLLM_CHECK(nbInputs == 4 + static_cast<int>(mClampValEnabled)); + if (pos < nbInputs) + { + if (pos < 3) + { + // activation, weight, bias + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == 3) + { + // scale + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == 4 && mClampValEnabled) + { + // clamp_max_v + return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + } + } + else if (pos == 4 + int(mClampValEnabled)) + { + // Quantized output + return (inOut[pos].type == mOutputType) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == 5 + int(mClampValEnabled)) + { + // Dynamic scaling if enabled + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (pos == 6 + int(mClampValEnabled)) + { + // Per-token activation sum if enabled + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + + // Never should be here + TLLM_CHECK_WITH_INFO(false, "The input/output is not supported."); + return false; +} + +void RmsnormQuantizationPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t RmsnormQuantizationPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return 0; +} + +template <typename T, typename QuantT> +void RmsnormQuantizationPlugin::dispatchDataType(void* out, void const* input, void const* gamma, void const* beta, + float const eps, int const tokens, int const hidden_dim, cudaStream_t stream, void const* clampValPtr, + void const* scale, void* dynamic_scale, void* sum_per_token, void* normed_output_quant) noexcept +{ + // inputs + // activation [dim0(*), dim1] + // clamp_value [2], contains min val, and max val (optional) + // outputs + // quant [dim0(*), dim1] + // scale_tokens [dim0(*), 1] + + invokeGeneralRmsNorm(reinterpret_cast<T*>(out), reinterpret_cast<T const*>(input), + reinterpret_cast<T const*>(gamma), reinterpret_cast<T const*>(beta), eps, tokens, hidden_dim, mQuantMode, + stream, reinterpret_cast<float const*>(clampValPtr), reinterpret_cast<float const*>(scale), + reinterpret_cast<float*>(dynamic_scale), reinterpret_cast<float*>(sum_per_token), + reinterpret_cast<QuantT*>(normed_output_quant)); +} + +int RmsnormQuantizationPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // input [M(*), N] + // weight [N, ] + // bias [N, ] + // scale_to_int [1] + // clamp_value [2], contains min val, and max val (optional) + // outputs + // output [M(*), N] Normalized activations, potentially with quantization applied. + // dynamic_scaling [M(*), 1] (Optional) Per-token scales if quantization is enabled. + // token_sums [M(*), 1] (Optional) Per-token sums of all the channels (before quantization). + + int64_t m64 = 1; + for (int i = 0; i < inputDesc[0].dims.nbDims - 1; ++i) + { + m64 *= inputDesc[0].dims.d[i]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); + + void const* input = inputs[0]; + void const* weight = inputs[1]; + void const* bias = inputs[2]; + void const* scale = inputs[3]; + void const* clampValPtr = mClampValEnabled ? inputs[4] : nullptr; + void* output = outputs[0]; + void* dynamic_scale = mDynActScaling ? outputs[1] : nullptr; + void* sum_per_token = mSumPerToken ? outputs[2] : nullptr; + + if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kINT8) + { + dispatchDataType<float, int8_t>( + nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kFLOAT && mOutputType == DataType::kFP8) + { + dispatchDataType<float, __nv_fp8_e4m3>( + nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#endif // ENABLE_FP8 + else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kINT8) + { + dispatchDataType<half, int8_t>( + nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kHALF && mOutputType == DataType::kFP8) + { + dispatchDataType<half, __nv_fp8_e4m3>( + nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#endif // ENABLE_FP8 +#ifdef ENABLE_BF16 + else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kINT8) + { + dispatchDataType<__nv_bfloat16, int8_t>( + nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#ifdef ENABLE_FP8 + else if (inputDesc[0].type == DataType::kBF16 && mOutputType == DataType::kFP8) + { + dispatchDataType<__nv_bfloat16, __nv_fp8_e4m3>( + nullptr, input, weight, bias, mEps, m, n, stream, clampValPtr, scale, dynamic_scale, sum_per_token, output); + } +#endif // ENABLE_FP8 +#endif // ENABLE_BF16 + sync_check_cuda_error(stream); + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType RmsnormQuantizationPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + assert(index <= 2); + + if (index == 0) + { + // Output 0 quantized output of layer norm + return mOutputType; + } + if (index == 1) + { + assert(mDynActScaling); + // Output 1 dynamic act scaling + return nvinfer1::DataType::kFLOAT; + } + // index == 2 + { + assert(mDynActScaling && mSumPerToken); + // Output 2 per token sum + return nvinfer1::DataType::kFLOAT; + } +} + +// IPluginV2 Methods + +char const* RmsnormQuantizationPlugin::getPluginType() const noexcept +{ + return RMSNORM_QUANTIZATION_PLUGIN_NAME; +} + +char const* RmsnormQuantizationPlugin::getPluginVersion() const noexcept +{ + return RMSNORM_QUANTIZATION_PLUGIN_VERSION; +} + +int RmsnormQuantizationPlugin::getNbOutputs() const noexcept +{ + return 1 + static_cast<int>(mDynActScaling) + static_cast<int>(mSumPerToken); +} + +int RmsnormQuantizationPlugin::initialize() noexcept +{ + return 0; +} + +void RmsnormQuantizationPlugin::terminate() noexcept {} + +size_t RmsnormQuantizationPlugin::getSerializationSize() const noexcept +{ + return sizeof(mOutputType) + sizeof(mClampValEnabled) + sizeof(mEps) + sizeof(mDynActScaling) + sizeof(mSumPerToken) + + sizeof(mType) + sizeof(mQuantMode); +} + +void RmsnormQuantizationPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mEps); + write(d, mDynActScaling); + write(d, mSumPerToken); + write(d, mClampValEnabled); + write(d, mQuantMode); + write(d, mType); + write(d, mOutputType); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void RmsnormQuantizationPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +RmsnormQuantizationPluginCreator::RmsnormQuantizationPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("eps", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("dyn_act_scaling", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("sum_per_token", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("clamp_enabled", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("quant_mode", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("out_type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* RmsnormQuantizationPluginCreator::getPluginName() const noexcept +{ + return RMSNORM_QUANTIZATION_PLUGIN_NAME; +} + +char const* RmsnormQuantizationPluginCreator::getPluginVersion() const noexcept +{ + return RMSNORM_QUANTIZATION_PLUGIN_VERSION; +} + +PluginFieldCollection const* RmsnormQuantizationPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* RmsnormQuantizationPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + nvinfer1::DataType outputType{}; + QuantMode quantMode; + bool clampValEnabled = false; + float eps{}; + nvinfer1::DataType type{}; + bool dynamicActivationScaling{}; + bool sumPerToken{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "quant_mode")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + quantMode = QuantMode(*(static_cast<int32_t const*>(fields[i].data))); + } + else if (!strcmp(attrName, "out_type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + outputType = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "clamp_enabled")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + clampValEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "eps")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + eps = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "dyn_act_scaling")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dynamicActivationScaling = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "sum_per_token")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + sumPerToken = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + } + try + { + auto* obj = new RmsnormQuantizationPlugin( + eps, dynamicActivationScaling, sumPerToken, clampValEnabled, quantMode, type, outputType); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* RmsnormQuantizationPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call RmsnormQuantizationPlugin::destroy() + try + { + auto* obj = new RmsnormQuantizationPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.h b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.h new file mode 100644 index 000000000000..762a9bb8de1b --- /dev/null +++ b/cpp/tensorrt_llm/plugins/rmsnormQuantizationPlugin/rmsnormQuantizationPlugin.h @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +class RmsnormQuantizationPlugin : public BasePlugin +{ +public: + RmsnormQuantizationPlugin(float eps, bool dynamicActivationScaling, bool sumPerToken, bool clampValEnabled, + tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, nvinfer1::DataType outputType); + + RmsnormQuantizationPlugin(void const* data, size_t length); + + ~RmsnormQuantizationPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + template <typename T, typename QuantT> + void dispatchDataType(void* out, void const* input, void const* gamma, void const* beta, float const eps, + int const tokens, int const hidden_dim, cudaStream_t stream, void const* clampValPtr, void const* scale, + void* dynamic_scale, void* normed_output_quant, void* act_sum) noexcept; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + float mEps; + bool mDynActScaling; + nvinfer1::DataType mType; + + std::string const mLayerName; + // The quantized output data type. + nvinfer1::DataType mOutputType; + // Do we clamp the input tensor ? + bool mClampValEnabled; + // The quantization mode. + tensorrt_llm::common::QuantMode mQuantMode; + // Should we output the sum of channels per-token? (Used by QServe GEMM) + bool mSumPerToken; +}; + +class RmsnormQuantizationPluginCreator : public BaseCreator +{ +public: + RmsnormQuantizationPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/CMakeLists.txt new file mode 100644 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.cpp b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.cpp new file mode 100644 index 000000000000..3e60182f28c2 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.cpp @@ -0,0 +1,594 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "selectiveScanPlugin.h" +#include "tensorrt_llm/common/assert.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::SelectiveScanPluginCreator; +using tensorrt_llm::plugins::SelectiveScanPlugin; + +static char const* SELECTIVE_SCAN_PLUGIN_VERSION{"1"}; +static char const* SELECTIVE_SCAN_PLUGIN_NAME{"SelectiveScan"}; +PluginFieldCollection SelectiveScanPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> SelectiveScanPluginCreator::mPluginAttributes; + +SelectiveScanPlugin::SelectiveScanPlugin(int dim, int dstate, int dtRank, int nHeads, int nGroups, int chunkSize, + bool deltaSoftplus, nvinfer1::DataType type, bool removePadding, bool pagedState, bool zEnabled, bool isMamba2) + : mDim(dim) + , mDState(dstate) + , mDtRank(dtRank) + , mNHeads(nHeads) + , mNGroups(nGroups) + , mChunkSize(chunkSize) + , mDeltaSoftplus(deltaSoftplus) + , mType(type) + , mRemovePadding(removePadding) + , mPagedState(pagedState) + , mZEnabled(zEnabled) + , mIsMamba2(isMamba2) + , mDriver(tensorrt_llm::common::CUDADriverWrapper::getInstance()) +{ + TLLM_CHECK_WITH_INFO( + (mChunkSize == 256 || mChunkSize == 128) || (!mIsMamba2), "Only support CHUNK_SIZE 256 or 128"); + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), + "Only support float, half, and bfloat16."); +} + +// Parameterized constructor +SelectiveScanPlugin::SelectiveScanPlugin(void const* data, size_t length) + : mDriver(tensorrt_llm::common::CUDADriverWrapper::getInstance()) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mDim); + read(d, mDState); + read(d, mDtRank); + read(d, mNHeads); + read(d, mNGroups); + read(d, mChunkSize); + read(d, mDeltaSoftplus); + read(d, mType); + read(d, mRemovePadding); + read(d, mPagedState); + read(d, mZEnabled); + read(d, mIsMamba2); + TLLM_CHECK(d == a + length); + TLLM_CHECK_WITH_INFO( + (mChunkSize == 256 || mChunkSize == 128) || (!mIsMamba2), "Only support CHUNK_SIZE 256 or 128"); + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF), + "Only support float, half, and bfloat16."); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* SelectiveScanPlugin::clone() const noexcept +{ + auto* plugin = new SelectiveScanPlugin(mDim, mDState, mDtRank, mNHeads, mNGroups, mChunkSize, mDeltaSoftplus, mType, + mRemovePadding, mPagedState, mZEnabled, mIsMamba2); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +// Outputs +// output_tensor: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// state: [batch_size, dstate, dim] +nvinfer1::DimsExprs SelectiveScanPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + if (outputIndex == 0) + { + if (mIsMamba2) + { + auto ret = inputs[getInputTensorIdx()]; + ret.d[mRemovePadding ? 1 : 2] = exprBuilder.constant(mDim); + return ret; + } + else + { + return inputs[getInputTensorIdx()]; + } + } + return inputs[getStateIdx()]; +} + +bool SelectiveScanPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + if (pos == getHostRequestTypesIdx() || pos == getLastTokenIdsIdx() + || (mRemovePadding && pos == getHostContextLengthIdx()) || (mPagedState && pos == getSlotMappingIdx())) + { + return inOut[pos].type == nvinfer1::DataType::kINT32; + } + else if (pos == getAIdx() || pos == getDeltaBiasIdx() || pos == getDIdx()) + { + return (inOut[pos].type == nvinfer1::DataType::kFLOAT) && (inOut[pos].format == TensorFormat::kLINEAR); + } + else if (mPagedState && pos == getStateIdx()) + { + return inOut[pos].type == nvinfer1::DataType::kINT64; + } + else + { + return (inOut[pos].type == mType) && (inOut[pos].format == TensorFormat::kLINEAR); + } +} + +void SelectiveScanPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t SelectiveScanPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + if (!mIsMamba2) + return 0; + + int const NUM_BUFFERS = 6; + size_t workspaces[NUM_BUFFERS]; + + if (mRemovePadding) + { + int B = inputs[getLastTokenIdsIdx()].dims.d[0]; + int BxL = inputs[getInputTensorIdx()].dims.d[0]; // num_tokens + int H = mNHeads; + int P = mDim / H; + int G = mNGroups; + int N = mDState; + int Q = mChunkSize; + int BxC = (BxL + Q - 1) / Q + B; + + workspaces[0] = long(BxC) * H * N * P * 2; // g_mxOs_ + workspaces[1] = long(BxC) * H * N * P * 4; // g_mxSt_ in float + workspaces[2] = long(BxC) * H * Q * 4; // g_mxdc_ in float + workspaces[3] = long(BxC) * H * Q * 4; // g_mxdA_ in float + workspaces[4] = long(BxC) * G * Q * Q * 2; // g_mxCB_ + workspaces[5] = 1024; // TMA descs + } + else + { + int B = inputs[getInputTensorIdx()].dims.d[0]; + int L = inputs[getInputTensorIdx()].dims.d[1]; + int H = mNHeads; + int P = mDim / H; + int G = mNGroups; + int N = mDState; + int Q = mChunkSize; + int C = (L + Q - 1) / Q; + + workspaces[0] = long(B * C) * H * N * P * 2; // g_mxOs_ + workspaces[1] = long(B * C) * H * N * P * 4; // g_mxSt_ in float + workspaces[2] = long(B * C) * H * Q * 4; // g_mxdc_ in float + workspaces[3] = long(B * C) * H * Q * 4; // g_mxdA_ in float + workspaces[4] = long(B * C) * G * Q * Q * 2; // g_mxCB_ + workspaces[5] = 1024; // TMA descs + } + + return calculateTotalWorkspaceSize(workspaces, NUM_BUFFERS); +} + +void SelectiveScanPlugin::setSSMParams(SSMParamsBase& params, const size_t batch, const size_t dim, + const size_t maxSeqLen, const size_t numTokens, const size_t dstate, const size_t dtRank, const size_t nHeads, + const size_t nGroups, const size_t chunkSize, void* statePtr, void const* x, void const* delta, + void const* deltaBias, void const* A, void const* BC, void const* D, void const* z, void* osPtr, void* stPtr, + void* dcPtr, void* dAPtr, void* cbPtr, void* descPtr, int const* lastTokenIds, int const* slotMapping, void* out, + bool deltaSoftplus, bool removePadding) +{ + // Reset the parameters + memset(¶ms, 0, sizeof(params)); + + params.batch = batch; + params.dim = dim; + params.max_seqlen = maxSeqLen; + params.num_tokens = numTokens; + params.dstate = dstate; + params.dt_rank = dtRank; + params.nheads = nHeads; + params.ngroups = nGroups; + params.chunk_size = chunkSize; + + params.delta_softplus = deltaSoftplus; + params.remove_padding = removePadding; + params.is_mamba2 = mIsMamba2; + + // Set the pointers and strides. + params.u_ptr = const_cast<void*>(x); + params.delta_ptr = const_cast<void*>(delta); + params.A_ptr = const_cast<void*>(A); + params.BC_ptr = const_cast<void*>(BC); + params.D_ptr = const_cast<void*>(D); + params.delta_bias_ptr = const_cast<void*>(deltaBias); + params.out_ptr = out; + params.x_ptr = statePtr; + params.z_ptr = const_cast<void*>(z); + params.Os_ptr = osPtr; + params.St_ptr = stPtr; + params.dc_ptr = dcPtr; + params.dA_ptr = dAPtr; + params.CB_ptr = cbPtr; + params.desc_ptr = descPtr; + params.last_token_ids_ptr = lastTokenIds; + params.slot_mapping_ptr = slotMapping; +} + +template <typename T> +int SelectiveScanPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) +{ + // inputs + // 0. input_tensor [batch_size, max_seq_len, dim] or [num_tokens, dim] + // 1. state mamba: [batch_size, dstate, dim] or host [1] containing only pointer for paged_state + // mamba2: [batch_size, nheads, dstate, dim] or host [1] containing only pointer for paged_state + // 2. delta, mamba: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding + // mamba2: [batch_size, seq_len, nheads] or [num_tokens, nheads] for remove_input_padding + // 3. delta_bias, [dim] for mamba, [nheads] for mamba2 + // 4. A, [dstate, dim] for mamba, [nheads] for mamba2 + // 5. BC, mamba: [batch_size, seq_len, dstate * 2] or [num_tokens, dstate * 2] for remove_input_padding + // mamba2: [batch_size, seq_len, ngroups * dstate * 2] or [num_tokens, ngroups * dstate * 2] for + // remove_input_padding + // 6. D, [dim] for mamba, [nheads] for mamba2 + // 7. host_request_types [batch_size] int32. 0: context; 1: generation. + // 8. last_token_ids [batch_size] int32 + // 9. host_context_lengths [batch_size] int32, optional for remove_input_padding + // 10. state_slot_mapping [batch_size] int32, optional for paged state + // 11. z [batch_size, max_seq_len, dim] or [num_tokens, dim] + // outputs + // 0. output_tensor [batch_size, max_seq_len, dim] or [num_tokens, dim] + // 1. state, [batch_size, dstate, dim] for mamba, [batch_size, nheads, dstate, dim] for mamba2 + auto const batch_size = inputDesc[getHostRequestTypesIdx()].dims.d[0]; + int max_seq_len; + if (mRemovePadding) + { + int const* host_context_length = static_cast<int const*>(inputs[getHostContextLengthIdx()]); + max_seq_len = *std::max_element(host_context_length, host_context_length + batch_size); + } + else + { + max_seq_len = inputDesc[getInputTensorIdx()].dims.d[1]; + } + + // only support context or generation, not for both of them + RequestType const* reqTypes = static_cast<RequestType const*>(inputs[getHostRequestTypesIdx()]); + + SSMParamsBase ssm_params; + + int const* slotMapping = mPagedState ? static_cast<int const*>(inputs[getSlotMappingIdx()]) : nullptr; + void const* z = mZEnabled ? inputs[getZIdx()] : nullptr; + + void* statePtr = mPagedState ? *reinterpret_cast<void**>(const_cast<void*>(inputs[getStateIdx()])) : outputs[1]; + + // Workspace pointer shift + int8_t* workspace_byte_ptr = reinterpret_cast<int8_t*>(workspace); + size_t offset = 0; + + T* mxOs = nullptr; + float* mxSt = nullptr; + float* mxdc = nullptr; + float* mxdA = nullptr; + T* mxCB = nullptr; + void* descs = nullptr; + + if (!mIsMamba2 || reqTypes[0] == RequestType::kGENERATION) /* no workspace needed */ + ; + else if (mRemovePadding) + { + int B = inputDesc[getLastTokenIdsIdx()].dims.d[0]; + int BxL = inputDesc[getInputTensorIdx()].dims.d[0]; // num_tokens + int H = mNHeads; + int P = mDim / H; + int G = mNGroups; + int N = mDState; + int Q = mChunkSize; + int BxC = (BxL + Q - 1) / Q + B; + + mxOs = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * H * N * P * 2)); + mxSt = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * H * N * P * 4)); + mxdc = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * H * Q * 4)); + mxdA = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * H * Q * 4)); + mxCB = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(BxC) * G * Q * Q * 2)); + descs = nextWorkspacePtr(workspace_byte_ptr, offset, 1024); + } + else + { + int B = inputDesc[getInputTensorIdx()].dims.d[0]; + int L = inputDesc[getInputTensorIdx()].dims.d[1]; + int H = mNHeads; + int P = mDim / H; + int G = mNGroups; + int N = mDState; + int Q = mChunkSize; + int C = (L + Q - 1) / Q; + + mxOs = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * H * N * P * 2)); + mxSt = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * H * N * P * 4)); + mxdc = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * H * Q * 4)); + mxdA = reinterpret_cast<float*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * H * Q * 4)); + mxCB = reinterpret_cast<T*>(nextWorkspacePtr(workspace_byte_ptr, offset, long(B * C) * G * Q * Q * 2)); + descs = nextWorkspacePtr(workspace_byte_ptr, offset, 1024); + } + + int numTokens = inputDesc[getInputTensorIdx()].dims.d[0]; + if (!mRemovePadding) + numTokens *= inputDesc[getInputTensorIdx()].dims.d[1]; + + setSSMParams(ssm_params, batch_size, mDim, max_seq_len, numTokens, mDState, mDtRank, mNHeads, mNGroups, mChunkSize, + statePtr, inputs[getInputTensorIdx()], inputs[getDeltaIdx()], inputs[getDeltaBiasIdx()], inputs[getAIdx()], + inputs[getBCIdx()], inputs[getDIdx()], z, mxOs, mxSt, mxdc, mxdA, mxCB, descs, + static_cast<int const*>(inputs[getLastTokenIdsIdx()]), slotMapping, outputs[0], mDeltaSoftplus, mRemovePadding); + + if (reqTypes[0] == RequestType::kCONTEXT) + { + if (mIsMamba2) + { + invokeChunkScan<T, float>(ssm_params, stream, mDriver.get()); + } + else + { + invokeSelectiveScan<T, float>(ssm_params, stream); + } + } + else if (reqTypes[0] == RequestType::kGENERATION) + { + invokeSelectiveScanUpdate<T, float>(ssm_params, stream); + } + sync_check_cuda_error(stream); + return 0; +} + +int SelectiveScanPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + if (isBuilding()) + { + return 0; + } + if (mType == DataType::kHALF) + { + return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kFLOAT) + { + return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16) + { + return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType SelectiveScanPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + if (index == 0) + { + return inputTypes[getInputTensorIdx()]; + } + else + { + return inputTypes[getStateIdx()]; + } +} + +// IPluginV2 Methods + +char const* SelectiveScanPlugin::getPluginType() const noexcept +{ + return SELECTIVE_SCAN_PLUGIN_NAME; +} + +char const* SelectiveScanPlugin::getPluginVersion() const noexcept +{ + return SELECTIVE_SCAN_PLUGIN_VERSION; +} + +int SelectiveScanPlugin::getNbOutputs() const noexcept +{ + return mPagedState ? 1 : 2; +} + +int SelectiveScanPlugin::initialize() noexcept +{ + return 0; +} + +void SelectiveScanPlugin::terminate() noexcept {} + +size_t SelectiveScanPlugin::getSerializationSize() const noexcept +{ + return sizeof(mDim) + sizeof(mDState) + sizeof(mDtRank) + sizeof(mNHeads) + sizeof(mNGroups) + sizeof(mChunkSize) + + sizeof(mDeltaSoftplus) + sizeof(mType) + sizeof(mRemovePadding) + sizeof(mPagedState) + sizeof(mZEnabled) + + sizeof(mIsMamba2); +} + +void SelectiveScanPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mDim); + write(d, mDState); + write(d, mDtRank); + write(d, mNHeads); + write(d, mNGroups); + write(d, mChunkSize); + write(d, mDeltaSoftplus); + write(d, mType); + write(d, mRemovePadding); + write(d, mPagedState); + write(d, mZEnabled); + write(d, mIsMamba2); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void SelectiveScanPlugin::destroy() noexcept +{ + delete this; +} + +/////////////// + +SelectiveScanPluginCreator::SelectiveScanPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("dim", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("dstate", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("dt_rank", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("nheads", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("ngroups", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("chunk_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("delta_softplus", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("remove_input_padding", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("paged_state", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("z_enabled", nullptr, PluginFieldType::kINT8)); + mPluginAttributes.emplace_back(PluginField("is_mamba2", nullptr, PluginFieldType::kINT8)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* SelectiveScanPluginCreator::getPluginName() const noexcept +{ + return SELECTIVE_SCAN_PLUGIN_NAME; +} + +char const* SelectiveScanPluginCreator::getPluginVersion() const noexcept +{ + return SELECTIVE_SCAN_PLUGIN_VERSION; +} + +PluginFieldCollection const* SelectiveScanPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* SelectiveScanPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int dim{}; + int dstate{}; + int dtRank{}; + int nHeads{}; + int nGroups{}; + int chunkSize{}; + bool deltaSoftplus{}; + bool removePadding{}; + bool pagedState{}; + bool zEnabled{}; + bool isMamab2{}; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "dim")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dim = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "dstate")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dstate = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "dt_rank")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + dtRank = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "nheads")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + nHeads = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "ngroups")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + nGroups = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "chunk_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + chunkSize = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "delta_softplus")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + deltaSoftplus = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "remove_input_padding")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + removePadding = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "paged_state")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + pagedState = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "z_enabled")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + zEnabled = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + else if (!strcmp(attrName, "is_mamba2")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT8); + isMamab2 = static_cast<bool>(*(static_cast<bool const*>(fields[i].data))); + } + } + try + { + auto* obj = new SelectiveScanPlugin(dim, dstate, dtRank, nHeads, nGroups, chunkSize, deltaSoftplus, type, + removePadding, pagedState, zEnabled, isMamab2); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* SelectiveScanPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call SelectiveScanPlugin::destroy() + try + { + auto* obj = new SelectiveScanPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.h b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.h new file mode 100644 index 000000000000..96cb86fc4cbb --- /dev/null +++ b/cpp/tensorrt_llm/plugins/selectiveScanPlugin/selectiveScanPlugin.h @@ -0,0 +1,218 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRT_SELECTIVE_SCAN_PLUGIN_H +#define TRT_SELECTIVE_SCAN_PLUGIN_H +#include "tensorrt_llm/kernels/selectiveScan/selectiveScan.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> + +namespace tensorrt_llm::plugins +{ +// batch_size = num_ctx_requests or num_gen_requests +// num_ctx_requests = number of context requests (single sequence per request). +// num_gen_requests = number of generation requests (single sequences per request). +// can not support beam search + +// inputs +// 0. input_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 1. state, mamba: [batch_size, dstate, dim] or host [1] containing only pointer for paged_state +// mamba2: [batch_size, nheads, dstate, dim] or host [1] containing only pointer for paged_state +// 2. delta, mamba: [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// mamba2: [batch_size, seq_len, nheads] or [num_tokens, nheads] for remove_input_padding +// 3. delta_bias, [dim] for mamba, [nheads] for mamba2 +// 4. A, [dstate, dim] for mamba, [nheads] for mamba2 +// 5. BC, mamba: [batch_size, seq_len, dstate * 2] or [num_tokens, dstate * 2] for remove_input_padding +// mamba2: [batch_size, seq_len, ngroups * dstate * 2] or [num_tokens, ngroups * dstate * 2] for +// remove_input_padding +// 6. D, [dim] for mamba, [nheads] for mamba2 +// 7. host_request_types [batch_size] int32. 0: context; 1: generation; 2: none. +// 8. last_token_ids [batch_size] int32 +// 9. host_context_lengths [batch_size] int32, optional for remove_input_padding +// 10. state_slot_mapping [batch_size] int32, optional for paged state +// 11. z [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// outputs +// 0. output_tensor [batch_size, seq_len, dim] or [num_tokens, dim] for remove_input_padding +// 1. state, [batch_size, dstate, dim] for mamba, [batch_size, nheads, dstate, dim] for mamba2 + +class SelectiveScanPlugin : public BasePlugin +{ +public: + SelectiveScanPlugin(int dim, int dstate, int dtRank, int nHeads, int nGroups, int chunkSize, bool deltaSoftplus, + nvinfer1::DataType type, bool removePadding, bool pagedState, bool zEnabled, bool isMamba2); + + SelectiveScanPlugin(void const* data, size_t length); + + ~SelectiveScanPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + template <typename T> + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + + enum class RequestType : int32_t + { + kCONTEXT = 0, + kGENERATION = 1 + }; + +private: + using IndexType = std::int32_t; + + IndexType getInputTensorIdx() const + { + return 0; + }; + + IndexType getStateIdx() const + { + return 1; + }; + + IndexType getDeltaIdx() const + { + return 2; + }; + + IndexType getDeltaBiasIdx() const + { + return 3; + }; + + IndexType getAIdx() const + { + return 4; + }; + + IndexType getBCIdx() const + { + return 5; + }; + + IndexType getDIdx() const + { + return 6; + }; + + IndexType getHostRequestTypesIdx() const + { + return 7; + }; + + IndexType getLastTokenIdsIdx() const + { + return 8; + }; + + IndexType getHostContextLengthIdx() const + { + if (mRemovePadding) + return 9; + else + return 8; + }; + + IndexType getSlotMappingIdx() const + { + if (mPagedState) + return getHostContextLengthIdx() + 1; + else + return getHostContextLengthIdx(); + }; + + IndexType getZIdx() const + { + if (mZEnabled) + return getSlotMappingIdx() + 1; + else + return getSlotMappingIdx(); + }; + + void setSSMParams(tensorrt_llm::kernels::SSMParamsBase& params, + // sizes + const size_t batch, const size_t dim, const size_t maxSeqLen, const size_t numTokens, const size_t dstate, + const size_t dtRank, const size_t nHeads, const size_t nGroups, const size_t chunkSize, + // device pointers + void* statePtr, void const* x, void const* delta, void const* deltaBias, void const* A, void const* BC, + void const* D, void const* z, void* osPtr, void* stPtr, void* dcPtr, void* dAPtr, void* cbPtr, void* descs, + int const* lastTokenIds, int const* slotMapping, void* out, bool deltaSoftplus, bool removePadding); + +private: + int mDim; + int mDState; + int mDtRank; + int mNHeads; + int mNGroups; + int mChunkSize; + bool mDeltaSoftplus; + nvinfer1::DataType mType; + bool mRemovePadding = false; + bool mPagedState = false; + bool mZEnabled = true; + bool mIsMamba2 = false; + std::shared_ptr<tensorrt_llm::common::CUDADriverWrapper> mDriver; +}; + +class SelectiveScanPluginCreator : public BaseCreator +{ +public: + SelectiveScanPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins + +#endif // TRT_SELECTIVE_SCAN_PLUGIN_H diff --git a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/CMakeLists.txt new file mode 100755 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.cpp b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.cpp new file mode 100644 index 000000000000..718d8b7e830d --- /dev/null +++ b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.cpp @@ -0,0 +1,431 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "smoothQuantGemmPlugin.h" +#include "tensorrt_llm/kernels/weightOnlyBatchedGemv/int8SQ.h" +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels::cutlass_kernels; +using tensorrt_llm::plugins::SmoothQuantGemmPluginCreator; +using tensorrt_llm::plugins::SmoothQuantGemmPlugin; +using tensorrt_llm::plugins::SmoothQuantGemmPluginProfiler; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +static char const* SQ_GEMM_PLUGIN_VERSION{"1"}; +static char const* SQ_GEMM_PLUGIN_NAME{"SmoothQuantGemm"}; +PluginFieldCollection SmoothQuantGemmPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> SmoothQuantGemmPluginCreator::mPluginAttributes; + +void SmoothQuantGemmPluginProfiler::runTactic(int m, int n, int k, SmoothQuantGemmPluginProfiler::Config const& tactic, + char* workspace, cudaStream_t const& stream) +{ + int8_t* aTmp = reinterpret_cast<int8_t*>(workspace); + int8_t* bTmp = nextWorkspacePtr(aTmp, m * k * sizeof(int8_t)); + void* cTmp = reinterpret_cast<void*>(nextWorkspacePtr(bTmp, n * k * sizeof(int8_t))); + float* alphaRowTmp = reinterpret_cast<float*>( + nextWorkspacePtr(reinterpret_cast<int8_t*>(cTmp), m * n * (mType == nvinfer1::DataType::kFLOAT ? 4 : 2))); + float* alphaColTmp + = reinterpret_cast<float*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(alphaRowTmp), m * sizeof(float))); + char* workspaceTmp + = reinterpret_cast<char*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(alphaColTmp), n * sizeof(float))); + + int const wsSize = mRunner->getWorkspaceSize(m, n, k); + + mRunner->gemm( + aTmp, bTmp, mQuantMode, alphaColTmp, alphaRowTmp, cTmp, m, n, k, tactic, workspaceTmp, wsSize, stream); +} + +void SmoothQuantGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + std::vector<size_t> workspaces = { + maxM * k * sizeof(int8_t), // A + n * k * sizeof(int8_t), // B + maxM * n * (mType == nvinfer1::DataType::kFLOAT ? 4u : 2u), // C + maxM * sizeof(float), // alphaRow + n * sizeof(float), // alphaCol + mRunner->getWorkspaceSize(maxM, n, k) // workspace + }; + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<SmoothQuantGemmPluginProfiler::Config> SmoothQuantGemmPluginProfiler::getTactics(int m, int n, int k) const +{ + return mRunner->getConfigs(); +} + +SmoothQuantGemmPlugin::SmoothQuantGemmPlugin( + QuantMode quantMode, nvinfer1::DataType type, SmoothQuantGemmPlugin::PluginProfilerPtr const& pluginProfiler) + : mQuantMode(quantMode) + , mPluginProfiler(pluginProfiler) +{ + init(type); +} + +// Parameterized constructor +SmoothQuantGemmPlugin::SmoothQuantGemmPlugin( + void const* data, size_t length, SmoothQuantGemmPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + nvinfer1::DataType type; + unsigned int quantMode; + read(d, quantMode); + read(d, type); + read(d, mDims); + + mQuantMode = QuantMode(quantMode); + + init(type); + + mPluginProfiler->deserialize(d, mDims, mGemmId); + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +void SmoothQuantGemmPlugin::init(nvinfer1::DataType type) +{ + mType = type; + if (mType == nvinfer1::DataType::kHALF) + { + m_sqGemmRunner = std::make_shared<CutlassInt8GemmRunner<half>>(); + } + else if (mType == nvinfer1::DataType::kFLOAT) + { + m_sqGemmRunner = std::make_shared<CutlassInt8GemmRunner<float>>(); + } + else if (mType == nvinfer1::DataType::kINT32) + { + m_sqGemmRunner = std::make_shared<CutlassInt8GemmRunner<int32_t>>(); + } +#ifdef ENABLE_BF16 + else if (mType == nvinfer1::DataType::kBF16) + { + m_sqGemmRunner = std::make_shared<CutlassInt8GemmRunner<__nv_bfloat16>>(); + } +#endif + + mPluginProfiler->setQuantMode(mQuantMode); + + mGemmId = GemmIdCore(mDims.n, mDims.k, mType); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* SmoothQuantGemmPlugin::clone() const noexcept +{ + auto* plugin = new SmoothQuantGemmPlugin(*this); + return plugin; +} + +nvinfer1::DimsExprs SmoothQuantGemmPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + try + { + TLLM_CHECK(nbInputs == 4); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + TLLM_CHECK(nbDimsA >= 2); + DimsExprs ret; + ret.nbDims = nbDimsA; + for (int ii = 0; ii < nbDimsA - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + ret.d[nbDimsA - 1] = inputs[1].d[0]; + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool SmoothQuantGemmPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + switch (pos) + { + case 0: + // activation + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + case 1: + // weights + // Weights stored in checkpoint must have int8 type + return inOut[pos].type == nvinfer1::DataType::kINT8 && inOut[pos].format == TensorFormat::kLINEAR; + case 2: + // scales channels + case 3: + // scales tokens + return inOut[pos].type == nvinfer1::DataType::kFLOAT && inOut[pos].format == TensorFormat::kLINEAR; + case 4: + // out + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + default: + // Never should be here + assert(false); + return false; + } +} + +void SmoothQuantGemmPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + int const maxN = in[1].max.d[0]; + int const minK = in[0].min.d[in[0].min.nbDims - 1]; + int const minN = in[1].min.d[0]; + + TLLM_CHECK_WITH_INFO(minN == maxN, "Variable out channels is not allowed"); + TLLM_CHECK_WITH_INFO(minK == maxK, "Variable in channels is not allowed"); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, maxN, maxK}; + } + mGemmId = {maxN, maxK, mType}; + + m_workspaceMaxSize = m_sqGemmRunner->getWorkspaceSize(maxM, maxN, maxK); +} + +size_t SmoothQuantGemmPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return m_workspaceMaxSize; +} + +int SmoothQuantGemmPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // mat1 [M(*), K] + // mat2 [N, K] + // scale_tokens [M, 1] if has_per_token_scaling else [1, 1] + // scale_channels [1, N] if has_per_channel_scaling else [1, 1] + // outputs + // mat [M(*), N] + int64_t m64 = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m64 *= inputDesc[0].dims.d[ii]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[0]); + int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); + int const wsSize = m_sqGemmRunner->getWorkspaceSize(m, n, k); + if (m <= 4) + { + tensorrt_llm::kernels::smooth_quant::Params params(reinterpret_cast<int8_t const*>(inputs[0]), + reinterpret_cast<int8_t const*>(inputs[1]), reinterpret_cast<float const*>(inputs[2]), + reinterpret_cast<float const*>(inputs[3]), reinterpret_cast<void*>(outputs[0]), m, n, k, mQuantMode); + if (mType == nvinfer1::DataType::kHALF) + { + tensorrt_llm::kernels::smooth_quant::int8_sq_launcher<half>(params, stream); + } + else if (mType == nvinfer1::DataType::kFLOAT) + { + tensorrt_llm::kernels::smooth_quant::int8_sq_launcher<float>(params, stream); + } +#ifdef ENABLE_BF16 + else if (mType == nvinfer1::DataType::kBF16) + { + tensorrt_llm::kernels::smooth_quant::int8_sq_launcher<__nv_bfloat16>(params, stream); + } +#endif + else if (mType == nvinfer1::DataType::kINT32) + { + tensorrt_llm::kernels::smooth_quant::int8_sq_launcher<int>(params, stream); + } + } + else + { + auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); + TLLM_CHECK_WITH_INFO(bestTactic, "No valid SQ GEMM tactic"); + m_sqGemmRunner->gemm(reinterpret_cast<int8_t const*>(inputs[0]), reinterpret_cast<int8_t const*>(inputs[1]), + mQuantMode, reinterpret_cast<float const*>(inputs[3]), reinterpret_cast<float const*>(inputs[2]), + reinterpret_cast<void*>(outputs[0]), m, n, k, *bestTactic, reinterpret_cast<char*>(workspace), wsSize, + stream); + } + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType SmoothQuantGemmPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* SmoothQuantGemmPlugin::getPluginType() const noexcept +{ + return SQ_GEMM_PLUGIN_NAME; +} + +char const* SmoothQuantGemmPlugin::getPluginVersion() const noexcept +{ + return SQ_GEMM_PLUGIN_VERSION; +} + +int SmoothQuantGemmPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int SmoothQuantGemmPlugin::initialize() noexcept +{ + configGemm(); + return 0; +} + +void SmoothQuantGemmPlugin::terminate() noexcept {} + +size_t SmoothQuantGemmPlugin::getSerializationSize() const noexcept +{ + return sizeof(unsigned int) + // QuantMode + sizeof(nvinfer1::DataType) + // dtype + sizeof(mDims) + // Dimensions + mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size +} + +void SmoothQuantGemmPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mQuantMode.value()); + write(d, mType); + write(d, mDims); + + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void SmoothQuantGemmPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +void SmoothQuantGemmPlugin::configGemm() +{ + mPluginProfiler->profileTactics(m_sqGemmRunner, mType, mDims, mGemmId); +} + +/////////////// + +SmoothQuantGemmPluginCreator::SmoothQuantGemmPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("has_per_channel_scaling", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("has_per_token_scaling", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* SmoothQuantGemmPluginCreator::getPluginName() const noexcept +{ + return SQ_GEMM_PLUGIN_NAME; +} + +char const* SmoothQuantGemmPluginCreator::getPluginVersion() const noexcept +{ + return SQ_GEMM_PLUGIN_VERSION; +} + +PluginFieldCollection const* SmoothQuantGemmPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* SmoothQuantGemmPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + bool perTokenScaling{}; + bool perChannelScaling{}; + nvinfer1::DataType type{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "has_per_channel_scaling")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + perChannelScaling = static_cast<bool>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "has_per_token_scaling")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + perTokenScaling = static_cast<bool>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + try + { + // SmoothQuantGemmPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); + QuantMode quantMode = QuantMode::fromDescription(true, true, perTokenScaling, perChannelScaling, false, false, + false, false, false, false, false, false, false, false, false, false); + auto* obj = new SmoothQuantGemmPlugin(quantMode, type, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* SmoothQuantGemmPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call SmoothQuantGemmPlugin::destroy() + try + { + // Create plugin profiler with private tactics map which is read from the serialized engine + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); + auto* obj = new SmoothQuantGemmPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.h b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.h new file mode 100644 index 000000000000..3cabf558076b --- /dev/null +++ b/cpp/tensorrt_llm/plugins/smoothQuantGemmPlugin/smoothQuantGemmPlugin.h @@ -0,0 +1,140 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/kernels/cutlass_kernels/int8_gemm/int8_gemm.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::plugins +{ + +using perfMapType = std::unordered_map<int, tensorrt_llm::cutlass_extensions::CutlassGemmConfig>; +using SqGemmRunnerPtr = std::shared_ptr<tensorrt_llm::kernels::cutlass_kernels::CutlassInt8GemmRunnerInterface>; + +class SmoothQuantGemmPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + SqGemmRunnerPtr, GemmIdCore, GemmIdCoreHash> +{ +public: + using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; + + void setQuantMode(tensorrt_llm::common::QuantMode const& quantMode) + { + mQuantMode = quantMode; + } + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + std::vector<Config> getTactics(int m, int n, int k) const override; + +private: + tensorrt_llm::common::QuantMode mQuantMode; +}; + +class SmoothQuantGemmPlugin : public BasePlugin +{ +public: + using PluginProfilerPtr = std::shared_ptr<SmoothQuantGemmPluginProfiler>; + + SmoothQuantGemmPlugin() = delete; + + SmoothQuantGemmPlugin( + tensorrt_llm::common::QuantMode quantMode, nvinfer1::DataType type, PluginProfilerPtr const& pluginProfiler); + + SmoothQuantGemmPlugin(void const* data, size_t length, PluginProfilerPtr const& pluginProfiler); + + ~SmoothQuantGemmPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + void init(nvinfer1::DataType type); + + void configGemm(); + +private: + const std::string mLayerName; + + SqGemmRunnerPtr m_sqGemmRunner; + tensorrt_llm::common::QuantMode mQuantMode; + size_t m_workspaceMaxSize; + + GemmDims mDims{}; + GemmIdCore mGemmId{}; + + PluginProfilerPtr mPluginProfiler; + + nvinfer1::DataType mType; +}; + +class SmoothQuantGemmPluginCreator : public BaseCreator +{ +public: + SmoothQuantGemmPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<SmoothQuantGemmPluginProfiler> gemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/CMakeLists.txt new file mode 100644 index 000000000000..6b4e3d8d9e0f --- /dev/null +++ b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/CMakeLists.txt @@ -0,0 +1,22 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.cpp b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.cpp new file mode 100644 index 000000000000..072bfc9c8fc4 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.cpp @@ -0,0 +1,316 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "topkLastDimPlugin.h" +#include "tensorrt_llm/common/assert.h" + +using namespace nvinfer1; +using namespace tensorrt_llm::kernels; +using namespace tensorrt_llm::common; +using tensorrt_llm::plugins::TopkLastDimPluginCreator; +using tensorrt_llm::plugins::TopkLastDimPlugin; + +static char const* TOPK_LAST_DIM_PLUGIN_VERSION{"1"}; +static char const* TOPK_LAST_DIM_PLUGIN_NAME{"TopkLastDim"}; +PluginFieldCollection TopkLastDimPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> TopkLastDimPluginCreator::mPluginAttributes; + +TopkLastDimPlugin::TopkLastDimPlugin(nvinfer1::DataType type, int32_t k, bool is_largest) + : mType(type) + , mK(k) // To avoid data-dependent shape, enforce K to be non-dynamic + , mIsLargest(is_largest) +{ + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF) + || (mType == DataType::kINT32), + "Only support int, float, half, and bfloat16."); +} + +// Parameterized constructor +TopkLastDimPlugin::TopkLastDimPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + read(d, mType); + read(d, mK); + read(d, mIsLargest); + TLLM_CHECK(d == a + length); + TLLM_CHECK_WITH_INFO((mType == DataType::kBF16) || (mType == DataType::kFLOAT) || (mType == DataType::kHALF) + || (mType == DataType::kINT32), + "Only support int, float, half, and bfloat16."); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* TopkLastDimPlugin::clone() const noexcept +{ + auto* plugin = new TopkLastDimPlugin(mType, mK, mIsLargest); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +// Outputs +// out_val or out_idx: [batch_size, K] +nvinfer1::DimsExprs TopkLastDimPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + TLLM_CHECK_WITH_INFO(outputIndex < 2, "Only 2 outputs."); + nvinfer1::DimsExprs output(inputs[0]); + int numDim = output.nbDims; + output.d[numDim - 1] = exprBuilder.constant(mK); + return output; +} + +bool TopkLastDimPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + bool res = inOut[pos].format == TensorFormat::kLINEAR; + if (pos < 2) // input and out_val tensor must be the same type as the plugin + { + res = res && inOut[pos].type == mType; + } + else if (pos == 2) // out_idx must be int32 + { + res = res && inOut[pos].type == DataType::kINT32; + } + return res; +} + +void TopkLastDimPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t TopkLastDimPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + // extract shape info and then call helper + auto const batchSize = inputs[getInputTensorIdx()].dims.d[0]; + auto const inputLength = inputs[getInputTensorIdx()].dims.d[1]; + size_t tempStorageBytes{}; + if (mType == DataType::kINT32) + { + tempStorageBytes = invokeComputeTopkLastDimWorkspaceSize<int>(batchSize, inputLength, mK, mIsLargest); + } + else if (mType == DataType::kHALF) + { + tempStorageBytes = invokeComputeTopkLastDimWorkspaceSize<half>(batchSize, inputLength, mK, mIsLargest); + } + else if (mType == DataType::kFLOAT) + { + tempStorageBytes = invokeComputeTopkLastDimWorkspaceSize<float>(batchSize, inputLength, mK, mIsLargest); + } +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16) + { + tempStorageBytes = invokeComputeTopkLastDimWorkspaceSize<__nv_bfloat16>(batchSize, inputLength, mK, mIsLargest); + } +#endif + return tempStorageBytes; +} + +template <typename T> +int TopkLastDimPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) +{ + // inputs + // 0. input_tensor [batch_size, inputLength] + // outputs + // 0. output_values [batch_size, k] + // 1. output_indices [batch_size, k] + auto const batchSize = inputDesc[getInputTensorIdx()].dims.d[0]; + auto const inputLength = inputDesc[getInputTensorIdx()].dims.d[1]; + if (batchSize == 0) + { + // nothing to do for empty tensor + return 0; + } + + invokeTopkLastDim<T>( + batchSize, inputLength, mK, mIsLargest, inputs[getInputTensorIdx()], outputs[0], outputs[1], workspace, stream); + + sync_check_cuda_error(stream); + return 0; +} + +int TopkLastDimPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + if (mType == DataType::kINT32) + { + return enqueueImpl<int>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kHALF) + { + return enqueueImpl<half>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kFLOAT) + { + return enqueueImpl<float>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#ifdef ENABLE_BF16 + else if (mType == DataType::kBF16) + { + return enqueueImpl<__nv_bfloat16>(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } +#endif + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType TopkLastDimPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK_WITH_INFO(index < 2, "Only 2 outputs."); + nvinfer1::DataType data_type; + if (index == 1) + { + data_type = DataType::kINT32; + } + else + { + data_type = inputTypes[getInputTensorIdx()]; + } + return data_type; +} + +// IPluginV2 Methods + +char const* TopkLastDimPlugin::getPluginType() const noexcept +{ + return TOPK_LAST_DIM_PLUGIN_NAME; +} + +char const* TopkLastDimPlugin::getPluginVersion() const noexcept +{ + return TOPK_LAST_DIM_PLUGIN_VERSION; +} + +int TopkLastDimPlugin::getNbOutputs() const noexcept +{ + return 2; +} + +int TopkLastDimPlugin::initialize() noexcept +{ + return 0; +} + +void TopkLastDimPlugin::terminate() noexcept {} + +size_t TopkLastDimPlugin::getSerializationSize() const noexcept +{ + return sizeof(mType) + sizeof(mK) + sizeof(mIsLargest); +} + +void TopkLastDimPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mK); + write(d, mIsLargest); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void TopkLastDimPlugin::destroy() noexcept +{ + delete this; +} + +/////////////// + +TopkLastDimPluginCreator::TopkLastDimPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("k", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("is_largest", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* TopkLastDimPluginCreator::getPluginName() const noexcept +{ + return TOPK_LAST_DIM_PLUGIN_NAME; +} + +char const* TopkLastDimPluginCreator::getPluginVersion() const noexcept +{ + return TOPK_LAST_DIM_PLUGIN_VERSION; +} + +PluginFieldCollection const* TopkLastDimPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* TopkLastDimPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + nvinfer1::DataType type{}; + int32_t k{}; + bool is_largest{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "k")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + k = static_cast<int32_t>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "is_largest")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + is_largest = static_cast<int32_t>(*(static_cast<int const*>(fields[i].data))) != 0; + } + } + try + { + auto* obj = new TopkLastDimPlugin(type, k, is_largest); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* TopkLastDimPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call TopkLastDimPlugin::destroy() + try + { + auto* obj = new TopkLastDimPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.h b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.h new file mode 100644 index 000000000000..0ca38ccfe105 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/topkLastDimPlugin/topkLastDimPlugin.h @@ -0,0 +1,99 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TRT_TOPK_LAST_DIM_PLUGIN_H +#define TRT_TOPK_LAST_DIM_PLUGIN_H + +#include "tensorrt_llm/kernels/topkLastDim.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include <cassert> + +namespace tensorrt_llm::plugins +{ +class TopkLastDimPlugin : public BasePlugin +{ +public: + TopkLastDimPlugin(nvinfer1::DataType type, int32_t k, bool largest); + TopkLastDimPlugin(void const* data, size_t length); + ~TopkLastDimPlugin() override = default; + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + template <typename T> + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + using IndexType = std::int32_t; + + IndexType getInputTensorIdx() const + { + return 0; + }; + +private: + nvinfer1::DataType mType; + int32_t mK; + bool mIsLargest; +}; + +class TopkLastDimPluginCreator : public BaseCreator +{ +public: + TopkLastDimPluginCreator(); + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins + +#endif diff --git a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/CMakeLists.txt new file mode 100755 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.cpp b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.cpp new file mode 100644 index 000000000000..85f0cf011293 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.cpp @@ -0,0 +1,657 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "weightOnlyGroupwiseQuantMatmulPlugin.h" + +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels::cutlass_kernels; +using tensorrt_llm::plugins::WeightOnlyGroupwiseQuantMatmulPluginCreator; +using tensorrt_llm::plugins::WeightOnlyGroupwiseQuantMatmulPlugin; +using tensorrt_llm::plugins::WeightOnlyGroupwiseQuantGemmPluginProfiler; +using tensorrt_llm::plugins::WeightOnlyGemmRunnerPtr; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +static char const* WOQ_GROUPWISE_MATMUL_PLUGIN_VERSION{"1"}; +static char const* WOQ_GROUPWISE_MATMUL_PLUGIN_NAME{"WeightOnlyGroupwiseQuantMatmul"}; +PluginFieldCollection WeightOnlyGroupwiseQuantMatmulPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> WeightOnlyGroupwiseQuantMatmulPluginCreator::mPluginAttributes; + +void WeightOnlyGroupwiseQuantGemmPluginProfiler::runTactic(int m, int n, int k, + WeightOnlyGroupwiseQuantGemmPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) +{ + // Quantized weights are packed in FP16 format (INT4*4 -> FP16, INT8*2 -> FP16) + int const originalN = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? n * FP16_INT8_RATIO : n * FP16_INT4_RATIO; + half* actPtr = reinterpret_cast<half*>(workspace); + void* weightPtr = nextWorkspacePtr(reinterpret_cast<int8_t*>(actPtr), m * k * sizeof(half)); + half* inputScalesPtr + = reinterpret_cast<half*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(weightPtr), n * k * sizeof(float))); + half* zerosPtr = reinterpret_cast<half*>( + nextWorkspacePtr(reinterpret_cast<int8_t*>(inputScalesPtr), k * originalN * sizeof(half) / mGroupSize)); + half* biasesPtr = reinterpret_cast<half*>( + nextWorkspacePtr(reinterpret_cast<int8_t*>(zerosPtr), k * originalN * sizeof(half) / mGroupSize)); + half* outputPtr = reinterpret_cast<half*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(biasesPtr), n * sizeof(half))); + char* workspacePtr + = reinterpret_cast<char*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(outputPtr), m * originalN * sizeof(half))); + if ((mQuantAlgo & GroupwiseQuantAlgo::ZERO) == 0) + { + zerosPtr = nullptr; + } + if ((mQuantAlgo & GroupwiseQuantAlgo::BIAS) == 0) + { + biasesPtr = nullptr; + } + + if (tactic.enableCudaKernel) + { + // run CUDA kernel + void const* pre_quant_scale_ptr = nullptr; + bool apply_alpha_in_advance = false; + float alpha = 1.0; + tensorrt_llm::kernels::weight_only::Params params{actPtr, pre_quant_scale_ptr, weightPtr, inputScalesPtr, + zerosPtr, biasesPtr, outputPtr, alpha, m, originalN, k, mGroupSize, mCudaKernelType, + apply_alpha_in_advance}; + tensorrt_llm::kernels::weight_only::kernel_launcher(mArch, params, stream); + } + else + { + // run CUTLASS kernel + int const wsSize = mRunner->getWorkspaceSize(m, originalN, k); + if (mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT) + { + mRunner->gemm(actPtr, reinterpret_cast<int8_t*>(weightPtr), inputScalesPtr, zerosPtr, biasesPtr, outputPtr, + m, originalN, k, mGroupSize, tactic, workspacePtr, wsSize, stream); + } + else + { + mRunner->gemm(actPtr, reinterpret_cast<cutlass::uint4b_t*>(weightPtr), inputScalesPtr, zerosPtr, biasesPtr, + outputPtr, m, originalN, k, mGroupSize, tactic, workspacePtr, wsSize, stream); + } + } +} + +void WeightOnlyGroupwiseQuantGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + // Quantized weights are packed in FP16 format (INT4*4 -> FP16, INT8*2 -> FP16) + int const originalN = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? n * FP16_INT8_RATIO : n * FP16_INT4_RATIO; + std::vector<size_t> workspaces = { + maxM * k * sizeof(half), // A + k * n * sizeof(float), // B + k * originalN * sizeof(half) / mGroupSize, // scales + k * originalN * sizeof(half) / mGroupSize, // zeros + originalN * sizeof(half), // biases + maxM * originalN * sizeof(half), // C + mRunner->getWorkspaceSize(maxM, originalN, k) // workspace + }; + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<WeightOnlyGroupwiseQuantGemmPluginProfiler::Config> WeightOnlyGroupwiseQuantGemmPluginProfiler::getTactics( + int m, int n, int k) const +{ + return mRunner->getConfigs(); +} + +bool WeightOnlyGroupwiseQuantGemmPluginProfiler::checkTactic(int m, int n, int k, Config const& tactic) const +{ + // stop to profile Cuda kernel for m >= 16 + if (tactic.enableCudaKernel) + { + return m < 16; + } + return true; +} + +WeightOnlyGroupwiseQuantMatmulPlugin::WeightOnlyGroupwiseQuantMatmulPlugin(nvinfer1::DataType type, int quant_algo, + int group_size, float alpha, WeightOnlyGroupwiseQuantMatmulPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + init(type, quant_algo, group_size, alpha); +} + +// Parameterized constructor +WeightOnlyGroupwiseQuantMatmulPlugin::WeightOnlyGroupwiseQuantMatmulPlugin( + void const* data, size_t length, WeightOnlyGroupwiseQuantMatmulPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + nvinfer1::DataType type; + int quant_algo = 0; + int group_size = 0; + float alpha = 1.0f; + read(d, type); + read(d, quant_algo); + read(d, group_size); + read(d, alpha); + read(d, mDims); + + init(type, quant_algo, group_size, alpha); + + mPluginProfiler->deserialize(d, mDims, mGemmId); + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +template <typename ActivationType, typename WeightType, typename OutputType, typename ScaleZeroType, + cutlass::WeightOnlyQuantOp QuantOp> +using GemmRunner = tensorrt_llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunner<ActivationType, WeightType, QuantOp, + ScaleZeroType, OutputType, OutputType>; + +template <typename ActivationType, typename WeightType, typename OutputType, typename ScaleZeroType = OutputType> +WeightOnlyGemmRunnerPtr selectGemmRunnerForZERO(int quant_algo) +{ + if (quant_algo & GroupwiseQuantAlgo::ZERO) + { + return std::make_shared<GemmRunner<ActivationType, WeightType, OutputType, ScaleZeroType, + cutlass::WeightOnlyQuantOp::FINEGRAINED_SCALE_AND_ZEROS>>(); + } + else + { + return std::make_shared<GemmRunner<ActivationType, WeightType, OutputType, ScaleZeroType, + cutlass::WeightOnlyQuantOp::FINEGRAINED_SCALE_ONLY>>(); + } +} + +template <typename ActivationType> +WeightOnlyGemmRunnerPtr selectGemmRunnerForWeightType(int quant_algo) +{ + if (quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) + { + return selectGemmRunnerForZERO<ActivationType, uint8_t, ActivationType>(quant_algo); + } + else + { + return selectGemmRunnerForZERO<ActivationType, cutlass::uint4b_t, ActivationType>(quant_algo); + } +} + +void WeightOnlyGroupwiseQuantMatmulPlugin::init(nvinfer1::DataType type, int quant_algo, int group_size, float alpha) +{ + mArch = tensorrt_llm::common::getSMVersion(); + mType = type; + mQuantAlgo = quant_algo; + mGroupSize = group_size; + + // quant_algo = int8_weight * 16 + fp8_alpha * 8 + pre_quant_scale * 4 + zero * 2 + bias + mPreQuantScaleInputIdx = (quant_algo & GroupwiseQuantAlgo::PRE_QUANT_SCALE) ? 1 : 0; + mWeightInputIdx = mPreQuantScaleInputIdx + 1; + mScalesInputIdx = mWeightInputIdx + 1; + mZerosInputIdx = (quant_algo & GroupwiseQuantAlgo::ZERO) ? mScalesInputIdx + 1 : mScalesInputIdx; + mBiasesInputIdx = (quant_algo & GroupwiseQuantAlgo::BIAS) ? mZerosInputIdx + 1 : mZerosInputIdx; + + if (mType == nvinfer1::DataType::kHALF) + { + // CUTLASS kernel selection + if (quant_algo & GroupwiseQuantAlgo::FP8_ALPHA) + { + mAlpha = alpha; + + // Ada & Hopper style kernels + if (mArch < 89) + { + TLLM_THROW("W4A(fp)8 kernel is unsupported on pre-Ada (sm<89) architectures!"); + } + assert(!(quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) && "W4A(fp)8 kernel requires INT4 weight!"); + m_weightOnlyGroupwiseGemmRunner + = selectGemmRunnerForZERO<__nv_fp8_e4m3, cutlass::uint4b_t, half>(quant_algo); + } + else + { + m_weightOnlyGroupwiseGemmRunner = selectGemmRunnerForWeightType<half>(quant_algo); + } + // CUDA kernel selection + if (quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) + { + // INT8 weight + mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( + mArch, tensorrt_llm::kernels::weight_only::KernelType::FP16Int8Groupwise); + mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::FP16Int8Groupwise; + } + else + { + // INT4 weight + mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( + mArch, tensorrt_llm::kernels::weight_only::KernelType::FP16Int4Groupwise); + mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::FP16Int4Groupwise; + } + } +#if defined(ENABLE_BF16) + else if (mType == nvinfer1::DataType::kBF16) + { + // CUTLASS kernel selection + if (quant_algo & GroupwiseQuantAlgo::FP8_ALPHA) + { + mAlpha = alpha; + + // FP8 requires at least sm89 devices + if (mArch < 89) + { + TLLM_THROW("W4A(fp)8 kernel is unsupported on pre-Ada (sm<89) architectures!"); + } + assert(!(quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) && "W4A(fp)8 kernel requires INT4 weight!"); + m_weightOnlyGroupwiseGemmRunner + = selectGemmRunnerForZERO<__nv_fp8_e4m3, cutlass::uint4b_t, __nv_bfloat16, half>(quant_algo); + } + else + { + m_weightOnlyGroupwiseGemmRunner = selectGemmRunnerForWeightType<__nv_bfloat16>(quant_algo); + } + // CUDA kernel selection + if (quant_algo & GroupwiseQuantAlgo::INT8_WEIGHT) + { + // INT8 weight + mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( + mArch, tensorrt_llm::kernels::weight_only::KernelType::BF16Int8Groupwise); + mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::BF16Int8Groupwise; + } + else + { + // INT4 weight + mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( + mArch, tensorrt_llm::kernels::weight_only::KernelType::BF16Int4Groupwise); + mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::BF16Int4Groupwise; + } + } +#endif + else + { + TLLM_THROW("Unsupported data type"); + } + mPluginProfiler->setQuantAlgo(mQuantAlgo); + mPluginProfiler->setGroupSize(mGroupSize); + if (mCudaKernelEnabled) + { + mPluginProfiler->setCudaKernelType(mCudaKernelType, mArch); + } + mGemmId = GemmIdCore(mDims.n, mDims.k, mType); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* WeightOnlyGroupwiseQuantMatmulPlugin::clone() const noexcept +{ + auto* plugin = new WeightOnlyGroupwiseQuantMatmulPlugin(*this); + return plugin; +} + +void WeightOnlyGroupwiseQuantMatmulPlugin::configGemm() +{ + mPluginProfiler->profileTactics(m_weightOnlyGroupwiseGemmRunner, mType, mDims, mGemmId, mCudaKernelEnabled); +} + +nvinfer1::DimsExprs WeightOnlyGroupwiseQuantMatmulPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + + // inputs + // 0 activations [M, K] + // 1 pre-quant scales [K] (optional) + // 2 weights [K, N/2] + // 3 scales [K // group_size, N] + // 4 zeros [K // group_size, N] (optional) + // 5 biases [N] (optional) + + try + { + TLLM_CHECK(nbInputs == mBiasesInputIdx + 1); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + int const nbDimsB = inputs[mWeightInputIdx].nbDims; + TLLM_CHECK(nbDimsA >= 2); + TLLM_CHECK(nbDimsB == 2); + DimsExprs ret; + ret.nbDims = nbDimsA; + for (int ii = 0; ii < nbDimsA - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + + // int4/int8 weight only quant (INT4*4 -> FP16, INT8*2 -> FP16) + int const weight_multiplier = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? FP16_INT8_RATIO : FP16_INT4_RATIO; + ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[mWeightInputIdx].d[1]->getConstantValue() * weight_multiplier); + + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool WeightOnlyGroupwiseQuantMatmulPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + if (pos < nbInputs + 1) + { + return inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + } + else + { + // Never should be here + assert(false); + return false; + } +} + +void WeightOnlyGroupwiseQuantMatmulPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + + // Quantized weights are packed in FP16 format (INT4*4 -> FP16, INT8*2 -> FP16) + int const weight_multiplier = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? FP16_INT8_RATIO : FP16_INT4_RATIO; + int const maxN = in[mWeightInputIdx].max.d[1] * weight_multiplier; + + auto const K = maxK; + auto const N = maxN / weight_multiplier; + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, N, K}; + } + mGemmId = {N, K, mType}; + + size_t smoothedActSize = static_cast<size_t>(maxM) * static_cast<size_t>(maxK) + * (in[0].desc.type == nvinfer1::DataType::kFLOAT ? sizeof(float) : sizeof(half)); + m_workspaceMaxSize = smoothedActSize + m_weightOnlyGroupwiseGemmRunner->getWorkspaceSize(maxM, maxN, maxK); +} + +size_t WeightOnlyGroupwiseQuantMatmulPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return m_workspaceMaxSize; +} + +template <typename ActType> +void pre_quant_scale_for_act(int const m, int const k, int const mQuantAlgo, int const mPreQuantScaleInputIdx, + void const* const* inputs, void* workspace, cudaStream_t stream) +{ + // Apply pre-quant per channel scale on activations + if (mQuantAlgo & GroupwiseQuantAlgo::FP8_ALPHA) + { + tensorrt_llm::kernels::apply_per_channel_scale_kernel_launcher<ActType, __nv_fp8_e4m3>( + reinterpret_cast<__nv_fp8_e4m3*>(workspace), reinterpret_cast<ActType const*>(inputs[0]), + reinterpret_cast<ActType const*>(inputs[mPreQuantScaleInputIdx]), m, k, nullptr, stream); + } + else + { + tensorrt_llm::kernels::apply_per_channel_scale_kernel_launcher<ActType, ActType>( + reinterpret_cast<ActType*>(workspace), reinterpret_cast<ActType const*>(inputs[0]), + reinterpret_cast<ActType const*>(inputs[mPreQuantScaleInputIdx]), m, k, nullptr, stream); + } +} + +int WeightOnlyGroupwiseQuantMatmulPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // 0 activations [M, K] + // 1 pre-quant scales [K] + // 2 weights [K, N/2] + // 3 scales [K // group_size, N] + // 4 zeros [K // group_size, N] + // 5 biases [N] + // outputs + // mat [M, N] + + int64_t m64 = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m64 *= inputDesc[0].dims.d[ii]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[mWeightInputIdx].dims.d[1]); + int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); + + // get best tactic and check if CUDA kernel should be used + bool use_cuda_kernel = false; + auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); + TLLM_CHECK_WITH_INFO(bestTactic, + "No valid weight only groupwise GEMM tactic(It is usually caused by the failure to execute all " + "candidate configurations of the CUTLASS kernel, please pay attention to the warning information " + "when building the engine.)"); + use_cuda_kernel = bestTactic->enableCudaKernel; + + bool use_pre_quant_scale = mQuantAlgo & GroupwiseQuantAlgo::PRE_QUANT_SCALE; + half const* zeros_ptr + = (mQuantAlgo & GroupwiseQuantAlgo::ZERO) ? reinterpret_cast<half const*>(inputs[mZerosInputIdx]) : nullptr; + half const* biases_ptr + = (mQuantAlgo & GroupwiseQuantAlgo::BIAS) ? reinterpret_cast<half const*>(inputs[mBiasesInputIdx]) : nullptr; + half const* act_ptr = reinterpret_cast<half const*>(inputs[0]); + + if (use_pre_quant_scale && !use_cuda_kernel) + { + // Apply pre-quant per channel scale on activations + act_ptr = reinterpret_cast<half const*>(workspace); + if (mType == nvinfer1::DataType::kHALF) + { + pre_quant_scale_for_act<half>(m, k, mQuantAlgo, mPreQuantScaleInputIdx, inputs, workspace, stream); + } +#if defined(ENABLE_BF16) + else if (mType == nvinfer1::DataType::kBF16) + { + pre_quant_scale_for_act<__nv_bfloat16>(m, k, mQuantAlgo, mPreQuantScaleInputIdx, inputs, workspace, stream); + } +#endif + } + +#if defined(ENABLE_BF16) + TLLM_CHECK_WITH_INFO(mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16, + "No valid weightOnlyGropwiseQuantMatmul configuration"); +#else + TLLM_CHECK_WITH_INFO(mType == nvinfer1::DataType::kHALF, "No valid weightOnlyGropwiseQuantMatmul configuration"); +#endif + + // Quantized weights are packed in FP16 format (INT4*4 -> FP16, INT8*2 -> FP16) + int real_n = mQuantAlgo & GroupwiseQuantAlgo::INT8_WEIGHT ? n * FP16_INT8_RATIO : n * FP16_INT4_RATIO; + + if (use_cuda_kernel) + { + // Apply CUDA kernel + void const* pre_quant_scale_ptr = nullptr; + if (use_pre_quant_scale) + pre_quant_scale_ptr = inputs[mPreQuantScaleInputIdx]; + void const* cuda_kernel_act_ptr = inputs[0]; + void const* cuda_kernel_weight_ptr = inputs[mWeightInputIdx]; + void const* cuda_kernel_scales_ptr = inputs[mScalesInputIdx]; + void* cuda_kernel_out_ptr = outputs[0]; + tensorrt_llm::kernels::weight_only::Params params{cuda_kernel_act_ptr, pre_quant_scale_ptr, + cuda_kernel_weight_ptr, cuda_kernel_scales_ptr, zeros_ptr, biases_ptr, cuda_kernel_out_ptr, mAlpha, m, + real_n, k, mGroupSize, mCudaKernelType, static_cast<bool>(mQuantAlgo & GroupwiseQuantAlgo::FP8_ALPHA)}; + tensorrt_llm::kernels::weight_only::kernel_launcher(mArch, params, stream); + } + else + { + // Apply CUTLASS kernel + int const ws_bytes = m_weightOnlyGroupwiseGemmRunner->getWorkspaceSize(m, real_n, k); + int32_t* weight_ptr = const_cast<int32_t*>(reinterpret_cast<int32_t const*>(inputs[mWeightInputIdx])); + m_weightOnlyGroupwiseGemmRunner->gemm(act_ptr, weight_ptr, inputs[mScalesInputIdx], zeros_ptr, biases_ptr, + mAlpha, outputs[0], m, real_n, k, mGroupSize, *bestTactic, + reinterpret_cast<char*>(workspace) + m * k * sizeof(half), ws_bytes, stream); + } + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType WeightOnlyGroupwiseQuantMatmulPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* WeightOnlyGroupwiseQuantMatmulPlugin::getPluginType() const noexcept +{ + return WOQ_GROUPWISE_MATMUL_PLUGIN_NAME; +} + +char const* WeightOnlyGroupwiseQuantMatmulPlugin::getPluginVersion() const noexcept +{ + return WOQ_GROUPWISE_MATMUL_PLUGIN_VERSION; +} + +int WeightOnlyGroupwiseQuantMatmulPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int WeightOnlyGroupwiseQuantMatmulPlugin::initialize() noexcept +{ + configGemm(); + return 0; +} + +void WeightOnlyGroupwiseQuantMatmulPlugin::terminate() noexcept {} + +size_t WeightOnlyGroupwiseQuantMatmulPlugin::getSerializationSize() const noexcept +{ + return sizeof(nvinfer1::DataType) + // mType + sizeof(int) + // mQuantAlgo + sizeof(int) + // mGroupSize + sizeof(float) + // mAlpha + sizeof(mDims) + // Dimensions + mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size +} + +void WeightOnlyGroupwiseQuantMatmulPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mQuantAlgo); + write(d, mGroupSize); + write(d, mAlpha); + write(d, mDims); + + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void WeightOnlyGroupwiseQuantMatmulPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +WeightOnlyGroupwiseQuantMatmulPluginCreator::WeightOnlyGroupwiseQuantMatmulPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("quant_algo", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("group_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("alpha", nullptr, PluginFieldType::kFLOAT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* WeightOnlyGroupwiseQuantMatmulPluginCreator::getPluginName() const noexcept +{ + return WOQ_GROUPWISE_MATMUL_PLUGIN_NAME; +} + +char const* WeightOnlyGroupwiseQuantMatmulPluginCreator::getPluginVersion() const noexcept +{ + return WOQ_GROUPWISE_MATMUL_PLUGIN_VERSION; +} + +PluginFieldCollection const* WeightOnlyGroupwiseQuantMatmulPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* WeightOnlyGroupwiseQuantMatmulPluginCreator::createPlugin( + char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + nvinfer1::DataType type{}; + int QuantAlgo{}; + int GroupSize{}; + float Alpha{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "quant_algo")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + QuantAlgo = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "group_size")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + GroupSize = static_cast<int>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + else if (!strcmp(attrName, "alpha")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kFLOAT32); + Alpha = static_cast<float>(*(static_cast<float const*>(fields[i].data))); + } + } + try + { + // WeightOnlyGroupwiseQuantMatmulPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); + auto* obj = new WeightOnlyGroupwiseQuantMatmulPlugin(type, QuantAlgo, GroupSize, Alpha, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* WeightOnlyGroupwiseQuantMatmulPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call weightOnlyGroupwiseQuantMatmulPlugin::destroy() + try + { + // Create plugin profiler with private tactics map which is read from the serialized engine + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); + auto* obj = new WeightOnlyGroupwiseQuantMatmulPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.h b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.h new file mode 100644 index 000000000000..94e98ce0f5c0 --- /dev/null +++ b/cpp/tensorrt_llm/plugins/weightOnlyGroupwiseQuantMatmulPlugin/weightOnlyGroupwiseQuantMatmulPlugin.h @@ -0,0 +1,186 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/kernels/cutlass_kernels/fpA_intB_gemm/fpA_intB_gemm.h" +#include "tensorrt_llm/kernels/preQuantScaleKernel.h" +#include "tensorrt_llm/kernels/weightOnlyBatchedGemv//kernelLauncher.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h" + +#include <cutlass/numeric_types.h> + +#include <cassert> +#include <cuda_runtime.h> +#include <memory> +#include <set> +#include <string> +#include <vector> + +// The blank line here is to avoid clang-format -sort-includes option reordering these two cutlass header files and +// breaking dependencies +#include "cutlass/integer_subbyte.h" + +namespace tensorrt_llm::plugins +{ + +using WeightOnlyGemmRunner = tensorrt_llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunnerInterface; +using WeightOnlyGemmRunnerPtr = std::shared_ptr<WeightOnlyGemmRunner>; +using KernelType = tensorrt_llm::kernels::weight_only::KernelType; + +class WeightOnlyGroupwiseQuantGemmPluginProfiler + : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, WeightOnlyGemmRunnerPtr, + GemmIdCore, GemmIdCoreHash> +{ +public: + using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; + + void setQuantAlgo(int quantAlgo) + { + mQuantAlgo = quantAlgo; + } + + void setGroupSize(int groupSize) + { + mGroupSize = groupSize; + } + + void setCudaKernelType(KernelType cudaKernelType, int arch) + { + mCudaKernelType = cudaKernelType; + mArch = arch; + } + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + std::vector<Config> getTactics(int m, int n, int k) const override; + + bool checkTactic(int m, int n, int k, Config const& tactic) const override; + +private: + int mQuantAlgo; + int mGroupSize; + KernelType mCudaKernelType; + int mArch; +}; + +class WeightOnlyGroupwiseQuantMatmulPlugin : public BasePlugin +{ +public: + using PluginProfilerPtr = std::shared_ptr<WeightOnlyGroupwiseQuantGemmPluginProfiler>; + + WeightOnlyGroupwiseQuantMatmulPlugin() = delete; + + WeightOnlyGroupwiseQuantMatmulPlugin( + nvinfer1::DataType type, int quant_algo, int group_size, float alpha, PluginProfilerPtr const& profiler); + + WeightOnlyGroupwiseQuantMatmulPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); + + ~WeightOnlyGroupwiseQuantMatmulPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + // group_size: 64, 128 + void init(nvinfer1::DataType type, int quant_algo, int group_size, float alpha); + + void configGemm(); + +private: + const std::string mLayerName; + + WeightOnlyGemmRunnerPtr m_weightOnlyGroupwiseGemmRunner; + size_t m_workspaceMaxSize; + nvinfer1::DataType mType; + bool mCudaKernelEnabled; + tensorrt_llm::kernels::weight_only::KernelType mCudaKernelType; + int mArch; + + // When M is smaller than this value, we trigger a fast path + // I.e. a tailored kernel instead of cutlass. + + int mQuantAlgo; + + int mGroupSize; + + float mAlpha = 1.0f; + + int mPreQuantScaleInputIdx; + int mWeightInputIdx; + int mScalesInputIdx; + int mZerosInputIdx; + int mBiasesInputIdx; + + GemmDims mDims{}; + GemmIdCore mGemmId{}; + + PluginProfilerPtr mPluginProfiler; +}; + +class WeightOnlyGroupwiseQuantMatmulPluginCreator : public BaseCreator +{ +public: + WeightOnlyGroupwiseQuantMatmulPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<WeightOnlyGroupwiseQuantGemmPluginProfiler> gemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/CMakeLists.txt b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/CMakeLists.txt new file mode 100755 index 000000000000..86876224fccd --- /dev/null +++ b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +file(GLOB SRCS *.cpp) +set(PLUGIN_SOURCES ${PLUGIN_SOURCES} ${SRCS}) +set(PLUGIN_SOURCES + ${PLUGIN_SOURCES} + PARENT_SCOPE) diff --git a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.cpp b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.cpp new file mode 100644 index 000000000000..f3ed07fafaff --- /dev/null +++ b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.cpp @@ -0,0 +1,507 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & + * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "weightOnlyQuantMatmulPlugin.h" + +#include <numeric> + +using namespace nvinfer1; +using namespace tensorrt_llm::common; +using namespace tensorrt_llm::kernels::cutlass_kernels; +using tensorrt_llm::plugins::WeightOnlyQuantMatmulPluginCreator; +using tensorrt_llm::plugins::WeightOnlyQuantMatmulPlugin; +using tensorrt_llm::plugins::WeightOnlyQuantGemmPluginProfiler; +using tensorrt_llm::plugins::read; +using tensorrt_llm::plugins::write; + +static char const* WOQ_MATMUL_PLUGIN_VERSION{"1"}; +static char const* WOQ_MATMUL_PLUGIN_NAME{"WeightOnlyQuantMatmul"}; +PluginFieldCollection WeightOnlyQuantMatmulPluginCreator::mFC{}; +std::vector<nvinfer1::PluginField> WeightOnlyQuantMatmulPluginCreator::mPluginAttributes; + +void WeightOnlyQuantGemmPluginProfiler::runTactic(int m, int n, int k, + WeightOnlyQuantGemmPluginProfiler::Config const& tactic, char* workspace, cudaStream_t const& stream) +{ + int const originalN = n * getWeightTypeMultiplier(mWeightTypeId); + half* actPtr = reinterpret_cast<half*>(workspace); + int8_t* weightPtr + = reinterpret_cast<int8_t*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(actPtr), m * k * sizeof(half))); + half* scalesPtr + = reinterpret_cast<half*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(weightPtr), n * k * sizeof(int8_t))); + half* outputPtr + = reinterpret_cast<half*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(scalesPtr), originalN * sizeof(half))); + char* workspacePtr + = reinterpret_cast<char*>(nextWorkspacePtr(reinterpret_cast<int8_t*>(outputPtr), m * originalN * sizeof(half))); + + int const wsSize = mRunner->getWorkspaceSize(m, originalN, k); + + if (tactic.enableCudaKernel) + { + // run CUDA kernel + tensorrt_llm::kernels::weight_only::Params params{actPtr, nullptr, weightPtr, scalesPtr, nullptr, nullptr, + outputPtr, 1.f, m, originalN, k, 0, mCudaKernelType}; + tensorrt_llm::kernels::weight_only::kernel_launcher(mArch, params, stream); + } + else + { + // run CUTLASS kernel + if (mWeightTypeId == WeightTypeId::INT8) + { + mRunner->gemm( + actPtr, weightPtr, scalesPtr, outputPtr, m, originalN, k, tactic, workspacePtr, wsSize, stream); + } + else + { + mRunner->gemm(actPtr, reinterpret_cast<cutlass::uint4b_t*>(weightPtr), scalesPtr, outputPtr, m, originalN, + k, tactic, workspacePtr, wsSize, stream); + } + } +} + +void WeightOnlyQuantGemmPluginProfiler::computeTmpSize(size_t maxM, size_t n, size_t k) +{ + int const originalN = n * getWeightTypeMultiplier(mWeightTypeId); + std::vector<size_t> workspaces = { + maxM * k * sizeof(half), // A + n * k * sizeof(int8_t), // B + originalN * sizeof(half), // scales + maxM * originalN * sizeof(half), // C + mRunner->getWorkspaceSize(maxM, originalN, k) // workspace + }; + size_t bytes = calculateTotalWorkspaceSize(workspaces.data(), workspaces.size()); + setTmpWorkspaceSizeInBytes(bytes); +} + +std::vector<WeightOnlyQuantGemmPluginProfiler::Config> WeightOnlyQuantGemmPluginProfiler::getTactics( + int m, int n, int k) const +{ + return mRunner->getConfigs(); +} + +bool WeightOnlyQuantGemmPluginProfiler::checkTactic(int m, int n, int k, Config const& tactic) const +{ + // stop to profile Cuda kernel for m >= 16 + if (tactic.enableCudaKernel) + { + return m < 16; + } + return true; +} + +WeightOnlyQuantMatmulPlugin::WeightOnlyQuantMatmulPlugin(nvinfer1::DataType type, WeightTypeId weightTypeId, + WeightOnlyQuantMatmulPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + init(type, weightTypeId); +} + +// Parameterized constructor +WeightOnlyQuantMatmulPlugin::WeightOnlyQuantMatmulPlugin( + void const* data, size_t length, WeightOnlyQuantMatmulPlugin::PluginProfilerPtr const& pluginProfiler) + : mPluginProfiler(pluginProfiler) +{ + char const *d = reinterpret_cast<char const*>(data), *a = d; + nvinfer1::DataType type; + WeightTypeId weightTypeId; + read(d, type); + read(d, weightTypeId); + read(d, mDims); + + init(type, weightTypeId); + + mPluginProfiler->deserialize(d, mDims, mGemmId); + + TLLM_CHECK_WITH_INFO(d == a + length, + "Expected length (%d) != real length (%d). This is often " + "caused by using different TensorRT LLM version to build " + "engine and run engine.", + (int) length, (int) (d - a)); +} + +void WeightOnlyQuantMatmulPlugin::init(nvinfer1::DataType type, WeightTypeId weightTypeId) +{ + mArch = tensorrt_llm::common::getSMVersion(); + mType = type; + mWeightTypeId = weightTypeId; + + if (mWeightTypeId == WeightTypeId::INT8) + { + if (mType == nvinfer1::DataType::kHALF) + { + m_weightOnlyGemmRunner = std::make_shared< + CutlassFpAIntBGemmRunner<half, uint8_t, cutlass::WeightOnlyQuantOp::PER_COLUMN_SCALE_ONLY>>(); + mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( + mArch, tensorrt_llm::kernels::weight_only::KernelType::FP16Int8PerChannel); + mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::FP16Int8PerChannel; + } +#if defined(ENABLE_BF16) + else if (mType == nvinfer1::DataType::kBF16) + { + m_weightOnlyGemmRunner = std::make_shared< + CutlassFpAIntBGemmRunner<__nv_bfloat16, uint8_t, cutlass::WeightOnlyQuantOp::PER_COLUMN_SCALE_ONLY>>(); + mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( + mArch, tensorrt_llm::kernels::weight_only::KernelType::BF16Int8PerChannel); + mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::BF16Int8PerChannel; + } +#endif + else + { + TLLM_CHECK(false); + } + } + else if (mWeightTypeId == WeightTypeId::INT4) + { + if (mType == nvinfer1::DataType::kHALF) + { + m_weightOnlyGemmRunner = std::make_shared< + CutlassFpAIntBGemmRunner<half, cutlass::uint4b_t, cutlass::WeightOnlyQuantOp::PER_COLUMN_SCALE_ONLY>>(); + mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( + mArch, tensorrt_llm::kernels::weight_only::KernelType::FP16Int4PerChannel); + mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::FP16Int4PerChannel; + } +#if defined(ENABLE_BF16) + else if (mType == nvinfer1::DataType::kBF16) + { + m_weightOnlyGemmRunner = std::make_shared<CutlassFpAIntBGemmRunner<__nv_bfloat16, cutlass::uint4b_t, + cutlass::WeightOnlyQuantOp::PER_COLUMN_SCALE_ONLY>>(); + mCudaKernelEnabled = tensorrt_llm::kernels::weight_only::is_supported( + mArch, tensorrt_llm::kernels::weight_only::KernelType::BF16Int4PerChannel); + mCudaKernelType = tensorrt_llm::kernels::weight_only::KernelType::BF16Int4PerChannel; + } +#endif + else + { + TLLM_CHECK(false); + } + } + else + { + TLLM_CHECK(false); + } + + mPluginProfiler->setWeightTypeId(mWeightTypeId); + if (mCudaKernelEnabled) + { + mPluginProfiler->setCudaKernelType(mCudaKernelType, mArch); + } + mGemmId = GemmIdCore(mDims.n, mDims.k, mType); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* WeightOnlyQuantMatmulPlugin::clone() const noexcept +{ + auto* plugin = new WeightOnlyQuantMatmulPlugin(*this); + return plugin; +} + +void WeightOnlyQuantMatmulPlugin::configGemm() +{ + mPluginProfiler->profileTactics(m_weightOnlyGemmRunner, mType, mDims, mGemmId, mCudaKernelEnabled); +} + +nvinfer1::DimsExprs WeightOnlyQuantMatmulPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + // input [m1, m2, m3, ... , k] + // weight [k, n] for int8, [k, n/2] for int4 + + try + { + TLLM_CHECK(nbInputs == 3); + TLLM_CHECK(outputIndex == 0); + int const nbDimsA = inputs[0].nbDims; + int const nbDimsB = inputs[1].nbDims; + TLLM_CHECK(nbDimsA >= 2); + TLLM_CHECK(nbDimsB == 2); + DimsExprs ret; + ret.nbDims = nbDimsA; + for (int ii = 0; ii < nbDimsA - 1; ++ii) + { + ret.d[ii] = inputs[0].d[ii]; + } + if (mWeightTypeId == WeightTypeId::INT8) + { + // int8 weight only quant + ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[1].d[1]->getConstantValue()); + } + else + { + // int4 weight only quant + ret.d[nbDimsA - 1] = exprBuilder.constant(inputs[1].d[1]->getConstantValue() * INT8_INT4_RATIO); + } + return ret; + } + catch (std::exception const& e) + { + caughtError(e); + } + return DimsExprs{}; +} + +bool WeightOnlyQuantMatmulPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + switch (pos) + { + case 0: + // activation + return inOut[0].type == mType && inOut[0].format == TensorFormat::kLINEAR; + case 1: + // weights + // Weights are required to be int8, but will be reinterpreted as int4 in enqueue if required + // Weights stored in checkpoint should have int8/int4 type + return inOut[1].type == nvinfer1::DataType::kINT8 && inOut[1].format == TensorFormat::kLINEAR; + case 2: + // scales channels + return inOut[2].type == mType && inOut[2].format == TensorFormat::kLINEAR; + case 3: + // out + return inOut[3].type == mType && inOut[3].format == TensorFormat::kLINEAR; + default: + // Never should be here + assert(false); + return false; + } +} + +void WeightOnlyQuantMatmulPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ + auto const minM = std::accumulate(in[0].min.d, in[0].min.d + in[0].min.nbDims - 1, 1, std::multiplies<int>()); + auto const maxM = std::accumulate(in[0].max.d, in[0].max.d + in[0].max.nbDims - 1, 1, std::multiplies<int>()); + + int const maxK = in[0].max.d[in[0].max.nbDims - 1]; + int const maxN = in[1].max.d[1] * getWeightTypeMultiplier(mWeightTypeId); + + auto const K = maxK; + auto const N = maxN / getWeightTypeMultiplier(mWeightTypeId); + + if (!mDims.isInitialized()) + { + mDims = {minM, maxM, N, K}; + } + + mGemmId = {N, K, mType}; + + m_workspaceMaxSize = m_weightOnlyGemmRunner->getWorkspaceSize(maxM, maxN, maxK); +} + +size_t WeightOnlyQuantMatmulPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + return m_workspaceMaxSize; +} + +int WeightOnlyQuantMatmulPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + // inputs + // mat1 [M1, M2,..., K] + // mat2 [K, N] for int8, [K, N/2] for int4 + // scale_channels [N] + // outputs + // mat [M, N] + + int64_t m64 = 1; + for (int ii = 0; ii < inputDesc[0].dims.nbDims - 1; ++ii) + { + m64 *= inputDesc[0].dims.d[ii]; + } + int const m = TLLM_INT32_CAST(m64); + int const n = TLLM_INT32_CAST(inputDesc[1].dims.d[1]); + int const k = TLLM_INT32_CAST(inputDesc[0].dims.d[inputDesc[0].dims.nbDims - 1]); + + if (m == 0) + return 0; + +#if defined(ENABLE_BF16) + TLLM_CHECK_WITH_INFO(mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16, + "No valid weightOnlyQuantMatmul configuration"); +#else + TLLM_CHECK_WITH_INFO(mType == nvinfer1::DataType::kHALF, "No valid weightOnlyQuantMatmul configuration"); +#endif + int real_n = mWeightTypeId == WeightTypeId::INT4 ? n * INT8_INT4_RATIO : n; + + // get best tactic and check if CUDA kernel should be used + bool use_cuda_kernel = false; + auto const& bestTactic = mPluginProfiler->getBestConfig(m, mGemmId); + TLLM_CHECK_WITH_INFO(bestTactic, + "No valid weight only per-channel GEMM tactic(It is usually caused by the failure to execute all candidate " + "configurations of the CUTLASS kernel, please pay attention to the warning information when building the " + "engine.)"); + use_cuda_kernel = bestTactic->enableCudaKernel; + if (use_cuda_kernel) + { + void const* cuda_kernel_act_ptr = inputs[0]; + void const* cuda_kernel_weight_ptr = inputs[1]; + void const* cuda_kernel_scales_ptr = inputs[2]; + void* cuda_kernel_out_ptr = outputs[0]; + tensorrt_llm::kernels::weight_only::Params params(cuda_kernel_act_ptr, nullptr, cuda_kernel_weight_ptr, + cuda_kernel_scales_ptr, nullptr, nullptr, cuda_kernel_out_ptr, 1.f, m, real_n, k, 0, mCudaKernelType); + tensorrt_llm::kernels::weight_only::kernel_launcher(mArch, params, stream); + } + else + { + int const ws_size = m_weightOnlyGemmRunner->getWorkspaceSize(m, real_n, k); + + m_weightOnlyGemmRunner->gemm(inputs[0], inputs[1], inputs[2], outputs[0], m, real_n, k, *bestTactic, + reinterpret_cast<char*>(workspace), ws_size, stream); + } + + return 0; +} + +// IPluginV2Ext Methods +nvinfer1::DataType WeightOnlyQuantMatmulPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + TLLM_CHECK(index == 0); + return mType; +} + +// IPluginV2 Methods + +char const* WeightOnlyQuantMatmulPlugin::getPluginType() const noexcept +{ + return WOQ_MATMUL_PLUGIN_NAME; +} + +char const* WeightOnlyQuantMatmulPlugin::getPluginVersion() const noexcept +{ + return WOQ_MATMUL_PLUGIN_VERSION; +} + +int WeightOnlyQuantMatmulPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int WeightOnlyQuantMatmulPlugin::initialize() noexcept +{ + configGemm(); + return 0; +} + +void WeightOnlyQuantMatmulPlugin::terminate() noexcept {} + +size_t WeightOnlyQuantMatmulPlugin::getSerializationSize() const noexcept +{ + return sizeof(mWeightTypeId) + // mWeightTypeId + sizeof(nvinfer1::DataType) + // mType + sizeof(mDims) + // Dimensions + mPluginProfiler->getSerializationSize(mGemmId); // selected tactics container size +} + +void WeightOnlyQuantMatmulPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast<char*>(buffer), *a = d; + write(d, mType); + write(d, mWeightTypeId); + write(d, mDims); + + mPluginProfiler->serialize(d, mGemmId); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void WeightOnlyQuantMatmulPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +/////////////// + +WeightOnlyQuantMatmulPluginCreator::WeightOnlyQuantMatmulPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("weight_type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* WeightOnlyQuantMatmulPluginCreator::getPluginName() const noexcept +{ + return WOQ_MATMUL_PLUGIN_NAME; +} + +char const* WeightOnlyQuantMatmulPluginCreator::getPluginVersion() const noexcept +{ + return WOQ_MATMUL_PLUGIN_VERSION; +} + +PluginFieldCollection const* WeightOnlyQuantMatmulPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* WeightOnlyQuantMatmulPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + nvinfer1::DataType type{}; + WeightTypeId weightTypeId{}; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "weight_type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + weightTypeId = static_cast<WeightTypeId>(*(static_cast<int const*>(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + TLLM_CHECK(fields[i].type == PluginFieldType::kINT32); + type = static_cast<nvinfer1::DataType>(*(static_cast<nvinfer1::DataType const*>(fields[i].data))); + } + } + try + { + // WeightOnlyGroupwiseQuantMatmulPluginCreator is unique and shared for an engine generation + // Create plugin profiler with shared tactics map + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ false); + auto* obj = new WeightOnlyQuantMatmulPlugin(type, weightTypeId, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} + +IPluginV2* WeightOnlyQuantMatmulPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call WeightOnlyQuantMatmulPlugin::destroy() + try + { + // Create plugin profiler with private tactics map which is read from the serialized engine + auto pluginProfiler = gemmPluginProfileManager.createGemmPluginProfiler(/* inference */ true); + auto* obj = new WeightOnlyQuantMatmulPlugin(serialData, serialLength, pluginProfiler); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + caughtError(e); + } + return nullptr; +} diff --git a/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h new file mode 100644 index 000000000000..3177d8297d2d --- /dev/null +++ b/cpp/tensorrt_llm/plugins/weightOnlyQuantMatmulPlugin/weightOnlyQuantMatmulPlugin.h @@ -0,0 +1,175 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/common/quantization.h" +#include "tensorrt_llm/kernels/cutlass_kernels/fpA_intB_gemm/fpA_intB_gemm.h" +#include "tensorrt_llm/kernels/weightOnlyBatchedGemv/kernelLauncher.h" +#include "tensorrt_llm/plugins/common/gemmPluginProfiler.h" +#include "tensorrt_llm/plugins/common/plugin.h" + +#include <cassert> +#include <cutlass/numeric_types.h> +#include <memory> +#include <set> +#include <string> +#include <vector> + +// The blank line here is to avoid clang-format -sort-includes option reordering these two cutlass header files and +// breaking dependencies +#include "cutlass/integer_subbyte.h" + +namespace tensorrt_llm::plugins +{ +enum class WeightTypeId +{ + INT8 = 1, + INT4 = 2, +}; + +constexpr int32_t FP16_BITS = 16; +constexpr int32_t INT8_BITS = 8; +constexpr int32_t INT4_BITS = 4; +constexpr int32_t INT8_INT4_RATIO = INT8_BITS / INT4_BITS; +constexpr int32_t FP16_INT4_RATIO = FP16_BITS / INT4_BITS; +constexpr int32_t FP16_INT8_RATIO = FP16_BITS / INT8_BITS; + +inline int32_t getWeightTypeMultiplier(WeightTypeId weightTypeId) +{ + return weightTypeId == WeightTypeId::INT8 ? 1 : INT8_INT4_RATIO; +} + +using WeightOnlyGemmRunner = tensorrt_llm::kernels::cutlass_kernels::CutlassFpAIntBGemmRunnerInterface; +using WeightOnlyGemmRunnerPtr = std::shared_ptr<WeightOnlyGemmRunner>; +using KernelType = tensorrt_llm::kernels::weight_only::KernelType; + +class WeightOnlyQuantGemmPluginProfiler : public GemmPluginProfiler<tensorrt_llm::cutlass_extensions::CutlassGemmConfig, + WeightOnlyGemmRunnerPtr, GemmIdCore, GemmIdCoreHash> +{ +public: + using Config = tensorrt_llm::cutlass_extensions::CutlassGemmConfig; + + void setWeightTypeId(WeightTypeId weightId) + { + mWeightTypeId = weightId; + } + + void setCudaKernelType(KernelType cudaKernelType, int arch) + { + mCudaKernelType = cudaKernelType; + mArch = arch; + } + +protected: + void runTactic(int m, int n, int k, Config const& tactic, char* workspace, cudaStream_t const& stream) override; + + void computeTmpSize(size_t maxM, size_t n, size_t k) override; + + std::vector<Config> getTactics(int m, int n, int k) const override; + + bool checkTactic(int m, int n, int k, Config const& tactic) const override; + +private: + WeightTypeId mWeightTypeId; + KernelType mCudaKernelType; + int mArch; +}; + +class WeightOnlyQuantMatmulPlugin : public BasePlugin +{ +public: + using PluginProfilerPtr = std::shared_ptr<WeightOnlyQuantGemmPluginProfiler>; + WeightOnlyQuantMatmulPlugin() = delete; + + WeightOnlyQuantMatmulPlugin(nvinfer1::DataType type, WeightTypeId weightTypeId, PluginProfilerPtr const& profiler); + + WeightOnlyQuantMatmulPlugin(void const* data, size_t length, PluginProfilerPtr const& profiler); + + ~WeightOnlyQuantMatmulPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + +private: + void init(nvinfer1::DataType type, WeightTypeId weightTypeId); + + void configGemm(); + +private: + const std::string mLayerName; + + WeightOnlyGemmRunnerPtr m_weightOnlyGemmRunner; + size_t m_workspaceMaxSize; + nvinfer1::DataType mType; + WeightTypeId mWeightTypeId; + bool mCudaKernelEnabled; + tensorrt_llm::kernels::weight_only::KernelType mCudaKernelType; + int mArch; + + GemmDims mDims{}; + GemmIdCore mGemmId{}; + + PluginProfilerPtr mPluginProfiler; +}; + +class WeightOnlyQuantMatmulPluginCreator : public BaseCreator +{ +public: + WeightOnlyQuantMatmulPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + +private: + GemmPluginProfilerManager<WeightOnlyQuantGemmPluginProfiler> gemmPluginProfileManager; + static nvinfer1::PluginFieldCollection mFC; + static std::vector<nvinfer1::PluginField> mPluginAttributes; +}; + +} // namespace tensorrt_llm::plugins diff --git a/cpp/tensorrt_llm/runtime/CMakeLists.txt b/cpp/tensorrt_llm/runtime/CMakeLists.txt index 11a9391c0e69..ca81fbb0f6cd 100644 --- a/cpp/tensorrt_llm/runtime/CMakeLists.txt +++ b/cpp/tensorrt_llm/runtime/CMakeLists.txt @@ -26,6 +26,7 @@ set(SRCS eagleBuffers.cpp explicitDraftTokensBuffers.cpp lookaheadBuffers.cpp + layerProfiler.cpp loraManager.cpp loraUtils.cpp loraModule.cpp @@ -50,6 +51,9 @@ set(SRCS promptTuningParams.cpp runtimeKernels.cu tllmBuffers.cpp + tllmRuntime.cpp + tllmStreamReaders.cpp + tllmLogger.cpp workerPool.cpp worldConfig.cpp virtualMemory.cpp) diff --git a/cpp/tensorrt_llm/runtime/bufferManager.cpp b/cpp/tensorrt_llm/runtime/bufferManager.cpp index 58257516a387..3de42a253158 100644 --- a/cpp/tensorrt_llm/runtime/bufferManager.cpp +++ b/cpp/tensorrt_llm/runtime/bufferManager.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tllmBuffers.h" #include <cstring> @@ -38,7 +37,7 @@ BufferManager::BufferManager(CudaStreamPtr stream, bool trimPool) mPool = CudaMemPool::getPrimaryPoolForDevice(mStream->getDevice()); } -BufferManager::IBufferPtr BufferManager::gpu(std::size_t size, tensorrt_llm::DataType type) const +BufferManager::IBufferPtr BufferManager::gpu(std::size_t size, nvinfer1::DataType type) const { if (auto vmAllocator = getVirtualMemoryAllocator()) { @@ -52,7 +51,7 @@ BufferManager::IBufferPtr BufferManager::gpu(std::size_t size, tensorrt_llm::Dat return gpuSync(size, type); } -BufferManager::ITensorPtr BufferManager::gpu(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) const +BufferManager::ITensorPtr BufferManager::gpu(nvinfer1::Dims dims, nvinfer1::DataType type) const { if (auto vmAllocator = getVirtualMemoryAllocator()) { @@ -66,7 +65,7 @@ BufferManager::ITensorPtr BufferManager::gpu(tensorrt_llm::Dims dims, tensorrt_l return gpuSync(dims, type); } -BufferManager::IBufferPtr BufferManager::gpuSync(std::size_t size, tensorrt_llm::DataType type) +BufferManager::IBufferPtr BufferManager::gpuSync(std::size_t size, nvinfer1::DataType type) { if (auto vmAllocator = getVirtualMemoryAllocator()) { @@ -75,7 +74,7 @@ BufferManager::IBufferPtr BufferManager::gpuSync(std::size_t size, tensorrt_llm: return std::make_unique<StaticDeviceBuffer>(size, type, CudaAllocator{}); } -BufferManager::ITensorPtr BufferManager::gpuSync(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) +BufferManager::ITensorPtr BufferManager::gpuSync(nvinfer1::Dims dims, nvinfer1::DataType type) { if (auto vmAllocator = getVirtualMemoryAllocator()) { @@ -84,48 +83,47 @@ BufferManager::ITensorPtr BufferManager::gpuSync(tensorrt_llm::Dims dims, tensor return std::make_unique<StaticDeviceTensor>(dims, type, CudaAllocator{}); } -BufferManager::IBufferPtr BufferManager::cpu(std::size_t size, tensorrt_llm::DataType type) +BufferManager::IBufferPtr BufferManager::cpu(std::size_t size, nvinfer1::DataType type) { return std::make_unique<HostBuffer>(size, type); } -BufferManager::ITensorPtr BufferManager::cpu(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) +BufferManager::ITensorPtr BufferManager::cpu(nvinfer1::Dims dims, nvinfer1::DataType type) { return std::make_unique<HostTensor>(dims, type); } -BufferManager::IBufferPtr BufferManager::pinned(std::size_t size, tensorrt_llm::DataType type) +BufferManager::IBufferPtr BufferManager::pinned(std::size_t size, nvinfer1::DataType type) { return std::make_unique<PinnedBuffer>(size, type); } -BufferManager::ITensorPtr BufferManager::pinned(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) +BufferManager::ITensorPtr BufferManager::pinned(nvinfer1::Dims dims, nvinfer1::DataType type) { return std::make_unique<PinnedTensor>(dims, type); } -BufferManager::IBufferPtr BufferManager::pinnedPool(std::size_t size, tensorrt_llm::DataType type) +BufferManager::IBufferPtr BufferManager::pinnedPool(std::size_t size, nvinfer1::DataType type) { return std::make_unique<PinnedPoolBuffer>(size, type); } -BufferManager::ITensorPtr BufferManager::pinnedPool(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) +BufferManager::ITensorPtr BufferManager::pinnedPool(nvinfer1::Dims dims, nvinfer1::DataType type) { return std::make_unique<PinnedPoolTensor>(dims, type); } -BufferManager::IBufferPtr BufferManager::managed(std::size_t size, tensorrt_llm::DataType type) +BufferManager::IBufferPtr BufferManager::managed(std::size_t size, nvinfer1::DataType type) { return std::make_unique<UVMBuffer>(size, type); } -BufferManager::ITensorPtr BufferManager::managed(tensorrt_llm::Dims dims, tensorrt_llm::DataType type) +BufferManager::ITensorPtr BufferManager::managed(nvinfer1::Dims dims, nvinfer1::DataType type) { return std::make_unique<UVMTensor>(dims, type); } -BufferManager::ITensorPtr BufferManager::ipcNvls( - std::set<int> ranks, tensorrt_llm::Dims dims, tensorrt_llm::DataType type) +BufferManager::ITensorPtr BufferManager::ipcNvls(std::set<int> ranks, nvinfer1::Dims dims, nvinfer1::DataType type) { return std::make_unique<MulticastTensor>(dims, type, ranks); } @@ -189,7 +187,7 @@ void BufferManager::copy(IBuffer const& src, IBuffer& dst) const } BufferManager::IBufferPtr BufferManager::allocate( - MemoryType memoryType, std::size_t size, tensorrt_llm::DataType type) const + MemoryType memoryType, std::size_t size, nvinfer1::DataType type) const { switch (memoryType) { @@ -204,7 +202,7 @@ BufferManager::IBufferPtr BufferManager::allocate( } BufferManager::ITensorPtr BufferManager::allocate( - MemoryType memoryType, tensorrt_llm::Dims dims, tensorrt_llm::DataType type) const + MemoryType memoryType, nvinfer1::Dims dims, nvinfer1::DataType type) const { switch (memoryType) { diff --git a/cpp/tensorrt_llm/runtime/bufferView.h b/cpp/tensorrt_llm/runtime/bufferView.h index a001d05f1f8b..236b89d7d455 100644 --- a/cpp/tensorrt_llm/runtime/bufferView.h +++ b/cpp/tensorrt_llm/runtime/bufferView.h @@ -17,7 +17,6 @@ #pragma once #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/iBuffer.h" #include <string> @@ -71,7 +70,7 @@ class BufferView : virtual public IBuffer return mBuffer->getCapacity() - mOffset; } - [[nodiscard]] tensorrt_llm::DataType getDataType() const override + [[nodiscard]] nvinfer1::DataType getDataType() const override { return mBuffer->getDataType(); } diff --git a/cpp/tensorrt_llm/runtime/decoderState.cpp b/cpp/tensorrt_llm/runtime/decoderState.cpp index 83037b2431cb..b5851dc1c2d2 100644 --- a/cpp/tensorrt_llm/runtime/decoderState.cpp +++ b/cpp/tensorrt_llm/runtime/decoderState.cpp @@ -16,7 +16,6 @@ #include "tensorrt_llm/runtime/decoderState.h" #include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -28,10 +27,10 @@ using TensorPtr = DecoderState::TensorPtr; BeamSearchBuffers::BeamSearchBuffers(BufferManager const& bufferManager) : mOutputBeamHypotheses{} - , mCumLogProbsTmp(bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT)) + , mCumLogProbsTmp(bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kFLOAT)) { mOutputBeamHypotheses.empty(bufferManager); - mCumLogProbsTmp = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); + mCumLogProbsTmp = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kFLOAT); int device; cudaGetDevice(&device); @@ -55,8 +54,8 @@ DecoderState::DecoderState() } void DecoderState::setup(SizeType32 maxNumSequences, SizeType32 maxBeamWidth, SizeType32 maxAttentionWindow, - SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, tensorrt_llm::DataType dtype, - ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager) + SizeType32 sinkTokenLength, SizeType32 maxSequenceLength, nvinfer1::DataType dtype, ModelConfig const& modelConfig, + WorldConfig const& worldConfig, BufferManager const& bufferManager) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); setupBuffers(dtype, bufferManager); @@ -65,7 +64,7 @@ void DecoderState::setup(SizeType32 maxNumSequences, SizeType32 maxBeamWidth, Si TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); } -void DecoderState::setupBuffers(tensorrt_llm::DataType dtype, BufferManager const& bufferManager) +void DecoderState::setupBuffers(nvinfer1::DataType dtype, BufferManager const& bufferManager) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); auto constexpr nvTokenIdType = TRTDataType<TokenIdType>::value; @@ -115,7 +114,7 @@ void DecoderState::setupBuffers(tensorrt_llm::DataType dtype, BufferManager cons } void DecoderState::setupSpeculativeDecoding(SpeculativeDecodingMode const& speculativeDecodingMode, - SizeType32 maxTokensPerEngineStep, tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, + SizeType32 maxTokensPerEngineStep, nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig, BufferManager const& bufferManager) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); @@ -125,8 +124,8 @@ void DecoderState::setupSpeculativeDecoding(SpeculativeDecodingMode const& specu TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); } -void DecoderState::setupSpeculativeDecodingBuffers(SpeculativeDecodingMode const speculativeDecodingMode, - tensorrt_llm::DataType dtype, BufferManager const& bufferManager) +void DecoderState::setupSpeculativeDecodingBuffers( + SpeculativeDecodingMode const speculativeDecodingMode, nvinfer1::DataType dtype, BufferManager const& bufferManager) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); @@ -152,13 +151,13 @@ void DecoderState::setupSpeculativeDecodingBuffers(SpeculativeDecodingMode const if (speculativeDecodingMode.predictsDraftTokens()) { speculativeDecodingOutputs.nextDraftTokens - = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); if (speculativeDecodingMode.variableDraftLength()) { speculativeDecodingOutputs.nextDraftTokensLen - = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); speculativeDecodingOutputs.prevDraftTokensLen - = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); } } if (speculativeDecodingMode.isLookaheadDecoding()) @@ -168,11 +167,11 @@ void DecoderState::setupSpeculativeDecodingBuffers(SpeculativeDecodingMode const if (speculativeDecodingMode.needsKVCacheRewind()) { speculativeDecodingOutputs.acceptedTokensLen - = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); speculativeDecodingOutputs.acceptedLengthsCumSum - = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); speculativeDecodingOutputs.pathsOffsets - = bufferManager.emptyTensor(MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + = bufferManager.emptyTensor(MemoryType::kGPU, nvinfer1::DataType::kINT32); } dOutput->speculativeDecodingOutputs = speculativeDecodingOutputs; diff --git a/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.cpp b/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.cpp index c5098bf777e0..f8a4fa4e7467 100644 --- a/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.cpp +++ b/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.cpp @@ -15,12 +15,11 @@ */ #include "tensorrt_llm/runtime/decodingLayerWorkspace.h" -#include "tensorrt_llm/common/tllmDataType.h" #include <utility> tensorrt_llm::runtime::DecodingLayerWorkspace::DecodingLayerWorkspace(std::shared_ptr<BufferManager> bufferManager, - tensorrt_llm::layers::DecoderDomain const& decoderDomain, tensorrt_llm::DataType logitsType, + tensorrt_llm::layers::DecoderDomain const& decoderDomain, nvinfer1::DataType logitsType, size_t workspaceBufferSizeInBytes) : mBufferManager(std::move(bufferManager)) , mBatchSlotsDevice( @@ -83,8 +82,7 @@ void tensorrt_llm::runtime::DecodingLayerWorkspace::resize(size_t minSize) } tensorrt_llm::runtime::DecodingLayerWorkspace::TensorPtr -tensorrt_llm::runtime::DecodingLayerWorkspace::getWorkspaceAsDeviceTensor( - ITensor::Shape shape, tensorrt_llm::DataType type) +tensorrt_llm::runtime::DecodingLayerWorkspace::getWorkspaceAsDeviceTensor(ITensor::Shape shape, nvinfer1::DataType type) { auto const sizeInBytes = ITensor::volume(shape) * BufferDataType(type).getSize(); return std::make_shared<GenericTensor<BorrowingAllocator<MemoryType::kGPU>>>( diff --git a/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.h b/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.h index 68d3d54124f5..c2688b51139f 100644 --- a/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.h +++ b/cpp/tensorrt_llm/runtime/decodingLayerWorkspace.h @@ -19,7 +19,6 @@ #include <memory> #include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/workspace.h" #include "tensorrt_llm/layers/decodingParams.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -40,7 +39,7 @@ class DecodingLayerWorkspace using BufferPtr = IBuffer::SharedPtr; DecodingLayerWorkspace(std::shared_ptr<BufferManager> bufferManager, layers::DecoderDomain const& decoderDomain, - tensorrt_llm::DataType logitsType, size_t workspaceBufferSizeInBytes); + nvinfer1::DataType logitsType, size_t workspaceBufferSizeInBytes); DecodingLayerWorkspace() = delete; @@ -72,7 +71,7 @@ class DecodingLayerWorkspace [[nodiscard]] TensorPtr getDeviceRuntimeLogits() const; ///@brief Gets a tensor with the given shape and type at the start of the device workspace. - TensorPtr getWorkspaceAsDeviceTensor(ITensor::Shape shape, tensorrt_llm::DataType type); + TensorPtr getWorkspaceAsDeviceTensor(ITensor::Shape shape, nvinfer1::DataType type); /// @brief A convenience function to copy the content of a standard vector to a device workspace. template <typename T, typename Alloc> @@ -113,7 +112,7 @@ class DecodingLayerWorkspace { size_t lastTensorOffset = 0; auto alignedSizeCalculator - = [&lastTensorOffset](std::pair<ITensor::Shape, tensorrt_llm::DataType> const& tensorDescriptor) + = [&lastTensorOffset](std::pair<ITensor::Shape, nvinfer1::DataType> const& tensorDescriptor) { auto const& [shape, type] = tensorDescriptor; auto const sizeInBytes = ITensor::volume(shape) * tensorrt_llm::common::getDTypeSize(type); diff --git a/cpp/tensorrt_llm/runtime/eagleBuffers.cpp b/cpp/tensorrt_llm/runtime/eagleBuffers.cpp index e0f2198c3e58..097fd95f49aa 100644 --- a/cpp/tensorrt_llm/runtime/eagleBuffers.cpp +++ b/cpp/tensorrt_llm/runtime/eagleBuffers.cpp @@ -19,7 +19,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/speculativeDecoding/eagleDecodingKernels.h" #include "tensorrt_llm/kernels/speculativeDecoding/explicitDraftTokensKernels.h" #include "tensorrt_llm/runtime/common.h" @@ -42,51 +41,50 @@ void EagleBuffers::Inputs::create(SizeType32 maxNumSequences, BufferManager cons auto const numEagleLayers = speculativeDecodingModule.getMaxDraftPathLen(); auto constexpr TRTTokenIdType = runtime::TRTDataType<runtime::TokenIdType>::value; - temperatures = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kFLOAT); - randomDataSample = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kFLOAT); + temperatures = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kFLOAT); + randomDataSample = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kFLOAT); randomDataValidation - = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingTokens}), tensorrt_llm::DataType::kFLOAT); + = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingTokens}), nvinfer1::DataType::kFLOAT); draftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), TRTTokenIdType); - draftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + draftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); draftPaths - = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), nvinfer1::DataType::kINT32); draftPathsHost = BufferManager::pinnedPool( - ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), tensorrt_llm::DataType::kINT32); - specDecodingGenerationLengths = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), nvinfer1::DataType::kINT32); + specDecodingGenerationLengths = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); specDecodingGenerationLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); specDecodingPackedMasks = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingTokens, common::ceilDiv(maxDecodingTokens, 32)}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); specDecodingPositionOffsets - = manager.gpu(ITensor::makeShape({maxNumSequences * maxDecodingTokens}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences * maxDecodingTokens}), nvinfer1::DataType::kINT32); eagleNetCtxRequestTypesHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); eagleNetCtxContextLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); eagleNetCtxPastKeyValueLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); eagleNetGenRequestTypesHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); eagleNetGenContextLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); eagleNetGenPastKeyValueLengthsHost - = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); inputGenTokensHost = BufferManager::pinnedPool( - ITensor::makeShape({maxNumSequences * maxDecodingTokens}), tensorrt_llm::DataType::kINT32); - chunkedContextNextTokens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); - useSpecDecoding = manager.cpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxNumSequences * maxDecodingTokens}), nvinfer1::DataType::kINT32); + chunkedContextNextTokens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + useSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); // Eagle-2 - useDynamicTreeHost = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); - dynamicTreeMaxTopKHost = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); - prevScores - = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), tensorrt_llm::DataType::kFLOAT); + useDynamicTreeHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + dynamicTreeMaxTopKHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + prevScores = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), nvinfer1::DataType::kFLOAT); currentExpandIndices = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), TRTTokenIdType); allLayersScores = manager.gpu( ITensor::makeShape({maxNumSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens}), - tensorrt_llm::DataType::kFLOAT); + nvinfer1::DataType::kFLOAT); allLayersDraftTokenIds = manager.gpu( ITensor::makeShape({maxNumSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens}), TRTTokenIdType); @@ -116,63 +114,58 @@ EagleBuffers::EagleBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, run auto constexpr TRTTokenIdType = runtime::TRTDataType<runtime::TokenIdType>::value; // input tensors - engineInputs.temperatures = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); - engineInputs.posteriorAlpha = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); - engineInputs.posteriorThreshold = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); - posteriorAlphaHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kFLOAT); - posteriorThresholdHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kFLOAT); - greedySamplingHost = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + engineInputs.temperatures = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); + engineInputs.posteriorAlpha = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); + engineInputs.posteriorThreshold = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); + posteriorAlphaHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kFLOAT); + posteriorThresholdHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kFLOAT); + greedySamplingHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); engineInputs.draftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), TRTTokenIdType); - engineInputs.draftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + engineInputs.draftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); engineInputs.draftPaths - = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), nvinfer1::DataType::kINT32); engineInputs.specDecodingGenerationLengths - = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); engineInputs.specDecodingPositionOffsets - = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineInputs.specDecodingPackedMasks - = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineInputs.specDecodingPackedMasks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); - engineInputs.randomDataSample = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); - engineInputs.randomDataValidation = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kFLOAT); + engineInputs.randomDataSample = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); + engineInputs.randomDataValidation = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kFLOAT); engineInputs.eagleNetCtxRequestTypesHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); engineInputs.eagleNetCtxContextLengthsHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); engineInputs.eagleNetCtxPastKeyValueLengthsHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); engineInputs.eagleNetGenRequestTypesHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); engineInputs.eagleNetGenContextLengthsHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); engineInputs.eagleNetGenPastKeyValueLengthsHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); - engineInputs.inputGenTokensHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); - engineInputs.chunkedContextNextTokens - = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineInputs.useSpecDecoding = BufferManager::cpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); + engineInputs.inputGenTokensHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); + engineInputs.chunkedContextNextTokens = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineInputs.useSpecDecoding = BufferManager::cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); bufferCast<SizeType32>(*engineInputs.useSpecDecoding)[0] = 1; - chunkedContextNextTokensHost - = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, tensorrt_llm::DataType::kINT32); + chunkedContextNextTokensHost = manager.emptyTensor(runtime::MemoryType::kPINNEDPOOL, nvinfer1::DataType::kINT32); // Eagle-2 - engineInputs.useDynamicTreeHost - = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + engineInputs.useDynamicTreeHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); engineInputs.dynamicTreeMaxTopKHost - = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); engineInputs.prevScores - = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), tensorrt_llm::DataType::kFLOAT); + = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), nvinfer1::DataType::kFLOAT); engineInputs.currentExpandIndices = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingDraftTokens}), TRTTokenIdType); engineInputs.allLayersScores = manager.gpu( ITensor::makeShape({maxNumSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens}), - tensorrt_llm::DataType::kFLOAT); + nvinfer1::DataType::kFLOAT); engineInputs.allLayersDraftTokenIds = manager.gpu( ITensor::makeShape({maxNumSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens}), TRTTokenIdType); @@ -183,24 +176,24 @@ EagleBuffers::EagleBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, run // output tensors engineOutputs.nextDraftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), TRTTokenIdType); - engineOutputs.nextDraftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + engineOutputs.nextDraftLens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); engineOutputs.nextDraftPaths - = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, numPaths, pathLen}), nvinfer1::DataType::kINT32); engineOutputs.acceptedTokens - = manager.gpu(ITensor::makeShape({maxNumSequences, pathLen}), tensorrt_llm::DataType::kINT32); - engineOutputs.acceptedLens = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); - engineOutputs.acceptedPaths = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, pathLen}), nvinfer1::DataType::kINT32); + engineOutputs.acceptedLens = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + engineOutputs.acceptedPaths = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); engineOutputs.chunkedContextNextTokens - = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); // helper tensors scanReduceTempStorageBytes = tksd::invokeScanReduceGenerationLengths( maxNumSequences, nullptr, nullptr, 0, nullptr, nullptr, manager.getStream().get()); scanReduceTempStorage = manager.gpu(scanReduceTempStorageBytes); - cumSumGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - maxGenerationLength = manager.gpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + cumSumGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + maxGenerationLength = manager.gpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); // pre-allocate empty tensors reshape(0, maxNumSequences, modelConfig); @@ -527,15 +520,15 @@ void EagleBuffers::setFromInputs(RequestVector const& contextRequests, RequestVe switch (dtype) { - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: setFromInputs<float>( contextRequests, genRequests, vocabSizePadded, seqSlots, draftBuffers, *eagleModule, manager); break; - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: setFromInputs<half>( contextRequests, genRequests, vocabSizePadded, seqSlots, draftBuffers, *eagleModule, manager); break; - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: setFromInputs<__nv_bfloat16>( contextRequests, genRequests, vocabSizePadded, seqSlots, draftBuffers, *eagleModule, manager); break; diff --git a/cpp/tensorrt_llm/runtime/explicitDraftTokensBuffers.cpp b/cpp/tensorrt_llm/runtime/explicitDraftTokensBuffers.cpp index 89c74e6f9349..ed205ca0e117 100644 --- a/cpp/tensorrt_llm/runtime/explicitDraftTokensBuffers.cpp +++ b/cpp/tensorrt_llm/runtime/explicitDraftTokensBuffers.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/speculativeDecoding/explicitDraftTokensKernels.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iBuffer.h" @@ -41,24 +40,23 @@ void ExplicitDraftTokensBuffers::Inputs::create(SizeType32 maxNumSequences, Buff auto constexpr TRTTokenIdType = runtime::TRTDataType<runtime::TokenIdType>::value; auto const dtype = modelConfig.getDataType(); - maxGenLengthHost = manager.pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + maxGenLengthHost = manager.pinned(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); temperatures = manager.gpu(ITensor::makeShape({maxNumSequences}), dtype); - positionIdsBase = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); - generationLengths = manager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); - generationLengthsHost = manager.pinned(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32); + positionIdsBase = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + generationLengths = manager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); + generationLengthsHost = manager.pinned(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32); randomDataSample = manager.gpu(ITensor::makeShape({maxNumSequences}), dtype); randomDataValidation = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxDraftPathLen}), dtype); draftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), TRTTokenIdType); draftIndices - = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxPathLen}), nvinfer1::DataType::kINT32); draftProbs = manager.gpu(ITensor::makeShape({maxNumSequences, maxNumPaths, maxDraftPathLen, vocabSizePadded}), dtype); packedMasks = manager.gpu(ITensor::makeShape({maxNumSequences, maxDecodingTokens, common::ceilDiv(maxDecodingTokens, 32)}), - tensorrt_llm::DataType::kINT32); - positionIds - = manager.gpu(ITensor::makeShape({maxNumSequences * maxDecodingTokens}), tensorrt_llm::DataType::kINT32); - useSpecDecoding = manager.cpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); + positionIds = manager.gpu(ITensor::makeShape({maxNumSequences * maxDecodingTokens}), nvinfer1::DataType::kINT32); + useSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); } ExplicitDraftTokensBuffers::ExplicitDraftTokensBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, @@ -83,53 +81,52 @@ ExplicitDraftTokensBuffers::ExplicitDraftTokensBuffers(SizeType32 maxBatchSize, auto const dtype = modelConfig.getDataType(); // input tensors - engineInputs.requestTypesDevice = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineInputs.requestTypesDevice = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); engineInputs.temperatures = manager.emptyTensor(runtime::MemoryType::kGPU, dtype); engineInputs.draftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), TRTTokenIdType); engineInputs.draftIndices - = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), nvinfer1::DataType::kINT32); engineInputs.draftProbs = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamDraftLength, vocabSizePadded}), dtype); - engineInputs.generationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineInputs.positionIds = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineInputs.positionOffsets = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineInputs.packedMasks = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineInputs.generationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineInputs.positionIds = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineInputs.positionOffsets = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineInputs.packedMasks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); engineInputs.randomDataSample = manager.emptyTensor(runtime::MemoryType::kGPU, dtype); engineInputs.randomDataValidation = manager.emptyTensor(runtime::MemoryType::kGPU, dtype); - engineInputs.positionIdsBase = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineInputs.useSpecDecoding = manager.cpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + engineInputs.positionIdsBase = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineInputs.useSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); bufferCast<SizeType32>(*engineInputs.useSpecDecoding)[0] = 1; // output tensors engineOutputs.nextDraftTokens = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), TRTTokenIdType); engineOutputs.nextDraftIndices - = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), tensorrt_llm::DataType::kINT32); + = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamLength}), nvinfer1::DataType::kINT32); engineOutputs.nextDraftProbs = manager.gpu(ITensor::makeShape({maxNumSequences, numBeams, beamDraftLength, vocabSizePadded}), dtype); - engineOutputs.maxGenToken = manager.gpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); - engineOutputs.totalGenToken = manager.gpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + engineOutputs.maxGenToken = manager.gpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + engineOutputs.totalGenToken = manager.gpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); - engineOutputs.nextGenerationLengths - = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineOutputs.nextPositionOffsets = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineOutputs.masks = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kBOOL); + engineOutputs.nextGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineOutputs.nextPositionOffsets = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineOutputs.masks = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kBOOL); engineOutputs.nextFlatTokens = manager.emptyTensor(runtime::MemoryType::kGPU, TRTTokenIdType); - engineOutputs.bestPathLengths = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineOutputs.bestPathIndices = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); - engineOutputs.packedPositionIds = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + engineOutputs.bestPathLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineOutputs.bestPathIndices = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); + engineOutputs.packedPositionIds = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); // helper tensors auto const& stream = manager.getStream(); scanTempStorageBytes = tksd::invokeScanGenerationLengths(nullptr, 0, nullptr, nullptr, maxNumSequences, stream.get()); scanTempStorage = manager.gpu(scanTempStorageBytes); - cumSumGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, tensorrt_llm::DataType::kINT32); + cumSumGenerationLengths = manager.emptyTensor(runtime::MemoryType::kGPU, nvinfer1::DataType::kINT32); // pre-allocate empty tensors reshape(0, maxNumSequences, modelConfig); @@ -298,15 +295,15 @@ void ExplicitDraftTokensBuffers::setFromInputs(SizeType32 numCtxSequences, SizeT switch (dtype) { - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: setFromInputs<float>(numCtxSequences, numGenSequences, vocabSizePadded, seqSlots, draftBuffers, contextPositionIds, *explicitDraftTokensModule, stream); break; - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: setFromInputs<half>(numCtxSequences, numGenSequences, vocabSizePadded, seqSlots, draftBuffers, contextPositionIds, *explicitDraftTokensModule, stream); break; - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: setFromInputs<__nv_bfloat16>(numCtxSequences, numGenSequences, vocabSizePadded, seqSlots, draftBuffers, contextPositionIds, *explicitDraftTokensModule, stream); break; diff --git a/cpp/tensorrt_llm/runtime/gptDecoder.cpp b/cpp/tensorrt_llm/runtime/gptDecoder.cpp index e1ac1717af45..930877206462 100644 --- a/cpp/tensorrt_llm/runtime/gptDecoder.cpp +++ b/cpp/tensorrt_llm/runtime/gptDecoder.cpp @@ -21,7 +21,7 @@ #include "tensorrt_llm/layers/dynamicDecodeLayer.h" #include "tensorrt_llm/runtime/decodingLayerWorkspace.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <memory> @@ -121,7 +121,7 @@ void GptDecoder<T>::disableLookahead( template <typename T> void GptDecoder<T>::setup(SamplingConfig const& samplingConfig, size_t batchSize, TensorConstPtr const& batchSlots, - std::optional<DecodingOutput> const& output, std::optional<tensorrt_llm::DataType> explicitDraftTokensDType, + std::optional<DecodingOutput> const& output, std::optional<nvinfer1::DataType> explicitDraftTokensDType, std::optional<std::vector<TensorConstPtr>> const& lookaheadPrompt, std::optional<std::vector<tle::LookaheadDecodingConfig>> const& lookaheadAlgoConfigs) { diff --git a/cpp/tensorrt_llm/runtime/gptDecoderBatched.cpp b/cpp/tensorrt_llm/runtime/gptDecoderBatched.cpp index 7b3a12ed7a2c..c55d02093afc 100644 --- a/cpp/tensorrt_llm/runtime/gptDecoderBatched.cpp +++ b/cpp/tensorrt_llm/runtime/gptDecoderBatched.cpp @@ -22,7 +22,6 @@ #include "tensorrt_llm/batch_manager/decoderBuffers.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/kernels/decodingKernels.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -75,7 +74,7 @@ void GptDecoderBatched::disableLookahead(RequestVector const& genRequests, Tenso } void GptDecoderBatched::setup(executor::DecodingMode const& mode, SizeType32 maxNumSequences, SizeType32 maxBeamWidth, - tensorrt_llm::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) + nvinfer1::DataType dtype, ModelConfig const& modelConfig, WorldConfig const& worldConfig) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); TLLM_CHECK(maxNumSequences > 0); diff --git a/cpp/tensorrt_llm/runtime/gptJsonConfig.cpp b/cpp/tensorrt_llm/runtime/gptJsonConfig.cpp index 47310a9a1282..311f63eaf1e7 100644 --- a/cpp/tensorrt_llm/runtime/gptJsonConfig.cpp +++ b/cpp/tensorrt_llm/runtime/gptJsonConfig.cpp @@ -20,7 +20,6 @@ #include "modelConfig.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/eagleModule.h" #include "tensorrt_llm/runtime/explicitDraftTokensModule.h" #include "tensorrt_llm/runtime/jsonSerialization.h" @@ -81,14 +80,14 @@ std::optional<FieldType> parseJsonFieldOptional(Json const& json, std::string_vi return value; } -tensorrt_llm::DataType strToDType(std::string type) +nvinfer1::DataType strToDType(std::string type) { - static std::map<std::string, tensorrt_llm::DataType> const typeMap = {{"int64", tensorrt_llm::DataType::kINT64}, - {"int32", tensorrt_llm::DataType::kINT32}, {"int", tensorrt_llm::DataType::kINT32}, - {"float32", tensorrt_llm::DataType::kFLOAT}, {"bfloat16", tensorrt_llm::DataType::kBF16}, - {"float16", tensorrt_llm::DataType::kHALF}, {"bool", tensorrt_llm::DataType::kBOOL}, - {"uint8", tensorrt_llm::DataType::kUINT8}, {"int8", tensorrt_llm::DataType::kINT8}, - {"fp8", tensorrt_llm::DataType::kFP8}, {"int4", tensorrt_llm::DataType::kINT4}}; + static std::map<std::string, nvinfer1::DataType> const typeMap = {{"int64", nvinfer1::DataType::kINT64}, + {"int32", nvinfer1::DataType::kINT32}, {"int", nvinfer1::DataType::kINT32}, + {"float32", nvinfer1::DataType::kFLOAT}, {"bfloat16", nvinfer1::DataType::kBF16}, + {"float16", nvinfer1::DataType::kHALF}, {"bool", nvinfer1::DataType::kBOOL}, + {"uint8", nvinfer1::DataType::kUINT8}, {"int8", nvinfer1::DataType::kINT8}, {"fp8", nvinfer1::DataType::kFP8}, + {"int4", nvinfer1::DataType::kINT4}}; TLLM_CHECK_WITH_INFO(typeMap.count(type) > 0, type + " not found in strToDtype."); return typeMap.at(type); @@ -141,14 +140,14 @@ std::vector<ModelConfig::LayerType> buildLayerTypes( return result; } -ModelConfig parseMultimodalConfig(Json const& json, tensorrt_llm::DataType dataType) +ModelConfig parseMultimodalConfig(Json const& json, nvinfer1::DataType dataType) { return ModelConfig{128, 10, 10, 0, 1, 128, dataType}; // use dummy values because vision engines of multimodal models does not record this info in config } ModelConfig createModelConfig(Json const& json, bool engineVersionNone, SizeType32 tensorParallelism, - SizeType32 contextParallelism, tensorrt_llm::DataType dataType) + SizeType32 contextParallelism, nvinfer1::DataType dataType) { auto const& config = engineVersionNone ? json.at("builder_config") : json.at("pretrained_config"); auto const multiModalName = parseJsonFieldOptional<std::string>(config, "model_name"); @@ -249,14 +248,14 @@ ModelConfig createModelConfig(Json const& json, bool engineVersionNone, SizeType modelConfig.setLayerTypes(layerTypes); // Set logits datatype - auto logitsDtype = tensorrt_llm::DataType::kFLOAT; + auto logitsDtype = nvinfer1::DataType::kFLOAT; if (logitsDtypeStr == "float32") { - logitsDtype = tensorrt_llm::DataType::kFLOAT; + logitsDtype = nvinfer1::DataType::kFLOAT; } else if (logitsDtypeStr == "float16") { - logitsDtype = tensorrt_llm::DataType::kHALF; + logitsDtype = nvinfer1::DataType::kHALF; } else { @@ -491,15 +490,15 @@ GptJsonConfig parseJson(InputType&& input) { if (precision == "float32") { - return tensorrt_llm::DataType::kFLOAT; + return nvinfer1::DataType::kFLOAT; } if (precision == "float16") { - return tensorrt_llm::DataType::kHALF; + return nvinfer1::DataType::kHALF; } if (precision == "bfloat16") { - return tensorrt_llm::DataType::kBF16; + return nvinfer1::DataType::kBF16; } TLLM_THROW("Model data type '%s' not supported", precision.c_str()); }(); diff --git a/cpp/tensorrt_llm/runtime/iBuffer.cpp b/cpp/tensorrt_llm/runtime/iBuffer.cpp index 82574b658b39..77707a0e4cf8 100644 --- a/cpp/tensorrt_llm/runtime/iBuffer.cpp +++ b/cpp/tensorrt_llm/runtime/iBuffer.cpp @@ -20,7 +20,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferView.h" #include <cuda_runtime_api.h> @@ -49,7 +48,7 @@ IBuffer::UniquePtr IBuffer::slice(IBuffer::SharedPtr buffer, std::size_t offset, return std::make_unique<BufferView>(std::move(buffer), offset, size); } -IBuffer::UniquePtr IBuffer::wrap(void* data, tensorrt_llm::DataType type, std::size_t size, std::size_t capacity) +IBuffer::UniquePtr IBuffer::wrap(void* data, nvinfer1::DataType type, std::size_t size, std::size_t capacity) { TLLM_CHECK_WITH_INFO(size <= capacity, "Requested size is larger than capacity"); auto memoryType = IBuffer::memoryType(data); @@ -92,17 +91,17 @@ char const* IBuffer::getDataTypeName(DataType dataType) { switch (dataType) { - case tensorrt_llm::DataType::kINT64: return DataTypeTraits<tensorrt_llm::DataType::kINT64>::name; - case tensorrt_llm::DataType::kINT32: return DataTypeTraits<tensorrt_llm::DataType::kINT32>::name; - case tensorrt_llm::DataType::kFLOAT: return DataTypeTraits<tensorrt_llm::DataType::kFLOAT>::name; - case tensorrt_llm::DataType::kBF16: return DataTypeTraits<tensorrt_llm::DataType::kBF16>::name; - case tensorrt_llm::DataType::kHALF: return DataTypeTraits<tensorrt_llm::DataType::kHALF>::name; - case tensorrt_llm::DataType::kBOOL: return DataTypeTraits<tensorrt_llm::DataType::kBOOL>::name; - case tensorrt_llm::DataType::kUINT8: return DataTypeTraits<tensorrt_llm::DataType::kUINT8>::name; - case tensorrt_llm::DataType::kINT8: return DataTypeTraits<tensorrt_llm::DataType::kINT8>::name; - case tensorrt_llm::DataType::kFP8: return DataTypeTraits<tensorrt_llm::DataType::kFP8>::name; - case tensorrt_llm::DataType::kINT4: [[fallthrough]] /* do nothing */; - case tensorrt_llm::DataType::kFP4: [[fallthrough]] /* do nothing */; + case nvinfer1::DataType::kINT64: return DataTypeTraits<nvinfer1::DataType::kINT64>::name; + case nvinfer1::DataType::kINT32: return DataTypeTraits<nvinfer1::DataType::kINT32>::name; + case nvinfer1::DataType::kFLOAT: return DataTypeTraits<nvinfer1::DataType::kFLOAT>::name; + case nvinfer1::DataType::kBF16: return DataTypeTraits<nvinfer1::DataType::kBF16>::name; + case nvinfer1::DataType::kHALF: return DataTypeTraits<nvinfer1::DataType::kHALF>::name; + case nvinfer1::DataType::kBOOL: return DataTypeTraits<nvinfer1::DataType::kBOOL>::name; + case nvinfer1::DataType::kUINT8: return DataTypeTraits<nvinfer1::DataType::kUINT8>::name; + case nvinfer1::DataType::kINT8: return DataTypeTraits<nvinfer1::DataType::kINT8>::name; + case nvinfer1::DataType::kFP8: return DataTypeTraits<nvinfer1::DataType::kFP8>::name; + case nvinfer1::DataType::kINT4: [[fallthrough]] /* do nothing */; + case nvinfer1::DataType::kFP4: [[fallthrough]] /* do nothing */; default: TLLM_THROW("Unknown data type"); } } diff --git a/cpp/tensorrt_llm/runtime/iTensor.cpp b/cpp/tensorrt_llm/runtime/iTensor.cpp index 70b31707130a..f78b25fdb19a 100644 --- a/cpp/tensorrt_llm/runtime/iTensor.cpp +++ b/cpp/tensorrt_llm/runtime/iTensor.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/stringUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/tensorView.h" #include "tensorrt_llm/runtime/tllmBuffers.h" @@ -73,22 +72,22 @@ ITensor::UniquePtr ITensor::slice(SharedPtr tensor, Shape const& offsetDims, ITe return std::make_unique<TensorView>(std::move(tensor), offset, volume(dims), dims); } -ITensor::UniquePtr ITensor::view(IBuffer::SharedPtr buffer, tensorrt_llm::Dims const& dims) +ITensor::UniquePtr ITensor::view(IBuffer::SharedPtr buffer, nvinfer1::Dims const& dims) { auto const size = buffer->getSize(); return std::make_unique<TensorView>(std::move(buffer), 0, size, dims); } -tensorrt_llm::Dims ITensor::makeShape(std::initializer_list<ITensor::DimType64> const& dims) +nvinfer1::Dims ITensor::makeShape(std::initializer_list<ITensor::DimType64> const& dims) { - TLLM_CHECK_WITH_INFO(dims.size() <= tensorrt_llm::Dims::MAX_DIMS, "Number of dimensions is too large"); - tensorrt_llm::Dims shape{}; + TLLM_CHECK_WITH_INFO(dims.size() <= nvinfer1::Dims::MAX_DIMS, "Number of dimensions is too large"); + nvinfer1::Dims shape{}; shape.nbDims = static_cast<decltype(Shape::nbDims)>(dims.size()); std::copy(dims.begin(), dims.end(), shape.d); return shape; } -std::string ITensor::toString(tensorrt_llm::Dims const& dims) +std::string ITensor::toString(nvinfer1::Dims const& dims) { if (dims.nbDims < 0) { @@ -104,8 +103,7 @@ std::string ITensor::toString(tensorrt_llm::Dims const& dims) } } -ITensor::UniquePtr ITensor::wrap( - void* data, tensorrt_llm::DataType type, tensorrt_llm::Dims const& shape, std::size_t capacity) +ITensor::UniquePtr ITensor::wrap(void* data, nvinfer1::DataType type, nvinfer1::Dims const& shape, std::size_t capacity) { auto const size = volumeNonNegative(shape); TLLM_CHECK_WITH_INFO(size <= capacity, "Requested size is larger than capacity"); @@ -232,18 +230,18 @@ std::ostream& tensorrt_llm::runtime::operator<<(std::ostream& out, ITensor const { switch (tensor.getDataType()) { - case tensorrt_llm::DataType::kFLOAT: printTensor<float>(tensor, out); break; - case tensorrt_llm::DataType::kHALF: printTensor<half, float>(tensor, out); break; - case tensorrt_llm::DataType::kBOOL: printTensor<bool>(tensor, out); break; - case tensorrt_llm::DataType::kINT8: printTensor<std::int8_t, std::int32_t>(tensor, out); break; - case tensorrt_llm::DataType::kINT32: printTensor<std::int32_t>(tensor, out); break; - case tensorrt_llm::DataType::kINT64: printTensor<std::int64_t>(tensor, out); break; - case tensorrt_llm::DataType::kUINT8: printTensor<std::uint8_t, std::int32_t>(tensor, out); break; + case nvinfer1::DataType::kFLOAT: printTensor<float>(tensor, out); break; + case nvinfer1::DataType::kHALF: printTensor<half, float>(tensor, out); break; + case nvinfer1::DataType::kBOOL: printTensor<bool>(tensor, out); break; + case nvinfer1::DataType::kINT8: printTensor<std::int8_t, std::int32_t>(tensor, out); break; + case nvinfer1::DataType::kINT32: printTensor<std::int32_t>(tensor, out); break; + case nvinfer1::DataType::kINT64: printTensor<std::int64_t>(tensor, out); break; + case nvinfer1::DataType::kUINT8: printTensor<std::uint8_t, std::int32_t>(tensor, out); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: printTensor<__nv_bfloat16, float>(tensor, out); break; + case nvinfer1::DataType::kBF16: printTensor<__nv_bfloat16, float>(tensor, out); break; #endif #ifdef ENABLE_FP8 - case tensorrt_llm::DataType::kFP8: printTensor<__nv_fp8_e4m3, float>(tensor, out); break; + case nvinfer1::DataType::kFP8: printTensor<__nv_fp8_e4m3, float>(tensor, out); break; #endif default: TLLM_THROW("Unsupported data type"); } diff --git a/cpp/tensorrt_llm/runtime/ipcUtils.cpp b/cpp/tensorrt_llm/runtime/ipcUtils.cpp index 48368844f850..23a7e28a4f27 100644 --- a/cpp/tensorrt_llm/runtime/ipcUtils.cpp +++ b/cpp/tensorrt_llm/runtime/ipcUtils.cpp @@ -20,7 +20,7 @@ #include "tensorrt_llm/common/workspace.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntimeBase.h> #include <cstddef> namespace tensorrt_llm::runtime @@ -83,7 +83,7 @@ void IpcMemory::allocateIpcMemory(std::size_t bufferSize, BufferManager const& m // IPC handles. If we want to support stream-ordered allocations here, we need to create another pool with the // correct handle type. auto const ipcAlignedBufferSize = common::alignSize(bufferSize, 1LU << 21); - mBuffer = BufferManager::gpuSync(ipcAlignedBufferSize, tensorrt_llm::DataType::kUINT8); + mBuffer = BufferManager::gpuSync(ipcAlignedBufferSize, nvinfer1::DataType::kUINT8); manager.setZero(*mBuffer); auto* bufferPtr = mBuffer->data(); @@ -149,7 +149,7 @@ AllReduceBuffers::AllReduceBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWi { auto const tpSize = worldConfig.getTensorParallelism(); mAllReduceCommPtrs = BufferManager::cpu( - ITensor::makeShape({static_cast<SizeType32>(7) * tpSize + 3}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({static_cast<SizeType32>(7) * tpSize + 3}), nvinfer1::DataType::kINT64); } else { @@ -178,7 +178,7 @@ AllReduceBuffers::AllReduceBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWi mAllReduceCommPtrs = BufferManager::cpu(ITensor::makeShape({static_cast<SizeType32>(mIpcMemoryHandles.size()) * tpSize + 3}), - tensorrt_llm::DataType::kINT64); + nvinfer1::DataType::kINT64); auto commPtrs = BufferRange<void*>(*mAllReduceCommPtrs); // Start from 1 since 0 represents released state for barrier at the beginning of the all_reduce. // The last element is the barrier flag counter. @@ -211,9 +211,9 @@ AllReduceBuffers::AllReduceBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWi void lamportInitializeAll(void* buffer_0, void* buffer_1, void* buffer_2, size_t size) { #if ENABLE_MULTI_DEVICE - tensorrt_llm::kernels::lamportInitialize(buffer_0, size / sizeof(half), tensorrt_llm::DataType::kHALF, 0); - tensorrt_llm::kernels::lamportInitialize(buffer_1, size / sizeof(half), tensorrt_llm::DataType::kHALF, 0); - tensorrt_llm::kernels::lamportInitialize(buffer_2, size / sizeof(half), tensorrt_llm::DataType::kHALF, 0); + tensorrt_llm::kernels::lamportInitialize(buffer_0, size / sizeof(half), nvinfer1::DataType::kHALF, 0); + tensorrt_llm::kernels::lamportInitialize(buffer_1, size / sizeof(half), nvinfer1::DataType::kHALF, 0); + tensorrt_llm::kernels::lamportInitialize(buffer_2, size / sizeof(half), nvinfer1::DataType::kHALF, 0); cudaDeviceSynchronize(); #endif } diff --git a/cpp/tensorrt_llm/runtime/layerProfiler.cpp b/cpp/tensorrt_llm/runtime/layerProfiler.cpp new file mode 100644 index 000000000000..4c3c9779cedb --- /dev/null +++ b/cpp/tensorrt_llm/runtime/layerProfiler.cpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/runtime/layerProfiler.h" +#include <iomanip> +#include <iostream> +#include <numeric> +#include <sstream> + +using namespace tensorrt_llm::runtime; + +void LayerProfiler::reportLayerTime(char const* layerName, float timeMs) noexcept +{ + if (mIterator == mLayers.end()) + { + bool const first = !mLayers.empty() && mLayers.begin()->name == layerName; + mUpdatesCount += mLayers.empty() || first; + if (first) + { + mIterator = mLayers.begin(); + } + else + { + mLayers.emplace_back(); + mLayers.back().name = layerName; + mIterator = mLayers.end() - 1; + } + } + + mIterator->timeMs.push_back(timeMs); + ++mIterator; +} + +float LayerProfiler::getTotalTime() const noexcept +{ + auto const plusLayerTime = [](float accumulator, LayerProfile const& lp) + { return accumulator + std::accumulate(lp.timeMs.begin(), lp.timeMs.end(), 0.F, std::plus<float>()); }; + return std::accumulate(mLayers.begin(), mLayers.end(), 0.0F, plusLayerTime); +} + +std::string LayerProfiler::getLayerProfile() noexcept +{ + std::string const nameHdr(" Layer"); + std::string const timeHdr(" Time(ms)"); + + float const totalTimeMs = getTotalTime(); + + auto const timeLength = timeHdr.size(); + + std::unordered_map<std::string, float> layer2times; + std::vector<std::string> layer_order; + for (auto const& p : mLayers) + { + if (!layer2times.count(p.name)) + { + layer2times[p.name] = 0; + layer_order.push_back(p.name); + } + for (auto const& t : p.timeMs) + { + layer2times[p.name] += t; + } + } + + std::stringstream ss; + ss << "\n=== Per-layer Profile ===\n" << timeHdr << nameHdr << "\n"; + + for (auto const& name : layer_order) + { + if (layer2times[name] == 0.0f) + { + continue; + } + ss << std::setw(timeLength) << std::fixed << std::setprecision(2) << layer2times[name] << " " << name << "\n"; + } + + ss << std::setw(timeLength) << std::fixed << std::setprecision(2) << totalTimeMs << " Total\n"; + ss << "\n"; + + // clear data + mLayers.clear(); + + return ss.str(); +} diff --git a/cpp/tensorrt_llm/runtime/layerProfiler.h b/cpp/tensorrt_llm/runtime/layerProfiler.h new file mode 100644 index 000000000000..bcae1546de3d --- /dev/null +++ b/cpp/tensorrt_llm/runtime/layerProfiler.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/runtime/common.h" +#include <vector> + +#include <NvInfer.h> + +namespace tensorrt_llm::runtime +{ +struct LayerProfile +{ + std::string name; + std::vector<float> timeMs; +}; + +class LayerProfiler : public nvinfer1::IProfiler +{ + +public: + void reportLayerTime(char const* layerName, float timeMs) noexcept override; + + std::string getLayerProfile() noexcept; + +private: + [[nodiscard]] float getTotalTime() const noexcept; + + std::vector<LayerProfile> mLayers; + std::vector<LayerProfile>::iterator mIterator{mLayers.begin()}; + int32_t mUpdatesCount{0}; +}; +} // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/lookaheadBuffers.cpp b/cpp/tensorrt_llm/runtime/lookaheadBuffers.cpp index 5e77046c47e0..ef800ef218e4 100644 --- a/cpp/tensorrt_llm/runtime/lookaheadBuffers.cpp +++ b/cpp/tensorrt_llm/runtime/lookaheadBuffers.cpp @@ -16,23 +16,208 @@ */ #include "tensorrt_llm/runtime/lookaheadBuffers.h" -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include "tensorrt_llm/layers/lookaheadDecodingUtils.h" namespace tensorrt_llm::runtime { LookaheadDecodingBuffers::LookaheadDecodingBuffers( SizeType32 maxNumSequences, SizeType32 maxTokensPerStep, BufferManager const& bufferManager) - : generationLengths(bufferManager.gpu(ITensor::makeShape({maxNumSequences}), tensorrt_llm::DataType::kINT32)) + : generationLengths(bufferManager.gpu(ITensor::makeShape({maxNumSequences}), nvinfer1::DataType::kINT32)) , positionOffsets( - bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep}), tensorrt_llm::DataType::kINT32)) + bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep}), nvinfer1::DataType::kINT32)) , packedMasks(bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep, static_cast<ITensor::DimType64>(common::divUp(maxTokensPerStep, 32))}), - tensorrt_llm::DataType::kINT32)) + nvinfer1::DataType::kINT32)) , positionIds( - bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep}), tensorrt_llm::DataType::kINT32)) + bufferManager.gpu(ITensor::makeShape({maxNumSequences, maxTokensPerStep}), nvinfer1::DataType::kINT32)) { } +LookaheadRuntimeBuffers::LookaheadRuntimeBuffers(SizeType32 maxBatchSize, SizeType32 maxBeamWidth, + BufferManager const& manager, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + executor::DecodingConfig const& /* decodingConfig */, TllmRuntime const& runtime) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + TLLM_CHECK_WITH_INFO(maxBeamWidth == 1, "Lookahead decoding does not support beam search"); + + auto const tokensPerStep = modelConfig.getMaxDecodingTokens(); + auto const numPackedMasks = static_cast<ITensor::DimType64>(tensorrt_llm::common::divUp(tokensPerStep, 32)); + + cumSumLength = manager.pinned(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + + packedMasksDevice + = manager.gpu(ITensor::makeShape({maxBatchSize * tokensPerStep, numPackedMasks}), nvinfer1::DataType::kINT32); + positionOffsetsDevice = manager.gpu(ITensor::makeShape({maxBatchSize, tokensPerStep}), nvinfer1::DataType::kINT32); + generationLengthsDevice = manager.gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + positionIdsDevice = manager.gpu(ITensor::makeShape({maxBatchSize, tokensPerStep}), nvinfer1::DataType::kINT32); + + packedMaskHost = manager.cpu(packedMasksDevice->getShape(), nvinfer1::DataType::kINT32); + positionOffsetsHost = manager.cpu(positionOffsetsDevice->getShape(), nvinfer1::DataType::kINT32); + generationLengthsHost = manager.cpu(generationLengthsDevice->getShape(), nvinfer1::DataType::kINT32); + positionIdsHost = manager.cpu(positionIdsDevice->getShape(), nvinfer1::DataType::kINT32); + + packedMaskHostCopy = manager.cpu(packedMasksDevice->getShape(), nvinfer1::DataType::kINT32); + positionOffsetsHostCopy = manager.cpu(positionOffsetsDevice->getShape(), nvinfer1::DataType::kINT32); + generationLengthsHostCopy = manager.cpu(generationLengthsDevice->getShape(), nvinfer1::DataType::kINT32); + positionIdsHostCopy = manager.cpu(positionIdsDevice->getShape(), nvinfer1::DataType::kINT32); + + batchSlotsHostCopy = manager.cpu(generationLengthsDevice->getShape(), nvinfer1::DataType::kINT32); + + useSpecDecoding = manager.cpu(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + bufferCast<SizeType32>(*useSpecDecoding)[0] = 1; + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void LookaheadRuntimeBuffers::setFromInputs(SizeType32 numCtxSequences, SizeType32 numGenSequences, + ITensor const& requestTypes, ITensor const& seqSlots, LookaheadDecodingBuffers const& decoderLookaheadBuffers, + TllmRuntime const& runtime, ModelConfig const& modelConfig, WorldConfig const& worldConfig) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const& manager = runtime.getBufferManager(); + + auto const tokensPerStep = modelConfig.getMaxDecodingTokens(); + + manager.copy(seqSlots, *batchSlotsHostCopy); + manager.copy(*decoderLookaheadBuffers.generationLengths, *generationLengthsHostCopy); + manager.copy(*decoderLookaheadBuffers.positionOffsets, *positionOffsetsHostCopy); + manager.copy(*decoderLookaheadBuffers.packedMasks, *packedMaskHostCopy); + manager.copy(*decoderLookaheadBuffers.positionIds, *positionIdsHostCopy); + + manager.getStream().synchronize(); + + BufferRange<SizeType32 const> batchSlotsRange(*batchSlotsHostCopy); + BufferRange<SizeType32> cumSumLengthRange(*cumSumLength); + + SizeType32 maxGenerationLength = 0; + for (SizeType32 bi = 0; bi < numGenSequences; bi++) + { + SizeType32 gbi = batchSlotsRange[bi + numCtxSequences]; + SizeType32 theLength = BufferRange<SizeType32>(*generationLengthsHostCopy)[gbi]; + maxGenerationLength = std::max(maxGenerationLength, theLength); + } + + auto positionOffsetShape = positionOffsetsHost->getShape(); + positionOffsetShape.d[1] = maxGenerationLength; + positionOffsetsHost->reshape(positionOffsetShape); + positionOffsetsDevice->reshape(positionOffsetShape); + + auto positionIdsShape = positionIdsHostCopy->getShape(); + auto positionIdsShape1D = ITensor::makeShape({ITensor::volume(positionIdsShape)}); + positionIdsHostCopy->reshape(positionIdsShape1D); + positionIdsHost->reshape(positionIdsShape1D); + + cumSumLengthRange[0] = 0; + for (SizeType32 bi = 0; bi < numGenSequences; bi++) + { + SizeType32 gbi = batchSlotsRange[bi + numCtxSequences]; + SizeType32 theLength = BufferRange<SizeType32>(*generationLengthsHostCopy)[gbi]; + + manager.copy(*ITensor::at(generationLengthsHostCopy, {gbi}), *ITensor::at(generationLengthsHost, {bi})); + + manager.copy(*ITensor::slice(positionOffsetsHostCopy, {gbi, 0}, theLength), + *ITensor::slice(positionOffsetsHost, {bi, 0}, theLength)); + + manager.copy(*ITensor::slice(packedMaskHostCopy, gbi * tokensPerStep, theLength), + *ITensor::slice(packedMaskHost, cumSumLengthRange[0], theLength)); + + manager.copy(*ITensor::slice(positionIdsHostCopy, gbi * tokensPerStep, theLength), + *ITensor::slice(positionIdsHost, cumSumLengthRange[0], theLength)); + + cumSumLengthRange[0] += theLength; + } + + positionIdsHostCopy->reshape(positionIdsShape); + positionIdsHost->reshape(positionIdsShape); + positionIdsDevice->reshape(positionIdsShape); + + manager.copy(*ITensor::slice(generationLengthsHost, 0, numGenSequences), + *ITensor::slice(generationLengthsDevice, 0, numGenSequences)); + manager.copy(*ITensor::slice(positionOffsetsHost, 0, numGenSequences), + *ITensor::slice(positionOffsetsDevice, 0, numGenSequences)); + manager.copy(*ITensor::slice(packedMaskHost, 0, numGenSequences * tokensPerStep), + *ITensor::slice(packedMasksDevice, 0, numGenSequences * tokensPerStep)); + manager.copy( + *ITensor::slice(positionIdsHost, 0, numGenSequences), *ITensor::slice(positionIdsDevice, 0, numGenSequences)); + positionIdsDevice->reshape(ITensor::makeShape({cumSumLengthRange[0]})); + + manager.getStream().synchronize(); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void LookaheadRuntimeBuffers::reshape(SizeType32 numCtxSequences, SizeType32 numGenSequences, SizeType32 tokensPerStep) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const numSequences = numGenSequences; + + auto packedMaskShape = packedMasksDevice->getShape(); + packedMaskShape.d[0] = numSequences * tokensPerStep; + packedMasksDevice->reshape(packedMaskShape); + packedMaskHost->reshape(packedMaskShape); + + auto generationLengthsShape = generationLengthsDevice->getShape(); + generationLengthsShape.d[0] = numSequences; + generationLengthsDevice->reshape(generationLengthsShape); + generationLengthsHost->reshape(generationLengthsShape); + + auto positionOffsetsShape = positionOffsetsDevice->getShape(); + positionOffsetsShape.d[0] = numSequences; + positionOffsetsDevice->reshape(positionOffsetsShape); + positionOffsetsHost->reshape(positionOffsetsShape); + + auto positionIdsShape = positionIdsDevice->getShape(); + positionIdsShape.d[0] = numSequences; + positionIdsDevice->reshape(positionIdsShape); + positionIdsHost->reshape(positionIdsShape); + + auto batchSlotsShape = batchSlotsHostCopy->getShape(); + batchSlotsShape.d[0] = numCtxSequences + numGenSequences; + batchSlotsHostCopy->reshape(batchSlotsShape); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void LookaheadRuntimeBuffers::enableLookaheadDecoding(SizeType32 maxBatchSize, SizeType32 tokensPerStep) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + auto const numPackedMasks = static_cast<ITensor::DimType64>(tensorrt_llm::common::divUp(tokensPerStep, 32)); + packedMasksDevice->reshape(ITensor::makeShape({maxBatchSize * tokensPerStep, numPackedMasks})); + generationLengthsDevice->reshape(ITensor::makeShape({maxBatchSize})); + positionOffsetsDevice->reshape(ITensor::makeShape({maxBatchSize, tokensPerStep})); + bufferCast<SizeType32>(*useSpecDecoding)[0] = 1; + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void LookaheadRuntimeBuffers::disableLookaheadDecoding() +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + packedMasksDevice->reshape(ITensor::makeShape({1, 1})); + generationLengthsDevice->reshape(ITensor::makeShape({1})); + positionOffsetsDevice->reshape(ITensor::makeShape({1, 1})); + bufferCast<SizeType32>(*useSpecDecoding)[0] = 0; + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void LookaheadRuntimeBuffers::insertInputTensors( + TensorMap& inputBuffers, TensorMap& /* outputBuffers */, WorldConfig const& /* worldConfig */) const +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + + inputBuffers.insert_or_assign("spec_decoding_packed_mask", packedMasksDevice); + inputBuffers.insert_or_assign("spec_decoding_generation_lengths", generationLengthsDevice); + inputBuffers.insert_or_assign("spec_decoding_position_offsets", positionOffsetsDevice); + inputBuffers.insert_or_assign("spec_decoding_use", useSpecDecoding); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + } // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/loraCache.cpp b/cpp/tensorrt_llm/runtime/loraCache.cpp index 36fb0363816f..3dbb814f058b 100644 --- a/cpp/tensorrt_llm/runtime/loraCache.cpp +++ b/cpp/tensorrt_llm/runtime/loraCache.cpp @@ -23,7 +23,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/loraUtils.h" #include <memory> #include <mutex> @@ -538,15 +537,15 @@ void LoraCache::splitTransposeCpu(ITensor& output, ITensor const& input, SizeTyp switch (input.getDataType()) { - case tensorrt_llm::DataType::kINT32: splitTransposeCpuInner<SizeType32>(output, input, tpSize, tpRank); break; - case tensorrt_llm::DataType::kFLOAT: splitTransposeCpuInner<float>(output, input, tpSize, tpRank); break; - case tensorrt_llm::DataType::kHALF: splitTransposeCpuInner<half>(output, input, tpSize, tpRank); break; - case tensorrt_llm::DataType::kINT8: splitTransposeCpuInner<int8_t>(output, input, tpSize, tpRank); break; + case nvinfer1::DataType::kINT32: splitTransposeCpuInner<SizeType32>(output, input, tpSize, tpRank); break; + case nvinfer1::DataType::kFLOAT: splitTransposeCpuInner<float>(output, input, tpSize, tpRank); break; + case nvinfer1::DataType::kHALF: splitTransposeCpuInner<half>(output, input, tpSize, tpRank); break; + case nvinfer1::DataType::kINT8: splitTransposeCpuInner<int8_t>(output, input, tpSize, tpRank); break; #ifdef ENABLE_FP8 - case tensorrt_llm::DataType::kFP8: splitTransposeCpuInner<__nv_fp8_e4m3>(output, input, tpSize, tpRank); break; + case nvinfer1::DataType::kFP8: splitTransposeCpuInner<__nv_fp8_e4m3>(output, input, tpSize, tpRank); break; #endif // ENABLE_FP8 #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: splitTransposeCpuInner<__nv_bfloat16>(output, input, tpSize, tpRank); break; + case nvinfer1::DataType::kBF16: splitTransposeCpuInner<__nv_bfloat16>(output, input, tpSize, tpRank); break; #endif // ENABLE_BF16 default: TLLM_CHECK_WITH_INFO(false, "data type not supported"); } diff --git a/cpp/tensorrt_llm/runtime/loraManager.cpp b/cpp/tensorrt_llm/runtime/loraManager.cpp index 1d25ea20c8e4..8d7ebe389853 100644 --- a/cpp/tensorrt_llm/runtime/loraManager.cpp +++ b/cpp/tensorrt_llm/runtime/loraManager.cpp @@ -26,6 +26,8 @@ #include "tensorrt_llm/runtime/utils/runtimeUtils.h" #include "tensorrt_llm/runtime/worldConfig.h" +#include <NvInferRuntime.h> + namespace tensorrt_llm::runtime { diff --git a/cpp/tensorrt_llm/runtime/loraUtils.cpp b/cpp/tensorrt_llm/runtime/loraUtils.cpp index da7f1475ddec..3c5e95162474 100644 --- a/cpp/tensorrt_llm/runtime/loraUtils.cpp +++ b/cpp/tensorrt_llm/runtime/loraUtils.cpp @@ -17,7 +17,6 @@ #include "tensorrt_llm/runtime/loraUtils.h" #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/modelConfig.h" @@ -58,7 +57,7 @@ void loraValidateRequestTensorDims(std::optional<ITensor::SharedPtr> const& optR keys->getShape().d[0] == expectedBatchSize, "Expected batch dimension to be 1 for each lora request"); TLLM_CHECK_WITH_INFO(weights->getMemoryType() != MemoryType::kGPU, "Expected lora weights to be in CPU memory"); TLLM_CHECK_WITH_INFO(keys->getMemoryType() != MemoryType::kGPU, "Expected lora weights to be in CPU memory"); - TLLM_CHECK_WITH_INFO(keys->getDataType() == tensorrt_llm::DataType::kINT32, + TLLM_CHECK_WITH_INFO(keys->getDataType() == nvinfer1::DataType::kINT32, "Expected lora keys to have TYPE_INT32 but was " + std::string(keys->getDataTypeName())); TLLM_CHECK_WITH_INFO(keys->getShape().d[1] == weights->getShape().d[1], diff --git a/cpp/tensorrt_llm/runtime/ncclCommunicator.cpp b/cpp/tensorrt_llm/runtime/ncclCommunicator.cpp index 7edfba42f935..b76efb75952a 100644 --- a/cpp/tensorrt_llm/runtime/ncclCommunicator.cpp +++ b/cpp/tensorrt_llm/runtime/ncclCommunicator.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/envUtils.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/ipcNvlsMemory.h" #include "tensorrt_llm/runtime/utils/multiDeviceUtils.h" @@ -103,19 +102,19 @@ void initNcclCommProbeWithTimeout(ncclUniqueId const& id, int worldSize, int ran } } -ncclDataType_t toNcclType(tensorrt_llm::DataType dataType) +ncclDataType_t toNcclType(nvinfer1::DataType dataType) { switch (dataType) { - case tensorrt_llm::DataType::kFLOAT: return ncclFloat32; - case tensorrt_llm::DataType::kHALF: return ncclHalf; - case tensorrt_llm::DataType::kINT8: return ncclInt8; - case tensorrt_llm::DataType::kINT32: return ncclInt32; - case tensorrt_llm::DataType::kUINT8: return ncclUint8; - case tensorrt_llm::DataType::kINT64: return ncclInt64; - case tensorrt_llm::DataType::kFP8: return ncclUint8; + case nvinfer1::DataType::kFLOAT: return ncclFloat32; + case nvinfer1::DataType::kHALF: return ncclHalf; + case nvinfer1::DataType::kINT8: return ncclInt8; + case nvinfer1::DataType::kINT32: return ncclInt32; + case nvinfer1::DataType::kUINT8: return ncclUint8; + case nvinfer1::DataType::kINT64: return ncclInt64; + case nvinfer1::DataType::kFP8: return ncclUint8; #if ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: return ncclBfloat16; + case nvinfer1::DataType::kBF16: return ncclBfloat16; #endif // ENABLE_BF16 default: TLLM_THROW("Unsupported data type: %d", static_cast<int>(dataType)); } @@ -124,7 +123,7 @@ ncclDataType_t toNcclType(tensorrt_llm::DataType dataType) } // namespace void NcclCommunicator::send( - void const* sendbuff, size_t count, tensorrt_llm::DataType dataType, int peer, CudaStream const& stream) const + void const* sendbuff, size_t count, nvinfer1::DataType dataType, int peer, CudaStream const& stream) const { #if ENABLE_MULTI_DEVICE TLLM_NCCL_CHECK(ncclSend(sendbuff, count, toNcclType(dataType), peer, mComm, stream.get())); @@ -134,7 +133,7 @@ void NcclCommunicator::send( } void NcclCommunicator::receive( - void* sendbuff, size_t count, tensorrt_llm::DataType dataType, int peer, CudaStream const& stream) const + void* sendbuff, size_t count, nvinfer1::DataType dataType, int peer, CudaStream const& stream) const { #if ENABLE_MULTI_DEVICE TLLM_NCCL_CHECK(ncclRecv(sendbuff, count, toNcclType(dataType), peer, mComm, stream.get())); diff --git a/cpp/tensorrt_llm/runtime/ncclCommunicator.h b/cpp/tensorrt_llm/runtime/ncclCommunicator.h index 21d7f116e95b..76cce4beab8a 100644 --- a/cpp/tensorrt_llm/runtime/ncclCommunicator.h +++ b/cpp/tensorrt_llm/runtime/ncclCommunicator.h @@ -16,7 +16,6 @@ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" @@ -58,10 +57,9 @@ class NcclCommunicator private: void send( - void const* sendbuff, size_t count, tensorrt_llm::DataType dataType, int peer, CudaStream const& stream) const; + void const* sendbuff, size_t count, nvinfer1::DataType dataType, int peer, CudaStream const& stream) const; - void receive( - void* sendbuff, size_t count, tensorrt_llm::DataType dataType, int peer, CudaStream const& stream) const; + void receive(void* sendbuff, size_t count, nvinfer1::DataType dataType, int peer, CudaStream const& stream) const; static ncclComm_t createComm(int worldSize, int rank, mpi::MpiComm const& mpiComm); diff --git a/cpp/tensorrt_llm/runtime/runtimeKernels.cu b/cpp/tensorrt_llm/runtime/runtimeKernels.cu index b22d36052370..3b3dbcac894a 100644 --- a/cpp/tensorrt_llm/runtime/runtimeKernels.cu +++ b/cpp/tensorrt_llm/runtime/runtimeKernels.cu @@ -21,7 +21,7 @@ #include "tensorrt_llm/kernels/speculativeDecoding/kvCacheUpdateKernels.h" #include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntimeBase.h> #include <cuda_runtime.h> using namespace tensorrt_llm::runtime; @@ -333,13 +333,13 @@ void invokeFillBatch(IBuffer& buffer, IBuffer const& slotIndices, std::size_t sl { switch (buffer.getDataType()) { - case tensorrt_llm::DataType::kINT32: + case nvinfer1::DataType::kINT32: invokeFillBatch<std::int32_t>(buffer, slotIndices, slotStride, values, stream); break; - case tensorrt_llm::DataType::kINT8: + case nvinfer1::DataType::kINT8: invokeFillBatch<std::int8_t>(buffer, slotIndices, slotStride, values, stream); break; - case tensorrt_llm::DataType::kFLOAT: invokeFillBatch<float>(buffer, slotIndices, slotStride, values, stream); break; + case nvinfer1::DataType::kFLOAT: invokeFillBatch<float>(buffer, slotIndices, slotStride, values, stream); break; default: TLLM_THROW("data type not supported"); } } @@ -349,15 +349,13 @@ void invokeGatherBatch(IBuffer& buffer, IBuffer const& values, IBuffer const& sl { switch (buffer.getDataType()) { - case tensorrt_llm::DataType::kINT32: + case nvinfer1::DataType::kINT32: invokeGatherBatch<std::int32_t>(buffer, values, slotIndices, slotStride, stream); break; - case tensorrt_llm::DataType::kINT8: + case nvinfer1::DataType::kINT8: invokeGatherBatch<std::int8_t>(buffer, values, slotIndices, slotStride, stream); break; - case tensorrt_llm::DataType::kFLOAT: - invokeGatherBatch<float>(buffer, values, slotIndices, slotStride, stream); - break; + case nvinfer1::DataType::kFLOAT: invokeGatherBatch<float>(buffer, values, slotIndices, slotStride, stream); break; default: TLLM_THROW("data type not supported"); } } @@ -410,12 +408,12 @@ void scatterTensor(ITensor& output, ITensor const& input, SizeType32 beamWidth, { switch (input.getDataType()) { - case tensorrt_llm::DataType::kINT32: invokeScatterTensor<SizeType32>(output, input, beamWidth, stream); break; - case tensorrt_llm::DataType::kFLOAT: invokeScatterTensor<float>(output, input, beamWidth, stream); break; - case tensorrt_llm::DataType::kHALF: invokeScatterTensor<half>(output, input, beamWidth, stream); break; - case tensorrt_llm::DataType::kINT8: invokeScatterTensor<int8_t>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kINT32: invokeScatterTensor<SizeType32>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kFLOAT: invokeScatterTensor<float>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kHALF: invokeScatterTensor<half>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kINT8: invokeScatterTensor<int8_t>(output, input, beamWidth, stream); break; #ifdef ENABLE_FP8 - case tensorrt_llm::DataType::kFP8: invokeScatterTensor<__nv_fp8_e4m3>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kFP8: invokeScatterTensor<__nv_fp8_e4m3>(output, input, beamWidth, stream); break; #endif // ENABLE_FP8 default: TLLM_THROW("data type not supported"); } @@ -425,15 +423,15 @@ void tileTensor(ITensor& output, ITensor const& input, SizeType32 beamWidth, Cud { switch (input.getDataType()) { - case tensorrt_llm::DataType::kINT32: invokeTileTensor<SizeType32>(output, input, beamWidth, stream); break; - case tensorrt_llm::DataType::kFLOAT: invokeTileTensor<float>(output, input, beamWidth, stream); break; - case tensorrt_llm::DataType::kHALF: invokeTileTensor<half>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kINT32: invokeTileTensor<SizeType32>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kFLOAT: invokeTileTensor<float>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kHALF: invokeTileTensor<half>(output, input, beamWidth, stream); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: invokeTileTensor<__nv_bfloat16>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kBF16: invokeTileTensor<__nv_bfloat16>(output, input, beamWidth, stream); break; #endif // ENABLE_BF16 - case tensorrt_llm::DataType::kINT8: invokeTileTensor<int8_t>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kINT8: invokeTileTensor<int8_t>(output, input, beamWidth, stream); break; #ifdef ENABLE_FP8 - case tensorrt_llm::DataType::kFP8: invokeTileTensor<__nv_fp8_e4m3>(output, input, beamWidth, stream); break; + case nvinfer1::DataType::kFP8: invokeTileTensor<__nv_fp8_e4m3>(output, input, beamWidth, stream); break; #endif // ENABLE_FP8 default: TLLM_THROW("data type not supported"); } @@ -446,22 +444,22 @@ void mergeLogitsFragments(BufferManager const& bufferManager, ITensor& output, { switch (output.getDataType()) { - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: invokeMergeLogitsFragments<float>(bufferManager, output, fragmentsVector, cachePointerDevice, cachePointerHost, firstBatchSlotIdx, microBatchSize, beamWidth, stream, stepOffset); break; - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: invokeMergeLogitsFragments<half>(bufferManager, output, fragmentsVector, cachePointerDevice, cachePointerHost, firstBatchSlotIdx, microBatchSize, beamWidth, stream, stepOffset); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: invokeMergeLogitsFragments<__nv_bfloat16>(bufferManager, output, fragmentsVector, cachePointerDevice, cachePointerHost, firstBatchSlotIdx, microBatchSize, beamWidth, stream, stepOffset); break; #endif // ENABLE_BF16 #ifdef ENABLE_FP8 - case tensorrt_llm::DataType::kFP8: + case nvinfer1::DataType::kFP8: invokeMergeLogitsFragments<__nv_fp8_e4m3>(bufferManager, output, fragmentsVector, cachePointerDevice, cachePointerHost, firstBatchSlotIdx, microBatchSize, beamWidth, stream, stepOffset); break; diff --git a/cpp/tensorrt_llm/runtime/tensorView.h b/cpp/tensorrt_llm/runtime/tensorView.h index d9e65b0efe23..17e7fb719415 100644 --- a/cpp/tensorrt_llm/runtime/tensorView.h +++ b/cpp/tensorrt_llm/runtime/tensorView.h @@ -16,7 +16,6 @@ #pragma once -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferView.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -46,19 +45,19 @@ class TensorView : virtual public ITensor, public BufferView mDims.d[0] = size; } - TensorView(IBuffer::SharedPtr const& buffer, size_t offset, size_t size, tensorrt_llm::Dims const& dims) + TensorView(IBuffer::SharedPtr const& buffer, size_t offset, size_t size, nvinfer1::Dims const& dims) : BufferView{buffer, offset, size} , mDims{dims} { Base::resize(ITensor::volumeNonNegative(dims)); } - [[nodiscard]] tensorrt_llm::Dims const& getShape() const override + [[nodiscard]] nvinfer1::Dims const& getShape() const override { return mDims; } - void reshape(tensorrt_llm::Dims const& dims) override + void reshape(nvinfer1::Dims const& dims) override { Base::resize(ITensor::volumeNonNegative(dims)); mDims = dims; @@ -82,6 +81,6 @@ class TensorView : virtual public ITensor, public BufferView return shape.nbDims > 0 && shape.d[0] > 0 ? ITensor::volume(shape) / shape.d[0] : 0; } - tensorrt_llm::Dims mDims{}; + nvinfer1::Dims mDims{}; }; } // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/tllmBuffers.cpp b/cpp/tensorrt_llm/runtime/tllmBuffers.cpp index 4876d5b87bf6..ff7ed04001d3 100644 --- a/cpp/tensorrt_llm/runtime/tllmBuffers.cpp +++ b/cpp/tensorrt_llm/runtime/tllmBuffers.cpp @@ -15,7 +15,6 @@ */ #include "tensorrt_llm/runtime/tllmBuffers.h" -#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::runtime { @@ -63,12 +62,12 @@ std::shared_ptr<MulticastBuffer> MulticastTensorView::lock() const /////////////////////////////////////// // MulticastTensorView ITensor methods /////////////////////////////////////// -tensorrt_llm::Dims const& MulticastTensorView::getShape() const +nvinfer1::Dims const& MulticastTensorView::getShape() const { return mDims; } -void MulticastTensorView::reshape(tensorrt_llm::Dims const& dims) +void MulticastTensorView::reshape(nvinfer1::Dims const& dims) { auto new_size = nonNegative(volume(dims)); if (new_size > getCapacity()) @@ -103,7 +102,7 @@ std::size_t MulticastTensorView::getCapacity() const return lock()->getCapacity(); } -tensorrt_llm::DataType MulticastTensorView::getDataType() const +nvinfer1::DataType MulticastTensorView::getDataType() const { return lock()->getDataType(); } diff --git a/cpp/tensorrt_llm/runtime/tllmBuffers.h b/cpp/tensorrt_llm/runtime/tllmBuffers.h index d023823de5b2..faed36537e5c 100644 --- a/cpp/tensorrt_llm/runtime/tllmBuffers.h +++ b/cpp/tensorrt_llm/runtime/tllmBuffers.h @@ -27,7 +27,7 @@ #include "tensorrt_llm/runtime/memoryCounters.h" #include "tensorrt_llm/runtime/virtualMemory.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_runtime_api.h> #include <algorithm> @@ -550,7 +550,7 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca //! //! \brief Construct an empty buffer. //! - explicit GenericBuffer(tensorrt_llm::DataType type, TAllocator allocator = {}) // NOLINT(*-pro-type-member-init) + explicit GenericBuffer(nvinfer1::DataType type, TAllocator allocator = {}) // NOLINT(*-pro-type-member-init) : GenericBuffer{0, type, std::move(allocator)} { } @@ -559,7 +559,7 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca //! \brief Construct a buffer with the specified allocation size in number of elements. //! explicit GenericBuffer( // NOLINT(*-pro-type-member-init) - std::size_t size, tensorrt_llm::DataType type, TAllocator allocator = {}) + std::size_t size, nvinfer1::DataType type, TAllocator allocator = {}) : GenericBuffer{size, size, type, std::move(allocator)} { } @@ -636,7 +636,7 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca //! //! \brief Returns the type of the buffer. //! - [[nodiscard]] tensorrt_llm::DataType getDataType() const override + [[nodiscard]] nvinfer1::DataType getDataType() const override { return mType; } @@ -687,8 +687,7 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca } protected: - explicit GenericBuffer( - std::size_t size, std::size_t capacity, tensorrt_llm::DataType type, TAllocator allocator = {}) + explicit GenericBuffer(std::size_t size, std::size_t capacity, nvinfer1::DataType type, TAllocator allocator = {}) : TAllocator{std::move(allocator)} , mSize{size} , mCapacity{capacity} @@ -701,14 +700,14 @@ class GenericBuffer : virtual public IBuffer, TAllocator // Inherit from TAlloca private: std::size_t mSize{0}, mCapacity{0}; - tensorrt_llm::DataType mType; + nvinfer1::DataType mType; void* mBuffer; }; class MulticastBuffer : virtual public IBuffer { public: - explicit MulticastBuffer(tensorrt_llm::DataType type, std::set<int> const& ranks) + explicit MulticastBuffer(nvinfer1::DataType type, std::set<int> const& ranks) : mSize(0) , mCapacity(0) , mType(type) @@ -717,7 +716,7 @@ class MulticastBuffer : virtual public IBuffer TLLM_CHECK(ranks.size() > 1); } - explicit MulticastBuffer(size_t size, tensorrt_llm::DataType type, std::set<int> const& ranks) + explicit MulticastBuffer(size_t size, nvinfer1::DataType type, std::set<int> const& ranks) : mSize(0) , mCapacity(0) , mType(type) @@ -818,7 +817,7 @@ class MulticastBuffer : virtual public IBuffer return mCapacity; } - [[nodiscard]] tensorrt_llm::DataType getDataType() const override + [[nodiscard]] nvinfer1::DataType getDataType() const override { return mType; } @@ -854,7 +853,7 @@ class MulticastBuffer : virtual public IBuffer private: std::size_t mSize = 0; std::size_t mCapacity = 0; - tensorrt_llm::DataType mType; + nvinfer1::DataType mType; std::set<int> mRanks; IpcNvlsHandle* mHandle; }; @@ -883,7 +882,7 @@ class GenericTensor : virtual public ITensor, public GenericBuffer<TAllocator> //! //! \brief Construct an empty tensor. //! - explicit GenericTensor(tensorrt_llm::DataType type, TAllocator allocator = {}) + explicit GenericTensor(nvinfer1::DataType type, TAllocator allocator = {}) : Base{type, std::move(allocator)} { mDims.nbDims = 0; @@ -892,14 +891,14 @@ class GenericTensor : virtual public ITensor, public GenericBuffer<TAllocator> //! //! \brief Construct a tensor with the specified allocation dimensions. //! - explicit GenericTensor(tensorrt_llm::Dims const& dims, tensorrt_llm::DataType type, TAllocator allocator = {}) + explicit GenericTensor(nvinfer1::Dims const& dims, nvinfer1::DataType type, TAllocator allocator = {}) : Base{nonNegative(volume(dims)), type, std::move(allocator)} , mDims{dims} { } explicit GenericTensor( - tensorrt_llm::Dims const& dims, std::size_t capacity, tensorrt_llm::DataType type, TAllocator allocator = {}) + nvinfer1::Dims const& dims, std::size_t capacity, nvinfer1::DataType type, TAllocator allocator = {}) : Base{nonNegative(volume(dims)), capacity, type, std::move(allocator)} , mDims{dims} { @@ -924,12 +923,12 @@ class GenericTensor : virtual public ITensor, public GenericBuffer<TAllocator> return *this; } - [[nodiscard]] tensorrt_llm::Dims const& getShape() const override + [[nodiscard]] nvinfer1::Dims const& getShape() const override { return mDims; } - void reshape(tensorrt_llm::Dims const& dims) override + void reshape(nvinfer1::Dims const& dims) override { Base::resize(nonNegative(volume(dims))); mDims = dims; @@ -947,7 +946,7 @@ class GenericTensor : virtual public ITensor, public GenericBuffer<TAllocator> } private: - tensorrt_llm::Dims mDims{}; + nvinfer1::Dims mDims{}; }; // Forward declaration @@ -972,9 +971,9 @@ class MulticastTensorView : virtual public ITensor ///////////////////// // ITensor methods ///////////////////// - [[nodiscard]] tensorrt_llm::Dims const& getShape() const override; + [[nodiscard]] nvinfer1::Dims const& getShape() const override; - void reshape(tensorrt_llm::Dims const& dims) override; + void reshape(nvinfer1::Dims const& dims) override; ///////////////////// // IBuffer methods @@ -984,7 +983,7 @@ class MulticastTensorView : virtual public ITensor [[nodiscard]] std::size_t getCapacity() const override; - [[nodiscard]] tensorrt_llm::DataType getDataType() const override; + [[nodiscard]] nvinfer1::DataType getDataType() const override; [[nodiscard]] MemoryType getMemoryType() const override; @@ -1017,7 +1016,7 @@ class MulticastTensorView : virtual public ITensor std::weak_ptr<MulticastTensor> mTensor; ViewType mViewType; - tensorrt_llm::Dims mDims{}; + nvinfer1::Dims mDims{}; }; class MulticastTensor : virtual public ITensor, @@ -1027,13 +1026,13 @@ class MulticastTensor : virtual public ITensor, public: using Base = MulticastBuffer; - explicit MulticastTensor(tensorrt_llm::DataType type, std::set<int> const& ranks) + explicit MulticastTensor(nvinfer1::DataType type, std::set<int> const& ranks) : Base(type, ranks) { mDims.nbDims = 0; } - explicit MulticastTensor(tensorrt_llm::Dims const& dims, tensorrt_llm::DataType type, std::set<int> const& ranks) + explicit MulticastTensor(nvinfer1::Dims const& dims, nvinfer1::DataType type, std::set<int> const& ranks) : Base(nonNegative(volume(dims)), type, ranks) , mDims(dims) { @@ -1069,12 +1068,12 @@ class MulticastTensor : virtual public ITensor, ///////////////////// // ITensor methods ///////////////////// - [[nodiscard]] tensorrt_llm::Dims const& getShape() const override + [[nodiscard]] nvinfer1::Dims const& getShape() const override { return mDims; } - void reshape(tensorrt_llm::Dims const& dims) override + void reshape(nvinfer1::Dims const& dims) override { Base::resize(nonNegative(volume(dims))); mDims = dims; @@ -1092,7 +1091,7 @@ class MulticastTensor : virtual public ITensor, } private: - tensorrt_llm::Dims mDims{}; + nvinfer1::Dims mDims{}; }; using DeviceTensor = GenericTensor<CudaAllocatorAsync>; diff --git a/cpp/tensorrt_llm/runtime/tllmLogger.cpp b/cpp/tensorrt_llm/runtime/tllmLogger.cpp new file mode 100644 index 000000000000..586ab2f4ae95 --- /dev/null +++ b/cpp/tensorrt_llm/runtime/tllmLogger.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "tllmLogger.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" + +using namespace tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; + +void TllmLogger::log(nvinfer1::ILogger::Severity severity, nvinfer1::AsciiChar const* msg) noexcept +{ + switch (severity) + { + case nvinfer1::ILogger::Severity::kINTERNAL_ERROR: + case nvinfer1::ILogger::Severity::kERROR: TLLM_LOG_ERROR(msg); break; + case nvinfer1::ILogger::Severity::kWARNING: TLLM_LOG_WARNING(msg); break; + case nvinfer1::ILogger::Severity::kINFO: TLLM_LOG_INFO(msg); break; + case nvinfer1::ILogger::Severity::kVERBOSE: TLLM_LOG_DEBUG(msg); break; + default: TLLM_LOG_TRACE(msg); break; + } +} + +nvinfer1::ILogger::Severity TllmLogger::getLevel() +{ + auto* const logger = tc::Logger::getLogger(); + switch (logger->getLevel()) + { + case tc::Logger::Level::ERROR: return nvinfer1::ILogger::Severity::kERROR; + case tc::Logger::Level::WARNING: return nvinfer1::ILogger::Severity::kWARNING; + case tc::Logger::Level::INFO: return nvinfer1::ILogger::Severity::kINFO; + case tc::Logger::Level::DEBUG: + case tc::Logger::Level::TRACE: return nvinfer1::ILogger::Severity::kVERBOSE; + default: return nvinfer1::ILogger::Severity::kINTERNAL_ERROR; + } +} + +void TllmLogger::setLevel(nvinfer1::ILogger::Severity level) +{ + auto* const logger = tc::Logger::getLogger(); + switch (level) + { + case nvinfer1::ILogger::Severity::kINTERNAL_ERROR: + case nvinfer1::ILogger::Severity::kERROR: logger->setLevel(tc::Logger::Level::ERROR); break; + case nvinfer1::ILogger::Severity::kWARNING: logger->setLevel(tc::Logger::Level::WARNING); break; + case nvinfer1::ILogger::Severity::kINFO: logger->setLevel(tc::Logger::Level::INFO); break; + case nvinfer1::ILogger::Severity::kVERBOSE: logger->setLevel(tc::Logger::Level::TRACE); break; + default: TLLM_THROW("Unsupported severity"); + } +} diff --git a/cpp/tensorrt_llm/runtime/tllmRuntime.cpp b/cpp/tensorrt_llm/runtime/tllmRuntime.cpp new file mode 100644 index 000000000000..7c2ca4747213 --- /dev/null +++ b/cpp/tensorrt_llm/runtime/tllmRuntime.cpp @@ -0,0 +1,831 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "tllmRuntime.h" +#include "common.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/nvtxUtils.h" +#include "tensorrt_llm/common/safetensors.h" +#include "tensorrt_llm/executor/tensor.h" +#include "tensorrt_llm/kernels/userbuffers/ub_interface.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tllmLogger.h" +#include "tllmStreamReaders.h" + +#include "nlohmann/json.hpp" +#include <NvInferRuntime.h> + +#include <algorithm> +#include <cstddef> +#include <limits> +#include <memory> +#include <optional> +#include <string> +#include <type_traits> +#include <utility> +#include <vector> + +using namespace tensorrt_llm::runtime; +using TensorMap = StringPtrMap<ITensor>; + +namespace +{ +static_assert(std::is_signed<SizeType32>::value, "SizeType32 must be signed"); + +nvinfer1::Dims shapeToDims(std::vector<std::size_t> const& shape) +{ + TLLM_CHECK(shape.size() <= nvinfer1::Dims::MAX_DIMS); + nvinfer1::Dims dims; + auto constexpr dim_max = std::numeric_limits<ITensor::DimType64>::max(); + dims.nbDims = static_cast<std::int32_t>(shape.size()); + for (std::size_t i = 0; i < shape.size(); ++i) + { + // shape[i] >= 0 because it has unsigned type. Check upper bound: + TLLM_CHECK(shape[i] <= static_cast<std::size_t>(dim_max)); + dims.d[i] = static_cast<ITensor::DimType64>(shape[i]); + } + return dims; +} + +std::vector<std::size_t> dimsToShape(nvinfer1::Dims const& dims) +{ + TLLM_CHECK(dims.nbDims >= 0); + std::vector<std::size_t> shape(dims.nbDims); + for (std::int32_t i = 0; i < dims.nbDims; ++i) + { + TLLM_CHECK(dims.d[i] >= 0); + shape[i] = static_cast<std::size_t>(dims.d[i]); + } + return shape; +} + +tensorrt_llm::runtime::TllmLogger defaultLogger{}; + +void setWeightStreaming(nvinfer1::ICudaEngine& engine, float const gpuWeightsPercent) +{ + if (gpuWeightsPercent < 1) + { + int64_t streamableSize = engine.getStreamableWeightsSize(); + int64_t budget = gpuWeightsPercent * streamableSize; + TLLM_LOG_INFO("Set gpu weights percent to %f, which is %lld bytes. Valid range: %lld bytes - %lld bytes.", + gpuWeightsPercent, budget, 0, streamableSize); + engine.setWeightStreamingBudgetV2(budget); + } +} + +class LayerInfo +{ +public: + LayerInfo(std::optional<std::string> name, std::string type) + : name(std::move(name)) + , type(std::move(type)){}; + std::optional<std::string> name; + std::string type; +}; + +void assessLikelihoodOfRuntimeAllocation( + nvinfer1::ICudaEngine const& engine, nvinfer1::IEngineInspector const& engineInspector) + +{ + TLLM_LOG_INFO("Inspecting the engine to identify potential runtime issues..."); + auto const profilingVerbosity = engine.getProfilingVerbosity(); + if (profilingVerbosity != nvinfer1::ProfilingVerbosity::kDETAILED) + { + TLLM_LOG_INFO( + "The profiling verbosity of the engine does not allow this analysis to proceed. Re-build the engine with " + "'detailed' profiling verbosity to get more diagnostics."); + return; + } + auto const* const layerTypeKey = "LayerType"; + auto const* const nameKey = "Name"; + auto const numLayers = engine.getNbLayers(); + TLLM_LOG_INFO("Model has %i layers.", numLayers); + std::vector<SizeType32> indexes(numLayers); + std::iota(indexes.begin(), indexes.end(), 0); + std::vector<std::optional<LayerInfo>> layerInfos(numLayers); + std::transform(indexes.cbegin(), indexes.cend(), layerInfos.begin(), + [&](SizeType32 const idx) + { + auto const* const layerInfo + = engineInspector.getLayerInformation(idx, nvinfer1::LayerInformationFormat::kJSON); + + // Needs to be copied explicitly, see documentation of `getLayerInformation`. + auto const layerInfoCopy = std::string(layerInfo); + auto const jsonLayerInfo = nlohmann::json::parse(layerInfoCopy); + auto const layerJsonType = jsonLayerInfo.type(); + if (layerJsonType != nlohmann::detail::value_t::object) + { + return std::optional<LayerInfo>{}; + } + if (!jsonLayerInfo.contains(layerTypeKey)) + { + return std::optional<LayerInfo>{}; + } + auto const& typeJson = jsonLayerInfo.at(layerTypeKey); + if (typeJson.type() != nlohmann::detail::value_t::string) + { + return std::optional<LayerInfo>{}; + } + std::optional<std::string> name{}; + if (jsonLayerInfo.contains(nameKey)) + { + auto const& nameJson = jsonLayerInfo.at(nameKey); + auto const nameJsonType = nameJson.type(); + if (nameJsonType == nlohmann::detail::value_t::string) + { + name = nameJson.get<std::string>(); + } + } + return std::make_optional(LayerInfo{name, typeJson.get<std::string>()}); + }); + auto const layersWithInfoEnd = std::partition( + layerInfos.begin(), layerInfos.end(), [](std::optional<LayerInfo> const& info) { return info.has_value(); }); + if (layersWithInfoEnd == layerInfos.begin()) + { + TLLM_LOG_INFO("Engine layer infos could not be parsed into useful information."); + return; + } + auto const allocateLayersEnd = std::partition(layerInfos.begin(), layersWithInfoEnd, + [](std::optional<LayerInfo> const& info) { return info.value().type == "allocate"; }); + auto numWarnings = 0; + for (auto layerInfo = layerInfos.begin(); layerInfo != allocateLayersEnd; layerInfo++) + { + auto constexpr maxNumWarnings = 25; + if (numWarnings < maxNumWarnings) + { + auto const layerName = layerInfo->value().name.value_or(""); + TLLM_LOG_WARNING( + "Layer '%s' has type '%s', which could lead to large runtime memory allocations. Performance " + "might be degraded and / or you might run out of memory.", + layerName.c_str(), layerInfo->value().type.c_str()); + } + numWarnings++; + } + if (numWarnings > 0) + { + TLLM_LOG_WARNING( + "There were a total of %i layers with type 'allocate'. Some warnings might have been silenced to keep the " + "output concise.", + numWarnings); + } +} + +} // namespace + +TllmRuntime::TllmRuntime(RawEngine const& rawEngine, nvinfer1::ILogger* logger, bool useGpuDirectStorage, + float gpuWeightsPercent, bool useShapeInference) + : mStream(std::make_shared<CudaStream>()) + , mBufferManager{mStream, true} // Ensure to trim the memory pool on destruction. + , mRuntime{nvinfer1::createInferRuntime(static_cast<bool>(logger) ? *logger : defaultLogger)} + , mUseShapeInference{useShapeInference} + , mUserBufferEnabled{false} +{ + auto const startTime = std::chrono::high_resolution_clock::now(); + + switch (rawEngine.getType()) + { + case RawEngine::Type::FilePath: + { + if (useGpuDirectStorage) + { + TLLM_LOG_INFO("GDS is used to load the engine!"); + auto reader = GDSStreamReader(rawEngine.getPath()); + mEngine.reset(mRuntime->deserializeCudaEngine(reader)); + } + else + { + auto reader = StreamReader(rawEngine.getPath()); + mEngine.reset(mRuntime->deserializeCudaEngine(reader)); + } + break; + } + case RawEngine::Type::AddressWithSize: + mEngine.reset(mRuntime->deserializeCudaEngine(rawEngine.getAddress(), rawEngine.getSize())); + break; + case RawEngine::Type::HostMemory: + mEngine.reset( + mRuntime->deserializeCudaEngine(rawEngine.getHostMemory()->data(), rawEngine.getHostMemory()->size())); + break; + default: TLLM_THROW("Unsupported raw engine type."); + } + + auto const elapsedMs + = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - startTime); + + TLLM_LOG_INFO("Engine load time %lld ms", elapsedMs); + + TLLM_CHECK_WITH_INFO(mEngine != nullptr, "Failed to deserialize cuda engine."); + mEngineInspector.reset(mEngine->createEngineInspector()); + assessLikelihoodOfRuntimeAllocation(*mEngine, *mEngineInspector); + setWeightStreaming(getEngine(), gpuWeightsPercent); + auto const devMemorySize = mEngine->getDeviceMemorySizeV2(); + mEngineBuffer = mBufferManager.gpu(devMemorySize); + // Print context memory size for CI/CD to track. + TLLM_LOG_INFO("[MemUsageChange] Allocated %.2f MiB for execution context memory.", + static_cast<double>(devMemorySize) / 1048576.0); + + cacheTensorNames(); +} + +void TllmRuntime::cacheTensorNames() +{ + for (std::int32_t i = 0; i < mEngine->getNbIOTensors(); ++i) + { + auto const* const name = mEngine->getIOTensorName(i); + if (mEngine->getTensorIOMode(name) == nvinfer1::TensorIOMode::kINPUT) + { + mInputTensorNames.emplace_back(name); + } + else if (mEngine->getTensorIOMode(name) == nvinfer1::TensorIOMode::kOUTPUT) + { + mOutputTensorNames.emplace_back(name); + } + } +} + +nvinfer1::IExecutionContext& TllmRuntime::addContext(std::int32_t profileIndex) +{ + TLLM_CHECK(0 <= profileIndex && profileIndex < mEngine->getNbOptimizationProfiles()); + mContexts.emplace_back(mEngine->createExecutionContextWithoutDeviceMemory()); + if (!mContexts.back()) + { + if (mEngine->getStreamableWeightsSize() > 0) + { + TLLM_THROW("Failed to allocate memory for weights. Please try reducing --gpu_weights_percent."); + } + else + { + TLLM_THROW("Internal Error: Failed to create an execution context."); + } + } + auto& context = *mContexts.back(); + context.setDeviceMemoryV2(mEngineBuffer->data(), static_cast<int64_t>(mEngineBuffer->getCapacity())); + + if (tensorrt_llm::common::Logger::getLogger()->isEnabled(tensorrt_llm::common::Logger::TRACE) + && mContexts.size() == 1) + { + // Print engine information only once + printEngineInfo(); + } + + context.setOptimizationProfileAsync(profileIndex, mStream->get()); + // If nvtx verbosity is DETAILED, print an info about potential perf overhead. + if (context.getNvtxVerbosity() == nvinfer1::ProfilingVerbosity::kDETAILED) + { + TLLM_LOG_INFO( + "The engine was built with kDETAILED profiling verbosity, which may result in small overheads at runtime."); + } + return context; +} + +void TllmRuntime::printEngineInfo() +{ + auto& context = *(mContexts[0]); + int const nIO = mEngine->getNbIOTensors(); // Count of input / output tensor + int const nOP = mEngine->getNbOptimizationProfiles(); // Count of Optimization Profile + std::size_t maxNameWidth = 0; + std::size_t maxShapeWidth = 0; + + // Get information of engine input / output + std::vector<std::string> tensorNameList{}; + tensorNameList.reserve(nIO); + for (int i = 0; i < nIO; ++i) + { + tensorNameList.emplace_back(mEngine->getIOTensorName(i)); + } + std::vector<std::map<std::string, std::string>> tensorInfo(nIO); // Tensor Information Vector + std::vector<std::vector<std::vector<nvinfer1::Dims64>>> profileInfo(nIO); // Tensor Optimization Profile Vector + for (int i = 0; i < nIO; ++i) + { + auto const& name = tensorNameList[i]; + char const* nameC{name.c_str()}; // name of C-style + maxNameWidth = std::max(maxNameWidth, name.size()); + tensorInfo[i]["mode"] = mEngine->getTensorIOMode(nameC) == nvinfer1::TensorIOMode::kINPUT ? "I" : "O"; + tensorInfo[i]["location"] + = mEngine->getTensorLocation(nameC) == nvinfer1::TensorLocation::kDEVICE ? "GPU" : "CPU"; + tensorInfo[i]["data_type"] = dataTypeToString(mEngine->getTensorDataType(nameC)); + tensorInfo[i]["build_shape"] = shapeToString(mEngine->getTensorShape(nameC)); + maxShapeWidth = std::max(maxShapeWidth, tensorInfo[i]["build_shape"].size()); + if (tensorInfo[i]["mode"] == "I") + { + std::vector<std::vector<nvinfer1::Dims64>> topPerTensor(nOP); + for (int k = 0; k < nOP; ++k) + { + if (tensorInfo[i]["location"] == std::string("GPU")) + { + std::vector<nvinfer1::Dims64> top(3); + top[0] = mEngine->getProfileShape(nameC, k, nvinfer1::OptProfileSelector::kMIN); + top[1] = mEngine->getProfileShape(nameC, k, nvinfer1::OptProfileSelector::kOPT); + top[2] = mEngine->getProfileShape(nameC, k, nvinfer1::OptProfileSelector::kMAX); + topPerTensor[k] = top; + maxShapeWidth = std::max(maxShapeWidth, shapeToString(top[2]).size()); + } + else + { + // Shape input tensor, not used in TRT-LLM support yet + std::vector<nvinfer1::Dims64> top(3); + int const nDim = mEngine->getTensorShape(nameC).nbDims; + nvinfer1::Dims64 tensorShape{nDim, {-1}}; + int const* pos = nullptr; + pos = mEngine->getProfileTensorValues(nameC, k, nvinfer1::OptProfileSelector::kMIN); + std::copy(pos, pos + nDim, tensorShape.d); + top[0] = tensorShape; + pos = mEngine->getProfileTensorValues(nameC, k, nvinfer1::OptProfileSelector::kOPT); + std::copy(pos, pos + nDim, tensorShape.d); + top[1] = tensorShape; + pos = mEngine->getProfileTensorValues(nameC, k, nvinfer1::OptProfileSelector::kMAX); + std::copy(pos, pos + nDim, tensorShape.d); + top[2] = tensorShape; + topPerTensor[k] = top; + } + } + profileInfo[i] = topPerTensor; + } + else + { + profileInfo[i] = std::vector<std::vector<nvinfer1::Dims64>>(nOP); + } + } + // Set input shape to get output shape + for (int k = 0; k < nOP; ++k) + { + for (int j = 0; j < 3; ++j) // Min, Opt, Max + { + for (int i = 0; i < nIO; ++i) + { + auto const& name = tensorNameList[i]; + char const* nameC = name.c_str(); + if (tensorInfo[i]["mode"] == "I") + { + if (tensorInfo[i]["location"] == std::string("GPU")) + { + context.setInputShape(nameC, profileInfo[i][k][j]); + } + else + { + // Shape input tensor, not used in TRT-LLM support yet + context.setInputTensorAddress(nameC, profileInfo[i][k][j].d); + } + } + else + { + TLLM_CHECK_WITH_INFO(context.allInputDimensionsSpecified(), "Input dimensions not specified"); + TLLM_CHECK_WITH_INFO(context.allInputShapesSpecified(), "Input shapes not specified"); + if (tensorInfo[i]["location"] == std::string("GPU")) + { + profileInfo[i][k].push_back(context.getTensorShape(nameC)); + } + else + { + // Shape input tensor, not used in TRT-LLM support yet + int const nDim = mEngine->getTensorShape(nameC).nbDims; + nvinfer1::Dims64 tensorShape{nDim, {}}; + int const* pos = reinterpret_cast<int const*>(context.getTensorAddress(nameC)); + std::copy(pos, pos + nDim, tensorShape.d); + profileInfo[i][k].push_back(tensorShape); + } + } + } + } + } + + // Print information of engine input / output + std::string info; + TLLM_LOG_TRACE("Information of engine input / output."); + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 24, '=')); + info = alignText("Name", maxNameWidth) + "|I/O|Location|DataType|" + alignText("Shape", maxShapeWidth) + "|"; + TLLM_LOG_TRACE(info.c_str()); + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 24, '-')); + for (int i = 0; i < nIO; ++i) + { + info = alignText(tensorNameList[i], maxNameWidth, false) + "|"; + info += alignText(tensorInfo[i]["mode"], 3) + "|"; + info += alignText(tensorInfo[i]["location"], 8) + "|"; + info += alignText(tensorInfo[i]["data_type"], 8) + "|"; + info += alignText(tensorInfo[i]["build_shape"], maxShapeWidth) + "|"; + TLLM_LOG_TRACE(info.c_str()); + } + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 24, '=')); + // Print information of optimization profile + TLLM_LOG_TRACE("Information of optimization profile."); + for (int k = 0; k < nOP; ++k) + { + TLLM_LOG_TRACE("Optimization Profile %d:", k); + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth * 3 + 4, '=')); + info = alignText("Name", maxNameWidth) + "|"; + info += alignText("Min", maxShapeWidth) + "|"; + info += alignText("Opt", maxShapeWidth) + "|"; + info += alignText("Max", maxShapeWidth) + "|"; + TLLM_LOG_TRACE(info.c_str()); + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth * 3 + 4, '-')); + for (int i = 0; i < nIO; ++i) + { + auto const& top = profileInfo[i][k]; + info = alignText(tensorNameList[i], maxNameWidth, false) + "|"; + info += alignText(shapeToString(top[0]), maxShapeWidth) + "|"; + info += alignText(shapeToString(top[1]), maxShapeWidth) + "|"; + info += alignText(shapeToString(top[2]), maxShapeWidth) + "|"; + TLLM_LOG_TRACE(info.c_str()); + } + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth * 3 + 4, '=')); + } +} + +void TllmRuntime::printContextInfo(SizeType32 contextIndex) +{ + auto const& context = *(mContexts[contextIndex]); + int const nIO = mEngine->getNbIOTensors(); // Count of input / output tensor + std::size_t maxNameWidth = 0; + std::size_t maxShapeWidth = 0; + std::vector<std::tuple<std::string, bool, std::string>> tensorInfo(nIO); + for (int i = 0; i < nIO; ++i) + { + auto const name = std::string(mEngine->getIOTensorName(i)); + bool const isInput = mEngine->getTensorIOMode(name.c_str()) == nvinfer1::TensorIOMode::kINPUT; + auto const shape = shapeToString(context.getTensorShape(name.c_str())); + tensorInfo[i] = std::make_tuple(name, isInput, shape); + maxNameWidth = std::max(maxNameWidth, name.size()); + maxShapeWidth = std::max(maxShapeWidth, shape.size()); + // Shape input tensor is not considered in TRT-LLM yet + } + + TLLM_LOG_TRACE("Information of context input / output."); + TLLM_LOG_TRACE("Using Optimization Profile: %d", contextIndex); + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 6, '=')); + std::string info = alignText("Name", maxNameWidth) + "|I/O|" + alignText("Shape", maxShapeWidth) + "|"; + TLLM_LOG_TRACE(info.c_str()); + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 6, '-')); + for (int i = 0; i < nIO; ++i) + { + auto const& [name, isInput, shape] = tensorInfo[i]; + info = alignText(name, maxNameWidth, false) + "|"; + info += alignText(isInput ? "I" : "O", 3) + "|"; + info += alignText(shape, maxShapeWidth) + "|"; + TLLM_LOG_TRACE(info.c_str()); + } + TLLM_LOG_TRACE(std::string(maxNameWidth + maxShapeWidth + 6, '=')); +} + +void TllmRuntime::clearContexts() +{ + for (auto& context : mContexts) + { + context.reset(); + } + mContexts.clear(); +} + +bool TllmRuntime::executeContext(SizeType32 contextIndex) const +{ + NVTX3_FUNC_RANGE(); + auto& context = getContext(contextIndex); + auto res = context.enqueueV3(mStream->get()); + sync_check_cuda_error(mStream->get()); + return res; +} + +void TllmRuntime::setInputTensorsImpl(SizeType32 contextIndex, TensorMap const& tensorMap, bool throwOnMiss) +{ + NVTX3_FUNC_RANGE(); + auto& context = getContext(contextIndex); + for (auto const& name : mInputTensorNames) + { + auto const pos = tensorMap.find(name); + if (pos == tensorMap.end()) + { + if (throwOnMiss) + { + auto expectedShape = mEngine->getTensorShape(name.c_str()); + TLLM_THROW("Input tensor '%s' not found; expected shape: %s", name.c_str(), + ITensor::toString(expectedShape).c_str()); + } + else + { + continue; + } + } + + auto const& tensor = pos->second; + auto const tensorDtype = tensor->getDataType(); + auto const engineDtype = mEngine->getTensorDataType(name.c_str()); + // WAR: TRT does not support mixed FP8 and FP16 input, so engine expects FP16 tensors. + TLLM_CHECK_WITH_INFO(tensorDtype == engineDtype + || (tensorDtype == nvinfer1::DataType::kFP8 && engineDtype == nvinfer1::DataType::kHALF), + "%s: expected type %d, provided type %d", name.c_str(), static_cast<std::int32_t>(engineDtype), + static_cast<std::int32_t>(tensorDtype)); + + auto tensorShape = tensor->getShape(); + + // Change shape of `cache_indirection` for Variable-Beam-Width-Search + // TODO: remove this hack if beamWidth of each request are passed into GptAttentionPlugin by input tensor + if (name == "cache_indirection" && mCurrentBeamWidths.size() > 0) + { + SizeType32 const beamWidth = getCurrentBeamWidth(); + if (tensorShape.d[1] != beamWidth) + { + tensorShape.d[1] = beamWidth; + TLLM_LOG_TRACE("Change shape of cache_indirection to %s", ITensor::toString(tensorShape).c_str()); + } + } + + auto const setInputShapeSuccess = context.setInputShape(name.c_str(), tensorShape); + if (!setInputShapeSuccess) + { + auto const minShape + = mEngine->getProfileShape(name.c_str(), contextIndex, nvinfer1::OptProfileSelector::kMIN); + auto const maxShape + = mEngine->getProfileShape(name.c_str(), contextIndex, nvinfer1::OptProfileSelector::kMAX); + + TLLM_THROW("Tensor '%s' has invalid shape %s, expected in range min %s, max %s", name.c_str(), + ITensor::toString(tensorShape).c_str(), ITensor::toString(minShape).c_str(), + ITensor::toString(maxShape).c_str()); + } + auto* const data = tensor->data(); + if (static_cast<bool>(data)) + { + context.setInputTensorAddress(name.c_str(), data); + } + else + { + TLLM_CHECK_WITH_INFO(tensor->getSize() == 0, std::string("Invalid data for tensor: ") + name); + // TensorRT runtime does not support nullptr. + if (!mDummyTensor) + { + mDummyTensor = mBufferManager.gpu(ITensor::makeShape({1})); + } + context.setInputTensorAddress(name.c_str(), mDummyTensor->data()); + } + } +} + +void TllmRuntime::setStaticInputTensors(TensorMap const& tensorMap) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_FUNC_RANGE(); + + TLLM_CHECK_WITH_INFO(getNbContexts() > 0, "Contexts should be created before calling setStaticInputTensors"); + for (auto contextIndex = 0; contextIndex < getNbContexts(); ++contextIndex) + { + setInputTensorsImpl(contextIndex, tensorMap, false); + } + + // move static input tensor names to separate vector + auto const begin = mInputTensorNames.begin(); + auto end = mInputTensorNames.end(); + for (auto const& [name, tensor] : tensorMap) + { + end = std::remove(begin, end, name); + } + mInputTensorNames.erase(end, mInputTensorNames.end()); + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TllmRuntime::setInputTensors(SizeType32 contextIndex, TensorMap const& tensorMap) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_FUNC_RANGE(); + setInputTensorsImpl(contextIndex, tensorMap, true); + + auto& context = getContext(contextIndex); + if (mUseShapeInference) + { + NVTX3_SCOPED_RANGE(infer_shapes); + char const* missing = nullptr; + auto const nbMissing = context.inferShapes(1, &missing); + if (nbMissing > 0) + { + TLLM_THROW("Input shape not specified: %s", missing); + } + else if (nbMissing < 0) + { + TLLM_THROW("Invalid input shape"); + } + } + + { + NVTX3_SCOPED_RANGE(final_checks); + TLLM_CHECK_WITH_INFO(context.allInputDimensionsSpecified(), "Input dimensions not specified"); + TLLM_CHECK_WITH_INFO(context.allInputShapesSpecified(), "Input shapes not specified"); + } + + // Print shape of input / output tensors for the TRT engine + if (tensorrt_llm::common::Logger::getLogger()->isEnabled(tensorrt_llm::common::Logger::TRACE)) + { + printContextInfo(contextIndex); + } + + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TllmRuntime::setOutputTensors(SizeType32 contextIndex, TensorMap& tensorMap) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_FUNC_RANGE(); + if (isUserBufferEnabled()) + { + // This function will identify the output tensors in the network that need to be bound as UB buffers + // and bind the corresponding buffers to them based on their names. + setUserBufferTensors(contextIndex, tensorMap); + } + + auto& context = getContext(contextIndex); + for (auto const& name : mOutputTensorNames) + { + auto const engineDtype = mEngine->getTensorDataType(name.c_str()); + auto const pos = tensorMap.find(name); + if (pos != tensorMap.end()) + { + auto const& tensor = pos->second; + auto const tensorDtype = tensor->getDataType(); + // WAR: TRT does not support mixed FP8 and FP16 input, so engine expects FP16 tensors. + TLLM_CHECK_WITH_INFO(tensorDtype == engineDtype + || (tensorDtype == nvinfer1::DataType::kFP8 && engineDtype == nvinfer1::DataType::kHALF), + "%s: expected type %d, provided type %d", name.c_str(), static_cast<std::int32_t>(engineDtype), + static_cast<std::int32_t>(tensorDtype)); + + if (mUseShapeInference) + { + auto const dims = context.getTensorShape(name.c_str()); + tensor->reshape(dims); + } + context.setTensorAddress(name.c_str(), tensor->data()); + } + else if (mUseShapeInference) + { + auto const dims = context.getTensorShape(name.c_str()); + auto tensor = ITensor::SharedPtr(mBufferManager.gpu(dims, engineDtype)); + tensorMap.insert(pos, std::make_pair(name, tensor)); + context.setTensorAddress(name.c_str(), tensor->data()); + } + else + { + TLLM_THROW("Tensor %s is not found in tensorMap and shape inference is not allowed", name.c_str()); + } + } + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} + +void TllmRuntime::setUserBufferTensors(SizeType32 contextIndex, TensorMap& tensorMap) +{ + auto startsWith = [](std::string const& str, std::string const& prefix) -> bool + { return str.size() > prefix.size() && str.compare(0, prefix.size(), prefix) == 0; }; + std::string const prefix(tensorrt_llm::runtime::ub::tensor_prefix); + auto& context = getContext(contextIndex); + for (auto const& name : mOutputTensorNames) + { + auto const pos = tensorMap.find(name); + if (pos != tensorMap.end() || !startsWith(name, prefix)) + { + continue; + } + auto const engineDtype = mEngine->getTensorDataType(name.c_str()); + auto const dims = context.getTensorShape(name.c_str()); + void* ubBuffer = nullptr; + if (name[prefix.size()] == '0') + { + ubBuffer = tensorrt_llm::runtime::ub::ub_get(0).addr; + } + else if (name[prefix.size()] == '1') + { + ubBuffer = tensorrt_llm::runtime::ub::ub_get(1).addr; + } + else if (name[prefix.size()] == '2') + { + ubBuffer = tensorrt_llm::runtime::ub::ub_get(2).addr; + } + else + { + TLLM_CHECK(false); + } + auto tensor = ITensor::SharedPtr(ITensor::wrap(ubBuffer, engineDtype, dims)); + tensorMap.insert(pos, std::make_pair(name, tensor)); + context.setTensorAddress(name.c_str(), ubBuffer); + } +} + +void TllmRuntime::initializeUserBuffer(tensorrt_llm::runtime::WorldConfig const& world_config, SizeType32 maxBatchSize, + SizeType32 maxBeamWidth, SizeType32 maxSequenceLength, SizeType32 hiddenSize, + std::optional<SizeType32> maxNumTokens) +{ + auto startsWith = [](std::string const& str, std::string const& prefix) -> bool + { return str.size() > prefix.size() && str.compare(0, prefix.size(), prefix) == 0; }; + std::string const prefix(tensorrt_llm::runtime::ub::tensor_prefix); + bool useNVFP4Model = false; + for (auto const& name : mOutputTensorNames) + { + if (startsWith(name, prefix)) + { + mUserBufferEnabled = true; + if (name[prefix.size()] == '2') + { + useNVFP4Model = true; + break; + } + } + } + if (!mUserBufferEnabled) + { + return; + } + // The hidden size returned by ModelConfig is the real hidden size divided by the TP size. + auto const tpSize = world_config.getTensorParallelism(); + size_t const realHiddenSize = hiddenSize * tpSize; + size_t const tokensNum = maxNumTokens.value_or(maxBatchSize * maxBeamWidth * maxSequenceLength); + TLLM_CHECK(tokensNum > 0); + size_t const elemNum = tokensNum * realHiddenSize; + TLLM_LOG_INFO("[UserBuffer] MaxBatchSize %d, maxBeamWidth %d, maxSequenceLength %d, maxNumTokens %d, select %lu", + maxBatchSize, maxBeamWidth, maxSequenceLength, maxNumTokens.has_value() ? maxNumTokens.value() : 0, tokensNum); + tensorrt_llm::runtime::ub::ub_initialize(world_config); + tensorrt_llm::runtime::ub::ub_allocate(elemNum * sizeof(half)); + tensorrt_llm::runtime::ub::ub_allocate(elemNum * sizeof(half)); + if (useNVFP4Model) + { + tensorrt_llm::runtime::ub::ub_allocate(elemNum * sizeof(uint8_t) / 16); + } +} + +CudaStream const& TllmRuntime::getStream() const +{ + return *mStream; +} + +bool TllmRuntime::hasLayerProfiler(SizeType32 contextId) const +{ + return mContexts[contextId]->getProfiler() != nullptr; +} + +void TllmRuntime::setLayerProfiler() +{ + mLayerProfiler = std::make_unique<LayerProfiler>(); + for (auto& context : mContexts) + { + context->setProfiler(mLayerProfiler.get()); + context->setEnqueueEmitsProfile(false); + } +} + +std::string TllmRuntime::getLayerProfileInfo() const +{ + TLLM_CHECK(mLayerProfiler); + return mLayerProfiler->getLayerProfile(); +} + +void TllmRuntime::reportToProfiler(SizeType32 contextId) +{ + mContexts[contextId]->reportToProfiler(); +} + +void TllmRuntime::loadManagedWeights(RawEngine const& rawEngine, int localRank) +{ + TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); + NVTX3_FUNC_RANGE(); + auto& engine = getEngine(); + auto& manager = getBufferManager(); + if (rawEngine.getManagedWeightsMapOpt().has_value()) + { + TLLM_LOG_DEBUG("Loading managed weights from raw engine"); + auto executorMap = rawEngine.getManagedWeightsMapOpt().value(); + for (auto const& [name, weight] : executorMap) + { + TLLM_LOG_DEBUG("Loading managed weight: %s", name.c_str()); + auto iTensor = tensorrt_llm::executor::detail::toITensor(weight); + auto weightsDevice = std::shared_ptr<ITensor>{manager.copyFrom(*iTensor, MemoryType::kGPU)}; + mManagedWeightsMap.insert(std::make_pair(name, weightsDevice)); + } + } + else + { + TLLM_LOG_DEBUG("Loading managed weights from file"); + auto const enginePath = rawEngine.getPathOpt(); + TLLM_CHECK_WITH_INFO(enginePath.has_value(), "Engine path is not set."); + auto weightPath + = enginePath->parent_path() / ("rank" + std::to_string(localRank) + "_managed_weights.safetensors"); + auto managed_weights = common::safetensors::ISafeTensor::open(weightPath.string().c_str()); + for (auto const& name : managed_weights->keys()) + { + TLLM_LOG_DEBUG("Loading managed weight: %s", name.c_str()); + auto const weight = managed_weights->getTensor(name.c_str()); + TLLM_CHECK(weight->dtype() == engine.getTensorDataType(name.c_str())); + auto weightsDevice + = std::shared_ptr<ITensor>{manager.allocate(MemoryType::kGPU, weight->trtDims(), weight->dtype())}; + manager.copy(weight->data(), *weightsDevice, MemoryType::kCPU); + mManagedWeightsMap.insert(std::make_pair(name, weightsDevice)); + } + } + setStaticInputTensors(mManagedWeightsMap); + TLLM_LOG_TRACE("%s stop", __PRETTY_FUNCTION__); +} diff --git a/cpp/tensorrt_llm/runtime/tllmRuntime.h b/cpp/tensorrt_llm/runtime/tllmRuntime.h new file mode 100644 index 000000000000..dfef06d8b45f --- /dev/null +++ b/cpp/tensorrt_llm/runtime/tllmRuntime.h @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/layerProfiler.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/worldConfig.h" +#include <NvInferRuntime.h> + +#include <cstdint> +#include <memory> +#include <set> +#include <string> +#include <vector> + +namespace tensorrt_llm::runtime +{ +class TllmRuntime +{ +public: + using TensorMap = StringPtrMap<ITensor>; + + explicit TllmRuntime(RawEngine const& rawEngine, nvinfer1::ILogger* logger, bool useGpuDirectStorage = false, + float gpuWeightsPercent = 1.0f, bool useShapeInference = true); + + SizeType32 getNbContexts() const + { + return static_cast<SizeType32>(mContexts.size()); + } + + nvinfer1::IExecutionContext& getContext(SizeType32 contextIndex) const + { + return *mContexts.at(contextIndex); + } + + SizeType32 getNbProfiles() const + { + return static_cast<SizeType32>(mEngine->getNbOptimizationProfiles()); + } + + /// @brief If multiple TensorRT optimization profiles are built in the engine, this function selects the + /// corresponding profile that is going to be used based on the runtime shape, for now, TensorRT LLM only split + /// multiple profiles on the num_tokens dimension, hence the profile index is selected based on which profile + /// handles the actual num_tokens + /// @return The index of the selected TensorRT optimization profile + [[nodiscard]] SizeType32 getOptProfileId(int numTokens, std::vector<SizeType32> const& splitPoints) const + { + if (getNbProfiles() == 1) + { + return 0; + } + auto const it = std::lower_bound(splitPoints.begin(), splitPoints.end(), numTokens); + auto const optProfileId = std::distance(splitPoints.begin(), it); + return optProfileId; + } + + nvinfer1::IExecutionContext& addContext(std::int32_t profileIndex); + + void clearContexts(); + + /// @brief Set input tensors from tensorMap for all contexts. + /// @details The function can be used to set static input tensors for all iterations. If a tensor was set this way, + /// it doesn't need to included in calls to setInputTensors anymore. + void setStaticInputTensors(TensorMap const& tensorMap); + + /// @brief Set input tensors from tensorMap for context at contextIndex. + /// @details The function expects that all input tensors (excluding the ones set by setStaticInputTensors) are + /// contained in the tensorMap. If a tensor is missing, has a bad shape or type, it will throw. + void setInputTensors(SizeType32 contextIndex, TensorMap const& tensorMap); + + /// @brief Set output tensors from tensorMap for context at contextIndex. + /// @details The function expects that all output tensors are contained in the tensorMap. If a tensor is missing and + /// shape inference is enabled, it will allocate the tensor on GPU and insert it into the tensorMap. Otherwise it + /// will throw. + void setOutputTensors(SizeType32 contextIndex, TensorMap& tensorMap); + + bool executeContext(SizeType32 contextIndex) const; + + CudaStream const& getStream() const; + + BufferManager::CudaStreamPtr getStreamPtr() + { + return mStream; + } + + nvinfer1::ICudaEngine& getEngine() + { + return *mEngine; + } + + nvinfer1::ICudaEngine const& getEngine() const + { + return *mEngine; + } + + nvinfer1::IEngineInspector& getEngineInspector() + { + return *mEngineInspector; + } + + nvinfer1::IEngineInspector const& getEngineInspector() const + { + return *mEngineInspector; + } + + BufferManager& getBufferManager() + { + return mBufferManager; + } + + BufferManager const& getBufferManager() const + { + return mBufferManager; + } + + void setLayerProfiler(); + bool hasLayerProfiler(SizeType32 contextId) const; + std::string getLayerProfileInfo() const; + void reportToProfiler(SizeType32 contextId); + void loadManagedWeights(RawEngine const& rawEngine, int localRank); + void initializeUserBuffer(tensorrt_llm::runtime::WorldConfig const& world_config, SizeType32 maxBatchSize, + SizeType32 maxBeamWidth, SizeType32 maxSequenceLength, SizeType32 hiddenSize, + std::optional<SizeType32> maxNumTokens); + + bool isUserBufferEnabled() const + { + return mUserBufferEnabled; + } + + void setCurrentBeamWidths(std::vector<SizeType32> const& beamWidth) noexcept + { + mCurrentBeamWidths = beamWidth; + } + + [[nodiscard]] SizeType32 const& getCurrentBeamWidth() const noexcept + { + // At present, all requests of a batch must have the same beam width in one generation step (or they will not + // be batched together). So, the beam widths in `mCurrentBeamWidths` are the same. + // Corresponding changes must be done if Diverse-Beam-Width-Search (DBWS, requests with diverse beam width in + // a batch in one generation step) is supported in the future. + TLLM_CHECK_WITH_INFO(mCurrentBeamWidths.size() > 0, "`mCurrentBeamWidths` is empty."); + bool const isEqual = std::all_of(mCurrentBeamWidths.begin(), mCurrentBeamWidths.end(), + [&](int elem) { return elem == mCurrentBeamWidths.front(); }); + TLLM_CHECK_WITH_INFO(isEqual, "beam widths in `mCurrentBeamWidths` are not all equal."); + return mCurrentBeamWidths.front(); + } + +private: + void cacheTensorNames(); + + void setInputTensorsImpl(SizeType32 contextIndex, TensorMap const& tensorMap, bool throwOnMiss); + + void setUserBufferTensors(SizeType32 contextIndex, TensorMap& tensorMap); + + void printEngineInfo(); + + void printContextInfo(SizeType32 contextIndex); + + // Tool functions for `printEngineInfo()`. + static std::string shapeToString(nvinfer1::Dims64 const& dim) + { + std::string output("("); + if (dim.nbDims == 0) + { + return output + ")"; + } + for (int i = 0; i < dim.nbDims - 1; ++i) + { + output += std::to_string(dim.d[i]) + ", "; + } + output += std::to_string(dim.d[dim.nbDims - 1]) + ")"; + return output; + } + + static std::string dataTypeToString(nvinfer1::DataType type) + { + switch (type) + { + case nvinfer1::DataType::kINT64: return "INT64"; + case nvinfer1::DataType::kINT32: return "INT32"; + case nvinfer1::DataType::kFLOAT: return "FP32"; + case nvinfer1::DataType::kBF16: return "BF16"; + case nvinfer1::DataType::kHALF: return "FP16"; + case nvinfer1::DataType::kBOOL: return "BOOL"; + case nvinfer1::DataType::kUINT8: return "UINT8"; + case nvinfer1::DataType::kINT8: return "INT8"; + case nvinfer1::DataType::kFP8: return "FP8"; + case nvinfer1::DataType::kINT4: return "INT4"; + case nvinfer1::DataType::kFP4: return "FP4"; + default: return "UNKNOWN"; + } + return ""; + } + + static std::string alignText( + std::string const& text, int const width, bool const bCenter = true, char const blank = ' ') + { + int textLen = text.size(); + int padLeft = 0; + int padRight = 0; + padLeft = bCenter ? (width - textLen) / 2 : 0; + padRight = width - padLeft - textLen; + return std::string(padLeft, blank) + text + std::string(padRight, blank); + } + + BufferManager::CudaStreamPtr mStream; + BufferManager mBufferManager; + std::unique_ptr<nvinfer1::IRuntime> mRuntime; + std::unique_ptr<nvinfer1::ICudaEngine> mEngine; + BufferManager::IBufferPtr mEngineBuffer; + std::vector<std::unique_ptr<nvinfer1::IExecutionContext>> mContexts; + std::unique_ptr<ITensor> mDummyTensor; + std::unique_ptr<nvinfer1::IEngineInspector> mEngineInspector; + std::unique_ptr<LayerProfiler> mLayerProfiler; + bool mUseShapeInference; + TensorMap mManagedWeightsMap; + // List of input tensor names. + // Names of static tensors are removed from this list when setStaticInputTensors is called. + std::vector<std::string> mInputTensorNames; + std::vector<std::string> mOutputTensorNames; + + bool mUserBufferEnabled; + // For Variable-Beam-Width-Search + std::vector<SizeType32> mCurrentBeamWidths; +}; +} // namespace tensorrt_llm::runtime diff --git a/cpp/tensorrt_llm/runtime/tllmStreamReaders.cpp b/cpp/tensorrt_llm/runtime/tllmStreamReaders.cpp new file mode 100644 index 000000000000..55440bbe714f --- /dev/null +++ b/cpp/tensorrt_llm/runtime/tllmStreamReaders.cpp @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tllmStreamReaders.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" + +#include <cufile.h> +#include <dlfcn.h> +#include <fcntl.h> +#include <filesystem> +#include <fstream> +#include <string> +#include <unistd.h> + +// Non-GDS StreamReader + +StreamReader::StreamReader(std::filesystem::path fp) +{ + mFile.open(fp.string(), std::ios::binary | std::ios::in); + TLLM_CHECK_WITH_INFO(mFile.good(), std::string("Error opening engine file: " + fp.string())); +} + +StreamReader::~StreamReader() +{ + if (mFile.is_open()) + { + mFile.close(); + } +} + +int64_t StreamReader::read(void* destination, int64_t nbBytes) +{ + if (!mFile.good()) + { + return -1; + } + + mFile.read(static_cast<char*>(destination), nbBytes); + + return mFile.gcount(); +} + +// StreamReader using GDS + +GDSStreamReader::GDSStreamReader(std::filesystem::path const& filePath) +{ + auto const start_time = std::chrono::high_resolution_clock::now(); + initializeDriver(); + auto const elapsed_ms + = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - start_time); + + TLLM_LOG_INFO("GDS driver initialization time %lld ms", elapsed_ms); + + open(filePath); +} + +bool GDSStreamReader::open(std::string const& filepath) +{ + if (!initializeDriver()) + { + TLLM_LOG_INFO("Failed to initialize cuFile driver"); + return false; + } + + int32_t const ret = ::open(filepath.c_str(), O_CREAT | O_RDWR | O_DIRECT, 0664); + + if (ret < 0) + { + TLLM_LOG_INFO("Failed to open engine file"); + return false; + } + + mFd = ret; + mFileSize = lseek(mFd, 0, SEEK_END); + lseek(mFd, 0, SEEK_SET); + + CUfileDescr_t fileDescr; + memset((void*) &fileDescr, 0, sizeof(fileDescr)); + fileDescr.handle.fd = mFd; + fileDescr.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD; + + CUfileError_t gdsStatus = cuFileHandleRegister(&mFileHandle, &fileDescr); + + if (gdsStatus.err != CU_FILE_SUCCESS) + { + TLLM_LOG_INFO("Failed to cuFileHandleRegister"); + ::close(mFd); + return false; + } + return true; +} + +void GDSStreamReader::close() +{ + if (mFd >= 0) + { + ::close(mFd); + mFd = -1; + } +} + +GDSStreamReader::~GDSStreamReader() +{ + if (mFileHandle) + { + cuFileHandleDeregister(mFileHandle); + mFileHandle = nullptr; + } + + if (mDriverInitialized) + { + cuFileDriverClose(); + } +} + +bool GDSStreamReader::seek(int64_t offset, nvinfer1::SeekPosition where) noexcept +{ + switch (where) + { + case nvinfer1::SeekPosition::kSET: mCursor = offset; return true; + case nvinfer1::SeekPosition::kCUR: mCursor += offset; return true; + case nvinfer1::SeekPosition::kEND: mCursor = -offset; return true; + default: return false; + } + return true; +} + +int64_t GDSStreamReader::read(void* dest, int64_t bytes, cudaStream_t stream) noexcept +{ + cudaPointerAttributes attributes{}; + if (cudaPointerGetAttributes(&attributes, dest) != cudaSuccess) + { + TLLM_LOG_INFO("cudaPointerGetAttributes failed"); + } + + off_t destOffset = 0; + void* destBase = dest; + + if (attributes.type == cudaMemoryTypeDevice) + { + CUdeviceptr cuDest = reinterpret_cast<CUdeviceptr>(dest); + CUdeviceptr cuBufBase = 0; + size_t cuBufSize = 0; + + cuMemGetAddressRange(&cuBufBase, &cuBufSize, cuDest); + destOffset += cuDest - cuBufBase; + destBase = reinterpret_cast<void*>(cuBufBase); + } + cuFileRead(this->mFileHandle, destBase, bytes, mCursor, destOffset); + + mCursor += bytes; + return bytes; +} + +void GDSStreamReader::reset() +{ + lseek(mFd, 0, SEEK_SET); + mCursor = 0; +} + +[[nodiscard]] bool GDSStreamReader::isOpen() const +{ + bool open = mFd >= 0; + return open; +} + +bool GDSStreamReader::initializeDriver() +{ + if (mDriverInitialized) + { + return true; + } + + mCuFileLibHandle = dlopen("libcufile.so", RTLD_LAZY | RTLD_GLOBAL); + if (!mCuFileLibHandle) + { + TLLM_LOG_INFO("Failed to dlopen libcufile.so"); + return false; + } + + // Load the required functions + *reinterpret_cast<void**>(&cuFileDriverOpen) = dlsym(mCuFileLibHandle, "cuFileDriverOpen"); + *reinterpret_cast<void**>(&cuFileHandleRegister) = dlsym(mCuFileLibHandle, "cuFileHandleRegister"); + *reinterpret_cast<void**>(&cuFileHandleDeregister) = dlsym(mCuFileLibHandle, "cuFileHandleDeregister"); + *reinterpret_cast<void**>(&cuFileDriverClose) = dlsym(mCuFileLibHandle, "cuFileDriverClose"); + *reinterpret_cast<void**>(&cuFileRead) = dlsym(mCuFileLibHandle, "cuFileRead"); + + if (!cuFileDriverOpen || !cuFileHandleRegister || !cuFileHandleDeregister || !cuFileDriverClose || !cuFileRead) + { + TLLM_LOG_INFO("Failed to dlsym libcufile.so"); + return false; + } + + CUfileError_t gdsStatus = cuFileDriverOpen(); + if (gdsStatus.err != CU_FILE_SUCCESS) + { + TLLM_LOG_INFO("cuFileDriverOpen failed"); + return false; + } + + mDriverInitialized = true; + return true; +} diff --git a/cpp/tensorrt_llm/runtime/tllmStreamReaders.h b/cpp/tensorrt_llm/runtime/tllmStreamReaders.h new file mode 100644 index 000000000000..943f0bb3e32e --- /dev/null +++ b/cpp/tensorrt_llm/runtime/tllmStreamReaders.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include <NvInferRuntime.h> + +#include <cufile.h> +#include <filesystem> +#include <fstream> + +class StreamReader final : public nvinfer1::IStreamReader +{ +public: + StreamReader(std::filesystem::path fp); + + virtual ~StreamReader(); + + int64_t read(void* destination, int64_t nbBytes) final; + +private: + std::ifstream mFile; +}; + +class GDSStreamReader final : public nvinfer1::IStreamReaderV2 +{ +public: + explicit GDSStreamReader(std::filesystem::path const& filePath); + + virtual ~GDSStreamReader(); + + void close(); + + [[nodiscard]] bool isOpen() const; + + bool open(std::string const& filepath); + + int64_t read(void* dest, int64_t bytes, cudaStream_t stream) noexcept final; + + void reset(); + + bool seek(int64_t offset, nvinfer1::SeekPosition where) noexcept final; + +private: + bool initializeDriver(); + + void* mCuFileLibHandle{}; + CUfileHandle_t mFileHandle{nullptr}; + bool mDriverInitialized{false}; + int32_t mFd{-1}; + int64_t mCursor{0}; + int64_t mFileSize{0}; + + CUfileError_t (*cuFileDriverOpen)(){}; + CUfileError_t (*cuFileHandleRegister)(CUfileHandle_t*, CUfileDescr_t*){}; + CUfileError_t (*cuFileHandleDeregister)(CUfileHandle_t){}; + CUfileError_t (*cuFileDriverClose)(){}; + ssize_t (*cuFileRead)(CUfileHandle_t, void*, size_t, int64_t, int64_t){}; +}; diff --git a/cpp/tensorrt_llm/runtime/utils/debugUtils.cu b/cpp/tensorrt_llm/runtime/utils/debugUtils.cu index d4aaa8244c11..661dacd9a7ac 100644 --- a/cpp/tensorrt_llm/runtime/utils/debugUtils.cu +++ b/cpp/tensorrt_llm/runtime/utils/debugUtils.cu @@ -26,7 +26,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include <cfloat> #include <string> @@ -168,7 +167,7 @@ template <typename T> bool tensorHasInvalid(ITensor const& tensor, BufferManager const& manager, std::string const& infoStr) { printLogitsKeyInfo<T>(tensor, infoStr); - auto foundInvalid = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + auto foundInvalid = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); auto foundInvalidPtr = bufferCast<int32_t>(*foundInvalid); foundInvalidPtr[0] = 0; auto const size = tensor.getSize(); @@ -185,24 +184,24 @@ template bool tensorHasInvalid<__nv_fp8_e4m3>( ITensor const& tensor, BufferManager const& manager, std::string const& infoStr); bool tensorHasInvalid( - size_t M, size_t K, tensorrt_llm::DataType type, void const* data, cudaStream_t stream, std::string const& infoStr) + size_t M, size_t K, nvinfer1::DataType type, void const* data, cudaStream_t stream, std::string const& infoStr) { auto tensorView = ITensor::wrap( const_cast<void*>(data), type, ITensor::makeShape({static_cast<int32_t>(M), static_cast<int32_t>(K)})); auto manager = BufferManager(std::make_shared<CudaStream>(stream)); - if (type == tensorrt_llm::DataType::kFLOAT) + if (type == nvinfer1::DataType::kFLOAT) { return tensorHasInvalid<float>(*tensorView, manager, infoStr); } - else if (type == tensorrt_llm::DataType::kHALF) + else if (type == nvinfer1::DataType::kHALF) { return tensorHasInvalid<half>(*tensorView, manager, infoStr); } - else if (type == tensorrt_llm::DataType::kBF16) + else if (type == nvinfer1::DataType::kBF16) { return tensorHasInvalid<__nv_bfloat16>(*tensorView, manager, infoStr); } - else if (type == tensorrt_llm::DataType::kFP8) + else if (type == nvinfer1::DataType::kFP8) { return tensorHasInvalid<__nv_fp8_e4m3>(*tensorView, manager, infoStr); } diff --git a/cpp/tensorrt_llm/runtime/utils/mpiUtils.cpp b/cpp/tensorrt_llm/runtime/utils/mpiUtils.cpp index ae508bbdbc3b..0f8f31082e96 100644 --- a/cpp/tensorrt_llm/runtime/utils/mpiUtils.cpp +++ b/cpp/tensorrt_llm/runtime/utils/mpiUtils.cpp @@ -597,16 +597,9 @@ MpiComm::~MpiComm() noexcept #if ENABLE_MULTI_DEVICE if (mFreeComm && mComm) { - // Calling MPI_Comm_free after MPI has been finalized is undefined behavior. - // We need this check to prevent heap corruption during program exit when - // static MpiComm objects are created. - int finalized = 0; - if (MPI_Finalized(&finalized) == MPI_SUCCESS && !finalized) + if (MPI_Comm_free(&mComm) != MPI_SUCCESS) { - if (MPI_Comm_free(&mComm) != MPI_SUCCESS) - { - TLLM_LOG_ERROR("MPI_Comm_free failed"); - } + TLLM_LOG_ERROR("MPI_Comm_free failed"); } } #endif // ENABLE_MULTI_DEVICE diff --git a/cpp/tensorrt_llm/runtime/utils/numpyUtils.cpp b/cpp/tensorrt_llm/runtime/utils/numpyUtils.cpp index 931fbf0203da..6f95704455d3 100644 --- a/cpp/tensorrt_llm/runtime/utils/numpyUtils.cpp +++ b/cpp/tensorrt_llm/runtime/utils/numpyUtils.cpp @@ -20,9 +20,9 @@ #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" #include "tensorrt_llm/common/stringUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" +#include <NvInferRuntime.h> #include <sstream> #include <stdexcept> @@ -34,9 +34,9 @@ namespace tc = tensorrt_llm::common; namespace tensorrt_llm::runtime::utils { -std::string getNumpyTypeDesc(tensorrt_llm::DataType type) +std::string getNumpyTypeDesc(nvinfer1::DataType type) { - using dt = tensorrt_llm::DataType; + using dt = nvinfer1::DataType; static std::unordered_map<dt, std::string> const type_map{{dt::kBOOL, "?"}, {dt::kUINT8, "u1"}, {dt::kINT8, "i1"}, {dt::kINT32, "i4"}, {dt::kINT64, "i8"}, {dt::kHALF, "f2"}, {dt::kFLOAT, "f4"}}; @@ -51,11 +51,11 @@ std::string getNumpyTypeDesc(tensorrt_llm::DataType type) return type_map.count(type) > 0 ? type_map.at(type) : "x"; } -tensorrt_llm::DataType typeFromNumpyDesc(std::string const& type) +nvinfer1::DataType typeFromNumpyDesc(std::string const& type) { TLLM_LOG_DEBUG("numpy type: %s", type.c_str()); - using dt = tensorrt_llm::DataType; + using dt = nvinfer1::DataType; static std::unordered_map<std::string, dt> const type_map{{"?", dt::kBOOL}, {"u1", dt::kUINT8}, {"i1", dt::kINT8}, {"i4", dt::kINT32}, {"i8", dt::kINT64}, {"f2", dt::kHALF}, {"f4", dt::kFLOAT}}; TLLM_CHECK_WITH_INFO(type_map.count(type) > 0, "numpy data type '" + type + "' not supported"); @@ -102,7 +102,7 @@ void parseNpyIntro(FILE*& f_ptr, uint32_t& header_len, uint32_t& start_data) start_data = 8 + 2 * npy_major + header_len; } -int parseNpyHeader(FILE*& f_ptr, uint32_t header_len, tensorrt_llm::DataType& type, std::vector<size_t>& shapeVec) +int parseNpyHeader(FILE*& f_ptr, uint32_t header_len, nvinfer1::DataType& type, std::vector<size_t>& shapeVec) { char* header_c = (char*) malloc(header_len * sizeof(char)); TLLM_CHECK_WITH_INFO(header_c != nullptr, "Failed to allocate memory for npy header"); @@ -168,11 +168,11 @@ int parseNpyHeader(FILE*& f_ptr, uint32_t header_len, tensorrt_llm::DataType& ty uint32_t header_len, start_data; utils::parseNpyIntro(f_ptr, header_len, start_data); - tensorrt_llm::DataType type; + nvinfer1::DataType type; std::vector<size_t> shape; utils::parseNpyHeader(f_ptr, header_len, type, shape); - tensorrt_llm::Dims dims; + nvinfer1::Dims dims; dims.nbDims = shape.size(); std::copy(shape.begin(), shape.end(), dims.d); @@ -203,10 +203,10 @@ void saveNpy(BufferManager const& manager, ITensor const& tensor, std::string co auto const dtype = tensor.getDataType(); #ifdef ENABLE_BF16 - if (dtype == tensorrt_llm::DataType::kBF16) + if (dtype == nvinfer1::DataType::kBF16) { TLLM_CHECK(where == MemoryType::kGPU); - auto tensorFp32 = manager.gpu(shape, tensorrt_llm::DataType::kFLOAT); + auto tensorFp32 = manager.gpu(shape, nvinfer1::DataType::kFLOAT); auto dataFp32 = bufferCast<float>(*tensorFp32); auto dataBf16 = bufferCast<__nv_bfloat16 const>(tensor); tc::invokeCudaD2DcpyConvert(dataFp32, dataBf16, tensorSize); diff --git a/cpp/tensorrt_llm/runtime/utils/runtimeUtils.h b/cpp/tensorrt_llm/runtime/utils/runtimeUtils.h index da94419eff64..f131ab3419bf 100644 --- a/cpp/tensorrt_llm/runtime/utils/runtimeUtils.h +++ b/cpp/tensorrt_llm/runtime/utils/runtimeUtils.h @@ -25,6 +25,7 @@ namespace tensorrt_llm::runtime { +class TllmRuntime; namespace utils { diff --git a/cpp/tensorrt_llm/runtime/virtualMemory.cpp b/cpp/tensorrt_llm/runtime/virtualMemory.cpp index c2ca710db9ac..0d08012a29d8 100644 --- a/cpp/tensorrt_llm/runtime/virtualMemory.cpp +++ b/cpp/tensorrt_llm/runtime/virtualMemory.cpp @@ -16,7 +16,6 @@ #include "tensorrt_llm/runtime/virtualMemory.h" #include "bufferManager.h" -#include "tensorrt_llm/common/tllmDataType.h" #include <forward_list> #include <shared_mutex> @@ -142,8 +141,8 @@ void OffloadConfigurator::teardown(CUmemGenericAllocationHandle, bool destructin { switch (mBackType) { - case MemoryType::kCPU: mBackedStorage = BufferManager::cpu(mSize, tensorrt_llm::DataType::kINT8); break; - case MemoryType::kPINNED: mBackedStorage = BufferManager::pinned(mSize, tensorrt_llm::DataType::kINT8); break; + case MemoryType::kCPU: mBackedStorage = BufferManager::cpu(mSize, nvinfer1::DataType::kINT8); break; + case MemoryType::kPINNED: mBackedStorage = BufferManager::pinned(mSize, nvinfer1::DataType::kINT8); break; default: TLLM_THROW("Unknown memory type: %d", static_cast<int32_t>(mBackType)); } } diff --git a/cpp/tensorrt_llm/testing/CMakeLists.txt b/cpp/tensorrt_llm/testing/CMakeLists.txt new file mode 100644 index 000000000000..646302929817 --- /dev/null +++ b/cpp/tensorrt_llm/testing/CMakeLists.txt @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +include(FetchContent) + +set(SRCS modelSpec.cpp) + +include_directories(${API_INCLUDE_DIR}/tensorrt_llm/runtime) + +if(NOT WIN32) + # additional warnings + # + # Ignore overloaded-virtual warning. We intentionally change parameters of + # some methods in derived class. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") + if(WARNING_IS_ERROR) + message(STATUS "Treating warnings as errors in GCC compilation") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") + endif() +else() # Windows + # warning level 4 + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") +endif() + +add_library(testing_src OBJECT ${SRCS}) +set_property(TARGET testing_src PROPERTY POSITION_INDEPENDENT_CODE ON) +set_property(TARGET testing_src PROPERTY CUDA_RESOLVE_DEVICE_SYMBOLS ON) diff --git a/cpp/tensorrt_llm/testing/modelSpec.cpp b/cpp/tensorrt_llm/testing/modelSpec.cpp new file mode 100644 index 000000000000..bc868a157a1c --- /dev/null +++ b/cpp/tensorrt_llm/testing/modelSpec.cpp @@ -0,0 +1,303 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "modelSpec.h" +#include "tensorrt_llm/common/dataType.h" + +#include <numeric> + +namespace tensorrt_llm::testing +{ + +std::string ModelSpec::getQuantMethodString() const +{ + switch (mQuantMethod) + { + case QuantMethod::kNONE: + // Bypass here. + break; + case QuantMethod::kSMOOTH_QUANT: return "sq"; break; + default: throw std::runtime_error("Unsupported quant method"); break; + } + + return ""; +} + +std::string ModelSpec::getKVCacheTypeString() const +{ + switch (mKVCacheType) + { + case KVCacheType::kDISABLED: return "no-cache"; break; + case KVCacheType::kPAGED: return "paged"; break; + case KVCacheType::kCONTINUOUS: return "continuous"; break; + default: throw std::runtime_error("Unsupported KV cache type"); break; + } + + return ""; +} + +std::string ModelSpec::getSpeculativeDecodingModeString() const +{ + if (mSpecDecodingMode.isLookaheadDecoding()) + { + return "la-decoding"; + } + else if (mSpecDecodingMode.isDraftTokensExternal()) + { + return "draft-tokens"; + } + else if (mSpecDecodingMode.isNone()) + { + // Bypass here. + } + else if (mSpecDecodingMode.isExplicitDraftTokens()) + { + return "explicit-draft-tokens"; + } + else if (mSpecDecodingMode.isMedusa()) + { + return "medusa"; + } + else if (mSpecDecodingMode.isEagle()) + { + return "eagle"; + } + else + { + throw std::runtime_error("Unsupported decoding mode"); + } + + return ""; +} + +std::string ModelSpec::getCapacitySchedulerString() const +{ + if (mCapacitySchedulerPolicy) + { + if (mCapacitySchedulerPolicy.value() == tensorrt_llm::executor::CapacitySchedulerPolicy::kMAX_UTILIZATION) + { + return "MaxUtilization"; + } + else if (mCapacitySchedulerPolicy.value() + == tensorrt_llm::executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT) + { + return "GuaranteedNoEvict"; + } + else if (mCapacitySchedulerPolicy.value() == tensorrt_llm::executor::CapacitySchedulerPolicy::kSTATIC_BATCH) + { + return "StaticBatch"; + } + else + { + throw std::runtime_error("Unsupported capacity scheduler"); + } + } + return ""; +} + +std::string ModelSpec::getInputFile() const +{ + return mInputFile; +} + +std::string ModelSpec::getModelPath() const +{ + std::vector<std::string> ret; + + ret.emplace_back(getDtypeString()); + + if (mUseGptAttentionPlugin || mUseMambaPlugin) + { + if (mUseGptAttentionPlugin && mUseMambaPlugin) + { + throw std::runtime_error("Cannot use both GPT attention plugin and MAMBA plugin"); + } + + ret.emplace_back("plugin"); + } + else + { + ret.emplace_back("default"); + } + + if (mUsePackedInput) + { + ret.emplace_back("packed"); + } + + ret.emplace_back(getKVCacheTypeString()); + + if (mMaxInputLength) + { + ret.emplace_back("in" + std::to_string(mMaxInputLength)); + } + + ret.emplace_back(getSpeculativeDecodingModeString()); + + if (mUseLoraPlugin) + { + ret.emplace_back("lora"); + } + + ret.emplace_back(getQuantMethodString()); + + if (mUseMultipleProfiles) + { + ret.emplace_back("nprofiles"); + } + + if (mGatherLogits) + { + ret.emplace_back("gather"); + } + + auto finalRet = std::accumulate(ret.begin(), ret.end(), std::string(), + [](std::string& a, std::string& b) + { + if (a.empty()) + { + return b; + } + else + { + return b.empty() ? a : a + "_" + b; + } + }); + + return finalRet; +} + +std::string ModelSpec::getResultsFileInternal(OutputContentType outputContentType) const +{ + std::vector<std::string> ret; + + if (mInputFile == "input_tokens_long.npy") + { + ret.emplace_back("output_tokens_long"); + } + else + { + ret.emplace_back("output_tokens"); + } + + if (mMaxOutputLength) + { + ret.emplace_back("out" + std::to_string(mMaxOutputLength)); + } + + ret.emplace_back(getDtypeString()); + + if (mUseGptAttentionPlugin || mUseMambaPlugin) + { + if (mUseGptAttentionPlugin && mUseMambaPlugin) + { + throw std::runtime_error("Cannot use both GPT attention plugin and MAMBA plugin"); + } + ret.emplace_back("plugin"); + } + + if (mUsePackedInput) + { + ret.emplace_back("packed"); + } + + ret.emplace_back(getKVCacheTypeString()); + + ret.emplace_back(getQuantMethodString()); + + if (mGatherLogits) + { + ret.emplace_back("gather"); + } + + ret.emplace_back("tp" + std::to_string(mTPSize)); + + ret.emplace_back("pp" + std::to_string(mPPSize)); + + ret.emplace_back("cp" + std::to_string(mCPSize)); + + if (mEnableContextFMHAFp32Acc) + { + ret.emplace_back("fmhafp32acc"); + } + + switch (outputContentType) + { + case OutputContentType::kNONE: + // Bypass here. + break; + case OutputContentType::kCONTEXT_LOGITS: ret.emplace_back("logits_context"); break; + case OutputContentType::kGENERATION_LOGITS: ret.emplace_back("logits_generation"); break; + case OutputContentType::kLOG_PROBS: ret.emplace_back("log_probs"); break; + case OutputContentType::kCUM_LOG_PROBS: ret.emplace_back("cum_log_probs"); break; + default: throw std::runtime_error("Unsupported output content type"); break; + } + + auto finalRet = std::accumulate(ret.begin(), ret.end(), std::string(), + [](std::string& a, std::string& b) + { + if (a.empty()) + { + return b; + } + else + { + return b.empty() ? a : a + "_" + b; + } + }); + return finalRet + ".npy"; +} + +std::string ModelSpec::getResultsFile() const +{ + return mOtherModelSpecToCompare ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kNONE) + : getResultsFileInternal(OutputContentType::kNONE); +} + +std::string ModelSpec::getGenerationLogitsFile() const +{ + return mOtherModelSpecToCompare + ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kGENERATION_LOGITS) + : getResultsFileInternal(OutputContentType::kGENERATION_LOGITS); +} + +std::string ModelSpec::getContextLogitsFile() const +{ + return mOtherModelSpecToCompare + ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kCONTEXT_LOGITS) + : getResultsFileInternal(OutputContentType::kCONTEXT_LOGITS); +} + +std::string ModelSpec::getCumLogProbsFile() const +{ + return mOtherModelSpecToCompare + ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kCUM_LOG_PROBS) + : getResultsFileInternal(OutputContentType::kCUM_LOG_PROBS); +} + +std::string ModelSpec::getLogProbsFile() const +{ + return mOtherModelSpecToCompare ? mOtherModelSpecToCompare->getResultsFileInternal(OutputContentType::kLOG_PROBS) + : getResultsFileInternal(OutputContentType::kLOG_PROBS); +} + +std::string ModelSpec::getDtypeString() const +{ + return tensorrt_llm::common::getDtypeString(mDataType); +} + +} // namespace tensorrt_llm::testing diff --git a/cpp/tensorrt_llm/testing/modelSpec.h b/cpp/tensorrt_llm/testing/modelSpec.h new file mode 100644 index 000000000000..5b6f88dcd135 --- /dev/null +++ b/cpp/tensorrt_llm/testing/modelSpec.h @@ -0,0 +1,342 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "NvInfer.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/speculativeDecodingMode.h" + +#include <filesystem> +#include <vector> + +namespace tensorrt_llm::testing +{ + +using tensorrt_llm::runtime::SizeType32; +using tensorrt_llm::runtime::SpeculativeDecodingMode; +using KVCacheType = tensorrt_llm::runtime::ModelConfig::KVCacheType; + +enum class QuantMethod +{ + kNONE, + kSMOOTH_QUANT, +}; + +enum class OutputContentType +{ + kNONE, + kCONTEXT_LOGITS, + kGENERATION_LOGITS, + kLOG_PROBS, + kCUM_LOG_PROBS +}; + +class ModelSpec +{ +public: + ModelSpec(std::string const& inputFile, nvinfer1::DataType dtype, + std::shared_ptr<ModelSpec> otherModelSpecToCompare = nullptr) + : mInputFile{std::move(inputFile)} + , mDataType{dtype} + , mOtherModelSpecToCompare(otherModelSpecToCompare) + { + } + + ModelSpec& setInputFile(std::string const& inputFile) + { + mInputFile = inputFile; + return *this; + } + + ModelSpec& useGptAttentionPlugin() + { + mUseGptAttentionPlugin = true; + return *this; + } + + ModelSpec& usePackedInput() + { + mUsePackedInput = true; + return *this; + } + + ModelSpec& setKVCacheType(KVCacheType kvCacheType) + { + mKVCacheType = kvCacheType; + return *this; + } + + ModelSpec& setKVCacheReuse(bool kvCacheReuse) + { + mKVCacheReuse = kvCacheReuse; + return *this; + } + + ModelSpec& useDecoderPerRequest() + { + mDecoderPerRequest = true; + return *this; + } + + ModelSpec& useTensorParallelism(int tensorParallelism) + { + mTPSize = tensorParallelism; + return *this; + } + + ModelSpec& usePipelineParallelism(int pipelineParallelism) + { + mPPSize = pipelineParallelism; + return *this; + } + + ModelSpec& useContextParallelism(int contextParallelism) + { + mCPSize = contextParallelism; + return *this; + } + + ModelSpec& setDraftTokens(SizeType32 maxDraftTokens) + { + mMaxDraftTokens = maxDraftTokens; + return *this; + } + + ModelSpec& useAcceptByLogits() + { + mAcceptDraftByLogits = true; + return *this; + } + + ModelSpec& useMambaPlugin() + { + mUseMambaPlugin = true; + return *this; + } + + ModelSpec& gatherLogits() + { + mGatherLogits = true; + return *this; + } + + ModelSpec& replaceLogits() + { + mReplaceLogits = true; + return *this; + } + + ModelSpec& returnLogProbs() + { + mReturnLogProbs = true; + return *this; + } + + ModelSpec& smokeTest() + { + mSmokeTest = true; + return *this; + } + + ModelSpec& useMedusa() + { + mSpecDecodingMode = SpeculativeDecodingMode::Medusa(); + return *this; + } + + ModelSpec& useEagle() + { + mSpecDecodingMode = SpeculativeDecodingMode::Eagle(); + return *this; + } + + ModelSpec& useLookaheadDecoding() + { + mSpecDecodingMode = SpeculativeDecodingMode::LookaheadDecoding(); + return *this; + } + + ModelSpec& useExplicitDraftTokensDecoding() + { + mSpecDecodingMode = SpeculativeDecodingMode::ExplicitDraftTokens(); + return *this; + } + + ModelSpec& useDraftTokensExternalDecoding() + { + mSpecDecodingMode = SpeculativeDecodingMode::DraftTokensExternal(); + return *this; + } + + [[nodiscard]] bool useLogits() const + { + return mGatherLogits || mReplaceLogits; + } + + ModelSpec& useMultipleProfiles() + { + mUseMultipleProfiles = true; + return *this; + } + + ModelSpec& enableContextFMHAFp32Acc() + { + mEnableContextFMHAFp32Acc = true; + return *this; + } + + [[nodiscard]] bool getEnableContextFMHAFp32Acc() const + { + return mEnableContextFMHAFp32Acc; + } + + ModelSpec& setMaxInputLength(SizeType32 maxInputLength) + { + mMaxInputLength = maxInputLength; + return *this; + } + + ModelSpec& setMaxOutputLength(SizeType32 maxOutputLength) + { + mMaxOutputLength = maxOutputLength; + return *this; + } + + ModelSpec& setQuantMethod(QuantMethod quantMethod) + { + mQuantMethod = quantMethod; + return *this; + } + + ModelSpec& useLoraPlugin() + { + mUseLoraPlugin = true; + return *this; + } + + ModelSpec& collectGenerationLogitsFile() + { + mCollectGenerationLogits = true; + return *this; + } + + ModelSpec& collectContextLogitsFile() + { + mCollectContextLogits = true; + return *this; + } + + ModelSpec& collectCumLogProbsFile() + { + mCollectCumLogProbs = true; + return *this; + } + + ModelSpec& collectLogProbsFile() + { + mCollectLogProbs = true; + return *this; + } + + ModelSpec& capacitySchedulerPolicy(tensorrt_llm::executor::CapacitySchedulerPolicy policy) + { + mCapacitySchedulerPolicy = policy; + return *this; + } + + friend std::ostream& operator<<(std::ostream& os, ModelSpec const& modelSpec) + { + return os << modelSpec.getModelPath(); + } + + // Computed properties + [[nodiscard]] std::string getInputFile() const; + + [[nodiscard]] std::string getModelPath() const; + + [[nodiscard]] std::string getResultsFileInternal( + OutputContentType outputContentType = OutputContentType::kNONE) const; + + [[nodiscard]] std::string getResultsFile() const; + [[nodiscard]] std::string getGenerationLogitsFile() const; + + [[nodiscard]] std::string getContextLogitsFile() const; + + [[nodiscard]] std::string getCumLogProbsFile() const; + + [[nodiscard]] std::string getLogProbsFile() const; + + [[nodiscard]] std::string getDtypeString() const; + + [[nodiscard]] std::string getQuantMethodString() const; + + [[nodiscard]] std::string getKVCacheTypeString() const; + + [[nodiscard]] std::string getSpeculativeDecodingModeString() const; + + [[nodiscard]] std::string getCapacitySchedulerString() const; + + static ModelSpec getDefaultModelSpec() + { + static ModelSpec modelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF}; + modelSpec.useGptAttentionPlugin().setKVCacheType(KVCacheType::kPAGED).usePackedInput(); + + return modelSpec; + } + + std::string mInputFile; + nvinfer1::DataType mDataType; + + bool mUseGptAttentionPlugin{false}; + bool mUsePackedInput{false}; + KVCacheType mKVCacheType{KVCacheType::kCONTINUOUS}; + bool mKVCacheReuse{false}; + bool mDecoderPerRequest{false}; + int mPPSize{1}; + int mTPSize{1}; + int mCPSize{1}; + int mMaxDraftTokens{0}; + bool mAcceptDraftByLogits{false}; + bool mUseMambaPlugin{false}; + bool mGatherLogits{false}; + bool mReplaceLogits{false}; + bool mReturnLogProbs{false}; + bool mSmokeTest{false}; + bool mUseMultipleProfiles{false}; + int mMaxInputLength{0}; + int mMaxOutputLength{0}; + bool mUseLoraPlugin{false}; + bool mEnableContextFMHAFp32Acc{false}; + + // Flags to store whether model spec wants collect these outputs, you could call getXXXFile() if you need the name. + bool mCollectGenerationLogits{false}; + bool mCollectContextLogits{false}; + bool mCollectCumLogProbs{false}; + bool mCollectLogProbs{false}; + QuantMethod mQuantMethod{QuantMethod::kNONE}; + + SpeculativeDecodingMode mSpecDecodingMode{SpeculativeDecodingMode::None()}; + + std::optional<tensorrt_llm::executor::CapacitySchedulerPolicy> mCapacitySchedulerPolicy{std::nullopt}; + + // Sometimes, we need to compare with another model spec for golden results. + std::shared_ptr<ModelSpec> mOtherModelSpecToCompare{nullptr}; +}; + +}; // namespace tensorrt_llm::testing diff --git a/cpp/tensorrt_llm/thop/CMakeLists.txt b/cpp/tensorrt_llm/thop/CMakeLists.txt index b95426f580fe..369bf721b099 100644 --- a/cpp/tensorrt_llm/thop/CMakeLists.txt +++ b/cpp/tensorrt_llm/thop/CMakeLists.txt @@ -31,11 +31,8 @@ endif() add_library(th_utils STATIC thUtils.cpp) set_property(TARGET th_utils PROPERTY POSITION_INDEPENDENT_CODE ON) set_property(TARGET th_utils PROPERTY CUDA_RESOLVE_DEVICE_SYMBOLS ON) -# Declare the dependency on the main shared library explicitly so consumers -# (e.g. thUtilsTest) place it after th_utils on the link line; this was -# previously satisfied transitively via the removed TensorRT plugin target. -target_link_libraries(th_utils PUBLIC ${SHARED_TARGET} ${TORCH_LIBRARIES} - ${CUBLAS_LIB} ${CURAND_LIB}) +target_link_libraries(th_utils PUBLIC ${TORCH_LIBRARIES} ${CUBLAS_LIB} + ${CURAND_LIB}) # TODO This does not compile with internal cutlass MOE gemm add_library( @@ -85,7 +82,6 @@ add_library( fusedAddRMSNormQuant.cpp fusedActivationQuant.cpp fusedGatedRMSNormQuant.cpp - rmsNormFp4Quant.cpp fusedTopkSoftmax.cpp gatherTreeOp.cpp groupRmsNormOp.cpp diff --git a/cpp/tensorrt_llm/thop/IndexerTopKOp.cpp b/cpp/tensorrt_llm/thop/IndexerTopKOp.cpp index fc29041bbf93..1981c417dbae 100644 --- a/cpp/tensorrt_llm/thop/IndexerTopKOp.cpp +++ b/cpp/tensorrt_llm/thop/IndexerTopKOp.cpp @@ -19,6 +19,7 @@ #include "tensorrt_llm/kernels/IndexerTopK.h" +// #include <NvInferRuntime.h> // #include <c10/cuda/CUDAStream.h> // #include <cassert> // #include <set> diff --git a/cpp/tensorrt_llm/thop/allgatherOp.cpp b/cpp/tensorrt_llm/thop/allgatherOp.cpp index 5f7d4571d258..0d92aa966901 100644 --- a/cpp/tensorrt_llm/thop/allgatherOp.cpp +++ b/cpp/tensorrt_llm/thop/allgatherOp.cpp @@ -20,6 +20,7 @@ #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" +#include <NvInferRuntime.h> #include <c10/cuda/CUDAStream.h> #include <cassert> #include <set> diff --git a/cpp/tensorrt_llm/thop/allreduceOp.cpp b/cpp/tensorrt_llm/thop/allreduceOp.cpp index 38cc79379f6d..4096116fdb42 100644 --- a/cpp/tensorrt_llm/thop/allreduceOp.cpp +++ b/cpp/tensorrt_llm/thop/allreduceOp.cpp @@ -23,7 +23,6 @@ #include "tensorrt_llm/common/ncclUtils.h" #include "tensorrt_llm/common/nvmlWrapper.h" #include "tensorrt_llm/common/opUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/MiniMaxReduceRMSKernel.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" #include "tensorrt_llm/kernels/communicationKernels/customLowPrecisionAllReduceKernels.h" @@ -62,6 +61,7 @@ #include <limits> #include <unordered_set> +// using namespace nvinfer1; using tensorrt_llm::kernels::AllReduceFusionOp; using tensorrt_llm::kernels::AllReduceStrategyType; using tensorrt_llm::mpi::MpiTag; @@ -234,8 +234,8 @@ std::set<int> getLocalGroupTorch(std::set<int> const& group) class AllreduceOp { public: - AllreduceOp(std::set<int> group, tensorrt_llm::DataType type, AllReduceStrategyType strategy, AllReduceFusionOp op, - float eps) + AllreduceOp( + std::set<int> group, nvinfer1::DataType type, AllReduceStrategyType strategy, AllReduceFusionOp op, float eps) : mGroup(std::move(group)) , mIsNVLINKSupported(false) , mIsP2PSupported(false) @@ -248,7 +248,7 @@ class AllreduceOp } AllreduceOp(std::set<int> group, c10::intrusive_ptr<c10d::ProcessGroup> const& process_group_, - tensorrt_llm::DataType type, AllReduceStrategyType strategy, AllReduceFusionOp op, float eps) + nvinfer1::DataType type, AllReduceStrategyType strategy, AllReduceFusionOp op, float eps) : mGroup(std::move(group)) , mIsNVLINKSupported(false) , mIsP2PSupported(false) @@ -348,7 +348,7 @@ class AllreduceOp { TORCH_CHECK(norm_weight, "norm_weight is required for residual rms norm allreduce"); TORCH_CHECK(!bias, "bias is not supported for residual rms norm allreduce"); - TORCH_CHECK(mType == tensorrt_llm::DataType::kHALF || mType == tensorrt_llm::DataType::kBF16); + TORCH_CHECK(mType == nvinfer1::DataType::kHALF || mType == nvinfer1::DataType::kBF16); auto [norm_out, ub_buffer1] = torch_ext::create_userbuffers_tensor(input.sizes(), input.scalar_type()); tensorrt_llm::kernels::ub::allreduce2_userbuff_rmsnorm_launcher(ub_buffer0.handle, 0, ub_buffer1.handle, 0, size, hidden_size, nullptr, norm_weight.value().data_ptr(), mEps, residual.value().data_ptr(), @@ -1461,7 +1461,7 @@ class AllreduceOp bool mIsNVLINKSupported; bool mIsP2PSupported; bool mIsMNNVLSupported; - tensorrt_llm::DataType mType; + nvinfer1::DataType mType; AllReduceStrategyType mStrategy; AllReduceFusionOp mOp; float mEps; diff --git a/cpp/tensorrt_llm/thop/attentionOp.cpp b/cpp/tensorrt_llm/thop/attentionOp.cpp index 811810a8d5a2..848554c512d6 100644 --- a/cpp/tensorrt_llm/thop/attentionOp.cpp +++ b/cpp/tensorrt_llm/thop/attentionOp.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/attentionOp.h" #include "tensorrt_llm/common/attentionWorkspace.h" #include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/flashMLA/flash_mla.h" #include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/mlaKernels.h" @@ -1085,8 +1084,7 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to std::optional<int64_t> compressed_kv_cache_pool_ptr, bool const is_cross, std::optional<torch::Tensor> cross_kv, std::optional<torch::Tensor> relative_attention_bias, int64_t relative_attention_max_distance, std::optional<int64_t> spec_decoding_target_max_draft_tokens, std::optional<torch::Tensor> quant_scale_qkv, - std::optional<torch::Tensor> dsv4_inv_rope_cos_sin_cache, bool enable_dsv4_epilogue_fusion, - bool const force_prepare_spec_dec_tree_mask) + std::optional<torch::Tensor> dsv4_inv_rope_cos_sin_cache, bool enable_dsv4_epilogue_fusion) { TLLM_LOG_TRACE("Attention op starts at layer %d", local_layer_idx); // Use these tensors to infer if the attention is using KV cache @@ -1125,7 +1123,7 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to bool const is_fp4_out = out_dtype == torch::kUInt8; RunnerPtr runner; - if (dtype == tensorrt_llm::DataType::kHALF) + if (dtype == nvinfer1::DataType::kHALF) { if (is_fp8_out) { @@ -1141,13 +1139,13 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to runner = std::make_shared<Runner<half>>(); } } - else if (dtype == tensorrt_llm::DataType::kFLOAT) + else if (dtype == nvinfer1::DataType::kFLOAT) { TLLM_CHECK(out_dtype == torch::kFloat32); runner = std::make_shared<Runner<float>>(); } #ifdef ENABLE_BF16 - else if (dtype == tensorrt_llm::DataType::kBF16) + else if (dtype == nvinfer1::DataType::kBF16) { if (is_fp8_out) { @@ -1170,7 +1168,7 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to auto op = std::make_shared<AttentionOp>(); op->mType = dtype; - op->mFMHAForceFP32Acc = dtype == tensorrt_llm::DataType::kBF16; + op->mFMHAForceFP32Acc = dtype == nvinfer1::DataType::kBF16; op->mLayerIdx = local_layer_idx; op->mNumHeads = num_heads; op->mNumKVHeads = num_kv_heads; @@ -1239,7 +1237,6 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to { op->mSpecDecodingTargetMaxGenLen = static_cast<int32_t>(spec_decoding_target_max_draft_tokens.value()) + 1; } - op->mForcePrepareSpecDecTreeMask = force_prepare_spec_dec_tree_mask; op->mUseSparseAttention = false; op->mUseTllmGenSparseAttentionPaged = false; @@ -1434,7 +1431,7 @@ bool attention_supports_nvfp4_output(int64_t const num_heads, int64_t const num_ } auto op = std::make_shared<AttentionOp>(); - op->mType = tensorrt_llm::DataType::kHALF; + op->mType = nvinfer1::DataType::kHALF; op->mNumHeads = num_heads; op->mNumKVHeads = num_kv_heads; op->mHeadSize = head_size; diff --git a/cpp/tensorrt_llm/thop/attentionOp.h b/cpp/tensorrt_llm/thop/attentionOp.h index f28209166e0d..b4f0e90bc9a7 100644 --- a/cpp/tensorrt_llm/thop/attentionOp.h +++ b/cpp/tensorrt_llm/thop/attentionOp.h @@ -95,8 +95,7 @@ void attention(torch::Tensor q, std::optional<torch::Tensor> k, std::optional<to std::optional<torch::Tensor> relative_attention_bias = std::nullopt, int64_t relative_attention_max_distance = 0, std::optional<int64_t> spec_decoding_target_max_draft_tokens = std::nullopt, std::optional<torch::Tensor> quant_scale_qkv = std::nullopt, - std::optional<torch::Tensor> dsv4_inv_rope_cos_sin_cache = std::nullopt, bool enable_dsv4_epilogue_fusion = false, - bool const force_prepare_spec_dec_tree_mask = false); + std::optional<torch::Tensor> dsv4_inv_rope_cos_sin_cache = std::nullopt, bool enable_dsv4_epilogue_fusion = false); struct KvCachePoolPointers { diff --git a/cpp/tensorrt_llm/thop/cublasFp4ScaledMM.cpp b/cpp/tensorrt_llm/thop/cublasFp4ScaledMM.cpp index 8e4da99dbadc..a9ad46ad8f04 100644 --- a/cpp/tensorrt_llm/thop/cublasFp4ScaledMM.cpp +++ b/cpp/tensorrt_llm/thop/cublasFp4ScaledMM.cpp @@ -16,6 +16,7 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/plugins/common/plugin.h" #include "tensorrt_llm/thop/outputTensor.h" #include "tensorrt_llm/thop/thUtils.h" #include "userbuffersTensor.h" diff --git a/cpp/tensorrt_llm/thop/cublasScaledMM.cpp b/cpp/tensorrt_llm/thop/cublasScaledMM.cpp index 62f51f7b06f9..dea6f51363e5 100644 --- a/cpp/tensorrt_llm/thop/cublasScaledMM.cpp +++ b/cpp/tensorrt_llm/thop/cublasScaledMM.cpp @@ -18,6 +18,8 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/kernels/userbuffers/ub_interface.h" +#include "tensorrt_llm/plugins/common/plugin.h" +#include "tensorrt_llm/plugins/gemmPlugin/gemmPlugin.h" #include "tensorrt_llm/runtime/torchUtils.h" #include "tensorrt_llm/thop/outputTensor.h" #include "tensorrt_llm/thop/thUtils.h" diff --git a/cpp/tensorrt_llm/thop/dynamicDecodeOp.cpp b/cpp/tensorrt_llm/thop/dynamicDecodeOp.cpp index 228b2c614ab7..8e9e817bbb51 100644 --- a/cpp/tensorrt_llm/thop/dynamicDecodeOp.cpp +++ b/cpp/tensorrt_llm/thop/dynamicDecodeOp.cpp @@ -16,7 +16,6 @@ #include "tensorrt_llm/thop/dynamicDecodeOp.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -55,7 +54,7 @@ FtDynamicDecode<T>::FtDynamicDecode(size_t const maxBatchSize, size_t const maxB auto bufferManager = std::make_shared<tensorrt_llm::runtime::BufferManager>(cudaStreamPtr); mFinishedSum = bufferManager->pinnedPool( - tr::ITensor::makeShape({static_cast<int32_t>(maxBatchSize)}), tensorrt_llm::DataType::kINT32); + tr::ITensor::makeShape({static_cast<int32_t>(maxBatchSize)}), nvinfer1::DataType::kINT32); mDynamicDecodeLayer = std::make_shared<tl::DynamicDecodeLayer<T>>(tle::DecodingMode::Auto(), decodingDomain, bufferManager); mBatchSlots = tr::getDefaultBatchSlots(maxBatchSize); diff --git a/cpp/tensorrt_llm/thop/dynamicTreeOp.cpp b/cpp/tensorrt_llm/thop/dynamicTreeOp.cpp index 0a236ced8c35..0b862108a4e6 100644 --- a/cpp/tensorrt_llm/thop/dynamicTreeOp.cpp +++ b/cpp/tensorrt_llm/thop/dynamicTreeOp.cpp @@ -25,6 +25,13 @@ namespace tk = tensorrt_llm::kernels::speculative_decoding; TRTLLM_NAMESPACE_BEGIN +namespace kernels::speculative_decoding +{ +th::Tensor computeProbsFromLogits(th::Tensor const& logits, th::Tensor const& temperatures, + th::optional<th::Tensor> const& topK, th::optional<th::Tensor> const& topP, bool skipTemperature, + runtime::SizeType32 kMax); +} // namespace kernels::speculative_decoding + namespace torch_ext { @@ -119,6 +126,26 @@ void verify_dynamic_tree_greedy_out_packed_op(th::Tensor& candidates, th::Tensor targetPredict.data_ptr<int32_t>(), treeValid.data_ptr<bool>(), batchSize, numDraftTokens, numSpecStep, stream); } +th::Tensor compute_probs_from_logits_op(th::Tensor logits, th::Tensor temperatures, th::optional<th::Tensor> topK, + th::optional<th::Tensor> topP, bool skipTemperature) +{ + TORCH_CHECK(logits.is_cuda(), "logits must be a CUDA tensor"); + TORCH_CHECK(temperatures.is_cuda(), "temperatures must be a CUDA tensor"); + TORCH_CHECK(logits.dim() == 2, "logits must be a 2D tensor"); + TORCH_CHECK(temperatures.dim() == 1, "temperatures must be a 1D tensor"); + TORCH_CHECK(logits.size(0) == temperatures.size(0), "logits and temperatures size mismatch"); + if (topK.has_value() && topK->defined()) + { + TORCH_CHECK(topK->is_cuda(), "top_k must be a CUDA tensor"); + } + if (topP.has_value() && topP->defined()) + { + TORCH_CHECK(topP->is_cuda(), "top_p must be a CUDA tensor"); + } + + return tk::computeProbsFromLogits(logits, temperatures, topK, topP, skipTemperature, /*kMax=*/0); +} + //! \brief Target-only rejection sampling verify op (no draft probabilities needed). void verify_dynamic_tree_rejection_out_op(th::Tensor& draftTokens, th::Tensor& targetProbs, th::Tensor& retrieveNextToken, th::Tensor& retrieveNextSibling, th::Tensor& treeValid, th::Tensor& acceptIndex, @@ -259,3 +286,16 @@ TORCH_LIBRARY_IMPL(trtllm, CUDA, m) { m.impl("verify_dynamic_tree_rejection_out_op", &tensorrt_llm::torch_ext::verify_dynamic_tree_rejection_out_op); } + +TORCH_LIBRARY_FRAGMENT(trtllm, m) +{ + m.def( + "compute_probs_from_logits_op(" + "Tensor logits, Tensor temperatures, Tensor? top_k=None, Tensor? top_p=None, " + "bool skip_temperature=False) -> Tensor"); +} + +TORCH_LIBRARY_IMPL(trtllm, CUDA, m) +{ + m.impl("compute_probs_from_logits_op", &tensorrt_llm::torch_ext::compute_probs_from_logits_op); +} diff --git a/cpp/tensorrt_llm/thop/fp8Quantize.cpp b/cpp/tensorrt_llm/thop/fp8Quantize.cpp index 43eea8cff838..35ba5c440e18 100644 --- a/cpp/tensorrt_llm/thop/fp8Quantize.cpp +++ b/cpp/tensorrt_llm/thop/fp8Quantize.cpp @@ -209,38 +209,6 @@ std::tuple<at::Tensor, at::Tensor> fp8_quantize_1x128_packed_ue8m0(at::Tensor co return {valueE4M3.slice(0, 0, m), packedScale}; } - -std::tuple<at::Tensor, at::Tensor> fp8_quantize_1x128_cutedsl_ue8m0(at::Tensor const& self) -{ - CHECK_TH_CUDA(self); - CHECK_CONTIGUOUS(self); - - TORCH_CHECK(self.scalar_type() == at::ScalarType::BFloat16, "Input matrix dtype must be BF16."); - TORCH_CHECK(self.dim() == 2, "input must be a matrix"); - TORCH_CHECK(tensorrt_llm::common::isSM100Family(), - "fp8_quantize_1x128_cutedsl_ue8m0 currently only supports SM100 (Blackwell)."); - - auto const m = self.sizes()[0]; - auto const k = self.sizes()[1]; - TORCH_CHECK(m <= std::numeric_limits<int32_t>::max(), "M must be within int32"); - TORCH_CHECK(k <= std::numeric_limits<int32_t>::max(), "K must be within int32"); - TORCH_CHECK(k % 128 == 0, "K must be divisible by the production FP8 block size 128, but got ", k); - - at::Tensor valueE4M3 - = at::detail::empty_cuda({m, k}, at::ScalarType::Float8_e4m3fn, self.device(), /* stride */ std::nullopt); - auto const paddedM = (m + 127) / 128 * 128; - auto const sfCols = (k / 32 + 3) / 4 * 4; - at::Tensor scaleE8M0 - = at::detail::empty_cuda({paddedM * sfCols}, at::ScalarType::Byte, self.device(), /* stride */ std::nullopt); - - auto stream = at::cuda::getCurrentCUDAStream(self.get_device()); - tensorrt_llm::kernels::fp8_blockscale_gemm::launch_fp8_quantize_1x128_cutedsl_bf16_e4m3( - reinterpret_cast<__nv_fp8_e4m3*>(valueE4M3.data_ptr()), scaleE8M0.data_ptr<uint8_t>(), - reinterpret_cast<__nv_bfloat16 const*>(self.data_ptr()), static_cast<int>(m), static_cast<int>(k), - static_cast<int>(paddedM), stream); - - return {valueE4M3, scaleE8M0}; -} } // namespace torch_ext TRTLLM_NAMESPACE_END @@ -250,7 +218,6 @@ TORCH_LIBRARY_FRAGMENT(trtllm, m) m.def("fp8_quantize_1x128(Tensor input, bool use_ue8m0=False) -> (Tensor, Tensor)"); m.def("fp8_batched_quantize_1x128_permute102(Tensor input) -> (Tensor, Tensor)"); m.def("fp8_quantize_1x128_packed_ue8m0(Tensor input) -> (Tensor, Tensor)"); - m.def("fp8_quantize_1x128_cutedsl_ue8m0(Tensor input) -> (Tensor, Tensor)"); } TORCH_LIBRARY_IMPL(trtllm, CUDA, m) @@ -258,5 +225,4 @@ TORCH_LIBRARY_IMPL(trtllm, CUDA, m) m.impl("fp8_quantize_1x128", &tensorrt_llm::torch_ext::fp8_quantize_1x128); m.impl("fp8_batched_quantize_1x128_permute102", &tensorrt_llm::torch_ext::fp8_batched_quantize_1x128_permute102); m.impl("fp8_quantize_1x128_packed_ue8m0", &tensorrt_llm::torch_ext::fp8_quantize_1x128_packed_ue8m0); - m.impl("fp8_quantize_1x128_cutedsl_ue8m0", &tensorrt_llm::torch_ext::fp8_quantize_1x128_cutedsl_ue8m0); } diff --git a/cpp/tensorrt_llm/thop/groupRmsNormOp.cpp b/cpp/tensorrt_llm/thop/groupRmsNormOp.cpp index c1f3f41c5a1c..c408a8c286fb 100644 --- a/cpp/tensorrt_llm/thop/groupRmsNormOp.cpp +++ b/cpp/tensorrt_llm/thop/groupRmsNormOp.cpp @@ -17,7 +17,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/dataType.h" #include "tensorrt_llm/common/opUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/groupRmsNormKernels/groupRmsNormKernels.h" #include "tensorrt_llm/runtime/torchUtils.h" #include "tensorrt_llm/thop/thUtils.h" @@ -101,9 +100,9 @@ void groupRMSNormBase(torch::TensorList const& inputs, torch::TensorList const& /* Handle dtype conversion */ \ switch (dtype) \ { \ - case torch::ScalarType::Half: params.dtype = tensorrt_llm::DataType::kHALF; break; \ - case torch::ScalarType::BFloat16: params.dtype = tensorrt_llm::DataType::kBF16; break; \ - case torch::ScalarType::Float: params.dtype = tensorrt_llm::DataType::kFLOAT; break; \ + case torch::ScalarType::Half: params.dtype = nvinfer1::DataType::kHALF; break; \ + case torch::ScalarType::BFloat16: params.dtype = nvinfer1::DataType::kBF16; break; \ + case torch::ScalarType::Float: params.dtype = nvinfer1::DataType::kFLOAT; break; \ default: TORCH_CHECK(false, "Unsupported data type"); \ } \ tensorrt_llm::kernels::group_rms_norm::GroupRMSNormBaseKernelLauncher<n>(params); \ @@ -182,9 +181,9 @@ void groupRMSNormLargeBatch(torch::TensorList const& inputs, torch::TensorList c // Handle dtype conversion switch (dtype) { - case torch::ScalarType::Half: params.dtype = tensorrt_llm::DataType::kHALF; break; - case torch::ScalarType::BFloat16: params.dtype = tensorrt_llm::DataType::kBF16; break; - case torch::ScalarType::Float: params.dtype = tensorrt_llm::DataType::kFLOAT; break; + case torch::ScalarType::Half: params.dtype = nvinfer1::DataType::kHALF; break; + case torch::ScalarType::BFloat16: params.dtype = nvinfer1::DataType::kBF16; break; + case torch::ScalarType::Float: params.dtype = nvinfer1::DataType::kFLOAT; break; default: TORCH_CHECK(false, "Unsupported data type"); } @@ -261,9 +260,9 @@ void groupRMSNormHeuristic(torch::TensorList const& inputs, torch::TensorList co /* Handle dtype conversion */ \ switch (dtype) \ { \ - case torch::ScalarType::Half: params.dtype = tensorrt_llm::DataType::kHALF; break; \ - case torch::ScalarType::BFloat16: params.dtype = tensorrt_llm::DataType::kBF16; break; \ - case torch::ScalarType::Float: params.dtype = tensorrt_llm::DataType::kFLOAT; break; \ + case torch::ScalarType::Half: params.dtype = nvinfer1::DataType::kHALF; break; \ + case torch::ScalarType::BFloat16: params.dtype = nvinfer1::DataType::kBF16; break; \ + case torch::ScalarType::Float: params.dtype = nvinfer1::DataType::kFLOAT; break; \ default: TORCH_CHECK(false, "Unsupported data type"); \ } \ \ diff --git a/cpp/tensorrt_llm/thop/loraOp.cpp b/cpp/tensorrt_llm/thop/loraOp.cpp index 6957987be6b8..b35ca2608625 100644 --- a/cpp/tensorrt_llm/thop/loraOp.cpp +++ b/cpp/tensorrt_llm/thop/loraOp.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/cublasMMWrapper.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/opUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/cuda_graph_grouped_gemm.h" #include "tensorrt_llm/kernels/lora/lora.h" #include "tensorrt_llm/kernels/lora/loraGroupGEMMParamFillRowReorderFusion.h" @@ -152,11 +151,11 @@ std::vector<th::Tensor> lora_grouped_gemm(th::Tensor const& input, th::Tensor co { outHiddenSizes[i] = output_hidden_sizes[i]; } - tensorrt_llm::DataType loraRuntimeDataType; + nvinfer1::DataType loraRuntimeDataType; switch (input.scalar_type()) { - case torch::kFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kHALF; break; - case torch::kBFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kBF16; break; + case torch::kFloat16: loraRuntimeDataType = nvinfer1::DataType::kHALF; break; + case torch::kBFloat16: loraRuntimeDataType = nvinfer1::DataType::kBF16; break; default: throw std::invalid_argument("Invalid dtype, only supports float16, bfloat16"); } @@ -222,11 +221,11 @@ void lora_grouped_gemm_cuda_graph(th::Tensor const& lora_in_sizes, // [layer_mod auto* splitk_offsets_gpu = reinterpret_cast<int64_t*>(const_cast<void*>(splitk_offsets.data_ptr())); // Get data type - tensorrt_llm::DataType loraRuntimeDataType; + nvinfer1::DataType loraRuntimeDataType; switch (dtype) { - case torch::kFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kHALF; break; - case torch::kBFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kBF16; break; + case torch::kFloat16: loraRuntimeDataType = nvinfer1::DataType::kHALF; break; + case torch::kBFloat16: loraRuntimeDataType = nvinfer1::DataType::kBF16; break; default: TORCH_CHECK(false, "Invalid dtype, only supports float16, bfloat16, got %s", c10::toString(dtype)); } @@ -302,11 +301,11 @@ void lora_group_gemm_param_fill_row_reorder_fusion(th::Tensor const& in_sizes, / int32_t const module_count = static_cast<int32_t>(in_sizes.size(0)); // Get data type info - tensorrt_llm::DataType loraRuntimeDataType; + nvinfer1::DataType loraRuntimeDataType; switch (dtype) { - case torch::kFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kHALF; break; - case torch::kBFloat16: loraRuntimeDataType = tensorrt_llm::DataType::kBF16; break; + case torch::kFloat16: loraRuntimeDataType = nvinfer1::DataType::kHALF; break; + case torch::kBFloat16: loraRuntimeDataType = nvinfer1::DataType::kBF16; break; default: TORCH_CHECK(false, "Invalid dtype, only supports float16, bfloat16, got %s", c10::toString(dtype)); } diff --git a/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp b/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp index e985eb943ee1..373f936c4c5d 100644 --- a/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp +++ b/cpp/tensorrt_llm/thop/moeAlltoAllOp.cpp @@ -15,7 +15,6 @@ */ #include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/moeAlltoAllKernels.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/thop/moeAlltoAllMeta.h" @@ -485,20 +484,20 @@ torch::Tensor moeA2ACombineOp(torch::Tensor const& payload, int64_t localNumToke TORCH_CHECK(epRank >= 0 && epRank < epSize, "epRank must be in the range [0, epSize)"); TORCH_CHECK(topK > 0 && topK <= kMaxTopK, "topK must be in the range (0, kMaxTopK]"); - // Map torch dtype to tensorrt_llm::DataType - tensorrt_llm::DataType nvDtype = tensorrt_llm::DataType::kFLOAT; + // Map torch dtype to nvinfer1::DataType + nvinfer1::DataType nvDtype = nvinfer1::DataType::kFLOAT; auto scalarType = payload.scalar_type(); if (scalarType == at::kHalf) { - nvDtype = tensorrt_llm::DataType::kHALF; + nvDtype = nvinfer1::DataType::kHALF; } else if (scalarType == at::kBFloat16) { - nvDtype = tensorrt_llm::DataType::kBF16; + nvDtype = nvinfer1::DataType::kBF16; } else if (scalarType == at::kFloat) { - nvDtype = tensorrt_llm::DataType::kFLOAT; + nvDtype = nvinfer1::DataType::kFLOAT; } else { diff --git a/cpp/tensorrt_llm/thop/moeOp.cpp b/cpp/tensorrt_llm/thop/moeOp.cpp index 81c975241a2e..4a938455488b 100644 --- a/cpp/tensorrt_llm/thop/moeOp.cpp +++ b/cpp/tensorrt_llm/thop/moeOp.cpp @@ -30,7 +30,6 @@ #include "tensorrt_llm/common/config.h" #include "tensorrt_llm/common/dataType.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/workspace.h" #include "tensorrt_llm/kernels/cuda_graph_grouped_gemm.h" #include "tensorrt_llm/kernels/cutlass_kernels/fp8_blockscale_gemm/fp8_blockscale_gemm.h" @@ -88,8 +87,7 @@ enum class MoeLoraRequestType : int32_t // --------------------------------------------------------------------------- inline void moeLoraGroupedGemmRunImpl(::tensorrt_llm::kernels::cutlass_kernels::MoeLoraGroupedGemmModule const& mod, int64_t num_permuted_tokens, int64_t in_hidden_size, int64_t max_lora_rank, int64_t dtype_bytes, - int64_t splitk_slices, void const* input_base, void* output_base, tensorrt_llm::DataType data_type, - cudaStream_t stream) + int64_t splitk_slices, void const* input_base, void* output_base, nvinfer1::DataType data_type, cudaStream_t stream) { TLLM_CHECK_WITH_INFO(mod.permuted_ranks_dev != nullptr, "Grouped-GEMM LoRA module is missing permuted ranks buffer (forgot to populate grouped_gemm?)."); @@ -1208,17 +1206,17 @@ class FusedMoeRunner : public torch::CustomClassHolder // ===== LoRA helpers ===== - // Map a torch dtype to the TRT-LLM tensorrt_llm::DataType used to size the + // Map a torch dtype to the TRT-LLM nvinfer1::DataType used to size the // grouped-GEMM low-rank scratch. Kept as a const member (not static) so the // FP8 case can read mOutputDtype to pick the fp16/bf16 LoRA compute dtype. - tensorrt_llm::DataType loraTypeFromActDtype(c10::ScalarType dtype) const + nvinfer1::DataType loraTypeFromActDtype(c10::ScalarType dtype) const { switch (dtype) { - case c10::ScalarType::Half: return tensorrt_llm::DataType::kHALF; - case c10::ScalarType::Float: return tensorrt_llm::DataType::kFLOAT; + case c10::ScalarType::Half: return nvinfer1::DataType::kHALF; + case c10::ScalarType::Float: return nvinfer1::DataType::kFLOAT; #ifdef ENABLE_BF16 - case c10::ScalarType::BFloat16: return tensorrt_llm::DataType::kBF16; + case c10::ScalarType::BFloat16: return nvinfer1::DataType::kBF16; #endif #ifdef ENABLE_FP8 case c10::ScalarType::Float8_e4m3fn: diff --git a/cpp/tensorrt_llm/thop/noAuxTcOp.cpp b/cpp/tensorrt_llm/thop/noAuxTcOp.cpp index 4dfb20072734..e445206e1d78 100644 --- a/cpp/tensorrt_llm/thop/noAuxTcOp.cpp +++ b/cpp/tensorrt_llm/thop/noAuxTcOp.cpp @@ -20,6 +20,7 @@ #include "tensorrt_llm/kernels/noAuxTcKernels.h" +// #include <NvInferRuntime.h> // #include <c10/cuda/CUDAStream.h> // #include <cassert> // #include <set> diff --git a/cpp/tensorrt_llm/thop/reducescatterOp.cpp b/cpp/tensorrt_llm/thop/reducescatterOp.cpp index a50ca1862f76..40f89e40ff75 100644 --- a/cpp/tensorrt_llm/thop/reducescatterOp.cpp +++ b/cpp/tensorrt_llm/thop/reducescatterOp.cpp @@ -20,6 +20,7 @@ #include "tensorrt_llm/runtime/utils/mpiUtils.h" #include "tensorrt_llm/runtime/utils/pgUtils.h" +#include <NvInferRuntime.h> #include <c10/cuda/CUDAStream.h> #include <torch/extension.h> #if ENABLE_MULTI_DEVICE diff --git a/cpp/tensorrt_llm/thop/rmsNormFp4Quant.cpp b/cpp/tensorrt_llm/thop/rmsNormFp4Quant.cpp deleted file mode 100644 index 420cb80df461..000000000000 --- a/cpp/tensorrt_llm/thop/rmsNormFp4Quant.cpp +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/kernels/quantization.h" -#include "tensorrt_llm/kernels/rmsNormFp4QuantKernels.h" -#include "tensorrt_llm/runtime/torchUtils.h" -#include "tensorrt_llm/thop/thUtils.h" - -#include <ATen/cuda/CUDAContext.h> -#include <ATen/cuda/EmptyTensor.h> - -#include <vector> - -TRTLLM_NAMESPACE_BEGIN - -namespace torch_ext -{ - -// Fused residual-add + RMSNorm + NVFP4 input-quantize in one kernel. -// Replaces the (flashinfer fused_add_rmsnorm + standalone fp4_quantize) pair on -// the no-allreduce / attention-DP path. Each rank operates on its local tokens. -// -// Inputs: -// hidden_states : [..., hidden_size] BF16/FP16 — read-only. The residual sum -// (hidden_states + residual) is returned as a fresh -// residual_out tensor; hidden_states itself is not mutated, so -// the op is functionalizable under torch.compile. -// residual : [..., hidden_size] same dtype, read-only. -// norm_weight : [hidden_size] same dtype, RMSNorm gamma. -// scale_factor : [] float32, = (448 * 6) / amax for static-quant Linear. -// eps : RMSNorm epsilon. -// return_norm_out : when true, also return the BF16 normed value (needed by -// DSA indexer's pre_indexer_proj). -// -// Returns: [quant_out, scale_out, residual_out] or -// [norm_out, quant_out, scale_out, residual_out] when return_norm_out. -std::vector<at::Tensor> fused_add_rmsnorm_fp4_quantize(at::Tensor const& hidden_states, at::Tensor residual, - at::Tensor const& norm_weight, at::Tensor const& scale_factor, double eps, bool return_norm_out) -{ - CHECK_TH_CUDA(hidden_states); - CHECK_CONTIGUOUS(hidden_states); - CHECK_TH_CUDA(residual); - CHECK_CONTIGUOUS(residual); - CHECK_TH_CUDA(norm_weight); - CHECK_CONTIGUOUS(norm_weight); - CHECK_INPUT(scale_factor, torch::kFloat32); - TORCH_CHECK( - hidden_states.scalar_type() == residual.scalar_type(), "hidden_states and residual must have matching dtype"); - TORCH_CHECK(hidden_states.scalar_type() == norm_weight.scalar_type(), - "hidden_states and norm_weight must have matching dtype"); - - auto const& input_shape = hidden_states.sizes(); - auto const rank = input_shape.size(); - TORCH_CHECK(rank >= 2, "hidden_states should be >=2D"); - int64_t m = 1; - for (size_t i = 0; i < rank - 1; i++) - { - m *= input_shape[i]; - } - auto const k = input_shape[rank - 1]; - int64_t const sf_vec_size = 16; - TORCH_CHECK(k % sf_vec_size == 0, "hidden_size must be divisible by 16"); - - std::vector<int64_t> quant_shape(input_shape.begin(), input_shape.end()); - quant_shape[rank - 1] = k / 2; - at::Tensor quant_out = at::detail::empty_cuda(quant_shape, FLOAT4_E2M1X2, hidden_states.device(), std::nullopt); - at::Tensor scale_out = at::detail::empty_cuda({tensorrt_llm::computeSwizzledLayoutSFSize(m, k / sf_vec_size)}, - SF_DTYPE, hidden_states.device(), std::nullopt); - - at::Tensor norm_out; - void* norm_out_ptr = nullptr; - if (return_norm_out) - { - norm_out = at::detail::empty_cuda( - input_shape.vec(), hidden_states.scalar_type(), hidden_states.device(), std::nullopt); - norm_out_ptr = norm_out.mutable_data_ptr(); - } - - // Freshly allocate residual_out (input + residual is written here by the - // kernel) the same way as quant_out/scale_out and the ws op - // (fusedAddRMSNormQuant.cpp): empty_cuda, not empty_like. empty_cuda is a - // plain allocation, whereas empty_like carries the input's layout and can be - // lowered to a separate node in the torch.compile trace. The kernel reads - // hidden_states (intermediate_buffer) read-only and writes the sum into this - // distinct buffer, so hidden_states is never mutated and no output aliases an - // input (the op stays functionalizable) -- with no pre-kernel copy. - at::Tensor residual_out - = at::detail::empty_cuda(input_shape.vec(), hidden_states.scalar_type(), hidden_states.device(), std::nullopt); - - tensorrt_llm::kernels::RmsNormFp4QuantParams params{}; - params.bias_buffer = nullptr; - params.residual_buffer = residual.data_ptr(); - params.weight_buffer = norm_weight.data_ptr(); - params.intermediate_buffer = hidden_states.data_ptr(); - params.scale_factor_ptr = static_cast<float const*>(scale_factor.data_ptr()); - params.quant_out = quant_out.mutable_data_ptr(); - params.scale_out = scale_out.mutable_data_ptr(); - params.norm_out = norm_out_ptr; - params.residual_out_buffer = residual_out.mutable_data_ptr(); - params.hidden_size = static_cast<int>(k); - params.eps = static_cast<float>(eps); - params.elts_total = hidden_states.numel(); - params.sf_layout = tensorrt_llm::QuantizationSFLayout::SWIZZLED; - - auto const stream = at::cuda::getCurrentCUDAStream(hidden_states.get_device()); - auto const dtype = tensorrt_llm::runtime::TorchUtils::dataType(hidden_states.scalar_type()); - - tensorrt_llm::kernels::residualRmsNormFp4Quant(params, dtype, stream); - - // residual_out holds the residual sum (= original hidden + original residual). - if (return_norm_out) - { - return {norm_out, quant_out, scale_out, residual_out}; - } - return {quant_out, scale_out, residual_out}; -} - -// Residual-less variant of fused_add_rmsnorm_fp4_quantize. Replaces the -// (flashinfer rmsnorm + standalone fp4_quantize) pair on intra-layer paths that -// have NO residual add — e.g. DSv3.2/Kimi-K2.5 MLA's q_a_layernorm feeding the -// static-NVFP4 q_b_proj. The kernel reads intermediate_buffer (=hidden_states), -// skips the residual add (residual_buffer == nullptr selects Residual=false in -// the launcher), RMSNorms it, and FP4-quantizes the result. hidden_states is -// NOT modified (no residual write-back happens when Residual=false). -// -// Inputs: -// hidden_states : [..., hidden_size] BF16/FP16 — read-only. -// norm_weight : [hidden_size] same dtype, RMSNorm gamma. -// scale_factor : [] float32, = (448 * 6) / amax for static-quant Linear. -// eps : RMSNorm epsilon. -// return_norm_out : when true, also return the BF16 normed value. -// -// Returns: [quant_out, scale_out] or [norm_out, quant_out, scale_out] when -// return_norm_out. -std::vector<at::Tensor> fused_rmsnorm_fp4_quantize(at::Tensor const& hidden_states, at::Tensor const& norm_weight, - at::Tensor const& scale_factor, double eps, bool return_norm_out) -{ - CHECK_TH_CUDA(hidden_states); - CHECK_TH_CUDA(norm_weight); - CHECK_CONTIGUOUS(norm_weight); - CHECK_INPUT(scale_factor, torch::kFloat32); - TORCH_CHECK(hidden_states.scalar_type() == norm_weight.scalar_type(), - "hidden_states and norm_weight must have matching dtype"); - - auto const& input_shape = hidden_states.sizes(); - auto const rank = input_shape.size(); - TORCH_CHECK(rank >= 2, "hidden_states should be >=2D"); - // hidden_states may be a column-slice of a wider projection (e.g. the leading - // q_lora_rank columns of kv_a_proj_with_mqa): its last dim is unit-stride but - // its row stride may exceed hidden_size. We read it in place via an input row - // stride and skip the otherwise-required contiguous copy. The kernel only - // reads with this stride; all outputs are written packed. - TORCH_CHECK(hidden_states.stride(rank - 1) == 1, "hidden_states last dim must be unit-stride"); - // All leading dims must be densely packed on top of the row pitch so that a - // single per-row element stride describes the flattened [m, k] layout. The - // only permitted non-packing is a row pitch larger than k (a column slice). - for (size_t i = 0; i + 2 < rank; i++) - { - TORCH_CHECK(hidden_states.stride(i) == hidden_states.stride(i + 1) * input_shape[i + 1], - "hidden_states leading dims must be densely packed"); - } - int64_t m = 1; - for (size_t i = 0; i < rank - 1; i++) - { - m *= input_shape[i]; - } - auto const k = input_shape[rank - 1]; - int64_t const sf_vec_size = 16; - TORCH_CHECK(k % sf_vec_size == 0, "hidden_size must be divisible by 16"); - // Element stride between consecutive logical rows. For a contiguous tensor - // this equals k, so input_row_stride==0 (packed) and behavior is identical. - int64_t const row_stride = hidden_states.stride(rank - 2); - int const input_row_stride = (row_stride == k) ? 0 : static_cast<int>(row_stride); - - std::vector<int64_t> quant_shape(input_shape.begin(), input_shape.end()); - quant_shape[rank - 1] = k / 2; - at::Tensor quant_out = at::detail::empty_cuda(quant_shape, FLOAT4_E2M1X2, hidden_states.device(), std::nullopt); - at::Tensor scale_out = at::detail::empty_cuda({tensorrt_llm::computeSwizzledLayoutSFSize(m, k / sf_vec_size)}, - SF_DTYPE, hidden_states.device(), std::nullopt); - - at::Tensor norm_out; - void* norm_out_ptr = nullptr; - if (return_norm_out) - { - // The kernel writes norm_out packed (stride hidden_size), so allocate a - // packed (contiguous) tensor rather than mirroring a possibly-strided - // input layout. - norm_out = at::detail::empty_cuda( - input_shape.vec(), hidden_states.scalar_type(), hidden_states.device(), std::nullopt); - norm_out_ptr = norm_out.mutable_data_ptr(); - } - - // residual_buffer == nullptr selects the Residual=false kernel path: the - // kernel RMSNorms intermediate_buffer (=hidden_states) directly without any - // add or write-back, so hidden_states is left unmodified. - tensorrt_llm::kernels::RmsNormFp4QuantParams params{}; - params.bias_buffer = nullptr; - params.residual_buffer = nullptr; - params.weight_buffer = norm_weight.data_ptr(); - params.intermediate_buffer = hidden_states.data_ptr(); - params.scale_factor_ptr = static_cast<float const*>(scale_factor.data_ptr()); - params.quant_out = quant_out.mutable_data_ptr(); - params.scale_out = scale_out.mutable_data_ptr(); - params.norm_out = norm_out_ptr; - params.hidden_size = static_cast<int>(k); - params.eps = static_cast<float>(eps); - params.elts_total = hidden_states.numel(); - params.sf_layout = tensorrt_llm::QuantizationSFLayout::SWIZZLED; - params.input_row_stride = input_row_stride; - - auto const stream = at::cuda::getCurrentCUDAStream(hidden_states.get_device()); - auto const dtype = tensorrt_llm::runtime::TorchUtils::dataType(hidden_states.scalar_type()); - - tensorrt_llm::kernels::residualRmsNormFp4Quant(params, dtype, stream); - - if (return_norm_out) - { - return {norm_out, quant_out, scale_out}; - } - return {quant_out, scale_out}; -} - -} // namespace torch_ext - -TRTLLM_NAMESPACE_END - -TORCH_LIBRARY_FRAGMENT(trtllm, m) -{ - m.def( - "fused_add_rmsnorm_fp4_quantize(" - "Tensor hidden_states," - "Tensor residual," - "Tensor norm_weight," - "Tensor scale_factor," - "float eps," - "bool return_norm_out) -> Tensor[]"); - m.def( - "fused_rmsnorm_fp4_quantize(" - "Tensor hidden_states," - "Tensor norm_weight," - "Tensor scale_factor," - "float eps," - "bool return_norm_out) -> Tensor[]"); -} - -TORCH_LIBRARY_IMPL(trtllm, CUDA, m) -{ - m.impl("fused_add_rmsnorm_fp4_quantize", &tensorrt_llm::torch_ext::fused_add_rmsnorm_fp4_quantize); - m.impl("fused_rmsnorm_fp4_quantize", &tensorrt_llm::torch_ext::fused_rmsnorm_fp4_quantize); -} diff --git a/cpp/tensorrt_llm/thop/thUtils.cpp b/cpp/tensorrt_llm/thop/thUtils.cpp index c151414127fa..97fe6acaab7b 100644 --- a/cpp/tensorrt_llm/thop/thUtils.cpp +++ b/cpp/tensorrt_llm/thop/thUtils.cpp @@ -15,7 +15,7 @@ */ #include "tensorrt_llm/thop/thUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <array> TRTLLM_NAMESPACE_BEGIN @@ -25,12 +25,12 @@ namespace torch_ext tensorrt_llm::runtime::ITensor::Shape convert_shape(torch::Tensor tensor) { - constexpr auto trtMaxDims = tensorrt_llm::Dims::MAX_DIMS; + constexpr auto trtMaxDims = nvinfer1::Dims::MAX_DIMS; auto const torchTensorNumDims = tensor.dim(); TLLM_CHECK_WITH_INFO(torchTensorNumDims <= trtMaxDims, "TensorRT supports at most %i tensor dimensions. Found a Torch tensor with %li dimensions.", trtMaxDims, torchTensorNumDims); - auto result = tensorrt_llm::Dims{}; + auto result = nvinfer1::Dims{}; result.nbDims = static_cast<int32_t>(torchTensorNumDims); for (int i = 0; i < torchTensorNumDims; i++) { diff --git a/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp b/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp index bc80d8446445..5720951e2720 100644 --- a/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp +++ b/cpp/tensorrt_llm/thop/trtllmGenQKVProcessOp.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/common/attentionOp.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/kernels/unfusedAttentionKernels.h" @@ -402,16 +401,16 @@ trtllmGenContextPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, tor switch (qkvDtype) { - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<float, KVBlockArray>&>(qkvParams), stream); break; - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<half, KVBlockArray>&>(qkvParams), stream); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<__nv_bfloat16, KVBlockArray>&>(qkvParams), stream); break; @@ -550,16 +549,16 @@ void trtllmGenContextPostprocess(torch::Tensor qkv_input, torch::Tensor workspac switch (qkvDtype) { - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: tensorrt_llm::kernels::invokeKvCachePostprocessing( reinterpret_cast<QKVPreprocessingParams<float, KVBlockArray>&>(qkvParams), stream); break; - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: tensorrt_llm::kernels::invokeKvCachePostprocessing( reinterpret_cast<QKVPreprocessingParams<half, KVBlockArray>&>(qkvParams), stream); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: tensorrt_llm::kernels::invokeKvCachePostprocessing( reinterpret_cast<QKVPreprocessingParams<__nv_bfloat16, KVBlockArray>&>(qkvParams), stream); break; @@ -734,16 +733,16 @@ trtllmGenGenerationPreprocess(torch::Tensor qkv_input, torch::Tensor workspace, switch (qkvDtype) { - case tensorrt_llm::DataType::kFLOAT: + case nvinfer1::DataType::kFLOAT: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<float, KVBlockArray>&>(qkvParams), stream); break; - case tensorrt_llm::DataType::kHALF: + case nvinfer1::DataType::kHALF: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<half, KVBlockArray>&>(qkvParams), stream); break; #ifdef ENABLE_BF16 - case tensorrt_llm::DataType::kBF16: + case nvinfer1::DataType::kBF16: tensorrt_llm::kernels::invokeQKVPreprocessing( reinterpret_cast<QKVPreprocessingParams<__nv_bfloat16, KVBlockArray>&>(qkvParams), stream); break; diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 99e16ea8b147..0a06d40ee85e 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -25,13 +25,8 @@ include_directories( ${PROJECT_SOURCE_DIR}/include ${cutlass_source_dir}/include ${cutlass_source_dir}/tools/util/include - ${PROJECT_SOURCE_DIR}/tests/batch_manager) - -# Tests previously inherited the MPI include dirs transitively through the -# removed TensorRT plugin target's PUBLIC includes. -if(ENABLE_MULTI_DEVICE) - include_directories(${MPI_C_INCLUDE_DIRS}) -endif() + ${PROJECT_SOURCE_DIR}/tests/batch_manager + ${PROJECT_SOURCE_DIR}/tests/utils) set(TOP_LEVEL_DIR "${PROJECT_SOURCE_DIR}/..") @@ -43,12 +38,13 @@ function(add_gtest test_name test_src) ${ARGN}) add_executable(${test_name} ${test_src}) - target_link_libraries(${test_name} PUBLIC gmock_main) + target_link_libraries(${test_name} PUBLIC gmock_main TensorRT::OnnxParser) if(NOT ARGS_NO_GTEST_MAIN) target_link_libraries(${test_name} PUBLIC gtest_main) endif() if(NOT ARGS_NO_TLLM_LINKAGE) - target_link_libraries(${test_name} PUBLIC ${SHARED_TARGET}) + target_link_libraries(${test_name} PUBLIC ${SHARED_TARGET} + nvinfer_plugin_tensorrt_llm) if(WIN32) target_link_libraries(${test_name} PRIVATE context_attention_src) endif() @@ -70,4 +66,6 @@ function(add_gtest test_name test_src) add_dependencies(google-tests ${test_name}) endfunction() +add_subdirectory(utils) add_subdirectory(unit_tests) +add_subdirectory(e2e_tests) diff --git a/cpp/tests/e2e_tests/CMakeLists.txt b/cpp/tests/e2e_tests/CMakeLists.txt new file mode 100644 index 000000000000..f5deb048a180 --- /dev/null +++ b/cpp/tests/e2e_tests/CMakeLists.txt @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +add_subdirectory(batch_manager) +add_subdirectory(executor) diff --git a/cpp/tests/e2e_tests/batch_manager/CMakeLists.txt b/cpp/tests/e2e_tests/batch_manager/CMakeLists.txt new file mode 100644 index 000000000000..875e12eb975f --- /dev/null +++ b/cpp/tests/e2e_tests/batch_manager/CMakeLists.txt @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# guidedDecoderTest requires model tokenizer info, so it's easier to run it with +# e2e tests instead of unit tests. +add_gtest(guidedDecoderTest guidedDecoderTest.cpp) +add_gtest(trtEncoderModelTest trtEncoderModelTest.cpp) +add_gtest(trtGptModelTest trtGptModelTest.cpp) +add_gtest(trtGptModelRealDecoderTest trtGptModelRealDecoderTest.cpp) +target_link_libraries(trtGptModelRealDecoderTest PRIVATE testingUtils) diff --git a/cpp/tests/e2e_tests/batch_manager/guidedDecoderTest.cpp b/cpp/tests/e2e_tests/batch_manager/guidedDecoderTest.cpp new file mode 100644 index 000000000000..7b262cacb27d --- /dev/null +++ b/cpp/tests/e2e_tests/batch_manager/guidedDecoderTest.cpp @@ -0,0 +1,227 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TOP_LEVEL_DIR +#error "Define TOP_LEVEL_DIR" +#endif + +#include <fstream> +#include <gtest/gtest.h> +#include <nlohmann/json.hpp> + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/batch_manager/decoderBuffers.h" +#include "tensorrt_llm/batch_manager/guidedDecoder.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/executor/executor.h" + +using namespace tensorrt_llm::runtime; +using namespace tensorrt_llm::batch_manager; +namespace texec = tensorrt_llm::executor; + +namespace +{ +auto const TEST_RESOURCE_PATH = std::filesystem::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; +auto const DATA_PATH = TEST_RESOURCE_PATH / "data"; +auto const GPT_XGRAMMAR_TOKENIZER_INFO_PATH = DATA_PATH / "gpt2" / "xgrammar_tokenizer_info.json"; +auto const LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH = DATA_PATH / "Llama-3.2-1B" / "xgrammar_tokenizer_info.json"; +} // namespace + +class GuidedDecoderTest : public ::testing::Test +{ +public: + using TensorPtr = ITensor::SharedPtr; + using VecTokens = std::vector<TokenIdType>; + using RequestIdType = std::uint64_t; + using RequestVector = std::vector<std::shared_ptr<LlmRequest>>; + + void SetUp() override + { + mStream = std::make_shared<CudaStream>(); + mRuntimeBufferManager = std::make_shared<BufferManager>(mStream); + } + + void TearDown() override {} + + void initData(std::filesystem::path tokenizerInfoPath, SizeType32 vocabSizePadded, VecTokens outputIds, + std::vector<int32_t> expectedNumRejected) + { + mLogitsDtype = nvinfer1::DataType::kFLOAT; + mMaxNumRequests = 16; + + mVocabSizePadded = vocabSizePadded; + auto const tokenizerInfo = nlohmann::json::parse(std::ifstream{tokenizerInfoPath}); + auto const encodedVocab = tokenizerInfo["encoded_vocab"].template get<std::vector<std::string>>(); + auto const tokenizerStr = tokenizerInfo["tokenizer_str"].template get<std::string>(); + auto const stopTokenIds = tokenizerInfo["stop_token_ids"].template get<std::vector<TokenIdType>>(); + texec::GuidedDecodingConfig guidedDecodingConfig( + texec::GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR, encodedVocab, tokenizerStr, stopTokenIds); + mGuidedDecoder = std::make_shared<GuidedDecoder>( + guidedDecodingConfig, mMaxNumRequests, mVocabSizePadded, mLogitsDtype, *mRuntimeBufferManager); + + mLogits.resize(mMaxNumRequests); + mLogitsHost.resize(mMaxNumRequests); + for (int i = 0; i < mMaxNumRequests; i++) + { + mLogits[i] = mRuntimeBufferManager->gpu(ITensor::makeShape({mVocabSizePadded}), mLogitsDtype); + mLogitsHost[i] = BufferManager::pinned(ITensor::makeShape({mVocabSizePadded}), mLogitsDtype); + } + + mOutputIds = outputIds; + mExpectedNumRejected = expectedNumRejected; + } + + void resetLogits() + { + for (int i = 0; i < mMaxNumRequests; i++) + { + auto logitsHostData = bufferCast<float>(*mLogitsHost[i]); + for (int j = 0; j < mVocabSizePadded; j++) + { + logitsHostData[j] = 0.0f; + } + mRuntimeBufferManager->copy(*(mLogitsHost[i]), *(mLogits[i])); + } + } + + void syncLogitsToHost() + { + for (int i = 0; i < mMaxNumRequests; i++) + { + mRuntimeBufferManager->copy(*(mLogits[i]), *(mLogitsHost[i])); + } + } + + int32_t countRejected(int i) + { + int32_t numRejected = 0; + for (int j = 0; j < mVocabSizePadded; j++) + { + auto logitsHostData = bufferCast<float>(*mLogitsHost[i]); + if (logitsHostData[j] < -1e6) + { + numRejected++; + } + } + return numRejected; + } + + void runTest() + { + auto llmReq1 = std::make_shared<LlmRequest>(1, 100, std::make_shared<VecTokens>(10), SamplingConfig(), false); + texec::GuidedDecodingParams guidedDecodingParams(texec::GuidedDecodingParams::GuideType::kJSON); + llmReq1->setGuidedDecodingParams(guidedDecodingParams); + llmReq1->mSeqSlot = 1; + + auto llmReq2 = std::make_shared<LlmRequest>(1, 100, std::make_shared<VecTokens>(10), SamplingConfig(), false); + llmReq2->mSeqSlot = 2; + + RequestVector contextRequests{llmReq1, llmReq2}; + RequestVector generationRequests{}; + ScheduledRequests scheduledRequests{contextRequests, generationRequests}; + DecoderInputBuffers decoderInputBuffers(mMaxNumRequests, 1, *mRuntimeBufferManager); + + for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + decoderInputBuffers.decoderRequests.push_back(llmReq); + } + } + decoderInputBuffers.decoderLogits = mLogits; + + // Context phase + resetLogits(); + mGuidedDecoder->build(scheduledRequests); + mGuidedDecoder->execute(decoderInputBuffers, *mRuntimeBufferManager); + syncLogitsToHost(); + mRuntimeBufferManager->getStream().synchronize(); + + // Move request to generation phase + contextRequests.pop_back(); + contextRequests.pop_back(); + llmReq1->setState(LlmRequestState::kGENERATION_IN_PROGRESS); + generationRequests.push_back(llmReq1); + llmReq2->setState(LlmRequestState::kGENERATION_IN_PROGRESS); + generationRequests.push_back(llmReq2); + + decoderInputBuffers.decoderRequests.clear(); + for (auto const& requests : {scheduledRequests.contextRequests, scheduledRequests.generationRequests}) + { + for (auto const& llmReq : requests) + { + decoderInputBuffers.decoderRequests.push_back(llmReq); + } + } + + EXPECT_EQ(countRejected(0), mExpectedNumRejected[0]); + EXPECT_EQ(countRejected(1), 0); + + // Generation phase + for (int i = 0; i < mOutputIds.size(); i++) + { + llmReq1->addNewToken(mOutputIds[i], 0); + llmReq2->addNewToken(mOutputIds[i], 0); + + resetLogits(); + mGuidedDecoder->build(scheduledRequests); + mGuidedDecoder->execute(decoderInputBuffers, *mRuntimeBufferManager); + syncLogitsToHost(); + mRuntimeBufferManager->getStream().synchronize(); + + EXPECT_EQ(countRejected(0), mExpectedNumRejected[i + 1]); + EXPECT_EQ(countRejected(1), 0); + } + } + +private: + SizeType32 mMaxNumRequests; + SizeType32 mVocabSizePadded; + nvinfer1::DataType mLogitsDtype; + + std::vector<TensorPtr> mLogits; // [mBatchSize, mVocabSizePadded] + std::vector<TensorPtr> mLogitsHost; // [mBatchSize, mVocabSizePadded] + + std::shared_ptr<BufferManager> mRuntimeBufferManager; + std::shared_ptr<CudaStream> mStream; + std::shared_ptr<GuidedDecoder> mGuidedDecoder; + + VecTokens mOutputIds; + std::vector<int32_t> mExpectedNumRejected; +}; + +TEST_F(GuidedDecoderTest, GptTokenizer) +{ + VecTokens outputIds{4895, 824, 312, 1298, 366, 27743, 7934, 49793, 1600, 366, 12961, 19703, 4668, 1298, 366, 54, + 4537, 17, 12, 17469, 7919, 1600, 366, 3903, 10394, 1298, 366, 1485, 405, 41022, 20662}; + std::vector<int32_t> expectedNumRejected{50251, 219, 219, 219, 48558, 219, 219, 219, 219, 50191, 219, 219, 219, 219, + 48558, 219, 219, 219, 219, 219, 219, 219, 50191, 219, 219, 219, 48558, 219, 219, 219, 219, 50256}; + initData(GPT_XGRAMMAR_TOKENIZER_INFO_PATH, 50257, outputIds, expectedNumRejected); + runTest(); +} + +TEST_F(GuidedDecoderTest, LlamaTokenizer) +{ + VecTokens outputIds{6377, 893, 333, 1115, 376, 27247, 6779, 7898, 545, 613, 376, 8926, 17830, 1115, 376, 29956, + 7228, 29906, 29899, 10399, 7734, 613, 376, 4980, 2103, 1115, 376, 29896, 29941, 29900, 29900, 341, 29890, 567, + 9092}; + std::vector<int32_t> expectedNumRejected{128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, + 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, + 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235, 128235}; + initData(LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH, 128256, outputIds, expectedNumRejected); + runTest(); +} diff --git a/cpp/tests/e2e_tests/batch_manager/trtEncoderModelTest.cpp b/cpp/tests/e2e_tests/batch_manager/trtEncoderModelTest.cpp new file mode 100644 index 000000000000..dce09539bd45 --- /dev/null +++ b/cpp/tests/e2e_tests/batch_manager/trtEncoderModelTest.cpp @@ -0,0 +1,219 @@ + +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TOP_LEVEL_DIR +#error "Define TOP_LEVEL_DIR" +#endif + +#include "tensorrt_llm/batch_manager/trtEncoderModel.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/gptJsonConfig.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/utils/numpyUtils.h" +#include "tensorrt_llm/runtime/utils/runtimeUtils.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +#include <filesystem> +#include <vector> + +using namespace tensorrt_llm::runtime; +namespace fs = std::filesystem; + +using TensorPtr = ITensor::SharedPtr; + +namespace +{ +auto const TEST_RESOURCE_PATH = fs::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; +auto const ENC_DEC_BASE = TEST_RESOURCE_PATH / "models/enc_dec/trt_engines"; +auto const ENC_DEC_ENGINE_BASE = TEST_RESOURCE_PATH / "models/enc_dec/trt_engines"; +auto const BART_TP1_PP1_ENCODER_RMPAD_DIR = "bart-large-cnn/1-gpu/float16/tp1/encoder"; +auto const BART_TP2_PP1_ENCODER_RMPAD_DIR = "bart-large-cnn/2-gpu/float16/tp2/encoder"; +auto const BART_TP2_PP2_ENCODER_RMPAD_DIR = "bart-large-cnn/4-gpu/float16/tp2/encoder"; +auto const T5_TP1_PP1_ENCODER_RMPAD_DIR = "t5-small/1-gpu/float16/tp1/encoder"; +auto const ENC_DEC_DATA_BASE = TEST_RESOURCE_PATH / "data/enc_dec"; +} // namespace + +namespace tensorrt_llm::batch_manager +{ + +class EncoderModelTestSingleGPU : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) +{ +protected: + EncoderModelTestSingleGPU(std::filesystem::path const& modelPath) + : mModelConfig(1, 2, 1, 1, 1, 1, nvinfer1::DataType::kFLOAT) + , mModelPath(modelPath) + { + } + + EncoderModelTestSingleGPU() + : EncoderModelTestSingleGPU(ENC_DEC_ENGINE_BASE / T5_TP1_PP1_ENCODER_RMPAD_DIR) + { + } + + void SetUp() override + { + std::filesystem::path trtEnginePath = mModelPath; + + mBeamWidth = 1; + + mLogger = std::make_shared<TllmLogger>(); + + initTrtLlmPlugins(mLogger.get()); + + auto const json = GptJsonConfig::parse(trtEnginePath / "config.json"); + mModelConfig = json.getModelConfig(); + mWorldConfig = WorldConfig::mpi(json.getGpusPerNode(), json.getTensorParallelism(), + json.getPipelineParallelism(), json.getContextParallelism()); + mVocabSizePadded = mModelConfig.getVocabSizePadded(mWorldConfig.getSize()); + + auto const enginePath = trtEnginePath / json.engineFilename(mWorldConfig); + auto const dtype = mModelConfig.getDataType(); + + ASSERT_TRUE(fs::exists(enginePath)); + mEngineBuffer = utils::loadEngine(enginePath.string()); + + mStream = std::make_unique<CudaStream>(); + mManager = std::make_unique<BufferManager>(mStream); + } + + void TearDown() override {} + + int32_t mMaxNumRequests; + int32_t mMaxSeqLen; + int32_t mBeamWidth; + int32_t mVocabSizePadded; + // SamplingConfig mSamplingConfig; + std::string mDataPath; + std::shared_ptr<nvinfer1::ILogger> mLogger; + ModelConfig mModelConfig; + WorldConfig mWorldConfig; + std::vector<std::uint8_t> mEngineBuffer; + std::unique_ptr<BufferManager> mManager; + BufferManager::CudaStreamPtr mStream; + std::filesystem::path mModelPath; +}; + +// test for TP2PP2 +class TrtEncoderModelTestMultiGPU : public EncoderModelTestSingleGPU +{ +protected: + TrtEncoderModelTestMultiGPU() + : EncoderModelTestSingleGPU(ENC_DEC_ENGINE_BASE / BART_TP2_PP2_ENCODER_RMPAD_DIR) + { + } +}; + +namespace +{ + +void runEncoderTest(std::unique_ptr<BufferManager>& bufferManager, ModelConfig const& modelConfig, + WorldConfig const& worldConfig, std::vector<std::uint8_t> const& engineBuffer, + std::shared_ptr<nvinfer1::ILogger>& logger) +{ + using VecTokens = LlmRequest::VecTokens; + using TokenIdType = LlmRequest::TokenIdType; + + auto inputsIdsHost + = utils::loadNpy(*bufferManager, (ENC_DEC_DATA_BASE / "input_ids.npy").string(), MemoryType::kCPU); + auto inputsIdsPtr = bufferCast<SizeType32>(*inputsIdsHost); + auto inputLengthsHost + = utils::loadNpy(*bufferManager, (ENC_DEC_DATA_BASE / "input_lengths.npy").string(), MemoryType::kCPU); + auto inputLengthsPtr = bufferCast<SizeType32>(*inputLengthsHost); + auto encoderOutput + = utils::loadNpy(*bufferManager, (ENC_DEC_DATA_BASE / "encoder_output.npy").string(), MemoryType::kCPU); + auto encoderOutputPrt = bufferCast<half>(*encoderOutput); + + SizeType32 const nbRequests = inputLengthsHost->getShape().d[0]; + SizeType32 const stride = inputsIdsHost->getShape().d[1]; + SizeType32 const hiddenSize = encoderOutput->getShape().d[1]; + ASSERT_EQ(nbRequests, inputsIdsHost->getShape().d[0]); + + // std::vector<std::shared_ptr<VecTokens>> inputIds(nbRequests); + RequestVector requestList; + for (SizeType32 i = 0; i < nbRequests; i++) + { + SizeType32 length = inputLengthsPtr[i]; + auto currentInputId = std::make_shared<VecTokens>(0); + currentInputId->insert(currentInputId->end(), inputsIdsPtr, inputsIdsPtr + length); + executor::Request req(*currentInputId, 1); + req.setEncoderInputTokenIds(*currentInputId); + auto request = std::make_shared<LlmRequest>(i, req); + inputsIdsPtr += stride; + requestList.push_back(request); + } + + tensorrt_llm::executor::ExecutorConfig executorConfig{}; + auto trtEncoderModel = std::make_shared<TrtEncoderModel>( + modelConfig, worldConfig, runtime::RawEngine(engineBuffer.data(), engineBuffer.size()), logger, executorConfig); + + trtEncoderModel->forward(requestList); + + if (worldConfig.isLastPipelineParallelRank() && worldConfig.getTensorParallelRank() == 0) + { + auto arrayEqual = [](auto it0, auto it1, SizeType32 length) + { + SizeType32 nbNotEqual = 0; + for (SizeType32 i = 0; i < length; i++) + { + auto v0 = static_cast<float>(*it0); + auto v1 = static_cast<float>(*it1); + if (std::abs(v0 - v1) > 1e-3) + { + nbNotEqual++; + } + it0++; + it1++; + } + return static_cast<double>(nbNotEqual) / length; + }; + ASSERT_EQ(requestList.size(), inputLengthsHost->getShape().d[0]); + { + auto curLengthPtr = inputLengthsPtr; + auto curOutPtr = encoderOutputPrt; + for (auto const& req : requestList) + { + ASSERT_TRUE(req->getEncoderOutputHost()) << "Encoder output is empty!"; + EXPECT_EQ(req->getState(), LlmRequestState::kCONTEXT_INIT); + auto actualOut = bufferCast<half>(*(req->getEncoderOutputHost())); + auto unequalFraction = arrayEqual(curOutPtr, actualOut, *curLengthPtr); + EXPECT_TRUE(unequalFraction == 0) + << "Req " << req->mRequestId << ": " << unequalFraction << " of outputs are different"; + curOutPtr += *curLengthPtr * hiddenSize; + curLengthPtr++; + } + } + } +} + +} // Anonymous namespace + +TEST_F(EncoderModelTestSingleGPU, Forward) +{ + runEncoderTest(mManager, mModelConfig, mWorldConfig, mEngineBuffer, mLogger); +} + +TEST_F(TrtEncoderModelTestMultiGPU, Forward) +{ + + runEncoderTest(mManager, mModelConfig, mWorldConfig, mEngineBuffer, mLogger); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tests/e2e_tests/batch_manager/trtGptModelRealDecoderTest.cpp b/cpp/tests/e2e_tests/batch_manager/trtGptModelRealDecoderTest.cpp new file mode 100644 index 000000000000..f401a31305d3 --- /dev/null +++ b/cpp/tests/e2e_tests/batch_manager/trtGptModelRealDecoderTest.cpp @@ -0,0 +1,1741 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/trtGptModel.h" +#include "tensorrt_llm/batch_manager/trtGptModelFactory.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tensorrt_llm/runtime/utils/numpyUtils.h" +#include "tensorrt_llm/testing/modelSpec.h" +#include "tests/utils/common.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +#include <cstdint> +#include <cstdlib> +#include <memory> +#include <optional> +#include <vector> + +using namespace tensorrt_llm::testing; +using namespace tensorrt_llm::runtime; +using namespace tensorrt_llm::runtime::utils; +using namespace tensorrt_llm::batch_manager; +namespace fs = std::filesystem; +namespace tc = tensorrt_llm::common; +namespace texec = tensorrt_llm::executor; +using tensorrt_llm::testing::ModelSpec; +using tensorrt_llm::testing::KVCacheType; +using tensorrt_llm::testing::QuantMethod; + +namespace +{ +using TensorPtr = tensorrt_llm::runtime::ITensor::SharedPtr; + +auto constexpr GPT_MODEL_DIR = "gpt2"; +auto constexpr GPTJ_MODEL_DIR = "gpt-j-6b"; +auto constexpr LLAMA_MODEL_DIR = "Llama-3.2-1B"; +auto constexpr MEDUSA_MODEL_DIR = "vicuna-7b-medusa"; +auto constexpr EAGLE_MODEL_DIR = "vicuna-7b-eagle"; +auto constexpr MAMBA_MODEL_DIR = "mamba-2.8b-hf"; +auto constexpr RECURRENTGEMMA_MODEL_DIR = "recurrentgemma-2b"; +auto constexpr EXPLICIT_DRAFT_MODEL_DIR = "vicuna-7b-redrafter"; +auto constexpr CHATGLM_MODEL_DIR = "chatglm-6b"; +auto constexpr GLM_MODEL_DIR = "glm-10b"; + +auto constexpr FP8_GPT_ATTENTION_PLUGIN_IFB_PACKED_PATH = "fp8-plugin"; + +auto constexpr INPUT_FILE = "input_tokens.npy"; +auto constexpr INPUT_LLAMA_FILE = "input_tokens_llama.npy"; +auto constexpr INPUT_VICUNA_FILE = "input_vicuna.npy"; +auto constexpr LONG_INPUT_FILE = "input_tokens_long.npy"; +auto constexpr CHATGLM_INPUT_FILE = "input_tokens_chatglm-6b.npy"; +auto constexpr GLM_INPUT_FILE = "input_tokens_glm-10b.npy"; + +auto constexpr LLAMA_END_ID = 128001; +auto constexpr LLAMA_PAD_ID = 128001; + +struct ModelParams +{ + char const* baseDir; + ModelIds ids; + + friend std::ostream& operator<<(std::ostream& os, ModelParams const& modelParams) + { + return os << "baseDir: " << modelParams.baseDir << ", ids: (" << modelParams.ids.padId << "," + << modelParams.ids.endId << ")"; + } +}; + +} // namespace + +class TrtModelRealDecoderTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) +{ +protected: + TrtModelRealDecoderTest() {} + + void SetUp() override + { + mDeviceCount = tc::getDeviceCount(); + if (mDeviceCount == 0) + { + GTEST_SKIP() << "No GPUs found"; + } + + mLogger = std::make_shared<TllmLogger>(); + + initTrtLlmPlugins(mLogger.get()); + } + + void TearDown() override {} + + int mDeviceCount{}; + std::shared_ptr<nvinfer1::ILogger> mLogger{}; +}; + +enum class TrtGptModelIfbTestType +{ + BULK, + WAVEFRONT, + RANDOM +}; + +namespace +{ + +void verifyOutput(RequestList const& finishedRequestList, + std::unordered_map<SizeType32, TestData> const& beamWidthTestData, std::vector<SizeType32> const& givenInputLengths, + SizeType32 nbGivenInputs, ModelSpec const& modelSpec) +{ + auto const checkRawLogits = modelSpec.mOtherModelSpecToCompare ? false : modelSpec.mGatherLogits; + auto const smokeTest = modelSpec.mSmokeTest; + auto const returnLogProbs = modelSpec.mReturnLogProbs; + auto const checkAcceptedTokenLogits = modelSpec.mAcceptDraftByLogits; + + if (smokeTest) + { + return; + } + + for (auto const& llmReqPtr : finishedRequestList) + { + auto const& llmReq = *llmReqPtr; + auto const requestId = llmReq.mRequestId; + auto const [givenInputIdx, givenInputLength] + = getRequestGivenInputIdxLength(requestId, nbGivenInputs, givenInputLengths); + auto const reqBeamWidth = llmReq.mSamplingConfig.beamWidth; + auto const& testData = beamWidthTestData.at(reqBeamWidth); + auto const* const expectedOutputData = bufferCast<TokenIdType const>(*testData.expectedOutputIds); + auto const expectedOutputLengths = testData.expectedOutputLengths; + auto const acceptedDraftTokensLengths = testData.acceptedDraftTokensLengths; + auto const endId = testData.endIds[givenInputIdx]; + auto const maxSeqLen = testData.maxSeqLen; + auto const draftLogits = testData.draftLogits; + auto const expectedGenerationLogits = testData.expectedGenerationLogits; + auto const expectedContextLogits = testData.expectedContextLogits; + auto const expectedCumLogProbs = testData.expectedCumLogProbs; + auto const expectedLogProbs = testData.expectedLogProbs; + auto const draftTokens = llmReq.getDraftTokens(); + auto const isDraftTokensExternal = modelSpec.mSpecDecodingMode.isDraftTokensExternal(); + auto const inputLength = givenInputLength + static_cast<SizeType32>(isDraftTokensExternal); + + for (auto beam = 0; beam < reqBeamWidth; ++beam) + { + auto const expectedOutputLength = expectedOutputLengths[givenInputIdx * reqBeamWidth + beam]; + auto const predictedTokens = llmReq.getTokens(beam); + + auto numPredTokens = static_cast<SizeType32>(predictedTokens.size() - inputLength); + if (isDraftTokensExternal && !draftTokens->empty()) + { + numPredTokens + = std::min(numPredTokens, acceptedDraftTokensLengths[givenInputIdx * reqBeamWidth + beam] + 1); + } + if (modelSpec.mSpecDecodingMode.isMedusa() || modelSpec.mSpecDecodingMode.isLookaheadDecoding() + || modelSpec.mSpecDecodingMode.isExplicitDraftTokens() || modelSpec.mSpecDecodingMode.isEagle()) + { + // WAR to ensure bulk execution of spec decoding. + // We hope that no request in batch can finish 2x faster than any other request. + // For the cases when BS < 8, some predicted tokens are mismatched to reference data. + numPredTokens /= 2; + } + + if (modelSpec.mKVCacheType == KVCacheType::kDISABLED) + { + EXPECT_EQ(numPredTokens, 1) << "b: " << requestId << " beam: " << beam; + } + else + { + EXPECT_EQ(predictedTokens.size(), expectedOutputLength) << "b: " << requestId << " beam: " << beam; + } + + bool anyMismatch = false; + for (auto i = 0; i < numPredTokens; ++i) + { + // Use the expected data for that beamWidth + auto const expectIndex = tc::flat_index3(givenInputIdx, beam, inputLength + i, reqBeamWidth, maxSeqLen); + + auto const expectedToken = expectedOutputData[expectIndex]; + if (expectedToken == endId) + { + break; + } + auto const predictIndex = inputLength + i; + auto const predictedToken = predictedTokens.at(predictIndex); + EXPECT_EQ(predictedToken, expectedToken) << "b: " << requestId << " beam: " << beam << " i: " << i; + anyMismatch |= (predictedToken != expectedToken); + } + EXPECT_FALSE(anyMismatch) << "b: " << requestId << " beam: " << beam; + + if (returnLogProbs) + { + auto cumLogProbs = llmReq.getCumLogProbs(); + auto* const reqExpectedCumLogProbs = bufferCast<float>(*expectedCumLogProbs[requestId]); + EXPECT_TRUE(almostEqual(reqExpectedCumLogProbs[beam], cumLogProbs[beam])); + + auto logProbs = llmReq.getLogProbs(beam); + auto expectedLogProbsBeam = std::shared_ptr(ITensor::slice(expectedLogProbs[requestId], beam, 1)); + expectedLogProbsBeam->squeeze(0); + auto* const reqExpectedLogProbs = bufferCast<float>(*expectedLogProbsBeam); + + for (auto i = 0; i < numPredTokens; ++i) + { + EXPECT_TRUE(almostEqual(reqExpectedLogProbs[inputLength + i], logProbs[i], 5e-2, 5e-2)) + << "expectedLogProbs : " << reqExpectedLogProbs[inputLength + i] + << " logProbs : " << logProbs[i]; + } + } + + if (checkAcceptedTokenLogits && llmReq.hasDraftTokens()) + { + TLLM_CHECK_WITH_INFO(reqBeamWidth == 1, "speculative decoding only works for beam width == 1"); + + TensorPtr const& acceptedTokensLogits = llmReq.getGenerationLogitsHost(); + auto const acceptedTokensLogitsShape = acceptedTokensLogits->getShape(); + + EXPECT_EQ(acceptedTokensLogitsShape.nbDims, 3); + EXPECT_EQ(1, acceptedTokensLogitsShape.d[0]); + EXPECT_EQ(numPredTokens, acceptedTokensLogitsShape.d[1]); + + TensorPtr const& expectedLogits = ITensor::slice(expectedGenerationLogits[requestId], 1, numPredTokens); + + // For hyperparameters + // Greater tolerance for the accepted logits of the target model. + float atol = 0.f; + float rtol = 0.01f; + EXPECT_TRUE(compareLogits(*expectedLogits, *acceptedTokensLogits, atol, rtol)); + } + + if (checkRawLogits) + { + // Check generation logits + TensorPtr const& expectedGenerationLogitsSliced + = ITensor::slice(expectedGenerationLogits[requestId], 0, numPredTokens); + + TensorPtr const& llmReqGeneration = llmReq.getGenerationLogitsHost(); + auto llmReqGenerationShape = llmReqGeneration->getShape(); + + TensorPtr generationLogitsBeam = nullptr; + if (llmReq.isStreaming()) + { + // Expect generation logits shape: [outputLength, beamWidth, vocabSizePad] + EXPECT_EQ(reqBeamWidth, llmReqGenerationShape.d[1]); + EXPECT_EQ(reqBeamWidth, 1); // Streaming mode does not support beam > 1 + llmReqGeneration->squeeze(1); // [outputLength, vocabSizePad] + generationLogitsBeam = llmReqGeneration; + } + else + { + // Expect generation logits shape: [beamWidth, outputLength, vocabSizePad] + EXPECT_EQ(reqBeamWidth, llmReqGenerationShape.d[0]); + generationLogitsBeam + = std::shared_ptr(ITensor::slice(llmReqGeneration, beam, 1)); // [1, outputLength, vocabSizePad] + generationLogitsBeam->squeeze(0); // [outputLength, vocabSizePad] + } + TensorPtr const& generationLogitsSliced = ITensor::slice(generationLogitsBeam, 0, numPredTokens); + EXPECT_TRUE(compareLogits(*expectedGenerationLogitsSliced, *generationLogitsSliced)); + } + } + + if (checkRawLogits) + { + // Check context logits + TensorPtr const& llmReqContext = llmReq.getContextLogitsHost(); + auto llmReqContextShape = llmReqContext->getShape(); + EXPECT_EQ(llmReqContextShape.nbDims, 2); + EXPECT_EQ(llmReq.mPromptLen, llmReqContextShape.d[0]); + EXPECT_TRUE(compareLogits(*expectedContextLogits[requestId], *llmReqContext)); + } + } +} + +// Pick a different endId at random from one of the expected tokens +std::vector<TokenIdType> pickRandomEndIds(TestData const& testData, std::vector<SizeType32> const& givenInputLengths, + SizeType32 const maxNewTokens, bool replaceLogits) +{ + auto const nbGivenInputs = testData.nbGivenInputs; + auto const beamWidth = testData.beamWidth; + auto* const expectedOutputData = bufferCast<TokenIdType>(*testData.expectedOutputIds); + + std::vector<TokenIdType> endIds; + + // For IFB, pick one of the output tokens as endId + for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) + { + TokenIdType skippedEndId0 = 0; + TokenIdType skippedEndId1 = 0; + SizeType32 endIdIndex = 0; + TokenIdType endId = 0; + auto const endIdRow = bi; + auto const inputLength = givenInputLengths.at(endIdRow); + do + { + auto const endIdBeam = std::rand() % beamWidth; + auto const firstOutputIndex + = tc::flat_index3(endIdRow, endIdBeam, inputLength, beamWidth, testData.maxSeqLen); + // We do not use the 1st token for EndId because of Speculative Decoding test design + // We skip 1st token because minLength is 1 + auto const endIdCol = 2 + (std::rand() % std::max(maxNewTokens - 2, 1)); + endIdIndex = firstOutputIndex + endIdCol; + skippedEndId0 = expectedOutputData[firstOutputIndex]; + skippedEndId1 = expectedOutputData[firstOutputIndex + 1]; + endId = expectedOutputData[endIdIndex]; + } while (endId == skippedEndId0 || endId == skippedEndId1); + // Workaround: The first example has endIdIndex 14, where the generation logits are almost same at + // token ids 257 and 373, which causes unstable generation results. Hence, we use the one previous + // token as endId. + if (bi == 0 && !replaceLogits) + { + endId = expectedOutputData[endIdIndex - 1]; + } + endIds.push_back(endId); + } + + return endIds; +} + +TestData loadTestData(ModelSpec const& modelSpec, ModelIds const modelIds, BeamResult const& beamResult, + ITensor const& givenInput, SizeType32 const maxBeamWidth, bool const useRandomEndId, bool const replaceLogits, + BufferManager& manager) +{ + auto const [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(givenInput, modelIds.padId); + auto const& [beamWidth, resultsFile, contextLogitsFile, genLogitsFile, cumLogProbsFile, logProbsFile] = beamResult; + + TestData testData{nbGivenInputs, beamWidth}; + testData.expectedOutputIds = loadNpy(manager, resultsFile.string(), MemoryType::kCPU); + + auto* const expectedOutputData = bufferCast<TokenIdType>(*testData.expectedOutputIds); + + auto const& outputShape = testData.expectedOutputIds->getShape(); + EXPECT_EQ(outputShape.nbDims, 2); + EXPECT_EQ(nbGivenInputs * beamWidth, outputShape.d[0]); + testData.maxSeqLen = static_cast<SizeType32>(outputShape.d[1]); + EXPECT_LE(maxInputLength, testData.maxSeqLen); + EXPECT_LE(beamWidth, maxBeamWidth); + + auto const maxNewTokens = testData.maxSeqLen - maxInputLength; + + std::srand(42); + + if (useRandomEndId) + { + testData.endIds = pickRandomEndIds(testData, givenInputLengths, maxNewTokens, replaceLogits); + } + else + { + testData.endIds.insert(testData.endIds.end(), nbGivenInputs, modelIds.endId); + } + + if (modelSpec.useLogits()) + { + testData.loadContextLogits(contextLogitsFile, givenInputLengths, manager); + } + if (modelSpec.useLogits() || modelSpec.mAcceptDraftByLogits) + { + testData.loadGenerationLogits(genLogitsFile, manager); + } + if (modelSpec.mReturnLogProbs) + { + testData.loadLogProbs(cumLogProbsFile, logProbsFile, manager); + } + + for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) + { + auto const endId = testData.endIds[bi]; + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + SizeType32 expectedLen = givenInputLengths[bi] + maxNewTokens; + for (SizeType32 si = givenInputLengths[bi]; si < testData.maxSeqLen; ++si) + { + auto const expectIndex = tc::flat_index2((bi * beamWidth + beam), si, testData.maxSeqLen); + if (expectedOutputData[expectIndex] == endId) + { + expectedLen = si; + break; + } + } + // Fill new EOS token to the expected data + for (SizeType32 si = expectedLen; si < testData.maxSeqLen; ++si) + { + auto const expectIndex = tc::flat_index2((bi * beamWidth + beam), si, testData.maxSeqLen); + expectedOutputData[expectIndex] = endId; + } + + testData.expectedOutputLengths[bi * beamWidth + beam] = expectedLen; + } + } + + if (modelSpec.mMaxDraftTokens > 0) + { + testData.makeDraft( + modelSpec.mMaxDraftTokens, modelSpec.mAcceptDraftByLogits, genLogitsFile, givenInputLengths, manager); + } + + return testData; +} + +std::tuple<std::vector<SizeType32>, std::unordered_map<SizeType32, TestData>> loadTestData(ModelSpec const& modelSpec, + ModelIds const modelIds, BeamResults const& resultsFilesBeamWidths, ITensor const& givenInput, + SizeType32 const maxBeamWidth, bool const useRandomEndId, bool const replaceLogits, BufferManager& manager) +{ + // Map between beam width, and expected results for that beam width + std::unordered_map<SizeType32, TestData> beamWidthTestData; + std::vector<SizeType32> beamWidths; + + for (auto const& beamResult : resultsFilesBeamWidths) + { + auto const beamWidth = beamResult.beamWidth; + + EXPECT_EQ(std::find(beamWidths.begin(), beamWidths.end(), beamWidth), beamWidths.end()); + beamWidths.push_back(beamWidth); + + auto testData = loadTestData( + modelSpec, modelIds, beamResult, givenInput, maxBeamWidth, useRandomEndId, replaceLogits, manager); + beamWidthTestData.emplace(beamWidth, std::move(testData)); + } + + return {std::move(beamWidths), std::move(beamWidthTestData)}; +} + +RequestList runGptModelInference(std::shared_ptr<TrtGptModel>& trtGptModel, std::vector<SizeType32> const& beamWidths, + std::unordered_map<SizeType32, TestData> const& beamWidthTestData, SizeType32 batchSize, SizeType32 nbGivenInputs, + SizeType32 maxInputLength, SizeType32 padId, std::vector<SizeType32> const& givenInputLengths, + TokenIdType const* givenInputData, ModelSpec const& modelSpec, TrtGptModelIfbTestType testType, int maxReqPerStep, + bool prepopulateKVCache, bool enableStreamingMode, bool enableBlockReuse) +{ + // Fill the requests using givenInput + // requestList will have batchSize requests + RequestList requestList; + + SizeType32 requestId = 0; + RequestList finishedRequestList; + std::vector<SizeType32> reqVec; + // Advance the requests until they are all finished + if (COMM_SESSION.getRank() == 0) + { + SizeType32 numReq = 0; + while (numReq < batchSize) + { + // Add appropriate number of requests in each iteration. For WAVEFRONT, this is always 1. + // For RANDOM, it could be any integer <= maxReqPerStep including 0. + SizeType32 reqThisStep{0}; + switch (testType) + { + case TrtGptModelIfbTestType::WAVEFRONT: reqThisStep = 1; break; + case TrtGptModelIfbTestType::RANDOM: reqThisStep = rand() % (maxReqPerStep + 1); break; + case TrtGptModelIfbTestType::BULK: [[fallthrough]]; + default: reqThisStep = batchSize; break; + } + reqThisStep = std::min(reqThisStep, (batchSize - numReq)); + reqVec.push_back(reqThisStep); + numReq += reqThisStep; + } + } + COMM_SESSION.bcast(reqVec, 0); + + SizeType32 reqVecIdx = 0; + while (requestId < batchSize || !requestList.empty()) + { + SizeType32 reqThisStep = reqVecIdx < reqVec.size() ? reqVec[reqVecIdx++] : 0; + for (SizeType32 req = 0; req < reqThisStep; req++) + { + // Alternate between beamWidths + SizeType32 beamWidth = beamWidths.at(requestId % beamWidths.size()); + auto const& testData = beamWidthTestData.at(beamWidth); + auto const* const expectedOutputData = bufferCast<TokenIdType const>(*testData.expectedOutputIds); + auto const maxSeqLen = testData.maxSeqLen; + + SamplingConfig samplingConfig{beamWidth}; + samplingConfig.temperature = std::vector{1.0f}; + samplingConfig.minLength = std::vector{1}; + samplingConfig.randomSeed = std::vector{static_cast<uint64_t>(42ull)}; + samplingConfig.topK = std::vector{1}; + samplingConfig.topP = std::vector{0.0f}; + samplingConfig.draftAcceptanceThreshold = std::vector{0.3f}; + samplingConfig.noRepeatNgramSize = std::vector{1 << 30}; + + auto const [givenInputIdx, inputLength] + = getRequestGivenInputIdxLength(requestId, nbGivenInputs, givenInputLengths); + SizeType32 endId = testData.endIds[givenInputIdx]; + + auto maxNewTokens = maxSeqLen - maxInputLength; + // Run model only to produce a single token and prepopulate KV cache + if (prepopulateKVCache || modelSpec.mKVCacheType == KVCacheType::kDISABLED) + { + maxNewTokens = 1; + } + auto const* const seqBegin = givenInputData + givenInputIdx * maxInputLength; + auto tokens = std::make_shared<std::vector<int32_t>>(seqBegin, seqBegin + inputLength); + if (!prepopulateKVCache && modelSpec.mMaxDraftTokens > 0) + { + // Append the 1st predicted token to the prompt to get the match with prepopulated KV cache + auto const expectIndex = tc::flat_index3(givenInputIdx, 0, inputLength, 1, maxSeqLen); + auto expectedToken = expectedOutputData[expectIndex]; + tokens->push_back(expectedToken); + // subtract this token from maxNewTokens + maxNewTokens -= 1; + } + auto r = std::make_shared<LlmRequest>(requestId, maxNewTokens, tokens, samplingConfig, false, endId, padId); + + auto const& draftTokens = testData.draftTokens[givenInputIdx]; + auto draftLogits = modelSpec.mAcceptDraftByLogits + ? std::make_optional<ITensor::SharedPtr>(testData.draftLogits[givenInputIdx]) + : std::nullopt; + if (!prepopulateKVCache && !draftTokens.empty()) + { + r->setDraftTokens(std::make_shared<std::vector<TokenIdType>>(draftTokens)); + r->setDraftLogits(draftLogits); + } + + SizeType32 maxDraftTokens{0}; + if (trtGptModel->getModelConfig().hasSpeculativeDecodingModule()) + { + maxDraftTokens + = trtGptModel->getModelConfig().getSpeculativeDecodingModulePtr()->getMaxDecodingDraftTokens(); + } + r->validate(trtGptModel->getMaxInputLen(), trtGptModel->getMaxSequenceLen(), maxDraftTokens, + trtGptModel->getVocabSizePadded(), std::nullopt, enableBlockReuse); + + if (enableStreamingMode) + { + r->setReturnAllGeneratedTokens(true); // Test allGeneratedTokens in this test + r->setStreaming(true); + } + + auto const vocabSizePadded + = trtGptModel->getModelConfig().getVocabSizePadded(trtGptModel->getWorldConfig().getSize()); + auto const logitDatatype = trtGptModel->getLogitDataType(); + if (modelSpec.mGatherLogits) + { + r->setReturnContextLogits(true); + r->setReturnGenerationLogits(true); + r->allocContextLogitsHost(vocabSizePadded, logitDatatype); + r->allocGenerationLogitsHost(vocabSizePadded, logitDatatype); + } + + if (!prepopulateKVCache && modelSpec.mAcceptDraftByLogits && !draftTokens.empty()) + { + r->allocTargetModelAcceptedTokenLogitsHost(vocabSizePadded, logitDatatype); + r->setReturnGenerationLogits(true); + } + + if (modelSpec.mReplaceLogits) + { + LlmRequest::LogitsPostProcessor logitsCb + = [&testData](uint64_t rId, tensorrt_llm::runtime::ITensor::SharedPtr& logits, + LlmRequest::BeamTokens const& tokens, + tensorrt_llm::runtime::BufferManager::CudaStreamPtr streamPtr, std::optional<uint64_t> cId) + { + auto const expectedGenerationLogits = testData.expectedGenerationLogits[rId]; + auto const expectedContextLogits = testData.expectedContextLogits[rId]; + auto const acceptedDraftTokensLengths = testData.acceptedDraftTokensLengths[rId]; + + auto const beamWidth = tokens.size(); + TLLM_CHECK_WITH_INFO(beamWidth == 1, "Logits substitution is not supported for beam search"); + + auto const genLogitsOffset = tokens[0].size() - expectedContextLogits->getShape().d[0]; + // TODO: Avoid static cast in TRT 10.0 + auto const numLogits = static_cast<SizeType32>(logits->getShape().d[0]); + auto const numVerifyLogits = std::min(numLogits, acceptedDraftTokensLengths + 1); + + TensorPtr logitsSlice = ITensor::slice(logits, 0, numVerifyLogits); + + auto manager = BufferManager(streamPtr); + TensorPtr logitsHost = manager.copyFrom(*logitsSlice, MemoryType::kCPU); + manager.getStream().synchronize(); + + TensorPtr refLogitsHost + = ITensor::slice(expectedGenerationLogits, genLogitsOffset, numVerifyLogits); + + EXPECT_TRUE(compareLogits(*refLogitsHost, *logitsHost, 0.f, 1e-2)) << "reqId: " << rId; + + manager.copy(*refLogitsHost, *logitsSlice); + }; + + r->mLogitsPostProcessor = logitsCb; + } + + if (modelSpec.mReturnLogProbs) + { + r->setReturnLogProbs(true); + } + requestList.push_back(r); + ++requestId; + } + + // Advance all active requests by one step + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + + // Check which requests are done, move them out + for (auto it = requestList.cbegin(); it != requestList.cend();) + { + if ((*it)->isGenerationCompleteState()) + { + finishedRequestList.push_back(*it); + requestList.erase(it++); + } + else + { + ++it; + } + } + } + return finishedRequestList; +} + +void runIfbTest(fs::path const& modelPath, ModelSpec const& modelSpec, ModelIds const modelIds, + TrtGptModelType modelType, std::vector<int32_t> const& batchSizes, BeamResults const& resultsFilesBeamWidths, + TrtGptModelIfbTestType testType, int maxReqPerStep, texec::ExecutorConfig const& executorConfig, + bool enableStreamingMode, bool useRandomEndId) +{ + auto manager = BufferManager(std::make_shared<CudaStream>()); + auto const padId = modelIds.padId; + + // Load input data + ASSERT_TRUE(fs::exists(DATA_PATH)); + auto const inputPath = DATA_PATH / modelSpec.mInputFile; + auto const& givenInput = loadNpy(manager, inputPath.string(), MemoryType::kCPU); + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, padId); + auto const* const givenInputData = bufferCast<TokenIdType const>(*givenInput); + + auto const& inputShape = givenInput->getShape(); + ASSERT_EQ(inputShape.nbDims, 2); + ASSERT_GT(inputShape.d[0], 0); + + auto const maxBeamWidth = executorConfig.getMaxBeamWidth(); + // Load expected outputs for each beam width value + auto [beamWidths, beamWidthTestData] = loadTestData(modelSpec, modelIds, resultsFilesBeamWidths, *givenInput, + maxBeamWidth, useRandomEndId, modelSpec.mReplaceLogits, manager); + + int const worldSize = modelSpec.mTPSize * modelSpec.mPPSize * modelSpec.mCPSize; + auto const worldConfig = WorldConfig::mpi(worldSize, modelSpec.mTPSize, modelSpec.mPPSize, modelSpec.mCPSize); + + ASSERT_TRUE(fs::exists(modelPath)); + + for (auto batchSize : batchSizes) + { + std::cout << "=== batchSize:" << batchSize << " ===\n"; + + auto trtGptModel = TrtGptModelFactory::create(modelPath, modelType, executorConfig, false); + + if (modelSpec.mKVCacheType == KVCacheType::kDISABLED) + { + ASSERT_FALSE(trtGptModel->hasKVCacheManager()); + } + + // Prepopulate KV cache for speculative decoding test + bool const prepopulateKVCache = modelSpec.mMaxDraftTokens > 0; + auto finishedRequestList = runGptModelInference(trtGptModel, beamWidths, beamWidthTestData, batchSize, + nbGivenInputs, maxInputLength, padId, givenInputLengths, givenInputData, modelSpec, testType, maxReqPerStep, + prepopulateKVCache, enableStreamingMode, modelSpec.mKVCacheReuse); + + if (prepopulateKVCache) + { + // Call the 2nd time with prefilled KV cache + finishedRequestList = runGptModelInference(trtGptModel, beamWidths, beamWidthTestData, batchSize, + nbGivenInputs, maxInputLength, padId, givenInputLengths, givenInputData, modelSpec, testType, + maxReqPerStep, false, enableStreamingMode, modelSpec.mKVCacheReuse); + } + + // WAR: disabled verification because of switched beams for different batch composition + if (worldConfig.isFirstPipelineParallelRank() + && (testType == TrtGptModelIfbTestType::BULK || maxBeamWidth == 1)) + { + bool shouldVerify = true; + + if (testType == TrtGptModelIfbTestType::BULK) + { + if (modelSpec.mKVCacheType == KVCacheType::kDISABLED && maxBeamWidth != 1) + { + // For disabled KV cache, only verify when maxBeamWidth is 1, the reason is we only compare with + // results with KV cache enabled case and usually, beams search results locate in last token while + // disabled KV cache only get exactly one new token. + shouldVerify = false; + } + } + + if (shouldVerify) + { + verifyOutput(finishedRequestList, beamWidthTestData, givenInputLengths, nbGivenInputs, modelSpec); + } + } + } +} + +struct BeamConfig +{ + SizeType32 maxBeamWidth; + std::vector<SizeType32> beamWidths; +}; + +} // namespace + +using ParamType = std::tuple<ModelParams, ModelSpec, TrtGptModelType, TrtGptModelIfbTestType, BeamConfig, // id: 0-4 + std::optional<int32_t>, // 5. maxTokensInPagedKvCache + std::optional<float>, // 6. freeGpuMemoryFraction + bool, // 7. enableTrtOverlap + bool, // 8. enableChunkedContext + bool, // 9. enableStreamingMode + bool, // 10. enableCudaGraphMode + std::optional<size_t>, // 11. hostCacheSize + bool, // 12. useRandomEndId + std::vector<SizeType32>, // 13. batchSizes + std::optional<SizeType32> // 14. maxNumTokens + >; + +std::string generateTestName(testing::TestParamInfo<ParamType> const& info) +{ + auto const modelSpec = std::get<1>(info.param); + std::string name; + switch (modelSpec.mDataType) + { + case nvinfer1::DataType::kFLOAT: name.append("Float"); break; + case nvinfer1::DataType::kHALF: name.append("Half"); break; + case nvinfer1::DataType::kINT8: name.append("Int8"); break; + case nvinfer1::DataType::kINT32: name.append("Int32"); + case nvinfer1::DataType::kBOOL: name.append("Bool"); break; + case nvinfer1::DataType::kUINT8: name.append("UInt8"); break; + case nvinfer1::DataType::kFP8: name.append("Float8"); break; + case nvinfer1::DataType::kBF16: name.append("BFloat16"); break; + case nvinfer1::DataType::kINT4: name.append("Int4"); break; + case nvinfer1::DataType::kFP4: name.append("Fp4"); break; + default: throw std::runtime_error("Unsupported DataType"); break; + } + + auto const modelType = std::get<2>(info.param); + switch (modelType) + { + case TrtGptModelType::InflightBatching: name.append("IbModel"); break; + case TrtGptModelType::InflightFusedBatching: name.append("FusedIbModel"); break; + default: name.append("DefaultModel"); break; + } + + switch (modelSpec.mKVCacheType) + { + case KVCacheType::kCONTINUOUS: name.append("ContinuousKVCache"); break; + case KVCacheType::kPAGED: name.append("PagedKVCache"); break; + case KVCacheType::kDISABLED: name.append("NoKVCache"); break; + default: throw std::runtime_error("Unknown KVCacheType"); break; + } + + auto const testType = std::get<3>(info.param); + switch (testType) + { + case TrtGptModelIfbTestType::BULK: name.append("Bulk"); break; + case TrtGptModelIfbTestType::WAVEFRONT: name.append("Wavefront"); break; + case TrtGptModelIfbTestType::RANDOM: name.append("Random"); break; + default: name.append("DefaultTest"); break; + } + BeamConfig const beamConfig = std::get<4>(info.param); + name.append("MaxBeamWidth" + std::to_string(beamConfig.maxBeamWidth)); + for (auto const beamWdith : beamConfig.beamWidths) + { + name.append("Bw" + std::to_string(beamWdith)); + } + + auto const maxTokensInPagedKvCache = std::get<5>(info.param); + if (maxTokensInPagedKvCache.has_value()) + { + name.append("KvCacheSize" + std::to_string(maxTokensInPagedKvCache.value())); + } + + auto const freeGpuMemoryFraction = std::get<6>(info.param); + if (freeGpuMemoryFraction.has_value()) + { + name.append("GpuFrac"); + } + + auto const enableTrtOverlap = std::get<7>(info.param); + if (enableTrtOverlap) + { + name.append("TrtOverlap"); + } + + auto const enableChunkedContext = std::get<8>(info.param); + if (enableChunkedContext) + { + name.append("Chunked"); + } + + if (modelSpec.mTPSize > 1) + { + name.append("TP" + std::to_string(modelSpec.mTPSize)); + } + + if (modelSpec.mPPSize > 1) + { + name.append("PP" + std::to_string(modelSpec.mPPSize)); + } + + if (modelSpec.mCPSize > 1) + { + name.append("CP" + std::to_string(modelSpec.mCPSize)); + } + + auto const useRandomEndId = std::get<12>(info.param); + if (useRandomEndId) + { + name.append("EndId"); + } + + if (modelSpec.mMaxDraftTokens > 0) + { + name.append("DraftTokens" + std::to_string(modelSpec.mMaxDraftTokens)); + } + + if (modelSpec.mAcceptDraftByLogits) + { + name.append("AcceptByLogits"); + } + + if (modelSpec.mCapacitySchedulerPolicy) + { + name.append(modelSpec.getCapacitySchedulerString()); + } + + auto const enableStreamingMode = std::get<9>(info.param); + if (enableStreamingMode) + { + name.append("Streaming"); + } + + auto const enableCudaGraphMode = std::get<10>(info.param); + if (enableCudaGraphMode) + { + name.append("CudaGraph"); + } + + auto const enableHostCache = std::get<11>(info.param); + if (enableHostCache) + { + name.append("SecondaryOffloading"); + } + + return name; +} + +class ParamTest : public TrtModelRealDecoderTest, public ::testing::WithParamInterface<ParamType> +{ +}; + +TEST_P(ParamTest, Test) +{ + + auto const& beamConfig = std::get<4>(GetParam()); + auto const& beamWidths = beamConfig.beamWidths; + + auto const modelParams = std::get<0>(GetParam()); + auto const modelIds = modelParams.ids; + auto const* const modelDir = modelParams.baseDir; + auto const modelSpec = std::get<1>(GetParam()); + + auto const useRandomEndId = std::get<12>(GetParam()); + + auto const batchSizes = std::get<13>(GetParam()); + + std::ostringstream gpuSizePath; + gpuSizePath << "tp" << modelSpec.mTPSize << "-pp" << modelSpec.mPPSize << "-cp" << modelSpec.mCPSize; + gpuSizePath << "-gpu"; + + auto const modelPath{ENGINE_PATH / modelDir / modelSpec.getModelPath() / gpuSizePath.str()}; + + auto const inputPath = DATA_PATH / modelSpec.mInputFile; + + BeamResults beamResults; + beamResults.reserve(beamWidths.size()); + for (auto beamWidth : beamWidths) + { + fs::path resultsPath + = DATA_PATH / modelDir / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + fs::path generationLogitsPath + = modelSpec.mCollectGenerationLogits ? (resultsPath / modelSpec.getGenerationLogitsFile()).string() : ""; + fs::path contextLogitsPath + = modelSpec.mCollectContextLogits ? (resultsPath / modelSpec.getContextLogitsFile()).string() : ""; + fs::path cumLogProbsPath + = modelSpec.mCollectCumLogProbs ? (resultsPath / modelSpec.getCumLogProbsFile()).string() : ""; + fs::path logProbsPath = modelSpec.mCollectLogProbs ? (resultsPath / modelSpec.getLogProbsFile()).string() : ""; + + beamResults.emplace_back(beamWidth, (resultsPath / modelSpec.getResultsFile()).string(), contextLogitsPath, + generationLogitsPath, cumLogProbsPath, logProbsPath); + } + + auto const modelType = std::get<2>(GetParam()); + auto const testType = std::get<3>(GetParam()); + auto const enableStreamingMode = std::get<9>(GetParam()); + auto const cudaGraphMode = std::get<10>(GetParam()); + + if (!(modelSpec.mUsePackedInput + && (modelSpec.mKVCacheType == KVCacheType::kPAGED || modelSpec.mKVCacheType == KVCacheType::kDISABLED))) + { + GTEST_SKIP() << "Inflight batching requires packed input and (paged KV cache or disabled KV cache)."; + } + + if (!modelSpec.mUsePackedInput && useRandomEndId) + { + GTEST_SKIP() << "Test does not support endId test with padded inputs"; + } + + for (auto beamWidth : beamWidths) + { + if (useRandomEndId && beamWidth > 1) + { + GTEST_SKIP() << "Test does not support endId test with beam search"; + } + + if (modelSpec.mMaxDraftTokens > 0 && beamWidth > 1) + { + GTEST_SKIP() << "Target model in speculative decoding does not support beam search"; + } + } + + auto executorConfig = texec::ExecutorConfig{}; + + auto const maxTokens = std::get<5>(GetParam()); + auto const enableBlockReuse = modelSpec.mMaxDraftTokens > 0 || modelSpec.mKVCacheReuse; + auto const freeGpuMemoryFraction = std::get<6>(GetParam()); + auto const hostCacheSize = std::get<11>(GetParam()); + auto const kvCacheConfig = texec::KvCacheConfig{ + enableBlockReuse, maxTokens, std::nullopt, std::nullopt, freeGpuMemoryFraction, hostCacheSize}; + executorConfig.setKvCacheConfig(kvCacheConfig); + + executorConfig.setEnableTrtOverlap(std::get<7>(GetParam())); + executorConfig.setEnableChunkedContext(std::get<8>(GetParam())); + auto const maxNumTokens = std::get<14>(GetParam()); + if (maxNumTokens.has_value()) + { + executorConfig.setMaxNumTokens(maxNumTokens.value()); + } + executorConfig.setNormalizeLogProbs(false); + executorConfig.setMaxBeamWidth(beamConfig.maxBeamWidth); + executorConfig.setGatherGenerationLogits(modelSpec.mCollectGenerationLogits); + auto extendedRuntimePerfKnobConfig = texec::ExtendedRuntimePerfKnobConfig{}; + extendedRuntimePerfKnobConfig.setCudaGraphMode(cudaGraphMode); + executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig); + + auto const capacitySchedulerPolicy + = modelSpec.mCapacitySchedulerPolicy.value_or(texec::CapacitySchedulerPolicy::kMAX_UTILIZATION); + executorConfig.setSchedulerConfig(texec::SchedulerConfig{capacitySchedulerPolicy}); + + if (modelSpec.mSpecDecodingMode == SpeculativeDecodingMode::LookaheadDecoding()) + { + auto decodingConfig = texec::DecodingConfig{}; + decodingConfig.setLookaheadDecodingConfig(texec::LookaheadDecodingConfig(5, 5, 5)); + executorConfig.setDecodingConfig(decodingConfig); + } + + for (auto beamWidth : beamWidths) + { + if (executorConfig.getEnableTrtOverlap() && beamWidth > 1) + { + GTEST_SKIP() << "TrtOverlap is not supported with beam search"; + } + } + + if (executorConfig.getEnableTrtOverlap() && modelSpec.mMaxDraftTokens > 0) + { + GTEST_SKIP() << "TrtOverlap is not supported with speculative decoding"; + } + + // Warning: This should be the last check before running the test. + // It will initialize MPI which can take significant time. + if (modelSpec.mTPSize * modelSpec.mPPSize * modelSpec.mCPSize != COMM_SESSION.getSize()) + { + GTEST_SKIP() << "Model's world size " << modelSpec.mPPSize * modelSpec.mTPSize * modelSpec.mCPSize + << " is not equal to the system world size"; + } + + runIfbTest(modelPath, modelSpec, modelIds, modelType, batchSizes, beamResults, testType, 2, executorConfig, + enableStreamingMode, useRandomEndId); +} + +auto constexpr gptModelParams = ModelParams{GPT_MODEL_DIR, ModelIds{50256, 50256}}; + +std::shared_ptr<ModelSpec> getGptDraftTestsCompareModelSpec() +{ + auto pModelSpec = std::make_shared<ModelSpec>(INPUT_FILE, nvinfer1::DataType::kHALF); + pModelSpec->useGptAttentionPlugin(); + pModelSpec->gatherLogits(); + pModelSpec->usePackedInput(); + pModelSpec->setKVCacheType(KVCacheType::kPAGED); + + return pModelSpec; +} + +std::shared_ptr<ModelSpec> getMedusaTestsCompareModelSpec() +{ + auto pModelSpec = std::make_shared<ModelSpec>(LONG_INPUT_FILE, nvinfer1::DataType::kHALF); + pModelSpec->useGptAttentionPlugin(); + pModelSpec->usePackedInput(); + pModelSpec->setKVCacheType(KVCacheType::kPAGED); + pModelSpec->setMaxOutputLength(128); + + return pModelSpec; +} + +std::shared_ptr<ModelSpec> getEagleTestsCompareModelSpec() +{ + auto pModelSpec = std::make_shared<ModelSpec>(LONG_INPUT_FILE, nvinfer1::DataType::kHALF); + pModelSpec->useGptAttentionPlugin(); + pModelSpec->usePackedInput(); + pModelSpec->setKVCacheType(KVCacheType::kPAGED); + pModelSpec->setMaxOutputLength(128); + + return pModelSpec; +} + +std::shared_ptr<ModelSpec> getGptChunkedContextTestsCompareModelSpec() +{ + auto pModelSpec = std::make_shared<ModelSpec>(LONG_INPUT_FILE, nvinfer1::DataType::kHALF); + pModelSpec->useGptAttentionPlugin(); + pModelSpec->usePackedInput(); + pModelSpec->setKVCacheType(KVCacheType::kPAGED); + pModelSpec->setMaxInputLength(128); + + return pModelSpec; +} + +INSTANTIATE_TEST_SUITE_P(GptTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput(), + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, + []() -> std::shared_ptr<ModelSpec> + { + auto pModelSpec = std::make_shared<ModelSpec>(INPUT_FILE, nvinfer1::DataType::kHALF); + pModelSpec->useGptAttentionPlugin().setKVCacheType(KVCacheType::kPAGED).usePackedInput(); + return pModelSpec; + }()} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kDISABLED) + .usePackedInput()), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values( + // TODO: enable more tests when mixed beam width is supported + BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} + ), + testing::Values(std::nullopt, 1280), // maxTokensInPagedKvCache + testing::Values(std::nullopt, 0.4), // freeGpuMemoryFraction + testing::Values(true), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptRandomEndIdTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput()), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values( + // TODO: enable more tests when mixed beam width is supported + BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} + ), + testing::Values(std::nullopt, 1280), // maxTokensInPagedKvCache + testing::Values(std::nullopt, 0.4), // freeGpuMemoryFraction + testing::Values(true), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(true), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptKVOffloadingTest, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // + ModelSpec{LONG_INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput() + .setKVCacheReuse(true)), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), + testing::Values(256), // maxTokensInPagedKvCache + testing::Values(std::nullopt, 0.4), // freeGpuMemoryFraction + testing::Values(true), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(100000000), // hostCacheSize + testing::Values(false, true), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptCudaGraphTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput() + .capacitySchedulerPolicy(texec::CapacitySchedulerPolicy::kSTATIC_BATCH), + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput() + .capacitySchedulerPolicy(texec::CapacitySchedulerPolicy::kMAX_UTILIZATION)), + testing::Values(TrtGptModelType::InflightBatching, TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values( + // TODO: enable more tests when mixed beam width is supported + BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} + ), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(true), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptSwitchBwTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput()), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values( + // TODO: enable more tests when mixed beam width is supported + BeamConfig{2, {1}} // , BeamConfig{2, {1, 2}} + ), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{4}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptNProfilesTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values(ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useMultipleProfiles()), + testing::Values(TrtGptModelType::InflightFusedBatching), testing::Values(TrtGptModelIfbTestType::BULK), + testing::Values( + // TODO: enable more tests when mixed beam width is supported + BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} + ), + testing::Values(std::nullopt, 1280), // maxTokensInPagedKvCache + testing::Values(std::nullopt, 0.4), // freeGpuMemoryFraction + testing::Values(true), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(true), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptSqTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values(ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .setQuantMethod(QuantMethod::kSMOOTH_QUANT), + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, + []() -> std::shared_ptr<ModelSpec> + { + auto pModelSpec = std::make_shared<ModelSpec>(INPUT_FILE, nvinfer1::DataType::kHALF); + pModelSpec->useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .setQuantMethod(QuantMethod::kSMOOTH_QUANT); + return pModelSpec; + }()} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kDISABLED) + .setQuantMethod(QuantMethod::kSMOOTH_QUANT)), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values( + // TODO: enable more tests when mixed beam width is supported + // FIXME: disabled flaky beam search tests (https://nvbugspro.nvidia.com/bug/4646234) + BeamConfig{1, {1}} //, BeamConfig{2, {2}} + ), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +// disabled because paused requests generate different tokens after resuming +INSTANTIATE_TEST_SUITE_P(DISABLED_GptChunkedContextTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, getGptChunkedContextTestsCompareModelSpec()} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .setMaxInputLength(128)), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values(TrtGptModelIfbTestType::BULK), // TrtGptModelIfbTestType + testing::Values(BeamConfig{1, {1}}), // beam config + testing::Values(257), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptChunkedLongContextTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // + ModelSpec{LONG_INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .setMaxInputLength(128), + ModelSpec{LONG_INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useDraftTokensExternalDecoding() + .setDraftTokens(5)), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values(TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, + TrtGptModelIfbTestType::RANDOM), // TrtGptModelIfbTestType + testing::Values(BeamConfig{1, {1}}), // beam config + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(true), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(64) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptDraftTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, getGptDraftTestsCompareModelSpec()} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useDraftTokensExternalDecoding() + .setDraftTokens(5) + .replaceLogits() + .collectGenerationLogitsFile() + .collectContextLogitsFile(), + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF, getGptDraftTestsCompareModelSpec()} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useDraftTokensExternalDecoding() + .setDraftTokens(5) + .useAcceptByLogits() + .replaceLogits() + .collectGenerationLogitsFile() + .collectContextLogitsFile()), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), // beamConfig + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false, true), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptLogitsTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // modelSpec + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .gatherLogits() + .collectGenerationLogitsFile() + .collectContextLogitsFile()), + testing::Values(TrtGptModelType::InflightBatching, TrtGptModelType::InflightFusedBatching), // modelType + testing::Values(TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, + TrtGptModelIfbTestType::RANDOM), // testType + testing::Values(BeamConfig{1, {1}}), // beamConfig + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false, true), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(true), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptLogProbsTests, ParamTest, + testing::Combine(testing::Values(gptModelParams), + testing::Values( + // modelSpec + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .returnLogProbs() + .collectCumLogProbsFile() + .collectLogProbsFile()), + testing::Values(TrtGptModelType::InflightFusedBatching), // modelType + testing::Values(TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, + TrtGptModelIfbTestType::RANDOM), // testType + testing::Values(BeamConfig{1, {1}}), // beamConfig + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptjTests, ParamTest, + testing::Combine(testing::Values(ModelParams{GPTJ_MODEL_DIR, {50256, 50256}}), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kCONTINUOUS) + .usePackedInput(), + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput() + + ), + testing::Values(TrtGptModelType::InflightFusedBatching), + // WAR: disable wavefront and random tests on because of switched beams + testing::Values(TrtGptModelIfbTestType::BULK + /* , TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM */), + testing::Values( + // TODO: enable more tests when mixed beam width is supported + BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} + ), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(MambaTests, ParamTest, + testing::Combine(testing::Values(ModelParams{MAMBA_MODEL_DIR, {0, 1}}), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kCONTINUOUS) + .usePackedInput(), + ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput() + + ), + testing::Values(TrtGptModelType::InflightBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(RecurrentGemmaTests, ParamTest, + testing::Combine(testing::Values(ModelParams{RECURRENTGEMMA_MODEL_DIR, {0, 1}}), + testing::Values(ModelSpec{INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput() + + ), + testing::Values(TrtGptModelType::InflightBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(LlamaTests, ParamTest, + testing::Combine(testing::Values(ModelParams{LLAMA_MODEL_DIR, {LLAMA_END_ID, LLAMA_PAD_ID}}), + testing::Values( + // + ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput(), + ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .usePipelineParallelism(4), + ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useTensorParallelism(4), + ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .usePipelineParallelism(2) + .useTensorParallelism(2) + + ), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values( + // TODO: enable more tests when mixed beam width is supported + BeamConfig{1, {1}}, BeamConfig{2, {2}} // , BeamConfig{2, {1, 2}} + ), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(ChatGlmTests, ParamTest, + testing::Combine(testing::Values(ModelParams{CHATGLM_MODEL_DIR, {130005, 3}}), + testing::Values( + // + ModelSpec{CHATGLM_INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED)), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(false, true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +// ChatGlm0Tests is for glm-10b. +INSTANTIATE_TEST_SUITE_P(ChatGlm0Tests, ParamTest, + testing::Combine(testing::Values(ModelParams{GLM_MODEL_DIR, {50258, 50256}}), + testing::Values( + // + ModelSpec{GLM_INPUT_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED)), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +// https://nvbugspro.nvidia.com/bug/4640177 +// WAVEFRONT and RANDOM are disabled because of the accuracy mismatch +INSTANTIATE_TEST_SUITE_P(MedusaTests, ParamTest, + testing::Combine(testing::Values(ModelParams{MEDUSA_MODEL_DIR, {2, 2}}), + testing::Values( + // + ModelSpec{INPUT_VICUNA_FILE, nvinfer1::DataType::kHALF, getMedusaTestsCompareModelSpec()} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useMedusa()), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(true, false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(EagleTests, ParamTest, + testing::Combine(testing::Values(ModelParams{EAGLE_MODEL_DIR, {2, 2}}), + testing::Values( + // + ModelSpec{INPUT_VICUNA_FILE, nvinfer1::DataType::kHALF, getEagleTestsCompareModelSpec()} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useEagle()), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(true, false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(LlamaLookaheadDecodingTests, ParamTest, + testing::Combine(testing::Values(ModelParams{LLAMA_MODEL_DIR, {LLAMA_END_ID, LLAMA_PAD_ID}}), + testing::Values( + // + ModelSpec{INPUT_LLAMA_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useLookaheadDecoding()), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), // beamConfig + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(false), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(true), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 16}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + + generateTestName); + +INSTANTIATE_TEST_SUITE_P(ExplicitDraftTokensDecodingTests, ParamTest, + testing::Combine(testing::Values(ModelParams{EXPLICIT_DRAFT_MODEL_DIR, {2, 2}}), + testing::Values( + // + ModelSpec{INPUT_VICUNA_FILE, nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useExplicitDraftTokensDecoding() + .setMaxOutputLength(128)), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values(BeamConfig{1, {1}}), // beamConfig + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + + generateTestName); + +#ifdef ENABLE_FP8 +// Using IFB-enabled engine +INSTANTIATE_TEST_SUITE_P(GptjFP8Tests, ParamTest, + testing::Combine(testing::Values(ModelParams{GPTJ_MODEL_DIR, {50256, 50256}}), + testing::Values( + // + ModelSpec{INPUT_FILE, nvinfer1::DataType::kFP8} + .useGptAttentionPlugin() + .setKVCacheType(KVCacheType::kPAGED) + .usePackedInput() + + ), + testing::Values(TrtGptModelType::InflightFusedBatching), + testing::Values( + TrtGptModelIfbTestType::BULK, TrtGptModelIfbTestType::WAVEFRONT, TrtGptModelIfbTestType::RANDOM), + testing::Values( + // TODO: enable more tests when supported + BeamConfig{1, {1}} // , BeamConfig{2, {2}}, BeamConfig{2, {1, 2}} + ), + testing::Values(std::nullopt), // maxTokensInPagedKvCache + testing::Values(0.4), // freeGpuMemoryFraction + testing::Values(false), // enableTrtOverlap + testing::Values(true), // enableChunkedContext + testing::Values(false), // enableStreamingMode + testing::Values(false), // enableCudaGraphMode + testing::Values(std::nullopt), // hostCacheSize + testing::Values(false), // useRandomEndId + testing::Values(std::vector<SizeType32>{1, 2, 8}), // batchSizes + testing::Values(std::nullopt) // maxNumTokens + ), + generateTestName); + +#endif diff --git a/cpp/tests/e2e_tests/batch_manager/trtGptModelTest.cpp b/cpp/tests/e2e_tests/batch_manager/trtGptModelTest.cpp new file mode 100644 index 000000000000..268e9bf9a238 --- /dev/null +++ b/cpp/tests/e2e_tests/batch_manager/trtGptModelTest.cpp @@ -0,0 +1,1328 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TOP_LEVEL_DIR +#error "Define TOP_LEVEL_DIR" +#endif + +#include "tensorrt_llm/batch_manager/trtGptModel.h" +#include "tensorrt_llm/batch_manager/kvCacheManager.h" +#include "tensorrt_llm/batch_manager/trtGptModelInflightBatching.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/gptJsonConfig.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/testing/modelSpec.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +#include <filesystem> +#include <memory> +#include <vector> + +using ::testing::ElementsAre; +using namespace tensorrt_llm::runtime; +namespace fs = std::filesystem; +using tensorrt_llm::testing::ModelSpec; +using tensorrt_llm::testing::KVCacheType; + +using TensorPtr = ITensor::SharedPtr; + +namespace +{ +auto const TEST_RESOURCE_PATH = fs::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; +auto const ENGINE_PATH = TEST_RESOURCE_PATH / "models/rt_engine"; +auto const GPT_MODEL_PATH = ENGINE_PATH / "gpt2"; +auto const LLAMA_MODEL_PATH = ENGINE_PATH / "Llama-3.2-1B"; +} // namespace + +namespace tensorrt_llm::batch_manager +{ + +class TrtGptModelTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) +{ +protected: + TrtGptModelTest(std::filesystem::path const& modelPath) + : mModelConfig(1, 1, 1, 0, 1, 1, nvinfer1::DataType::kFLOAT) + , mModelPath(modelPath) + { + } + + TrtGptModelTest() + : TrtGptModelTest(GPT_MODEL_PATH / GetModelSpec().getModelPath() / "tp1-pp1-cp1-gpu") + { + } + + static ModelSpec& GetModelSpec() + { + static ModelSpec modelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF}; + modelSpec.useGptAttentionPlugin().usePackedInput().setKVCacheType(KVCacheType::kPAGED); + return modelSpec; + } + + void SetUp() override + { + std::filesystem::path trtEnginePath = mModelPath; + + mBeamWidth = 1; + + mLogger = std::make_shared<TllmLogger>(); + + initTrtLlmPlugins(mLogger.get()); + + auto const json = GptJsonConfig::parse(trtEnginePath / "config.json"); + mModelConfig = json.getModelConfig(); + mMaxNumRequests = mModelConfig.getMaxBatchSize(); + mMaxSeqLen = mModelConfig.getMaxSequenceLen(); + mWorldConfig = WorldConfig::mpi(); + mVocabSizePadded = mModelConfig.getVocabSizePadded(mWorldConfig.getSize()); + + auto const enginePath = trtEnginePath / json.engineFilename(mWorldConfig); + auto const dtype = mModelConfig.getDataType(); + + mRawEngine.reset(new RawEngine(enginePath)); + + mSamplingConfig.temperature = std::vector{1.0f}; + mSamplingConfig.minLength = std::vector{1}; + mSamplingConfig.randomSeed = std::vector{static_cast<uint64_t>(42ul)}; + mSamplingConfig.topK = std::vector{0}; + mSamplingConfig.topP = std::vector{0.0f}; + mSamplingConfig.noRepeatNgramSize = std::vector{1 << 30}; + + mStream = std::make_unique<CudaStream>(); + mManager = std::make_unique<BufferManager>(mStream); + } + + void TearDown() override {} + + // Thin wrapper around the private TrtGptModelInflightBatching::changeBeamWidth(). + static void changeBeamWidth(std::shared_ptr<TrtGptModelInflightBatching> const& model, SizeType32 beamWidth) + { + model->changeBeamWidth(beamWidth); + } + + void forwardRequestsToCompletion( + std::shared_ptr<TrtGptModel> const& trtGptModel, RequestList& requestList, SizeType32 maxNumIterations) + { + SizeType32 numFinished = 0; + SizeType32 numIterations = 0; + while (numFinished < requestList.size() && numIterations < maxNumIterations) + { + if (numIterations > maxNumIterations) + { + FAIL() << "Iterations never finished"; + } + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + numFinished = 0; + for (auto& request : requestList) + { + if (request->isGenerationCompleteState()) + { + ++numFinished; + } + } + ++numIterations; + } + } + + int32_t mMaxNumRequests; + int32_t mMaxSeqLen; + int32_t mBeamWidth; + int32_t mVocabSizePadded; + SamplingConfig mSamplingConfig; + std::string mDataPath; + std::shared_ptr<nvinfer1::ILogger> mLogger; + ModelConfig mModelConfig; + WorldConfig mWorldConfig; + std::unique_ptr<RawEngine> mRawEngine; + std::unique_ptr<BufferManager> mManager; + BufferManager::CudaStreamPtr mStream; + std::filesystem::path mModelPath; +}; + +class TrtGptModelLoraTest : public TrtGptModelTest +{ +protected: + TrtGptModelLoraTest() + : TrtGptModelTest(GPT_MODEL_PATH / GetModelSpec().getModelPath() / "tp1-pp1-cp1-gpu") + { + } + + static ModelSpec& GetModelSpec() + { + static ModelSpec modelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF}; + modelSpec.useGptAttentionPlugin().usePackedInput().setKVCacheType(KVCacheType::kPAGED).useLoraPlugin(); + return modelSpec; + } +}; + +TEST_F(TrtGptModelTest, Forward) +{ + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 4; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + + RequestList requestList{llmRequest}; + + auto& manager = *mManager; + std::vector<int32_t> newTokensHost(mMaxNumRequests, 5); + TensorPtr const fakeNewTokens + = manager.copyFrom(newTokensHost, ITensor::makeShape({mMaxNumRequests, 1}), MemoryType::kGPU); + + std::vector<bool> finished(mMaxNumRequests, false); + + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + // Generate one token for the requests in request_table + // We need to sync with decoder + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + + EXPECT_EQ(requestList.size(), 1); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); + EXPECT_EQ(requestList.front()->getNumTokens(0), 5); + EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2)); +} + +TEST_F(TrtGptModelTest, ChangeBeamWidthClearsCudaGraphCache) +{ + if (mModelConfig.getMaxBeamWidth() < 2) + { + GTEST_SKIP() << "Engine was built with max_beam_width < 2; cannot exercise changeBeamWidth()."; + } + + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + // Configure the executor for max beam width = 2 so we can transition between + // operating beam widths 1 and 2. + executorConfig.setMaxBeamWidth(2); + executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); + + auto extendedRuntimePerfKnobConfig = executor::ExtendedRuntimePerfKnobConfig{}; + extendedRuntimePerfKnobConfig.setCudaGraphMode(true); + extendedRuntimePerfKnobConfig.setCudaGraphCacheSize(8); + executorConfig.setExtendedRuntimePerfKnobConfig(extendedRuntimePerfKnobConfig); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + // Run a single beam=1 request to completion. After at least one generation step + // the model captures and caches a CUDA graph for the subsequent batch state. + SamplingConfig samplingConfig; + samplingConfig.beamWidth = 1; + samplingConfig.temperature = std::vector{1.0f}; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); + auto llmRequest = std::make_shared<LlmRequest>( + /*requestId=*/0, /*maxNewTokens=*/4, tokens, samplingConfig, /*isStreaming=*/false); + RequestList requestList{llmRequest}; + + forwardRequestsToCompletion(trtGptModel, requestList, /*maxNumIterations=*/8); + + // Cache must have been populated by the captured generation graph(s). + EXPECT_GT(trtGptModel->numCachedCudaGraphs(), 0) + << "Expected the CUDA graph executor cache to be populated after running a " + "beam=1 request to completion."; + + // Drop the completed request before changing beam width (changeBeamWidth requires + // no in-flight requests). + requestList.clear(); + + // Switch operating beam width via the fixture's friend-access helper. + changeBeamWidth(trtGptModel, 2); + + EXPECT_EQ(trtGptModel->numCachedCudaGraphs(), 0) + << "changeBeamWidth() must invalidate the CUDA graph executor cache. Stale " + "cudaGraphExec_t instances captured against the previous decoder state " + "would otherwise be replayed against freshly allocated memory."; +} + +TEST_F(TrtGptModelLoraTest, Forward) +{ + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 4; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + + RequestList requestList{llmRequest}; + + auto& manager = *mManager; + std::vector<int32_t> newTokensHost(mMaxNumRequests, 5); + TensorPtr const fakeNewTokens + = manager.copyFrom(newTokensHost, ITensor::makeShape({mMaxNumRequests, 1}), MemoryType::kGPU); + + std::vector<bool> finished(mMaxNumRequests, false); + + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + // Generate one token for the requests in request_table + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + + EXPECT_EQ(requestList.size(), 1); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); + EXPECT_EQ(requestList.front()->getNumTokens(0), 5); + EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2)); +} + +TEST_F(TrtGptModelTest, ForwardMaxNewTokens) +{ + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setMaxTokens(10000); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 4; + auto tokens = std::make_shared<std::vector<int32_t>>(256); + std::iota(std::begin(*tokens), std::end(*tokens), 1); + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + + int correlationId2 = 2; + auto maxNewTokens2 = 8; + auto llmRequest2 = std::make_shared<LlmRequest>(correlationId2, maxNewTokens2, tokens, inSamplingConfig, false); + + RequestList requestList{llmRequest, llmRequest2}; + + auto& manager = *mManager; + std::vector<bool> finished(mMaxNumRequests, false); + + // Generate one token for the requests in request_table + // We call forward twice because the first call doesn't sync with decoder + SizeType32 maxNumIterations = 13; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + + for (auto& request : requestList) + { + auto outputTokens = request->getTokens(0); + if (request->mRequestId == correlationId) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + } + if (request->mRequestId == correlationId2) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens2); + } + } +} + +TEST_F(TrtGptModelTest, MaxNumTokensInChunked) +{ + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setEnableChunkedContext(true); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); + + auto modelConfig = mModelConfig; + mModelConfig.setMaxNumTokens(200); + + auto trtGptModelIfb = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + std::vector<std::shared_ptr<TrtGptModel>> trtGptModels{trtGptModelIfb}; + + for (auto trtGptModel : trtGptModels) + { + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 4; + auto tokens = std::make_shared<std::vector<int32_t>>(256); + std::iota(std::begin(*tokens), std::end(*tokens), 1); + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + + int correlationId2 = 2; + auto maxNewTokens2 = 8; + auto llmRequest2 = std::make_shared<LlmRequest>(correlationId2, maxNewTokens2, tokens, inSamplingConfig, false); + + RequestList requestList{llmRequest, llmRequest2}; + + auto& manager = *mManager; + std::vector<bool> finished(mMaxNumRequests, false); + + // Generate one token for the requests in request_table + // We call forward twice because the first call doesn't sync with decoder + SizeType32 maxNumIterations = 13; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + + for (auto& request : requestList) + { + auto outputTokens = request->getTokens(0); + if (request->mRequestId == correlationId) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + } + if (request->mRequestId == correlationId2) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens2); + } + } + } +} + +TEST_F(TrtGptModelTest, ForwardEndId) +{ + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setMaxTokens(10000); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 4; + auto endId = 107; + auto tokens = std::make_shared<std::vector<int32_t>>(256); + std::iota(std::begin(*tokens), std::end(*tokens), 1); + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, endId); + + int correlationId2 = 2; + auto maxNewTokens2 = 8; + auto llmRequest2 + = std::make_shared<LlmRequest>(correlationId2, maxNewTokens2, tokens, inSamplingConfig, false, endId); + + RequestList requestList{llmRequest, llmRequest2}; + + auto& manager = *mManager; + std::vector<bool> finished(mMaxNumRequests, false); + + // Generate one token for the requests in request_table + // We call forward twice because the first call doesn't sync with decoder + SizeType32 maxNumIterations = 13; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + + for (auto& request : requestList) + { + auto outputTokens = request->getTokens(0); + // endId token is generated at 2nd iteration, so expect 1 output token + if (request->mRequestId == correlationId) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + 1); + } + if (request->mRequestId == correlationId2) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + 1); + } + } +} + +TEST_F(TrtGptModelTest, ForwardNoEoS) +{ + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kSTATIC_BATCH}); + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setMaxTokens(10000); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + SamplingConfig inSamplingConfig; + inSamplingConfig.topP = {0.9}; + inSamplingConfig.temperature = {0.6}; + inSamplingConfig.minLength = {5}; + + auto tokens = std::make_shared<std::vector<int32_t>>(256); + std::iota(std::begin(*tokens), std::end(*tokens), 1); + + RequestList requestList; + for (auto requestIdx = 0; requestIdx < mMaxNumRequests; requestIdx++) + { + auto llmRequest = std::make_shared<LlmRequest>(requestIdx, 8, tokens, inSamplingConfig, false, -1); + requestList.push_back(llmRequest); + } + + auto& manager = *mManager; + std::vector<bool> finished(mMaxNumRequests, false); + + // Generate one token for the requests in request_table + // We call forward twice because the first call doesn't sync with decoder + SizeType32 maxNumIterations = 13; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); +} + +TEST_F(TrtGptModelTest, ForwardFinished) +{ + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 2; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{10, 9, 8, 7, 6}); + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + + RequestList requestList{llmRequest}; + + int mForwardCount = 0; + + auto& manager = *mManager; + std::vector<int32_t> newTokensHost(mMaxNumRequests, 5); + TensorPtr const fakeNewTokens + = manager.copyFrom(newTokensHost, ITensor::makeShape({mMaxNumRequests, 1}), MemoryType::kGPU); + + std::vector<int32_t> newTokensHost2(mMaxNumRequests, 4); + TensorPtr const fakeNewTokens2 + = manager.copyFrom(newTokensHost2, ITensor::makeShape({mMaxNumRequests, 1}), MemoryType::kGPU); + + // Below are only used if beam > 1 + // So we are just returning tensors with the correct shape, content is not important + std::vector<int32_t> outputIdsHost(mMaxNumRequests * (5 + 2), 5); + TensorPtr const fakeOutputIds + = manager.copyFrom(outputIdsHost, ITensor::makeShape({mMaxNumRequests, 1, 5 + 2}), MemoryType::kGPU); + + std::vector<bool> finishedFalse(mMaxNumRequests, false); + std::vector<bool> finishedTrue(mMaxNumRequests, true); + + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + // Generate one token for the requests in request_table + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + + EXPECT_EQ(requestList.size(), 1); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); + EXPECT_EQ(requestList.front()->getNumTokens(0), 6); + EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10)); + + // Generate one more token + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + + EXPECT_EQ(requestList.size(), 1); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + EXPECT_EQ(requestList.front()->getNumTokens(0), 7); + EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 2); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6)); +} + +TEST_F(TrtGptModelTest, ForwardStopWords) +{ + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setMaxTokens(10000); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 4; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{10, 9, 8, 7, 6}); + std::optional<SizeType32> endId(std::nullopt); + std::optional<SizeType32> padId(std::nullopt); + std::optional<TensorPtr> embeddingBias(std::nullopt); + std::optional<TensorPtr> badWordsList(std::nullopt); + + auto& manager = *mManager; + // No stop words + { + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + RequestList requestList{llmRequest}; + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); + } + // With stop words + { + TensorPtr stopWordsList = manager.cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); + auto stopWordsPtr = bufferCast<int32_t>(*stopWordsList); + // make 10, 6 10 the tokens for the stop word: + stopWordsPtr[0] = 10; + stopWordsPtr[1] = 6; + stopWordsPtr[2] = 10; + stopWordsPtr[3] = 3; + stopWordsPtr[4] = -1; + stopWordsPtr[5] = -1; + + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, + endId, padId, embeddingBias, badWordsList, stopWordsList); + RequestList requestList{llmRequest}; + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10)); + } + + // With stop words + { + TensorPtr stopWordsList = manager.cpu(ITensor::makeShape({1, 2, 1}), nvinfer1::DataType::kINT32); + auto stopWordsPtr = bufferCast<int32_t>(*stopWordsList); + // make 10 is the token for the stop word: + stopWordsPtr[0] = 10; + stopWordsPtr[1] = 1; + + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, + endId, padId, embeddingBias, badWordsList, stopWordsList); + RequestList requestList{llmRequest}; + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10)); + } + + // Multiple requests, each with different stop words + { + // Request w/o stop words + auto llmRequest = std::make_shared<LlmRequest>(1, maxNewTokens, tokens, inSamplingConfig, false); + + TensorPtr stopWordsList2 = manager.cpu(ITensor::makeShape({1, 2, 1}), nvinfer1::DataType::kINT32); + { + auto stopWordsPtr = bufferCast<int32_t>(*stopWordsList2); + stopWordsPtr[0] = 10; + stopWordsPtr[1] = 1; + } + auto llmRequest2 = std::make_shared<LlmRequest>(2, maxNewTokens, tokens, inSamplingConfig, false, endId, padId, + embeddingBias, badWordsList, stopWordsList2); + + TensorPtr stopWordsList3 = manager.cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); + { + auto stopWordsPtr = bufferCast<int32_t>(*stopWordsList3); + stopWordsPtr[0] = 10; + stopWordsPtr[1] = 6; + stopWordsPtr[2] = 10; + stopWordsPtr[3] = 3; + stopWordsPtr[4] = -1; + stopWordsPtr[5] = -1; + } + auto llmRequest3 = std::make_shared<LlmRequest>(3, maxNewTokens, tokens, inSamplingConfig, false, endId, padId, + embeddingBias, badWordsList, stopWordsList3); + + RequestList requestList{llmRequest, llmRequest2, llmRequest3}; + + SizeType32 maxNumIterations(5); + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + + for (auto& request : requestList) + { + auto outputTokens = request->getTokens(0); + if (request->mRequestId == 1) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); + } + if (request->mRequestId == 2) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + 1); + EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10)); + } + if (request->mRequestId == 3) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + 3); + EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10)); + } + } + } +} + +TEST_F(TrtGptModelTest, ForwardBadWords) +{ + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setMaxTokens(10000); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 4; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{10, 9, 8, 7, 6}); + std::optional<SizeType32> endId(std::nullopt); + std::optional<SizeType32> padId(std::nullopt); + std::optional<TensorPtr> embeddingBias(std::nullopt); + std::optional<TensorPtr> stopWordsList(std::nullopt); + + auto& manager = *mManager; + // No bad words + { + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + RequestList requestList{llmRequest}; + + SizeType32 maxNumIterations = 5; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); + } + // With bad words, multiple tokens + { + TensorPtr badWordsList = manager.cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); + auto badWordsPtr = bufferCast<int32_t>(*badWordsList); + // make 10, 6 10 the tokens for the bad word: + badWordsPtr[0] = 10; + badWordsPtr[1] = 6; + badWordsPtr[2] = 10; + badWordsPtr[3] = 3; + badWordsPtr[4] = -1; + badWordsPtr[5] = -1; + + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, + endId, padId, embeddingBias, badWordsList, stopWordsList); + RequestList requestList{llmRequest}; + SizeType32 maxNumIterations = 5; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + // Token at position 7 should be different than 10 + EXPECT_NE(requestList.front()->getTokens(0).at(7), 10); + } + + // With bad words single token + { + TensorPtr badWordsList = manager.cpu(ITensor::makeShape({1, 2, 1}), nvinfer1::DataType::kINT32); + auto badWordsPtr = bufferCast<int32_t>(*badWordsList); + // make 10 is the token for the bad word: + badWordsPtr[0] = 10; + badWordsPtr[1] = 1; + + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, + endId, padId, embeddingBias, badWordsList, stopWordsList); + RequestList requestList{llmRequest}; + SizeType32 maxNumIterations = 5; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + EXPECT_NE(requestList.front()->getTokens(0).at(5), 10); + } + + // Multiple requests, each with different bad words + { + // Request w/o bad words + auto llmRequest = std::make_shared<LlmRequest>(1, maxNewTokens, tokens, inSamplingConfig, false); + + TensorPtr badWordsList2 = manager.cpu(ITensor::makeShape({1, 2, 1}), nvinfer1::DataType::kINT32); + { + auto badWordsPtr = bufferCast<int32_t>(*badWordsList2); + badWordsPtr[0] = 10; + badWordsPtr[1] = 1; + } + auto llmRequest2 = std::make_shared<LlmRequest>(2, maxNewTokens, tokens, inSamplingConfig, false, endId, padId, + embeddingBias, badWordsList2, stopWordsList); + + TensorPtr badWordsList3 = manager.cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); + { + auto badWordsPtr = bufferCast<int32_t>(*badWordsList3); + badWordsPtr[0] = 10; + badWordsPtr[1] = 6; + badWordsPtr[2] = 10; + badWordsPtr[3] = 3; + badWordsPtr[4] = -1; + badWordsPtr[5] = -1; + } + auto llmRequest3 = std::make_shared<LlmRequest>(3, maxNewTokens, tokens, inSamplingConfig, false, endId, padId, + embeddingBias, badWordsList3, stopWordsList); + + RequestList requestList{llmRequest, llmRequest2, llmRequest3}; + + SizeType32 maxNumIterations(6); + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + + for (auto& request : requestList) + { + auto outputTokens = request->getTokens(0); + if (request->mRequestId == 1) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); + } + if (request->mRequestId == 2) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + EXPECT_NE(request->getTokens(0).at(5), 10); + } + if (request->mRequestId == 3) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + EXPECT_NE(request->getTokens(0).at(7), 10); + } + } + } +} + +TEST_F(TrtGptModelTest, ForwardEmbeddingBias) +{ + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setMaxTokens(10000); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto trtGptModelIfb = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + std::vector<std::shared_ptr<TrtGptModel>> trtGptModels{trtGptModelIfb}; + + for (auto& trtGptModel : trtGptModels) + { + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 4; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{10, 9, 8, 7, 6}); + std::optional<SizeType32> endId(std::nullopt); + std::optional<SizeType32> padId(std::nullopt); + std::optional<TensorPtr> badWordsList(std::nullopt); + std::optional<TensorPtr> stopWordsList(std::nullopt); + + auto& manager = *mManager; + // No bad words + { + auto llmRequest + = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + RequestList requestList{llmRequest}; + + SizeType32 maxNumIterations = 5; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); + } + // With embedding bias + { + TensorPtr embeddingBias + = manager.cpu(ITensor::makeShape({1, mVocabSizePadded}), nvinfer1::DataType::kFLOAT); + auto embeddingBiasPtr = bufferCast<float>(*embeddingBias); + for (SizeType32 vi = 0; vi < mVocabSizePadded; ++vi) + { + embeddingBiasPtr[vi] = 0.f; + } + // bias all words to the 10th token + embeddingBiasPtr[10] = std::numeric_limits<float>::max(); + + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, + endId, padId, embeddingBias, badWordsList, stopWordsList); + RequestList requestList{llmRequest}; + SizeType32 maxNumIterations = 5; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + // All tokens should become 10 after applying bias + EXPECT_EQ(requestList.front()->getTokens(0).at(5), 10); + EXPECT_EQ(requestList.front()->getTokens(0).at(6), 10); + EXPECT_EQ(requestList.front()->getTokens(0).at(7), 10); + EXPECT_EQ(requestList.front()->getTokens(0).at(8), 10); + } + + // Multiple requests, each with different bias + { + // Request w/o bias + auto llmRequest = std::make_shared<LlmRequest>(1, maxNewTokens, tokens, inSamplingConfig, false); + + TensorPtr embeddingBias1 + = manager.cpu(ITensor::makeShape({1, mVocabSizePadded}), nvinfer1::DataType::kFLOAT); + auto embeddingBias1Ptr = bufferCast<float>(*embeddingBias1); + for (SizeType32 vi = 0; vi < mVocabSizePadded; ++vi) + { + embeddingBias1Ptr[vi] = 0.f; + } + // bias all words to the 10th token + embeddingBias1Ptr[10] = std::numeric_limits<float>::max(); + + auto llmRequest2 = std::make_shared<LlmRequest>(2, maxNewTokens, tokens, inSamplingConfig, false, endId, + padId, embeddingBias1, badWordsList, stopWordsList); + + TensorPtr embeddingBias2 + = manager.cpu(ITensor::makeShape({1, mVocabSizePadded}), nvinfer1::DataType::kFLOAT); + auto embeddingBias2Ptr = bufferCast<float>(*embeddingBias2); + for (SizeType32 vi = 0; vi < mVocabSizePadded; ++vi) + { + embeddingBias2Ptr[vi] = 0.f; + } + // bias all words to the 100th token + embeddingBias2Ptr[100] = std::numeric_limits<float>::max(); + + auto llmRequest3 = std::make_shared<LlmRequest>(3, maxNewTokens, tokens, inSamplingConfig, false, endId, + padId, embeddingBias2, badWordsList, stopWordsList); + + RequestList requestList{llmRequest, llmRequest2, llmRequest3}; + + SizeType32 maxNumIterations(6); + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + + for (auto& request : requestList) + { + auto outputTokens = request->getTokens(0); + if (request->mRequestId == 1) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 6, 10, 6)); + } + if (request->mRequestId == 2) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 10, 10, 10, 10)); + } + if (request->mRequestId == 3) + { + EXPECT_EQ(outputTokens.size(), tokens->size() + maxNewTokens); + EXPECT_THAT(request->getTokens(0), ElementsAre(10, 9, 8, 7, 6, 100, 100, 100, 100)); + } + } + } + } +} + +class TrtGptModelIfbHelper : public TrtGptModelInflightBatching +{ +public: + using TrtGptModelInflightBatching::TrtGptModelInflightBatching; + + [[nodiscard]] std::shared_ptr<kv_cache_manager::BaseKVCacheManager const> getKVCacheManager() const + { + return TrtGptModelInflightBatching::getKVCacheManager(); + } + + [[nodiscard]] SizeType32 getMaxAttentionWindow() const + { + return TrtGptModelInflightBatching::getMaxAttentionWindow(); + } +}; + +TEST_F(TrtGptModelTest, KVCacheReuseChunked) +{ + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setEnableChunkedContext(true); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setEnableBlockReuse(true); + executorConfig.setKvCacheConfig(kvCacheConfig); + + mModelConfig.setMaxNumTokens(384); + + for (int const numBlocksExpectedReused : {1, 2}) + { + auto trtGptModelIfb = std::make_shared<TrtGptModelIfbHelper>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + auto const cacheManager = trtGptModelIfb->getKVCacheManager(); + auto const tokensPerBlock = cacheManager->getTokensPerBlock(); + constexpr int numPrefillBlocks = 2; + + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + constexpr int correlationId = 0; + constexpr int maxNewTokens = 4; + + auto tokens = std::make_shared<std::vector<int32_t>>(tokensPerBlock * numPrefillBlocks); + std::iota(std::begin(*tokens), std::end(*tokens), 1); + auto subTokens = std::make_shared<std::vector<int32_t>>( + tokens->begin(), tokens->begin() + numBlocksExpectedReused * tokensPerBlock); + // Add new token to "start" a new block. + subTokens->push_back(0); + { + auto llmRequest + = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + RequestList requests{llmRequest}; + forwardRequestsToCompletion(trtGptModelIfb, requests, 6); + EXPECT_EQ(llmRequest->isGenerationCompleteState(), true); + } + for (size_t i = 1; i <= 2; ++i) + { + auto llmRequest + = std::make_shared<LlmRequest>(correlationId, maxNewTokens, subTokens, inSamplingConfig, false); + RequestList req{llmRequest}; + forwardRequestsToCompletion(trtGptModelIfb, req, 5); + EXPECT_EQ(cacheManager->getBlockManager().getNumReusedBlocks(), i * numBlocksExpectedReused); + } + } +} + +TEST_F(TrtGptModelTest, PauseRequestStats) +{ + SamplingConfig inSamplingConfig; + inSamplingConfig.temperature = std::vector{2.0f}; + int correlationId = 0; + auto maxNewTokens = 3; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); + auto llmRequest = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, std::nullopt, false, false, false, std::nullopt, std::nullopt, false, + std::nullopt, false, std::nullopt, false, std::nullopt, executor::Request::kDefaultPriority, std::nullopt, + std::nullopt, std::nullopt, LlmRequestType::LLMREQUEST_TYPE_CONTEXT_AND_GENERATION, std::nullopt, 1, + std::nullopt, std::nullopt, true /* returnPerfMetrics */); + + RequestList requestList{llmRequest}; + + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setSchedulerConfig(executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + // Generate one token for the requests in request_table + // We need to sync with decoder + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + + EXPECT_EQ(requestList.size(), 1); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); + EXPECT_EQ(requestList.front()->getNumTokens(0), 5); + EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2)); + + auto perfMetrics = requestList.front()->getPerfMetrics(); + auto zero = executor::RequestPerfMetrics::TimePoint{}; + + EXPECT_NE(perfMetrics.timingMetrics.arrivalTime, zero); + EXPECT_NE(perfMetrics.timingMetrics.firstScheduledTime, zero); + EXPECT_NE(perfMetrics.timingMetrics.firstTokenTime, zero); + EXPECT_EQ(perfMetrics.timingMetrics.lastTokenTime, zero); + EXPECT_EQ(perfMetrics.firstIter, 0); + EXPECT_EQ(perfMetrics.iter, 0); + EXPECT_EQ(perfMetrics.lastIter, std::nullopt); + + // Pause the request + trtGptModel->terminateRequest(llmRequest, true); + + // Resume work + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + + // Generate one more token + EXPECT_EQ(requestList.size(), 1); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_IN_PROGRESS); + EXPECT_EQ(requestList.front()->getNumTokens(0), 6); + EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2, 4)); + + auto newPerfMetrics = requestList.front()->getPerfMetrics(); + EXPECT_EQ(newPerfMetrics.firstIter, 0); + EXPECT_EQ(newPerfMetrics.iter, 1); + EXPECT_EQ(newPerfMetrics.lastIter, std::nullopt); + + // Check that firstScheduledTime and firstTokenTime are the same + EXPECT_EQ(perfMetrics.timingMetrics.firstScheduledTime, newPerfMetrics.timingMetrics.firstScheduledTime); + EXPECT_EQ(perfMetrics.timingMetrics.firstTokenTime, newPerfMetrics.timingMetrics.firstTokenTime); + + // Pause the request + trtGptModel->terminateRequest(llmRequest, true); + + // Resume work + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + + // Generate last token + EXPECT_EQ(requestList.size(), 1); + EXPECT_EQ(requestList.front()->getState(), LlmRequestState::kGENERATION_COMPLETE); + EXPECT_EQ(requestList.front()->getNumTokens(0), 7); + EXPECT_EQ(requestList.front()->getMaxNumGeneratedTokens(), 1); + EXPECT_THAT(requestList.front()->getTokens(0), ElementsAre(1, 2, 3, 4, 2, 4, 2)); + + auto endPerfMetrics = requestList.front()->getPerfMetrics(); + EXPECT_EQ(endPerfMetrics.firstIter, 0); + EXPECT_EQ(endPerfMetrics.iter, 2); + EXPECT_EQ(endPerfMetrics.lastIter, 2); + + // Check that firstScheduledTime and firstTokenTime are the same + EXPECT_EQ(perfMetrics.timingMetrics.firstScheduledTime, endPerfMetrics.timingMetrics.firstScheduledTime); + EXPECT_EQ(perfMetrics.timingMetrics.firstTokenTime, endPerfMetrics.timingMetrics.firstTokenTime); +} + +class TrtGptModelLogitsTest : public TrtGptModelTest +{ +protected: + TrtGptModelLogitsTest() + : TrtGptModelTest(GPT_MODEL_PATH / GetModelSpec().getModelPath() / "tp1-pp1-cp1-gpu") + { + } + + static ModelSpec& GetModelSpec() + { + static ModelSpec modelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF}; + modelSpec.useGptAttentionPlugin().usePackedInput().setKVCacheType(KVCacheType::kPAGED).gatherLogits(); + return modelSpec; + } +}; + +TEST_F(TrtGptModelLogitsTest, ReturnContextLogitsWithChunkedContext) +{ + // General config + int correlationId = 0; + auto maxNewTokens = 4; + int const worldSize = 1; + auto const vocabSizePadded = mModelConfig.getVocabSizePadded(worldSize); + + SamplingConfig inSamplingConfig; + + // Different prompt length + for (int const promptLength : {10, 128, 200, 250, 256}) + { + RequestList finishList; + for (bool enableChunkedContext : {false, true}) + { + auto modelConfig = mModelConfig; + if (enableChunkedContext) + { + modelConfig.setMaxNumTokens(128); + } + + executor::ExecutorConfig executorConfig; + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(mBeamWidth); + executorConfig.setEnableChunkedContext(enableChunkedContext); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT}); + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setEnableBlockReuse(true); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto trtGptModelIfb = std::make_shared<TrtGptModelIfbHelper>( + mLogger, modelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + // Prepare input tokens + std::vector<int32_t> input_ids; + for (int i = 1; i <= promptLength; i++) + { + input_ids.push_back(i); + } + auto tokens = std::make_shared<std::vector<int32_t>>(input_ids); + + auto llmRequest + = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + TensorPtr contextLogitsHost = BufferManager::cpu( + ITensor::makeShape({llmRequest->mPromptLen, vocabSizePadded}), nvinfer1::DataType::kFLOAT); + + llmRequest->setContextLogitsHost(contextLogitsHost); + llmRequest->setReturnContextLogits(true); + + RequestList requestList{llmRequest}; + forwardRequestsToCompletion(trtGptModelIfb, requestList, 6); + + finishList.push_back(llmRequest); + } + EXPECT_EQ(finishList.size(), 2); + + float const* const disableChunkedContextLogits + = bufferCast<float>(*(finishList.front()->getContextLogitsHost())); + float const* const enableChunkedContextLogits = bufferCast<float>(*(finishList.back()->getContextLogitsHost())); + + for (int tokenIdx = 0; tokenIdx < promptLength; tokenIdx++) + { + for (int vocabIdx = 0; vocabIdx < vocabSizePadded; vocabIdx++) + { + size_t idx = tokenIdx * vocabSizePadded + vocabIdx; + EXPECT_NEAR(disableChunkedContextLogits[idx], enableChunkedContextLogits[idx], 1e-0) + << "tokenIdx=" << tokenIdx << " vocabIdx=" << vocabIdx; + } + } + finishList.clear(); + } +} + +class LlamaModelLADTest : public TrtGptModelTest +{ +protected: + LlamaModelLADTest() + : TrtGptModelTest(LLAMA_MODEL_PATH / GetModelSpec().getModelPath() / "tp1-pp1-cp1-gpu") + { + } + + static ModelSpec& GetModelSpec() + { + static ModelSpec modelSpec = ModelSpec{"input_tokens.npy", nvinfer1::DataType::kHALF} + .useGptAttentionPlugin() + .usePackedInput() + .setKVCacheType(KVCacheType::kPAGED) + .useLookaheadDecoding(); + return modelSpec; + } +}; + +TEST_F(LlamaModelLADTest, SeamlessLookaheadDecoding) +{ + GTEST_SKIP() << "Will enable this test when we have a force LAD support."; + SizeType32 requestId = 0; + for (bool const initLADConfig : {true, false}) + { + RequestList requestList{}; + for (SizeType32 i = 0; i < 8; ++i) + { + SamplingConfig inSamplingConfig; + int correlationId = requestId; + auto maxNewTokens = 8; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); + auto llmRequest + = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + requestList.emplace_back(std::move(llmRequest)); + requestId += 1; + } + + executor::ExecutorConfig executorConfig; + executorConfig.setEnableChunkedContext(false); + executorConfig.setEnableTrtOverlap(false); + executorConfig.setMaxBeamWidth(1); + executorConfig.setSchedulerConfig( + executor::SchedulerConfig{executor::CapacitySchedulerPolicy::kMAX_UTILIZATION}); + if (initLADConfig) + { + executor::DecodingConfig decodingConfig; + decodingConfig.setLookaheadDecodingConfig(executor::LookaheadDecodingConfig(5, 5, 5)); + executorConfig.setDecodingConfig(decodingConfig); + } + + auto trtGptModel = std::make_shared<TrtGptModelInflightBatching>( + mLogger, mModelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + + // Generate tokens for the requests in request_table + // We need to sync with decoder + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + EXPECT_EQ(trtGptModel->getSpeculativeDecodingMode().isLookaheadDecoding(), true); + + // Add new requests + for (SizeType32 i = 0; i < 4; ++i) + { + SamplingConfig inSamplingConfig; + int correlationId = requestId; + auto maxNewTokens = 8; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); + auto llmRequest + = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + requestList.emplace_back(std::move(llmRequest)); + requestId += 1; + } + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + EXPECT_EQ(trtGptModel->getSpeculativeDecodingMode().isLookaheadDecoding(), false); + + // Complete all of the requests + SizeType32 maxNumIterations = 8; + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + + // Run new requests with lookahead + requestList.clear(); + for (SizeType32 i = 0; i < 4; ++i) + { + SamplingConfig inSamplingConfig; + int correlationId = requestId; + auto maxNewTokens = 8; + auto tokens = std::make_shared<std::vector<int32_t>>(std::initializer_list<int32_t>{1, 2, 3, 4}); + auto llmRequest + = std::make_shared<LlmRequest>(correlationId, maxNewTokens, tokens, inSamplingConfig, false); + requestList.emplace_back(std::move(llmRequest)); + requestId += 1; + } + trtGptModel->forwardAsync(requestList); + trtGptModel->forwardSync(); + EXPECT_EQ(trtGptModel->getSpeculativeDecodingMode().isLookaheadDecoding(), true); + forwardRequestsToCompletion(trtGptModel, requestList, maxNumIterations); + requestList.clear(); + } +} + +TEST_F(TrtGptModelTest, ClampSeqLenToAttentionWindow) +{ + auto constexpr maxAttentionWindow = 65536; + auto constexpr maxSequenceLen = maxAttentionWindow + 1; + + executor::KvCacheConfig kvCacheConfig; + kvCacheConfig.setMaxAttentionWindowVec(std::vector<SizeType32>{maxAttentionWindow}); + kvCacheConfig.setFreeGpuMemoryFraction(0.0001); // minuscule amount of memory to force a clamp + + executor::ExecutorConfig executorConfig; + executorConfig.setKvCacheConfig(kvCacheConfig); + executorConfig.setMaxBeamWidth(mBeamWidth); + + auto modelConfig = mModelConfig; + modelConfig.setMaxSequenceLen(maxSequenceLen); + + auto trtGptModel = std::make_shared<TrtGptModelIfbHelper>( + mLogger, modelConfig, mWorldConfig, *mRawEngine, true, executorConfig, false); + EXPECT_LT(trtGptModel->getMaxAttentionWindow(), maxAttentionWindow); + EXPECT_EQ(trtGptModel->getMaxSequenceLen(), trtGptModel->getMaxAttentionWindow()); +} + +} // namespace tensorrt_llm::batch_manager diff --git a/cpp/tests/e2e_tests/executor/CMakeLists.txt b/cpp/tests/e2e_tests/executor/CMakeLists.txt new file mode 100644 index 000000000000..4813c92584fc --- /dev/null +++ b/cpp/tests/e2e_tests/executor/CMakeLists.txt @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +add_gtest(executorMockTest executorMockTest.cpp) +add_gtest(executorTest executorTest.cpp) +target_link_libraries(executorTest PRIVATE testingUtils) +add_gtest(encDecTest encDecTest.cpp) +target_link_libraries(encDecTest PRIVATE testingUtils) +add_gtest(disaggExecutorTest disaggExecutorTest.cpp) +target_link_libraries(disaggExecutorTest PRIVATE testingUtils) diff --git a/cpp/tests/e2e_tests/executor/disaggExecutor.h b/cpp/tests/e2e_tests/executor/disaggExecutor.h new file mode 100644 index 000000000000..6b3a529ca16e --- /dev/null +++ b/cpp/tests/e2e_tests/executor/disaggExecutor.h @@ -0,0 +1,840 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/stringUtils.h" +#include "tensorrt_llm/common/utils.h" +#include "tensorrt_llm/executor/dataTransceiverState.h" +#include "tensorrt_llm/executor/disaggServerUtil.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/requestWithId.h" +#include "tensorrt_llm/executor/serializeUtils.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <nlohmann/json.hpp> + +#include <algorithm> +#include <chrono> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <mutex> +#include <queue> +#include <string> +#include <thread> +#include <unordered_map> +#include <vector> + +using namespace tensorrt_llm::executor; +using namespace tensorrt_llm::executor::disagg_executor; +namespace su = tensorrt_llm::executor::serialize_utils; + +namespace tensorrt_llm::testing::disaggexecutor +{ + +constexpr int32_t kM_INSTANCE_ID_TAG{12024}; +constexpr int32_t kM_CONTROLLER_ID_TAG{22024}; +constexpr int32_t kM_INSTANCE_DATA_TAG{32024}; +constexpr int32_t kM_CONTROLLER_DATA_TAG{42024}; + +enum class MessageID : uint64_t +{ + PENDING_CONTEXT_REQUEST = 1, + PENDING_GENERATION_REQUEST = 2, + PENDING_FULL_REQUEST = 3, + CONTEXT_RESPONSE = 4, + GENERATION_RESPONSE = 5, + + TERMINATION = 6, +}; + +enum DisaggRole : uint32_t +{ + DISAGG_CONTEXT = 1, + DISAGG_GENERATION = 2, + DISAGG_MIXED = DISAGG_CONTEXT | DISAGG_GENERATION, + DISAGG_LEADER = 4, + DISAGG_CONTROLLER = 8, +}; + +struct RequestsData +{ + std::vector<RequestWithId> requests; +}; + +static std::vector<char> serializeResponseWithIds(std::vector<ResponseWithId> const& responseWithIds) +{ + size_t totalSize = 0; + totalSize += sizeof(size_t); + for (auto const& responseWithId : responseWithIds) + { + totalSize += su::serializedSize(responseWithId.gid); + totalSize += su::serializedSize(responseWithId.response); + } + + std::vector<char> buffer(totalSize); + std::stringbuf strbuf{std::ios_base::out | std::ios_base::in}; + strbuf.pubsetbuf(buffer.data(), static_cast<std::streamsize>(buffer.size())); + std::ostream ostream{&strbuf}; + + su::serialize(responseWithIds.size(), ostream); + for (auto const& responseWithId : responseWithIds) + { + su::serialize(responseWithId.gid, ostream); + su::serialize(responseWithId.response, ostream); + } + return buffer; +} + +static std::vector<ResponseWithId> deserializeResponseWithIds(std::vector<char>& buffer) +{ + std::vector<ResponseWithId> responseWithIds; + su::VectorWrapBuf<char> strbuf{buffer}; + std::istream istream{&strbuf}; + auto numReq = su::deserialize<std::int64_t>(istream); + for (int64_t req = 0; req < numReq; ++req) + { + auto const id = su::deserialize<std::uint64_t>(istream); + responseWithIds.emplace_back(ResponseWithId{Serialization::deserializeResponse(istream), id}); + } + return responseWithIds; +} + +struct ResponsesData +{ + std::vector<ResponseWithId> response; +}; + +using MessageData = std::variant<RequestsData, ResponsesData>; + +struct Message +{ + MessageID id; + MessageData data; +}; + +class MessageQueue +{ +public: + void push(Message&& message) + { + std::lock_guard<std::mutex> lock(mMutex); + mQueue.push(std::move(message)); + mCv.notify_one(); + } + + Message pop() + { + std::unique_lock<std::mutex> lock(mMutex); + mCv.wait(lock, [this] { return !mQueue.empty(); }); + Message message = std::move(mQueue.front()); + mQueue.pop(); + return message; + } + +private: + std::queue<Message> mQueue; + std::mutex mMutex; + std::condition_variable mCv; +}; + +class DisaggExecutorLeader +{ +public: + DisaggExecutorLeader(std::filesystem::path const& modelPath, ModelType modelType, + ExecutorConfig const& executorConfig, bool isController, bool isContext, bool isGeneration, int numRequests, + std::vector<int>& participatIds, std::vector<int> const& participantDeviceIdsThisInstance, int worldRank) + : mNumRequests(numRequests) + , mWorldRanksInstances(participatIds) + , mDeviceIdsThisInstance(participantDeviceIdsThisInstance) + , mWorldRank(worldRank) + , mShutdown(false) + , mWorldComm(tensorrt_llm::mpi::MpiComm::world()) + + { + +#if ENABLE_MULTI_DEVICE + + auto world_size = mWorldComm.getSize(); + mRolesPerRank.resize(world_size); + + if (isContext) + { + mRole |= DisaggRole::DISAGG_CONTEXT; + } + if (isGeneration) + { + mRole |= DisaggRole::DISAGG_GENERATION; + } + + if (!mWorldRanksInstances.empty() && mWorldRank == mWorldRanksInstances.front()) + { + mRole |= DisaggRole::DISAGG_LEADER; + } + + if (isController) + { + mRole |= DisaggRole::DISAGG_CONTROLLER; + } + + bool needExecutor = (std::find(mWorldRanksInstances.begin(), mWorldRanksInstances.end(), worldRank) + != mWorldRanksInstances.end()); + if (needExecutor) + { + ExecutorConfig executorConfigC = executorConfig; + + auto parallelConfig = executorConfigC.getParallelConfig().value_or(ParallelConfig{}); + std::vector<int> participantIds = mWorldRanksInstances; + + parallelConfig.setParticipantIds(participantIds); + TLLM_CHECK(parallelConfig.getCommunicationMode() == tensorrt_llm::executor::CommunicationMode::kLEADER); + parallelConfig.setCommunicationType(tensorrt_llm::executor::CommunicationType::kMPI); + parallelConfig.setDeviceIds(mDeviceIdsThisInstance); + executorConfigC.setParallelConfig(parallelConfig); + + mExecutor = std::make_unique<Executor>(modelPath, modelType, executorConfigC); + } + + TLLM_CHECK(mWorldRanksInstances.size() == mDeviceIdsThisInstance.size()); + + mWorldComm.allgather(&mRole, mRolesPerRank.data(), 1, tensorrt_llm::mpi::MpiType::kUINT32); + + generateRoles(); + + if (isController) + { + mControllerSendThread = std::thread(&DisaggExecutorLeader::ControllerSendThread, this); + mControllerRecvThread = std::thread(&DisaggExecutorLeader::ControllerRecvThread, this); + } + if (isLeaderInstance()) + { + mInstanceRecvThread = std::thread(&DisaggExecutorLeader::InstanceLeaderRecvThread, this); + mInstanceSendThread = std::thread(&DisaggExecutorLeader::InstanceLeaderSendThread, this); + mInstanceLoopThread = std::thread(&DisaggExecutorLeader::InstanceLeaderLoopThread, this); + } +#else + TLLM_THROW("DisaggExecutor only support being compiled with ENABLE_MULTI_DEVICE"); + +#endif + } + + bool isControllerRank() const + { + return mRole & DISAGG_CONTROLLER; + } + + bool isContextRank() const + { + return mRole & DISAGG_CONTEXT; + } + + bool isGenerationRank() const + { + return mRole & DISAGG_GENERATION; + } + + bool isLeaderInstance() const + { + return mRole & DISAGG_LEADER; + } + + std::vector<IdType> enqueueRequests(std::vector<Request> const& llmRequests) + + { + if (!isControllerRank()) + { + return {}; + } + + std::vector<RequestWithId> requestWithIds; + std::vector<RequestWithId> requestWithIdsFull; // full request, not disaggregated + std::vector<IdType> reqIds; + for (auto const& req : llmRequests) + { + IdType id = generatedControlId(); + reqIds.push_back(id); + + RequestWithId reqWithId{req, id}; + if (req.getRequestType() == RequestType::REQUEST_TYPE_CONTEXT_ONLY) + { + requestWithIds.push_back(std::move(reqWithId)); + } + else + { + TLLM_CHECK(req.getRequestType() == RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION); + requestWithIdsFull.push_back(std::move(reqWithId)); + } + + mRequestMap.insert(std::make_pair(id, req)); + } + + if (!requestWithIds.empty()) + { + Message message{MessageID::PENDING_CONTEXT_REQUEST, MessageData{RequestsData{requestWithIds}}}; + mControllerSendQueue.push(std::move(message)); + } + if (!requestWithIdsFull.empty()) + { + Message message{MessageID::PENDING_FULL_REQUEST, MessageData{RequestsData{requestWithIdsFull}}}; + mControllerSendQueue.push(std::move(message)); + } + + return reqIds; + } + + std::vector<Response> awaitResponses(std::optional<std::chrono::milliseconds> const& timeout) + { + // wait for responseQueue , modify reqid- + std::vector<Response> responses; + std::unique_lock<std::mutex> lck(mResponsesMtx); + auto pred = [&mShutdown = mShutdown, &resp = this->mResponses]() -> bool { return !resp.empty() || mShutdown; }; + auto storeResponses = [this, &resp = this->mResponses, &responses]() + { + for (auto it = resp.cbegin(); it != resp.cend();) + { + responses.insert(responses.end(), it->second.begin(), it->second.end()); + resp.erase(it++); + } + }; + + if (timeout) + { + if (mResponsesCv.wait_for(lck, timeout.value(), pred)) + { + storeResponses(); + } + } + else + { + mResponsesCv.wait(lck, pred); + storeResponses(); + } + return responses; + } + + std::deque<RequestStatsPerIteration> getLatestRequestStats() + { + if (mExecutor && mExecutor->canEnqueueRequests()) + { + return mExecutor->getLatestRequestStats(); + } + return {}; + } + + void shutDown() + { + if (mShutdown) + { + return; + } + + if (isControllerRank()) + { + std::call_once(mHasSendTerminFlag, + [&]() + { + MessageID terminationMessage = MessageID::TERMINATION; + std::vector<bool> isSend(mWorldComm.getSize(), false); + for (auto&& leaderRanks : {mContextLeaderRanks, mGenerationLeaderRanks}) + { + for (auto&& leaderRank : leaderRanks) + { + if (isSend[leaderRank]) + { + continue; + } + mWorldComm.sendRawTag(&terminationMessage, 1, tensorrt_llm::mpi::MpiType::kUINT64, + leaderRank, kM_CONTROLLER_ID_TAG); + isSend[leaderRank] = true; + } + } + + mWorldComm.sendRawTag(&terminationMessage, 1, tensorrt_llm::mpi::MpiType::kUINT64, mControllerRank, + kM_INSTANCE_ID_TAG); + }); + // end recv thread; + } + mShutdown = true; + + // end send thread + if (isControllerRank()) + { + mControllerSendQueue.push({MessageID::TERMINATION, {}}); + } + mInstanceSendQueue.push({MessageID::TERMINATION, {}}); + } + + ~DisaggExecutorLeader() + { + + if (isControllerRank()) + { + shutDown(); + } + + if (isLeaderInstance()) + { + if (mInstanceSendThread.joinable()) + { + mInstanceSendThread.join(); + } + if (mInstanceRecvThread.joinable()) + { + mInstanceRecvThread.join(); + } + if (mInstanceLoopThread.joinable()) + { + mInstanceLoopThread.join(); + } + } + + if (isControllerRank()) + { + if (mControllerSendThread.joinable()) + { + mControllerSendThread.join(); + } + if (mControllerRecvThread.joinable()) + { + mControllerRecvThread.join(); + } + } + + if (!isControllerRank()) + { + mExecutor->shutdown(); + } + if (isControllerRank() && isLeaderInstance()) + { + mExecutor->shutdown(); + } + + shutDown(); + } + +private: + tensorrt_llm::mpi::MpiComm const& mWorldComm; + std::unique_ptr<Executor> mExecutor; + std::thread mInstanceSendThread; + std::thread mInstanceRecvThread; + std::thread mInstanceLoopThread; + std::thread mControllerSendThread; + std::thread mControllerRecvThread; + int mNumRequests; + std::map<std::uint64_t, Request> mRequestMap; + std::map<IdType, DataTransceiverState> mGenIdToContextPhase; + std::unordered_map<IdType, IdType> mInstanceIdToGlobalId; + std::mutex mIdToGlbalMutex; + + std::vector<int> mWorldRanksInstances; + + int mWorldRank; + int mControllerRank = 0; + uint32_t mRole = 0; + std::vector<uint32_t> mRolesPerRank; + std::vector<int> mContextLeaderRanks; + std::vector<int> mGenerationLeaderRanks; + + IdType mLastId = 1; + MessageQueue mControllerSendQueue; + MessageQueue mInstanceSendQueue; + + std::atomic<bool> mShutdown; + + // Ready responses + std::unordered_map<IdType, std::vector<Response>> mResponses; + mutable std::mutex mResponsesMtx; + std::condition_variable mResponsesCv; + + std::vector<int> mDeviceIdsThisInstance; + std::once_flag mHasSendTerminFlag; + + void appendNewResponses(std::vector<ResponseWithId>& newResponses) + { + { + std::scoped_lock<std::mutex> lck(mResponsesMtx); + for (auto& responseWithId : newResponses) + { + // global id to Result + responseWithId.response = Response(responseWithId.gid, responseWithId.response.getResult()); + + mResponses[responseWithId.gid].emplace_back(responseWithId.response); + } + } + mResponsesCv.notify_all(); + } + + void generateRoles() + { + int contextNum = 0; + int genrationNum = 0; + int controllerNum = 0; + for (int rank = 0; rank < mRolesPerRank.size(); rank++) + { + uint32_t role = mRolesPerRank[rank]; + if (role & DISAGG_LEADER) + { + if (role & DISAGG_CONTEXT) + { + contextNum++; + mContextLeaderRanks.push_back(rank); + } + if (role & DISAGG_GENERATION) + { + genrationNum++; + mGenerationLeaderRanks.push_back(rank); + } + } + if (role & DISAGG_CONTROLLER) + { + controllerNum++; + mControllerRank = rank; + } + } + TLLM_CHECK_WITH_INFO(controllerNum == 1, "only one rank is controller but get %d controllerNum", controllerNum); + TLLM_LOG_INFO("leader ctx: %s, gen: %s", common::vec2str(mContextLeaderRanks).c_str(), + common::vec2str(mGenerationLeaderRanks).c_str()); + } + + IdType generatedControlId() + { + return (mLastId++ % UINT64_MAX); + } + + int selectContextLeaderRank() + { + static int leaderRank = 0; + leaderRank = (leaderRank + 1) % mContextLeaderRanks.size(); + return mContextLeaderRanks[leaderRank]; + } + + int selectGenerationLeaderRank() + { + + // TODO: for same reqId , need select specific generationLeader + static int leaderRank = 0; + leaderRank = (leaderRank + 1) % mGenerationLeaderRanks.size(); + return mGenerationLeaderRanks[leaderRank]; + } + + void ControllerSendThread() + { + // send request to context reqid + // and send context pahse to generation + + TLLM_CUDA_CHECK( + cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); + tensorrt_llm::common::setThreadName("ControllerSendThread"); + + while (!mShutdown) + { + auto message = mControllerSendQueue.pop(); + if (message.id == MessageID::TERMINATION) + { + + TLLM_LOG_DEBUG("controller get termination message in sendQueue"); + break; + } + if (message.id == MessageID::PENDING_CONTEXT_REQUEST) + { + + auto& reqWithIds = std::get<RequestsData>(message.data); + auto packed = RequestWithId::serializeReqWithIds(reqWithIds.requests); + int contextRank = selectContextLeaderRank(); + + mWorldComm.sendRawTag( + &message.id, 1, tensorrt_llm::mpi::MpiType::kUINT64, contextRank, kM_CONTROLLER_ID_TAG); + + mWorldComm.sendRawTag(packed.data(), packed.size(), tensorrt_llm::mpi::MpiType::kCHAR, contextRank, + kM_CONTROLLER_DATA_TAG); + } + else if (message.id == MessageID::PENDING_GENERATION_REQUEST + || message.id == MessageID::PENDING_FULL_REQUEST) + { + + auto& reqWithIds = std::get<RequestsData>(message.data); + auto packed = RequestWithId::serializeReqWithIds(reqWithIds.requests); + int generationRank = selectGenerationLeaderRank(); + + mWorldComm.sendRawTag( + &message.id, 1, tensorrt_llm::mpi::MpiType::kUINT64, generationRank, kM_CONTROLLER_ID_TAG); + + mWorldComm.sendRawTag(packed.data(), packed.size(), tensorrt_llm::mpi::MpiType::kCHAR, generationRank, + kM_CONTROLLER_DATA_TAG); + } + else + { + TLLM_THROW("rank:%d, size:%d controller send Invalid message id:%ld", mWorldComm.getRank(), + mWorldComm.getSize(), static_cast<uint64_t>(message.id)); + } + } + } + + void ControllerRecvThread() + { +#if ENABLE_MULTI_DEVICE + tensorrt_llm::common::setThreadName("ControllerRecvThread"); + + // recv response from context and push to sendQueue + // recv response from generation and push to responseQueue and notify awaitResponse + TLLM_CUDA_CHECK( + cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); + + while (!mShutdown) + { + + MPI_Message msg = nullptr; + MPI_Status status; + + mWorldComm.mprobeRawTag(MPI_ANY_SOURCE, kM_INSTANCE_ID_TAG, &msg, &status); + + auto sourceRank{status.MPI_SOURCE}; + int32_t count = 0; + MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); + TLLM_CHECK(count == 1); + + MessageID messageId; + MPICHECK(MPI_Mrecv(&messageId, count, MPI_UINT64_T, &msg, &status)); + + if (messageId == MessageID::TERMINATION) + { + TLLM_LOG_DEBUG("controller received termination message***************\n"); + break; + } + if (messageId == MessageID::CONTEXT_RESPONSE) + { + mWorldComm.mprobeRawTag(sourceRank, kM_INSTANCE_DATA_TAG, &msg, &status); + MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); + std::vector<char> buffer(count); + MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); + auto responseWithIds = deserializeResponseWithIds(buffer); + // enqueueTo sendQueue like enqueuRequest. . modify requestType and set ContextPhaseParams + // and push to sendQueue. + std::vector<RequestWithId> requestWithIds; + for (auto&& responseWithId : responseWithIds) + { + auto reqId = responseWithId.gid; + auto& request = mRequestMap.at(reqId); + + request.setRequestType(RequestType::REQUEST_TYPE_GENERATION_ONLY); + request.setContextPhaseParams(responseWithId.response.getResult().contextPhaseParams.value()); + requestWithIds.push_back(RequestWithId{request, reqId}); + } + mControllerSendQueue.push({MessageID::PENDING_GENERATION_REQUEST, RequestsData{requestWithIds}}); + } + + else if (messageId == MessageID::GENERATION_RESPONSE) + { + + mWorldComm.mprobeRawTag(sourceRank, kM_INSTANCE_DATA_TAG, &msg, &status); + MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); + std::vector<char> buffer(count); + MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); + + auto responseWithIds = deserializeResponseWithIds(buffer); + appendNewResponses(responseWithIds); + } + else + { + TLLM_THROW("rank:%d, size:%d controller recv Invalid message id:%ld", mWorldComm.getRank(), + mWorldComm.getSize(), static_cast<uint64_t>(messageId)); + } + } +#endif + } + + void InstanceLeaderSendThread() + { + tensorrt_llm::common::setThreadName("InstanceLeaderSendThread"); + + TLLM_CUDA_CHECK( + cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); + + // pop senQueue and send response to controller + + while (!mShutdown) + { + auto message = mInstanceSendQueue.pop(); + if (message.id == MessageID::CONTEXT_RESPONSE || message.id == MessageID::GENERATION_RESPONSE) + { + auto& responseWithIds = std::get<ResponsesData>(message.data); + auto packed = serializeResponseWithIds(responseWithIds.response); + + mWorldComm.sendRawTag( + &message.id, 1, tensorrt_llm::mpi::MpiType::kUINT64, mControllerRank, kM_INSTANCE_ID_TAG); + mWorldComm.sendRawTag(packed.data(), packed.size(), tensorrt_llm::mpi::MpiType::kCHAR, mControllerRank, + kM_INSTANCE_DATA_TAG); + } + else if (message.id == MessageID::TERMINATION) + { + // break; no send + TLLM_LOG_DEBUG( + "ranK:%d ,size:%d ,isContext:%d... Context or Generation leader get termination message in " + "sendQueue***************\n", + mWorldComm.getRank(), mWorldComm.getSize(), int(isContextRank())); + break; + } + else + { + TLLM_THROW("rank:%d, size:%d InstanceLeaderSendThread send Invalid message id:%ld", + mWorldComm.getRank(), mWorldComm.getSize(), static_cast<uint64_t>(message.id)); + } + } + } + + void InstanceLeaderRecvThread() + { + +#if ENABLE_MULTI_DEVICE + tensorrt_llm::common::setThreadName("InstanceLeaderRecvThread"); + + TLLM_CUDA_CHECK( + cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); + + // recv request from controller and enqueRequest to executor + while (!mShutdown) + { + MPI_Message msg; + MPI_Status status; + auto sourceRank{mControllerRank}; + mWorldComm.mprobeRawTag(sourceRank, kM_CONTROLLER_ID_TAG, &msg, &status); + + int32_t count; + MPICHECK(MPI_Get_count(&status, MPI_UINT64_T, &count)); + TLLM_CHECK(count == 1); + + MessageID messageId; + MPICHECK(MPI_Mrecv(&messageId, count, MPI_UINT64_T, &msg, &status)); + + if (messageId == MessageID::TERMINATION) + { + TLLM_LOG_DEBUG( + "ranK:%d ,size:%d ,isContext:%d ... Context or Generation leader recv termination message in " + "InstanceLeaderRecvThread***************\n", + mWorldComm.getRank(), mWorldComm.getSize(), int(isContextRank())); + shutDown(); + break; + } + if (messageId == MessageID::PENDING_CONTEXT_REQUEST || messageId == MessageID::PENDING_GENERATION_REQUEST + || messageId == MessageID::PENDING_FULL_REQUEST) + { + mWorldComm.mprobeRawTag(sourceRank, kM_CONTROLLER_DATA_TAG, &msg, &status); + MPICHECK(MPI_Get_count(&status, MPI_CHAR, &count)); + std::vector<char> buffer(count); + MPICHECK(MPI_Mrecv(buffer.data(), count, MPI_CHAR, &msg, &status)); + auto requestWithIds = RequestWithId::deserializeReqWithIds(buffer); + for (auto&& requestWithId : requestWithIds) + { + + auto globalReqId = requestWithId.id; + if (isContextRank() && messageId == MessageID::PENDING_CONTEXT_REQUEST) + { + TLLM_CHECK(requestWithId.req.getRequestType() == RequestType::REQUEST_TYPE_CONTEXT_ONLY); + } + else if (isGenerationRank() + && (messageId == MessageID::PENDING_GENERATION_REQUEST + || messageId == MessageID::PENDING_FULL_REQUEST)) + { + if (messageId == MessageID::PENDING_GENERATION_REQUEST) + { + TLLM_CHECK(requestWithId.req.getRequestType() == RequestType::REQUEST_TYPE_GENERATION_ONLY); + } + else // PENDING_FULL_REQUEST + { + TLLM_CHECK( + requestWithId.req.getRequestType() == RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION); + } + } + else + { + TLLM_THROW("rank:%d, size:%d InstanceLeaderRecvThread recv Invalid message id:%ld", + mWorldComm.getRank(), mWorldComm.getSize(), static_cast<uint64_t>(messageId)); + } + auto reqId = mExecutor->enqueueRequest(requestWithId.req); + { + std::scoped_lock<std::mutex> lock{mIdToGlbalMutex}; + mInstanceIdToGlobalId[reqId] = globalReqId; + } + } + } + else + { + TLLM_THROW("rank:%d, size:%d InstanceLeaderRecvThread send Invalid message id:%ld", + mWorldComm.getRank(), mWorldComm.getSize(), static_cast<uint64_t>(messageId)); + } + } +#endif + } + + void InstanceLeaderLoopThread() + { + + tensorrt_llm::common::setThreadName("InstanceLeaderLoopThread"); + + TLLM_CUDA_CHECK( + cudaSetDevice(mDeviceIdsThisInstance.at(COMM_SESSION.getRank() % (mDeviceIdsThisInstance.size())))); + + // loop awaitResponse and enqueue into sendQueue + while (!mShutdown) + { + std::chrono::milliseconds waitTime(1); + + auto responses = mExecutor->awaitResponses(waitTime); + if (responses.empty()) + { + continue; + } + std::vector<ResponseWithId> responseWithIdsContext; + std::vector<ResponseWithId> responseWithIdsGeneration; + for (auto&& response : responses) + { + auto reqId = response.getRequestId(); + IdType globalId{0}; + { + std::scoped_lock<std::mutex> lock{mIdToGlbalMutex}; + globalId = mInstanceIdToGlobalId[reqId]; + } + TLLM_CHECK(globalId != 0); + auto const& result = response.getResult(); + if (result.contextPhaseParams.has_value()) + { + responseWithIdsContext.emplace_back(response, globalId); + } + else + { + responseWithIdsGeneration.emplace_back(response, globalId); + } + } + + if (isContextRank()) + { + mInstanceSendQueue.push({MessageID::CONTEXT_RESPONSE, ResponsesData{responseWithIdsContext}}); + } + if (isGenerationRank()) + { + mInstanceSendQueue.push({MessageID::GENERATION_RESPONSE, ResponsesData{responseWithIdsGeneration}}); + } + } + } +}; +} // namespace tensorrt_llm::testing::disaggexecutor diff --git a/cpp/tests/e2e_tests/executor/disaggExecutorTest.cpp b/cpp/tests/e2e_tests/executor/disaggExecutorTest.cpp new file mode 100644 index 000000000000..0eb05d2cc807 --- /dev/null +++ b/cpp/tests/e2e_tests/executor/disaggExecutorTest.cpp @@ -0,0 +1,1437 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "disaggExecutor.h" +#include "executorTest.h" +#include "tensorrt_llm/common/envUtils.h" +#include "tensorrt_llm/runtime/utils/numpyUtils.h" +#include "tests/utils/common.h" + +#include <cstddef> +#include <unordered_set> + +namespace tr = tensorrt_llm::runtime; + +using namespace tensorrt_llm::testing; + +namespace +{ +auto constexpr LLAMA_INPUT_FILE = "input_tokens_llama.npy"; +auto constexpr LLAMA_VOCAB_SIZE_PADDED = 128256; +auto constexpr LLAMA_END_ID = 128001; +auto constexpr LLAMA_PAD_ID = 128001; + +using CondDisaggParamsType = std::tuple<std::string>; // modelName + +enum class InstanceRole : int +{ + kCONTEXT = 1, + kGENERATION = 0, + kMIXED = 2 +}; + +using DisaggParamsType = std::tuple< // + int, // processNum + std::vector<std::string>, // modelNames + std::vector<std::vector<int>>, // participantIdsEachInstance + std::vector<std::vector<int>>, // participantDeviceIdsEachInstance + std::vector<InstanceRole>, // instanceRoles + int // controllerRank + >; + +std::string convertToString(std::vector<std::vector<int>> const& vec) +{ + std::ostringstream oss; + oss << "XX"; + + for (size_t i = 0; i < vec.size(); ++i) + { + for (size_t j = 0; j < vec[i].size(); ++j) + { + oss << vec[i][j]; + if (j < vec[i].size() - 1) + { + oss << "_"; + } + } + if (i < vec.size() - 1) + { + oss << "X_X"; + } + } + + oss << "XX"; + return oss.str(); +}; + +std::string convertToString(std::vector<InstanceRole> const& vec) +{ + std::ostringstream oss; + oss << "XX"; + + for (size_t j = 0; j < vec.size(); ++j) + { + oss << static_cast<int>(vec[j]); + if (j < vec.size() - 1) + { + oss << "_"; + } + } + + oss << "XX"; + return oss.str(); +}; + +std::string generateTestNameDisaggParams(testing::TestParamInfo<DisaggParamsType> const& info) +{ + auto const processNum = std::get<0>(info.param); + auto const modelNames = std::get<1>(info.param); + auto const participantIdsEachInstance = std::get<2>(info.param); // std::vector<std::vector<int>> + auto const participantDeviceIdsEachInstance = std::get<3>(info.param); // std::vector<std::vector<int>>; + auto const instanceRoles = std::get<4>(info.param); // std::vector<int> ; //1 is context , 0 is generation + auto const controllerRank = std::get<5>(info.param); + + std::string name = "DisaggExecutorTest_"; + + name.append("ProcessNum_" + std::to_string(processNum)); + // name.append("_contextModel_" + contextModel + "_genModel_" + genModel); + name.append("_modelNames_"); + for (auto&& modelName : modelNames) + { + name.append(modelName).append("_"); + } + + name.append("_controllerRank_" + std::to_string(controllerRank)); + + name.append("_ranks_").append(convertToString(participantIdsEachInstance)); + name.append("_devices_").append(convertToString(participantDeviceIdsEachInstance)); + name.append("_roles_").append(convertToString(instanceRoles)); + name.append("_controllerRank_" + std::to_string(controllerRank)); + + return name; +} + +std::string generateTestNameCondDisaggParams(testing::TestParamInfo<CondDisaggParamsType> const& info) +{ + auto const modelName = std::get<0>(info.param); + return "Model_" + modelName; +} + +class DisaggParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<DisaggParamsType> +{ +}; + +class DisaggOrchestratorParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<DisaggParamsType> +{ +}; + +class ConditionalDisaggParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<CondDisaggParamsType> +{ +}; + +void verifyGenerateDistStats(std::deque<RequestStatsPerIteration> const& iterationStats) +{ + for (auto const& iteration : iterationStats) + { + for (auto const& requestStats : iteration.requestStats) + { + // exclude context only requests for mixed server + if (requestStats.stage == RequestStage::kGENERATION_COMPLETE && requestStats.numGeneratedTokens > 1) + { + EXPECT_TRUE(requestStats.disServingStats.has_value()); + EXPECT_GT(requestStats.disServingStats.value().kvCacheTransferMS, 0.0); + } + if (requestStats.stage != RequestStage::kQUEUED) + { + EXPECT_TRUE(requestStats.disServingStats.has_value()); + } + else + { + EXPECT_FALSE(requestStats.disServingStats.has_value()); + } + } + } +} +} // namespace + +void runDisaggTest(tensorrt_llm::testing::disaggexecutor::DisaggExecutorLeader& executor, + tensorrt_llm::runtime::BufferManager& manager, ITensor const& givenInput, ModelIds const& modelIds, + FlakyTestInfo const& flakyTestInfo, bool streaming, SizeType32 const vocabSizePadded, BeamResult const& beamResult, + OutputConfig const& outConfig, bool isSpeculativeDecoding, int maxWaitMs, BatchingType batchingType, + bool returnAllGeneratedTokens) +{ + + auto& comm = tensorrt_llm::mpi::MpiComm::world(); + auto const worldRank = comm.getRank(); + auto const worldSize = comm.getSize(); + auto const beamWidth = beamResult.beamWidth; + + std::unordered_map<IdType, SizeType32> reqIdToBatchId; + std::unordered_map<SizeType32, std::vector<BeamTokens>> tokens; + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(givenInput, modelIds.padId); + auto const* const givenInputData = tr::bufferCast<TokenIdType const>(givenInput); + + auto const& inputShape = givenInput.getShape(); + ASSERT_EQ(inputShape.nbDims, 2); + ASSERT_GT(inputShape.d[0], 0); + + // Load expected outputs for each beam width value + auto testData = TestData::loadTestData(beamResult, givenInput, beamWidth, manager, outConfig, modelIds); + auto const maxSeqLen = testData.maxSeqLen; + + // Load expected outputs and inputs + SizeType32 numRequests = static_cast<SizeType32>(givenInputLengths.size()); + SizeType32 maxRequests = numRequests; + std::vector<Request> requests; + std::vector<SizeType32> reqMaxNewTokens; + SizeType32 const numReturnSequences = 1; + + for (SizeType32 req = 0; req < maxRequests; ++req) + { + SizeType32 inputLen = givenInputLengths.at(req); + auto maxNewTokens = maxSeqLen - maxInputLength; + reqMaxNewTokens.push_back(maxNewTokens); + SizeType32 endId = -1; + auto const* const seqBegin = givenInputData + req * maxInputLength; + VecTokens tokens(seqBegin, seqBegin + inputLen); + auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); + samplingConfig.setNumReturnSequences(numReturnSequences); + auto request = Request( + VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, samplingConfig, outConfig, endId); + request.setReturnAllGeneratedTokens(returnAllGeneratedTokens); + request.setRequestType(RequestType::REQUEST_TYPE_CONTEXT_ONLY); + requests.emplace_back(std::move(request)); + } + + if (executor.isControllerRank()) + { + std::vector<IdType> reqIds; + + for (int i = 0; i < requests.size(); ++i) + { + std::vector<BeamTokens> resultTokens; + resultTokens.reserve(numReturnSequences); + for (SizeType32 seqIdx = 0; seqIdx < numReturnSequences; ++seqIdx) + { + resultTokens.emplace_back(beamWidth); + } + auto retReqId = executor.enqueueRequests({requests[i]}); + reqIds.push_back(retReqId.front()); + tokens[i] = std::move(resultTokens); + reqIdToBatchId[retReqId.front()] = i; + } + + // Get the new tokens for each requests + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + while (numFinished < maxRequests && iter < maxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + auto batchId = reqIdToBatchId.at(response.getRequestId()); + auto seqIdx = result.sequenceIndex; + + auto& contextLogits = result.contextLogits; + auto& genLogits = result.generationLogits; + auto& outputTokenIds = result.outputTokenIds; + + EXPECT_EQ(result.finishReasons.size(), beamWidth); + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto& newTokens = outputTokenIds.at(beam); + auto& reqTokens = tokens.at(batchId).at(seqIdx).at(beam); + + reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); + // FinishReason is only supported for bw=1 and inflight batching. + if (beamWidth == 1 && batchingType == BatchingType::kINFLIGHT) + { + EXPECT_EQ(result.finishReasons.at(beam), + result.isFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); + } + } + + auto& cumLogProbs = result.cumLogProbs; + auto& logProbs = result.logProbs; + auto& beamTokens = tokens.at(batchId).at(seqIdx); + testData.verifyLogProbs(outConfig.returnLogProbs, streaming, outConfig.excludeInputFromOutput, + givenInputLengths.at(batchId), beamWidth, beamTokens, cumLogProbs, logProbs, batchId, + flakyTestInfo); + + testData.validateContextLogits(outConfig.returnContextLogits, givenInputLengths.at(batchId), + beamWidth, contextLogits, vocabSizePadded, batchId); + testData.validateGenerationLogits(outConfig.returnGenerationLogits, result.isFinal, streaming, + outConfig.excludeInputFromOutput, givenInputLengths.at(batchId), reqMaxNewTokens.at(batchId), + beamWidth, beamTokens, genLogits, vocabSizePadded, batchId, returnAllGeneratedTokens); + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + ++iter; + } + EXPECT_LT(iter, maxWaitMs); + testData.verifyOutput(tokens, givenInputLengths, streaming, outConfig.excludeInputFromOutput, flakyTestInfo, + isSpeculativeDecoding, beamWidth, numReturnSequences, false); + } + comm.barrier(); + if (executor.isGenerationRank()) + { + verifyGenerateDistStats(executor.getLatestRequestStats()); + } +} + +void runDisaggTest(DisaggExecutorOrchestrator& executor, tensorrt_llm::runtime::BufferManager& manager, + ITensor const& givenInput, ModelIds const& modelIds, FlakyTestInfo const& flakyTestInfo, bool streaming, + SizeType32 const vocabSizePadded, BeamResult const& beamResult, OutputConfig const& outConfig, + bool isSpeculativeDecoding, int maxWaitMs, BatchingType batchingType, bool returnAllGeneratedTokens) +{ + + auto& comm = tensorrt_llm::mpi::MpiComm::world(); + auto const worldRank = comm.getRank(); + auto const worldSize = comm.getSize(); + auto const beamWidth = beamResult.beamWidth; + + std::unordered_map<IdType, SizeType32> reqIdToBatchId; + std::unordered_map<SizeType32, std::vector<BeamTokens>> tokens; + // std::unordered_map<IdType, IdType> gGenIdIdTogContextId; + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(givenInput, modelIds.padId); + auto const* const givenInputData = tr::bufferCast<TokenIdType const>(givenInput); + + auto const& inputShape = givenInput.getShape(); + ASSERT_EQ(inputShape.nbDims, 2); + ASSERT_GT(inputShape.d[0], 0); + + // Load expected outputs for each beam width value + auto testData = TestData::loadTestData(beamResult, givenInput, beamWidth, manager, outConfig, modelIds); + auto const maxSeqLen = testData.maxSeqLen; + + // Load expected outputs and inputs + SizeType32 numRequests = static_cast<SizeType32>(givenInputLengths.size()); + SizeType32 maxRequests = numRequests; + std::vector<Request> requests; + std::vector<SizeType32> reqMaxNewTokens; + SizeType32 const numReturnSequences = 1; + + for (SizeType32 req = 0; req < maxRequests; ++req) + { + SizeType32 inputLen = givenInputLengths.at(req); + auto maxNewTokens = maxSeqLen - maxInputLength; + reqMaxNewTokens.push_back(maxNewTokens); + SizeType32 endId = -1; + auto const* const seqBegin = givenInputData + req * maxInputLength; + VecTokens tokens(seqBegin, seqBegin + inputLen); + auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); + samplingConfig.setNumReturnSequences(numReturnSequences); + auto request = Request( + VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, samplingConfig, outConfig, endId); + request.setReturnAllGeneratedTokens(returnAllGeneratedTokens); + request.setRequestType(RequestType::REQUEST_TYPE_CONTEXT_ONLY); + requests.emplace_back(std::move(request)); + } + + if (worldRank == 0) + { + std::vector<IdType> reqIds; + + for (int i = 0; i < requests.size(); ++i) + { + std::vector<BeamTokens> resultTokens; + resultTokens.reserve(numReturnSequences); + for (SizeType32 seqIdx = 0; seqIdx < numReturnSequences; ++seqIdx) + { + resultTokens.emplace_back(beamWidth); + } + auto retReqId = executor.enqueueContext({requests[i]}, std::nullopt); + reqIds.push_back(retReqId.front()); + tokens[i] = std::move(resultTokens); + reqIdToBatchId[retReqId.front()] = i; + } + + int32_t numContextFinished = 0; + int contextIter = 0; + while (numContextFinished < maxRequests && contextIter < maxWaitMs) + { + std::chrono::milliseconds waitTime(1); + + auto contextResponses = executor.awaitContextResponses(waitTime); + contextIter++; + numContextFinished += contextResponses.size(); + + for (auto&& responseWithId : contextResponses) + { + auto contextGid = responseWithId.gid; + int batchId = reqIdToBatchId[contextGid]; + auto&& request = requests[batchId]; + request.setRequestType(RequestType::REQUEST_TYPE_GENERATION_ONLY); + request.setContextPhaseParams(responseWithId.response.getResult().contextPhaseParams.value()); + executor.enqueueGeneration({request}, {responseWithId.gid}, std::nullopt); + } + } + // Get the new tokens for each requests + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + while (numFinished < maxRequests && iter < maxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitGenerationResponses(waitTime); + for (auto& responseWithId : responses) + { + numResponses++; + if (!responseWithId.response.hasError()) + { + auto result = responseWithId.response.getResult(); + numFinished += result.isFinal; + auto batchId = reqIdToBatchId.at(responseWithId.gid); + auto seqIdx = result.sequenceIndex; + + auto& contextLogits = result.contextLogits; + auto& genLogits = result.generationLogits; + auto& outputTokenIds = result.outputTokenIds; + + EXPECT_EQ(result.finishReasons.size(), beamWidth); + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto& newTokens = outputTokenIds.at(beam); + auto& reqTokens = tokens.at(batchId).at(seqIdx).at(beam); + + reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); + // FinishReason is only supported for bw=1 and inflight batching. + if (beamWidth == 1 && batchingType == BatchingType::kINFLIGHT) + { + EXPECT_EQ(result.finishReasons.at(beam), + result.isFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); + } + } + + auto& cumLogProbs = result.cumLogProbs; + auto& logProbs = result.logProbs; + auto& beamTokens = tokens.at(batchId).at(seqIdx); + testData.verifyLogProbs(outConfig.returnLogProbs, streaming, outConfig.excludeInputFromOutput, + givenInputLengths.at(batchId), beamWidth, beamTokens, cumLogProbs, logProbs, batchId, + flakyTestInfo); + + testData.validateContextLogits(outConfig.returnContextLogits, givenInputLengths.at(batchId), + beamWidth, contextLogits, vocabSizePadded, batchId); + testData.validateGenerationLogits(outConfig.returnGenerationLogits, result.isFinal, streaming, + outConfig.excludeInputFromOutput, givenInputLengths.at(batchId), reqMaxNewTokens.at(batchId), + beamWidth, beamTokens, genLogits, vocabSizePadded, batchId, returnAllGeneratedTokens); + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(responseWithId.gid) + + " has already been processed and was terminated."; + EXPECT_EQ(responseWithId.response.getErrorMsg(), err); + } + } + ++iter; + } + EXPECT_LT(iter, maxWaitMs); + testData.verifyOutput(tokens, givenInputLengths, streaming, outConfig.excludeInputFromOutput, flakyTestInfo, + isSpeculativeDecoding, beamWidth, numReturnSequences, false); + } + comm.barrier(); +} + +TEST_P(DisaggParamsTest, DisaggTokenComparison) +{ + +#if ENABLE_MULTI_DEVICE + + if (!(tensorrt_llm::common::getEnvUseUCXKvCache())) + { + setenv("UCX_TLS", "^cuda_ipc", 1); // disable cuda_ipc for testing for mpi + } + else + { + setenv("UCX_TCP_CM_REUSEADDR", "y", + 1); // tests creates and destroies ucxCacheCommunicatoers frequently, so listener ports must be reused + } + auto const processNum = std::get<0>(GetParam()); + auto const modelNames = std::get<1>(GetParam()); + auto const participantIdsEachInstance = std::get<2>(GetParam()); // std::vector<std::vector<int>> + auto const participantDeviceIdsEachInstance = std::get<3>(GetParam()); // std::vector<std::vector<int>>; + auto const instanceRoles + = std::get<4>(GetParam()); // std::vector<int> ; //1 is context , 0 is generation, 2 is mixed + auto const controllerRank = std::get<5>(GetParam()); + + // params_check + auto const& world_comm = tensorrt_llm::mpi::MpiComm::world(); + int const commRank = world_comm.getRank(); + int const commSize = world_comm.getSize(); + if (commSize != processNum) + { + GTEST_SKIP() << " need " << processNum << " processes but got " << commSize << " mpi processes, skip test."; + } + ASSERT_EQ(participantIdsEachInstance.size(), participantDeviceIdsEachInstance.size()); + SizeType32 instanceNum = participantIdsEachInstance.size(); + ASSERT_EQ(instanceNum, instanceRoles.size()); + ASSERT_EQ(instanceNum, modelNames.size()); + + std::unordered_set<int> deviceIdsSet; + for (auto const& ids : participantDeviceIdsEachInstance) + { + for (auto const& id : ids) + { + deviceIdsSet.insert(id); + } + } + if (mDeviceCount < deviceIdsSet.size()) + { + GTEST_SKIP() << " need " << deviceIdsSet.size() << " devices but got " << mDeviceCount + << " devices, skip test."; + } + + ASSERT_GE(controllerRank, 0); + ASSERT_LT(controllerRank, commSize); + int ranksNum = 0; + std::unordered_map<SizeType32, SizeType32> rankCounter; + std::unordered_map<SizeType32, SizeType32> deviceCounter; + SizeType32 deviceRuseNum = 1; + bool isContext = false; + bool isGeneration = false; + std::vector<int> participatntIds; + std::vector<int> deviceIds; + std::string modelName; + bool isController = (commRank == controllerRank); + for (SizeType32 i = 0; i < instanceNum; i++) + { + auto const& ranksThisInstance = participantIdsEachInstance[i]; + auto const& devicesThisInstance = participantDeviceIdsEachInstance[i]; + + ASSERT_EQ(ranksThisInstance.size(), devicesThisInstance.size()); + SizeType32 rankNumThisInstance = ranksThisInstance.size(); + ASSERT_GT(rankNumThisInstance, 0); + ranksNum += rankNumThisInstance; + for (SizeType32 j = 0; j < rankNumThisInstance; j++) + { + rankCounter[ranksThisInstance[j]]++; + deviceCounter[devicesThisInstance[j]]++; + ASSERT_GE(rankCounter[ranksThisInstance[j]], 1); + deviceRuseNum = std::max(deviceCounter[devicesThisInstance[j]], deviceRuseNum); + ASSERT_GE(ranksThisInstance[j], 0); + ASSERT_LT(ranksThisInstance[j], commSize); + + if (commRank == ranksThisInstance[j]) + { + participatntIds = ranksThisInstance; + deviceIds = devicesThisInstance; + isContext = instanceRoles[i] == InstanceRole::kCONTEXT || instanceRoles[i] == InstanceRole::kMIXED; + isGeneration + = instanceRoles[i] == InstanceRole::kGENERATION || instanceRoles[i] == InstanceRole::kMIXED; + // modelName = isContext ? contextModel : genModel; + modelName = modelNames[i]; + } + } + } + ASSERT_GE(ranksNum, commSize); + + OutputConfig outConfig; + int const beamWidth = 1; + BeamResult beamResult{beamWidth}; + + bool streaming = false; + int const maxBeamWidth = 1; + ASSERT_TRUE(fs::exists(DATA_PATH)); + + fs::path modelPath; + // set defaults and adjust if needed by different models + fs::path inputPath = DATA_PATH / "input_tokens.npy"; + ModelIds modelIds{50256, 50256}; + SizeType32 vocabSizePadded{50257}; // gpt vocabSizePadded + bool isSpeculativeDecoding{false}; + + // NOTE: This can be used to disable checks for certain prompt batch entries + FlakyTestInfo flakyTestInfo; + + if (modelName == "gpt") + { + auto const resultsPath + = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + if (outConfig.returnContextLogits || outConfig.returnGenerationLogits) + { + modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"; + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); + beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); + beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); + if (outConfig.returnLogProbs) + { + beamResult.cumLogProbsFile + = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CUM_LOG_PROBS_FILE(); + beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_LOG_PROBS_FILE(); + } + } + else + { + modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); + if (outConfig.returnLogProbs) + { + beamResult.cumLogProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_FILE(); + beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_FILE(); + } + } + } + else if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1" + || modelName == "llama_tp1_pp2_cp1" || modelName == "llama_tp2_pp1_cp1" || modelName == "llama_tp1_pp1_cp1") + { + inputPath = DATA_PATH / LLAMA_INPUT_FILE; + vocabSizePadded = LLAMA_VOCAB_SIZE_PADDED; + + auto const resultsPath + = LLAMA_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + modelIds.padId = LLAMA_PAD_ID; + modelIds.endId = LLAMA_END_ID; + if (modelName == "llama_tp4_pp1_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp4_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp2_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp2-cp1-gpu"; + } + else if (modelName == "llama_tp2_pp1_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP1_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp2_pp2_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp1_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + } + } + else + { + TLLM_THROW("Unrecognized modelName"); + } + + // Warning: This should be the last check before running the test. + // It will initialize MPI which can take significant time. + if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1" + || modelName == "llama_tp1_pp2_cp1" || modelName == "llama_tp2_pp1_cp1") + { + if (outConfig.returnLogProbs || outConfig.returnContextLogits || outConfig.returnGenerationLogits) + { + GTEST_SKIP() << "Skipping logits and log probs tests for mpi runs"; + } + } + + // Returning logits will bring higher latency + if (streaming && (outConfig.returnContextLogits || outConfig.returnGenerationLogits)) + { + mMaxWaitMs = 20000; + } + + auto executorConfig = ExecutorConfig(maxBeamWidth); + FloatType freeGpuMemoryFraction = 0.9f / (deviceRuseNum); // context and gen instance run on same device + KvCacheConfig kvCacheConfig{true, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction}; + executorConfig.setKvCacheConfig(kvCacheConfig); + executorConfig.setRequestStatsMaxIterations(1000); + executorConfig.setCacheTransceiverConfig( + texec::CacheTransceiverConfig(texec::CacheTransceiverConfig::BackendType::DEFAULT)); + auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); + auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); + world_comm.barrier(); + auto disaggExecutor = tensorrt_llm::testing::disaggexecutor::DisaggExecutorLeader(modelPath, + ModelType::kDECODER_ONLY, executorConfig, isController, isContext, isGeneration, givenInputLengths.size(), + participatntIds, deviceIds, commRank); + + runDisaggTest(disaggExecutor, manager, *givenInput, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, + outConfig, isSpeculativeDecoding, mMaxWaitMs, executorConfig.getBatchingType(), false); + +#else + + GTEST_SKIP() << "Skipping DisaggExecutor Test"; + +#endif +} + +TEST_P(DisaggOrchestratorParamsTest, DisaggTokenComparison) +{ + +#if ENABLE_MULTI_DEVICE + + if (!(tensorrt_llm::common::getEnvUseUCXKvCache())) + { + setenv("UCX_TLS", "^cuda_ipc", 1); // disable cuda_ipc for testing for mpi + } + else + { + setenv("UCX_TCP_CM_REUSEADDR", "y", + 1); // tests creates and destroies ucxCacheCommunicatoers frequently, so listener ports must be reused + } + auto const processNum = std::get<0>(GetParam()); + auto const modelNames = std::get<1>(GetParam()); + auto const participantIdsEachInstance = std::get<2>(GetParam()); // std::vector<std::vector<int>> + auto const participantDeviceIdsEachInstance = std::get<3>(GetParam()); // std::vector<std::vector<int>>; + auto const instanceRoles = std::get<4>(GetParam()); // std::vector<int> ; //1 is context , 0 is generation + auto const controllerRank = std::get<5>(GetParam()); + + // params_check + auto const& world_comm = tensorrt_llm::mpi::MpiComm::world(); + int const commRank = world_comm.getRank(); + int const commSize = world_comm.getSize(); + if (commSize != processNum) + { + GTEST_SKIP() << " need " << processNum << " processes but got " << commSize << " mpi processes, skip test."; + } + + bool spawnProcess = false; + if (commSize == 1) + { + spawnProcess = true; + if (mDeviceCount < 4) + { + GTEST_SKIP() << "DisaggExecutorTest requires at least 4 GPUs"; + } + ASSERT_TRUE(tensorrt_llm::common::getEnvUseUCXKvCache() || tensorrt_llm::common::getEnvUseNixlKvCache()); + } + + ASSERT_EQ(participantIdsEachInstance.size(), participantDeviceIdsEachInstance.size()); + SizeType32 instanceNum = participantIdsEachInstance.size(); + ASSERT_EQ(instanceNum, instanceRoles.size()); + ASSERT_EQ(instanceNum, modelNames.size()); + + std::unordered_set<int> deviceIdsSet; + for (auto const& ids : participantDeviceIdsEachInstance) + { + for (auto const& id : ids) + { + deviceIdsSet.insert(id); + } + } + if (mDeviceCount < deviceIdsSet.size()) + { + GTEST_SKIP() << " need " << deviceIdsSet.size() << " devices but got " << mDeviceCount + << " devices, skip test."; + } + + ASSERT_GE(controllerRank, 0); + ASSERT_LT(controllerRank, commSize); + std::string modelName = modelNames[0]; + bool isController = (commRank == controllerRank); + std::vector<fs::path> contextModels; + std::vector<fs::path> genModels; + + auto getModelPath = [=](std::string modelNN) + { + fs::path retPath; + if (modelNN == "llama_tp4_pp1") + { + retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + } + else if (modelNN == "llama_tp1_pp4") + { + retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + } + else if (modelNN == "llama_tp1_pp2") + { + retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp2-cp1-gpu"; + } + else if (modelNN == "llama_tp2_pp1") + { + retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp1-cp1-gpu"; + } + else if (modelNN == "llama_tp2_pp2") + { + retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + } + else if (modelNN == "llama_tp1_pp1") + { + retPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + } + return retPath; + }; + for (SizeType32 i = 0; i < instanceNum; i++) + { + if (instanceRoles[i] == InstanceRole::kCONTEXT) + { + contextModels.push_back(getModelPath(modelNames[i])); + } + else + { + genModels.push_back(getModelPath(modelNames[i])); + } + } + + OutputConfig outConfig; + int const beamWidth = 1; + BeamResult beamResult{beamWidth}; + + bool streaming = false; + int const maxBeamWidth = 1; + ASSERT_TRUE(fs::exists(DATA_PATH)); + + fs::path modelPath; + // set defaults and adjust if needed by different models + fs::path inputPath = DATA_PATH / "input_tokens.npy"; + ModelIds modelIds{50256, 50256}; + SizeType32 vocabSizePadded{50257}; // gpt vocabSizePadded + bool isSpeculativeDecoding{false}; + + // NOTE: This can be used to disable checks for certain prompt batch entries + FlakyTestInfo flakyTestInfo; + if (modelName == "llama_tp4_pp1" || modelName == "llama_tp1_pp4" || modelName == "llama_tp2_pp2" + || modelName == "llama_tp1_pp2" || modelName == "llama_tp2_pp1" || modelName == "llama_tp1_pp1") + { + inputPath = DATA_PATH / LLAMA_INPUT_FILE; + vocabSizePadded = LLAMA_VOCAB_SIZE_PADDED; + + auto const resultsPath + = LLAMA_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + modelIds.padId = LLAMA_PAD_ID; + modelIds.endId = LLAMA_END_ID; + if (modelName == "llama_tp4_pp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp4") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp2") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp2-cp1-gpu"; + } + else if (modelName == "llama_tp2_pp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP1_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp2_pp2") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + } + } + + else + { + TLLM_THROW("Unrecognized modelName"); + } + + // Warning: This should be the last check before running the test. + // It will initialize MPI which can take significant time. + if (modelName == "llama_tp4_pp1" || modelName == "llama_tp1_pp4" || modelName == "llama_tp2_pp2" + || modelName == "llama_tp1_pp2" || modelName == "llama_tp2_pp1") + { + if (outConfig.returnLogProbs || outConfig.returnContextLogits || outConfig.returnGenerationLogits) + { + GTEST_SKIP() << "Skipping logits and log probs tests for mpi runs"; + } + } + + // Returning logits will bring higher latency + if (streaming && (outConfig.returnContextLogits || outConfig.returnGenerationLogits)) + { + mMaxWaitMs = 20000; + } + + auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); + auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); + world_comm.barrier(); + auto contextNum = contextModels.size(); + auto genNum = genModels.size(); + // int deviceCount = -1; + // TLLM_CUDA_CHECK(cudaGetDeviceCount(&deviceCount)); + bool isOrchestrator = commRank == 0; + std::vector<ExecutorConfig> ctxExecutorConfigs; + std::vector<ExecutorConfig> genExecutorConfigs; + for (int in = 0; in < instanceNum; in++) + { + tensorrt_llm::executor::SchedulerConfig schedulerConfig(CapacitySchedulerPolicy::kMAX_UTILIZATION); + KvCacheConfig kvCacheConfig{true, std::nullopt, std::nullopt, std::nullopt, 0.2}; + + tensorrt_llm::executor::ExecutorConfig executorConfig(maxBeamWidth, schedulerConfig, kvCacheConfig); + tensorrt_llm::executor::OrchestratorConfig orchestratorConfig{ + isOrchestrator, PathUtil::EXECUTOR_WORKER_PATH(), nullptr, spawnProcess}; + + tensorrt_llm::executor::ParallelConfig parallelConfig{tensorrt_llm::executor::CommunicationType::kMPI, + tensorrt_llm::executor::CommunicationMode::kORCHESTRATOR, participantDeviceIdsEachInstance.at(in), + spawnProcess ? std::nullopt : std::optional<std::vector<SizeType32>>(participantIdsEachInstance.at(in)), + orchestratorConfig}; + executorConfig.setParallelConfig(parallelConfig); + executorConfig.setCacheTransceiverConfig( + texec::CacheTransceiverConfig(texec::CacheTransceiverConfig::BackendType::DEFAULT)); + if (in < contextNum) + { + ctxExecutorConfigs.push_back(executorConfig); + } + else + { + genExecutorConfigs.push_back(executorConfig); + } + } + auto disaggExecutor + = DisaggExecutorOrchestrator(contextModels, genModels, ctxExecutorConfigs, genExecutorConfigs, true, true); + + runDisaggTest(disaggExecutor, manager, *givenInput, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, + outConfig, isSpeculativeDecoding, mMaxWaitMs, BatchingType::kINFLIGHT, false); + +#else + + GTEST_SKIP() << "Skipping DisaggExecutor Test"; + +#endif +} + +TEST_P(ConditionalDisaggParamsTest, DisaggTokenComparison) +{ +#if ENABLE_MULTI_DEVICE + if (!tensorrt_llm::common::getEnvUseUCXKvCache()) + { + setenv("UCX_TLS", "^cuda_ipc", 1); // disable cuda_ipc for testing for mpi + } + auto constexpr processNum = 2; + auto constexpr deviceNum = 2; + auto const& modelName = std::get<0>(GetParam()); + auto constexpr controllerRank = 0; + + // params_check + auto const& world_comm = tensorrt_llm::mpi::MpiComm::world(); + int const commRank = world_comm.getRank(); + int const commSize = world_comm.getSize(); + if (commSize != processNum) + { + GTEST_SKIP() << " need " << processNum << " processes but got " << commSize << " mpi processes, skip test."; + } + if (mDeviceCount < deviceNum) + { + GTEST_SKIP() << " need " << deviceNum << " devices but got " << mDeviceCount << " devices, skip test."; + } + + bool isContext = commRank == 0; + bool isGeneration = commRank == 1; + std::vector<int> participatntIds = {commRank}; + std::vector<int> deviceIds = {commRank}; + bool isController = (commRank == controllerRank); + + OutputConfig outConfig(false, false, false, false, false, false); + int const beamWidth = 1; + BeamResult beamResult{beamWidth}; + + bool streaming = false; + int const maxBeamWidth = 1; + ASSERT_TRUE(fs::exists(DATA_PATH)); + + fs::path modelPath; + // set defaults and adjust if needed by different models + fs::path inputPath = DATA_PATH / "input_tokens.npy"; + ModelIds modelIds{50256, 50256}; + SizeType32 vocabSizePadded{50257}; // gpt vocabSizePadded + bool isSpeculativeDecoding{false}; + + // NOTE: This can be used to disable checks for certain prompt batch entries + FlakyTestInfo flakyTestInfo; + + if (modelName == "gpt") + { + auto const resultsPath + = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); + } + else if (modelName == "llama_tp1_pp1_cp1") + { + inputPath = DATA_PATH / LLAMA_INPUT_FILE; + vocabSizePadded = LLAMA_VOCAB_SIZE_PADDED; + + auto const resultsPath + = LLAMA_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + modelIds.padId = LLAMA_PAD_ID; + modelIds.endId = LLAMA_END_ID; + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP1_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + } + else + { + TLLM_THROW("Unrecognized modelName"); + } + + auto executorConfig = ExecutorConfig(maxBeamWidth); + FloatType freeGpuMemoryFraction = 0.9f; + KvCacheConfig kvCacheConfig{true, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction}; + executorConfig.setKvCacheConfig(kvCacheConfig); + executorConfig.setRequestStatsMaxIterations(1000); + executorConfig.setCacheTransceiverConfig( + texec::CacheTransceiverConfig(CacheTransceiverConfig::BackendType::DEFAULT)); + auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); + auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); + world_comm.barrier(); + auto executor = tensorrt_llm::testing::disaggexecutor::DisaggExecutorLeader(modelPath, ModelType::kDECODER_ONLY, + executorConfig, isController, isContext, isGeneration, givenInputLengths.size(), participatntIds, deviceIds, + commRank); + + std::unordered_map<IdType, SizeType32> reqIdToBatchId; + std::unordered_map<SizeType32, std::vector<BeamTokens>> tokens; + auto const* const givenInputData = tr::bufferCast<TokenIdType const>(*givenInput); + + auto const& inputShape = givenInput->getShape(); + ASSERT_EQ(inputShape.nbDims, 2); + ASSERT_GT(inputShape.d[0], 0); + + // Load expected outputs for each beam width value + auto testData = TestData::loadTestData(beamResult, *givenInput, beamWidth, manager, outConfig, modelIds); + auto const maxSeqLen = testData.maxSeqLen; + + // Load expected outputs and inputs + SizeType32 numRequests = static_cast<SizeType32>(givenInputLengths.size()); + SizeType32 maxRequests = numRequests; + std::vector<Request> requests; + std::vector<SizeType32> reqMaxNewTokens; + SizeType32 const numReturnSequences = 1; + + for (SizeType32 req = 0; req < maxRequests; ++req) + { + SizeType32 inputLen = givenInputLengths.at(req); + auto maxNewTokens = maxSeqLen - maxInputLength; + reqMaxNewTokens.push_back(maxNewTokens); + SizeType32 endId = -1; + auto const* const seqBegin = givenInputData + req * maxInputLength; + VecTokens tokens(seqBegin, seqBegin + inputLen); + auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); + samplingConfig.setNumReturnSequences(numReturnSequences); + auto request = Request( + VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, samplingConfig, outConfig, endId); + request.setReturnAllGeneratedTokens(false); + // setting request type to context/full by condition + if (req % 2 == 0) + { + request.setRequestType(RequestType::REQUEST_TYPE_CONTEXT_ONLY); + } + else + { + request.setRequestType(RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION); + } + requests.emplace_back(std::move(request)); + } + + if (isController) + { + std::vector<IdType> reqIds; + + for (int i = 0; i < requests.size(); ++i) + { + std::vector<BeamTokens> resultTokens; + resultTokens.reserve(numReturnSequences); + for (SizeType32 seqIdx = 0; seqIdx < numReturnSequences; ++seqIdx) + { + resultTokens.emplace_back(beamWidth); + } + auto retReqId = executor.enqueueRequests({requests[i]}); + reqIds.push_back(retReqId.front()); + tokens[i] = std::move(resultTokens); + reqIdToBatchId[retReqId.front()] = i; + } + + // Get the new tokens for each requests + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + while (numFinished < maxRequests && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + auto batchId = reqIdToBatchId.at(response.getRequestId()); + auto seqIdx = result.sequenceIndex; + + auto& outputTokenIds = result.outputTokenIds; + + EXPECT_EQ(result.finishReasons.size(), beamWidth); + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto& newTokens = outputTokenIds.at(beam); + auto& reqTokens = tokens.at(batchId).at(seqIdx).at(beam); + + reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); + // FinishReason is only supported for bw=1 and inflight batching. + if (beamWidth == 1 && executorConfig.getBatchingType() == BatchingType::kINFLIGHT) + { + EXPECT_EQ(result.finishReasons.at(beam), + result.isFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); + } + } + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + testData.verifyOutput(tokens, givenInputLengths, streaming, outConfig.excludeInputFromOutput, flakyTestInfo, + isSpeculativeDecoding, beamWidth, numReturnSequences, false); + } + world_comm.barrier(); +#else + GTEST_SKIP() << "Skipping DisaggExecutor Test"; +#endif +} + +INSTANTIATE_TEST_SUITE_P(GptDisaggSymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(2), // processNum + testing::Values(std::vector<std::string>{"gpt", "gpt"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0, 1) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(GptDisaggSymmetricExecutorMixedTest, DisaggParamsTest, + testing::Combine( // + testing::Values(2), // processNum + testing::Values(std::vector<std::string>{"gpt", "gpt"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kMIXED, InstanceRole::kMIXED}), // instanceRoles + testing::Values(1) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(GptSingleDeviceDisaggSymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(2), // processNum + testing::Values(std::vector<std::string>{"gpt", "gpt"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {0}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(GptSingleDeviceDisaggSymmetricExecutorMixedTest, DisaggParamsTest, + testing::Combine( // + testing::Values(2), // processNum + testing::Values(std::vector<std::string>{"gpt", "gpt"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0}, {1}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {0}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kMIXED, InstanceRole::kMIXED}), // instanceRoles + testing::Values(1) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(GptConditionalDisaggSymmetricExecutorTest, ConditionalDisaggParamsTest, + testing::Combine(testing::Values("gpt")), generateTestNameCondDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaConditionalDisaggSymmetricExecutorTest, ConditionalDisaggParamsTest, + testing::Combine(testing::Values("llama_tp1_pp1_cp1")), generateTestNameCondDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaTP2DisaggSymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(4), // processNum + testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1", "llama_tp2_pp1_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaPP2DisaggSymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(4), // processNum + testing::Values(std::vector<std::string>{"llama_tp1_pp2_cp1", "llama_tp1_pp2_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{1, 0}, {3, 2}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaTP2DisaggSymmetricExecutorMixedTest, DisaggParamsTest, + testing::Combine( // + testing::Values(2), // processNum + testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0, 1}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0, 1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kMIXED}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaPP2DisaggSymmetricExecutorMixedTest, DisaggParamsTest, + testing::Combine( // + testing::Values(2), // processNum + testing::Values(std::vector<std::string>{"llama_tp1_pp2_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0, 1}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0, 1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kMIXED}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaTP2PP2DisaggSymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(8), // processNum + testing::Values(std::vector<std::string>{"llama_tp2_pp2_cp1", "llama_tp2_pp2_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0, 1, 2, 3}, {4, 5, 6, 7}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{2, 3, 0, 1}, {2, 3, 0, 1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaConPP2GenTP2DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(4), // processNum + testing::Values(std::vector<std::string>{"llama_tp1_pp2_cp1", "llama_tp2_pp1_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // (1,0) (2,3) // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{1, 0}, {2, 3}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaConTP2GenPP2DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(4), // processNum + testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1", "llama_tp1_pp2_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}}), // (0,1), (3,2)// participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0, 1}, {3, 2}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaConTP2PP2GenPP2DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(6), // processNum + testing::Values(std::vector<std::string>{"llama_tp2_pp2_cp1", "llama_tp1_pp2_cp1"}), // modelNames + testing::Values( + std::vector<std::vector<int>>{{0, 1, 2, 3}, {4, 5}}), // (2,3,0,1) , (5,4)// participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{2, 3, 0, 1}, {1, 0}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaConTP2PP2GenTP2DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(6), // processNum + testing::Values(std::vector<std::string>{"llama_tp2_pp2_cp1", "llama_tp2_pp1_cp1"}), // modelNames + testing::Values( + std::vector<std::vector<int>>{{0, 1, 2, 3}, {4, 5}}), // (2,3,0,1), (4,5)// participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{2, 3, 0, 1}, {0, 1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); +INSTANTIATE_TEST_SUITE_P(LlamaConTP2PP1GenTP2PP2DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(6), // processNum + testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1", "llama_tp2_pp2_cp1"}), // modelNames + testing::Values( + std::vector<std::vector<int>>{{0, 1}, {2, 3, 4, 5}}), // (0,1) , (4,5,2,3)%4// participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0, 1}, {0, 1, 2, 3}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaConTP2GenPP4DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(6), // processNum + testing::Values(std::vector<std::string>{"llama_tp2_pp1_cp1", "llama_tp1_pp4_cp1"}), // modelNames + testing::Values( + std::vector<std::vector<int>>{{4, 5}, {0, 1, 2, 3}}), // (4,5) ,(3,2,1,0)// participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0, 1}, {3, 2, 1, 0}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon4TP1Gen1TP4DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(8), // processNum + testing::Values(std::vector<std::string>{"llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", + "llama_tp1_pp1_cp1", "llama_tp4_pp1_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2}, {3}, {4, 5, 6, 7}}), // participantIdsEachInstance + testing::Values( + std::vector<std::vector<int>>{{0}, {1}, {2}, {3}, {0, 1, 2, 3}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(4) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2TP2AndPP2DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(6), // processNum + testing::Values(std::vector<std::string>{ + "llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", "llama_tp2_pp1_cp1", "llama_tp1_pp2_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2, 3}, {4, 5}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2, 3}, {1, 0}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2PP2DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(6), // processNum + testing::Values(std::vector<std::string>{ + "llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", "llama_tp1_pp2_cp1", "llama_tp1_pp2_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2, 3}, {4, 5}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {3, 2}, {1, 0}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon4TP1Gen1TP2PP2DisaggAsymmetricExecutorTest, DisaggParamsTest, + testing::Combine( // + testing::Values(8), // processNum + testing::Values(std::vector<std::string>{"llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", "llama_tp1_pp1_cp1", + "llama_tp1_pp1_cp1", "llama_tp2_pp2_cp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2}, {3}, {4, 5, 6, 7}}), // participantIdsEachInstance + testing::Values( + std::vector<std::vector<int>>{{0}, {1}, {2}, {3}, {2, 3, 0, 1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(4) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2TP2DisaaggOrchestrator, DisaggOrchestratorParamsTest, + testing::Combine( // + testing::Values(7), // processNum + testing::Values( + std::vector<std::string>{"llama_tp1_pp1", "llama_tp1_pp1", "llama_tp2_pp1", "llama_tp2_pp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{1}, {2}, {3, 4}, {5, 6}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {2, 3}, {0, 1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); +// for disaggOrchestrator 1->0, 2->1, 3->2, 4->3, 5->0, 6->1 + +INSTANTIATE_TEST_SUITE_P(LlamaCon2TP2Gen2TP1DisaaggOrchestrator, DisaggOrchestratorParamsTest, + testing::Combine( // + testing::Values(7), // processNum + testing::Values( + std::vector<std::string>{"llama_tp2_pp1", "llama_tp2_pp1", "llama_tp1_pp1", "llama_tp1_pp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{1, 2}, {3, 4}, {5}, {6}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}, {0}, {1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2PP2DisaaggOrchestrator, DisaggOrchestratorParamsTest, + testing::Combine( // + testing::Values(7), // processNum + testing::Values( + std::vector<std::string>{"llama_tp1_pp1", "llama_tp1_pp1", "llama_tp1_pp2", "llama_tp1_pp2"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{1}, {2}, {3, 4}, {5, 6}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {3, 2}, {1, 0}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen1TP2PP2DisaaggOrchestrator, DisaggOrchestratorParamsTest, + testing::Combine( // + testing::Values(7), // processNum + testing::Values(std::vector<std::string>{"llama_tp1_pp1", "llama_tp1_pp1", "llama_tp2_pp2"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{1}, {2}, {3, 4, 5, 6}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {0, 1, 2, 3}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{ + InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon2TP2Gen2TP1DisaggSpawnOrchestrator, DisaggOrchestratorParamsTest, + testing::Combine( // + testing::Values(1), // processNum + testing::Values( + std::vector<std::string>{"llama_tp2_pp1", "llama_tp2_pp1", "llama_tp1_pp1", "llama_tp1_pp1"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{1, 2}, {3, 4}, {5}, {6}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0, 1}, {2, 3}, {0}, {1}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); + +INSTANTIATE_TEST_SUITE_P(LlamaCon2TP1Gen2PP2DisaggSpawnOrchestrator, DisaggOrchestratorParamsTest, + testing::Combine( // + testing::Values(1), // processNum + testing::Values( + std::vector<std::string>{"llama_tp1_pp1", "llama_tp1_pp1", "llama_tp1_pp2", "llama_tp1_pp2"}), // modelNames + testing::Values(std::vector<std::vector<int>>{{1}, {2}, {3, 4}, {5, 6}}), // participantIdsEachInstance + testing::Values(std::vector<std::vector<int>>{{0}, {1}, {3, 2}, {1, 0}}), // participantDeviceIdsEachInstance + testing::Values(std::vector<InstanceRole>{InstanceRole::kCONTEXT, InstanceRole::kCONTEXT, + InstanceRole::kGENERATION, InstanceRole::kGENERATION}), // instanceRoles + testing::Values(0) // controllerRank + ), + generateTestNameDisaggParams); diff --git a/cpp/tests/e2e_tests/executor/encDecTest.cpp b/cpp/tests/e2e_tests/executor/encDecTest.cpp new file mode 100644 index 000000000000..0095ae30ad40 --- /dev/null +++ b/cpp/tests/e2e_tests/executor/encDecTest.cpp @@ -0,0 +1,387 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "executorTest.h" + +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tensorrt_llm/runtime/utils/numpyUtils.h" +#include "tensorrt_llm/testing/modelSpec.h" +#include "tests/utils/common.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <nlohmann/json.hpp> + +#include <algorithm> +#include <chrono> +#include <memory> +#include <string> +#include <vector> + +namespace tr = tensorrt_llm::runtime; + +using namespace tensorrt_llm::testing; +using namespace tensorrt_llm::executor; +using namespace std::chrono_literals; +using tensorrt_llm::testing::KVCacheType; + +namespace +{ + +std::string getEncDecEnginePath(std::string const& modelName, SizeType32 tp, SizeType32 pp, SizeType32 cp) +{ + return modelName + '/' + std::to_string(tp * pp * cp) + "-gpu/float16"; +} + +TokenIdType getDecTokenFromJsonConfig(std::filesystem::path decEnginePath, std::string const& token_name) +{ + TokenIdType tokenId = 0; + try + { + std::ifstream decoderJsonConfigPath(decEnginePath / "config.json"); + auto const decoderPretrainedConfig + = nlohmann::json::parse(decoderJsonConfigPath, nullptr, true, true).at("pretrained_config"); + tokenId = decoderPretrainedConfig.at(token_name).template get<int32_t>(); + } + catch (nlohmann::json::out_of_range& e) + { + TLLM_LOG_ERROR( + "Parameter %s cannot be found from decoder config.json in pretrained_config. Using default id 0.", + token_name.c_str()); + } + catch (nlohmann::json::type_error const& e) + { + TLLM_LOG_ERROR( + "Parameter %s has a different type from decoder config.json in pretrained_config. Using default id 0.", + token_name.c_str()); + } + return tokenId; +} + +} // namespace + +using EncDecParamsType = std::tuple<std::string, SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, SizeType32, + std::vector<SizeType32>>; + +std::string generateTestNameEncDec(testing::TestParamInfo<EncDecParamsType> const& info) +{ + auto modelName = std::get<0>(info.param); + auto const beamWidth = std::get<1>(info.param); + auto const maxNewTokens = std::get<2>(info.param); + auto const tp = std::get<3>(info.param); + auto const pp = std::get<4>(info.param); + + // GTEST does not allow '-' in its test name + for (auto& c : modelName) + { + if (c == '-') + { + c = '_'; + } + } + + std::string name = "EncDecTest"; + name.append("_" + modelName); + name.append("_BeamWidth" + std::to_string(beamWidth)); + name.append("_MaxNewTokens" + std::to_string(maxNewTokens)); + name.append("_TP" + std::to_string(tp)); + name.append("_PP" + std::to_string(pp)); + return name; +} + +bool isLanguageAdapterName(std::string const& modelName) +{ + return modelName == LANGUAGE_ADAPTER_NAME; +} + +class EncDecParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<EncDecParamsType> +{ +}; + +TEST_P(EncDecParamsTest, validEncDecCtor) +{ + auto const modelName = std::get<0>(GetParam()); + SizeType32 const beamWidth = std::get<1>(GetParam()); + SizeType32 const maxNewTokens = std::get<2>(GetParam()); + SizeType32 const tp = std::get<3>(GetParam()); + SizeType32 const pp = std::get<4>(GetParam()); + SizeType32 const cp = std::get<5>(GetParam()); + + auto const enginePathName = getEncDecEnginePath(modelName, tp, pp, cp); + std::filesystem::path encEnginePath = ENC_DEC_ENGINE_BASE / enginePathName / "encoder"; + std::filesystem::path decEnginePath = ENC_DEC_ENGINE_BASE / enginePathName / "decoder"; + ExecutorConfig executorConfig{}; + FloatType freeGpuMemoryFraction = 0.4f; + FloatType crossKvCacheFraction = 0.4f; + KvCacheConfig kvCacheConfig{false, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction}; + kvCacheConfig.setCrossKvCacheFraction(crossKvCacheFraction); + executorConfig.setKvCacheConfig(kvCacheConfig); + auto executor = Executor(encEnginePath, decEnginePath, ModelType::kENCODER_DECODER, executorConfig); +} + +TEST_P(EncDecParamsTest, Forward) +{ + bool constexpr VERBOSE = false; + auto const modelName = std::get<0>(GetParam()); + SizeType32 const beamWidth = std::get<1>(GetParam()); + SizeType32 const maxNewTokens = std::get<2>(GetParam()); + SizeType32 const tp = std::get<3>(GetParam()); + SizeType32 const pp = std::get<4>(GetParam()); + SizeType32 const cp = std::get<5>(GetParam()); + + // Parameters for language adapter test + SizeType32 const numLanguages = std::get<6>(GetParam()); + std::vector<SizeType32> languageAdapterUids = std::get<7>(GetParam()); + + bool const streaming = false; + + auto const enginePathName = getEncDecEnginePath(modelName, tp, pp, cp); + std::filesystem::path encEnginePath = ENC_DEC_ENGINE_BASE / enginePathName / "encoder"; + std::filesystem::path decEnginePath = ENC_DEC_ENGINE_BASE / enginePathName / "decoder"; + + // load ground truth input & output data + auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); + auto inputsIdsHost + = tr::utils::loadNpy(manager, (ENC_DEC_DATA_BASE / "input_ids.npy").string(), tr::MemoryType::kCPU); + auto inputsIdsPtr = tr::bufferCast<TokenIdType>(*inputsIdsHost); + auto inputLengthsHost + = tr::utils::loadNpy(manager, (ENC_DEC_DATA_BASE / "input_lengths.npy").string(), tr::MemoryType::kCPU); + auto inputLengthsPtr = tr::bufferCast<SizeType32>(*inputLengthsHost); + auto encoderOutputHost + = tr::utils::loadNpy(manager, (ENC_DEC_DATA_BASE / "encoder_output.npy").string(), tr::MemoryType::kCPU); + auto encoderOutputPtr = tr::bufferCast<half>(*encoderOutputHost); + auto decoderOutputHost = tr::utils::loadNpy(manager, + (ENC_DEC_DATA_BASE / "output_ids_beam").string() + std::to_string(beamWidth) + ".npy", tr::MemoryType::kCPU); + auto decoderOutputPtr = tr::bufferCast<TokenIdType>(*decoderOutputHost); + + // Rank and size info + auto& comm = tensorrt_llm::mpi::MpiComm::world(); + auto const worldRank = comm.getRank(); + auto const worldSize = comm.getSize(); + + // create executor + BatchingType const batchingType = BatchingType::kINFLIGHT; + FloatType freeGpuMemoryFraction = 0.5f; + FloatType crossKvCacheFraction = 0.5f; + KvCacheConfig kvCacheConfig{false, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction}; + kvCacheConfig.setCrossKvCacheFraction(crossKvCacheFraction); + + ExecutorConfig executorConfig{beamWidth}; + executorConfig.setBatchingType(batchingType); + executorConfig.setKvCacheConfig(kvCacheConfig); + executorConfig.setNormalizeLogProbs(false); + + // TODO: OrchestratorMode test does not pass + bool const useOrchestratorMode = (tp * pp) > worldSize; + std::optional<OrchestratorConfig> orchestratorConfig = std::nullopt; + if (useOrchestratorMode) + { + orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + } + auto parallelConfig = ParallelConfig(CommunicationType::kMPI, + useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, std::nullopt, + orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + auto executor = Executor(encEnginePath, decEnginePath, ModelType::kENCODER_DECODER, executorConfig); + + OutputConfig outConfig; + outConfig.excludeInputFromOutput = false; + outConfig.returnLogProbs = false; + outConfig.returnGenerationLogits = false; + outConfig.returnContextLogits = false; + outConfig.returnEncoderOutput = false; + + TokenIdType bosId = getDecTokenFromJsonConfig(decEnginePath, "bos_token_id"); + TokenIdType padId = getDecTokenFromJsonConfig(decEnginePath, "pad_token_id"); + TokenIdType eosId = getDecTokenFromJsonConfig(decEnginePath, "eos_token_id"); + TokenIdType decoderStartTokenId = getDecTokenFromJsonConfig(decEnginePath, "decoder_start_token_id"); + + bool const isLanguageAdapterTest = isLanguageAdapterName(modelName); + // create requests + SizeType32 const nbRequests = inputLengthsHost->getShape().d[0]; + std::vector<Request> requests; + for (int i = 0, cumInputLen = 0; i < nbRequests; i++) + { + auto encoderInput = VecTokens(&inputsIdsPtr[cumInputLen], + &inputsIdsPtr[cumInputLen] + inputLengthsPtr[i]); // assume inputIds is flattened / no-padding + cumInputLen += inputLengthsPtr[i]; + auto decoderInput = VecTokens{decoderStartTokenId}; + Request req(decoderInput, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, + eosId, padId); + req.setEncoderInputTokenIds(encoderInput); + if (isLanguageAdapterTest) + { + req.setLanguageAdapterUid(languageAdapterUids[i]); + } + requests.emplace_back(req); + } + + using namespace std::chrono; + + // enqueue requests + if (worldRank == 0) + { + auto tik = high_resolution_clock::now(); + std::vector<IdType> reqIds = executor.enqueueRequests(std::move(requests)); + + // get responses + milliseconds waitTime(5000); + auto responsesAll = executor.awaitResponses(reqIds, waitTime); + auto tok = high_resolution_clock::now(); + TLLM_LOG_DEBUG("TRT-LLM C++ E2E time %d ms", duration_cast<milliseconds>(tok - tik).count()); + TLLM_LOG_DEBUG("Number of responses: %d", responsesAll.size()); + + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + std::unordered_map<IdType, std::vector<VecTokens>> outputTokens; + for_each(reqIds.begin(), reqIds.end(), + [&outputTokens, &beamWidth](auto const& id) + { + TLLM_LOG_DEBUG("Request IDs: %d", id); + outputTokens[id] = {}; + for (int i = 0; i < beamWidth; i++) + { + outputTokens[id].emplace_back(VecTokens{}); + } + }); + for (int i = 0; i < reqIds.size(); i++) + { + auto& responses = responsesAll[i]; + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + for (int beam = 0; beam < beamWidth; beam++) + { + auto& resTokens = result.outputTokenIds.at(beam); + auto& outTokens = outputTokens.at(response.getRequestId()).at(beam); + outTokens.insert(outTokens.end(), std::make_move_iterator(resTokens.begin()), + std::make_move_iterator(resTokens.end())); + } + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + } + + // print output & check correctness with ground truth + for (auto const& [reqId, tokens] : outputTokens) + { + SizeType32 gtMaxLength = decoderOutputHost->getShape().d[1]; + auto gtOutput = decoderOutputPtr + (reqId - 1) * gtMaxLength; + + if constexpr (VERBOSE) + { + std::cout << ">>> Request ID: " << reqId << std::endl; + for (int beam = 0; beam < beamWidth; beam++) + { + std::cout << "output tokens, beam " << beam << ", output length " << tokens[beam].size() << ": " + << std::endl; + for_each(tokens[beam].begin(), tokens[beam].end(), + [](auto const& token) { std::cout << token << ", "; }); + std::cout << std::endl; + } + std::cout << "ground truth tokens: " << std::endl; + + SizeType32 gtLength = 0; + for (int i = 0; i < gtMaxLength; i++) + { + if (gtOutput[i] != eosId) + { + std::cout << gtOutput[i] << ", "; + gtLength++; + } + } + std::cout << std::endl; + std::cout << "ground truth length: " << gtLength << std::endl; + } + + // check token-by-token match between beam 0 & ground truth + ASSERT_TRUE(tokens.size() <= gtMaxLength) + << "Request ID " << reqId << "'s generated length is longer than ground truth length " << gtMaxLength; + for (int i = 0; i < gtMaxLength; i++) + { + if (outConfig.excludeInputFromOutput) + { + // if results exclude decoder start token, skip it in ground truth too + continue; + } + if (i < tokens[0].size()) + { + ASSERT_EQ(tokens[0][i], gtOutput[i]) + << "Generated token id: " << tokens[0][i] << " v.s. ground truth: " << gtOutput[i]; + } + else + { + ASSERT_EQ(gtOutput[i], eosId) << "Request ID " << reqId << "'s generated length " << tokens.size() + << " is shorter than ground truth length " << gtMaxLength; + } + } + } + } +} + +INSTANTIATE_TEST_SUITE_P(T5BasicTest, EncDecParamsTest, + testing::Combine(testing::Values(T5_NAME), testing::Values(1), testing::Values(64), testing::Values(1), + testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), + generateTestNameEncDec); + +INSTANTIATE_TEST_SUITE_P(T5Beam2Test, EncDecParamsTest, + testing::Combine(testing::Values(T5_NAME), testing::Values(2), testing::Values(64), testing::Values(1), + testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), + generateTestNameEncDec); + +INSTANTIATE_TEST_SUITE_P(T5MultiGPUTest, EncDecParamsTest, + testing::Combine(testing::Values(T5_NAME), testing::Values(1), testing::Values(64), testing::Values(4), + testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), + generateTestNameEncDec); + +INSTANTIATE_TEST_SUITE_P(BartBasicTest, EncDecParamsTest, + testing::Combine(testing::Values(BART_NAME), testing::Values(1), testing::Values(64), testing::Values(1), + testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), + generateTestNameEncDec); + +INSTANTIATE_TEST_SUITE_P(BartBeam2Test, EncDecParamsTest, + testing::Combine(testing::Values(BART_NAME), testing::Values(2), testing::Values(64), testing::Values(1), + testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), + generateTestNameEncDec); + +INSTANTIATE_TEST_SUITE_P(BartMultiGPUTest, EncDecParamsTest, + testing::Combine(testing::Values(BART_NAME), testing::Values(1), testing::Values(64), testing::Values(4), + testing::Values(1), testing::Values(1), testing::Values(0), testing::Values(std::vector<SizeType32>{})), + generateTestNameEncDec); + +INSTANTIATE_TEST_SUITE_P(LanguageAdapterBasicTest, EncDecParamsTest, + testing::Combine(testing::Values(LANGUAGE_ADAPTER_NAME), testing::Values(1), testing::Values(64), + testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), + testing::Values(std::vector<SizeType32>{2, 3})), + generateTestNameEncDec); diff --git a/cpp/tests/e2e_tests/executor/executorMockTest.cpp b/cpp/tests/e2e_tests/executor/executorMockTest.cpp new file mode 100644 index 000000000000..0f0176f1ed6b --- /dev/null +++ b/cpp/tests/e2e_tests/executor/executorMockTest.cpp @@ -0,0 +1,1028 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "executorTest.h" + +#include "tensorrt_llm/batch_manager/trtGptModel.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tensorrt_llm/testing/modelSpec.h" +#include "tests/utils/common.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <nlohmann/json.hpp> + +#include <algorithm> +#include <chrono> +#include <memory> +#include <numeric> +#include <string> +#include <thread> +#include <vector> + +using ::testing::_; +using ::testing::Invoke; + +namespace tr = tensorrt_llm::runtime; +namespace tb = tensorrt_llm::batch_manager; + +using namespace tensorrt_llm::testing; +using namespace tensorrt_llm::executor; +using namespace std::chrono_literals; +using tensorrt_llm::testing::KVCacheType; + +class MockedModel : public Model +{ + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + +public: + MOCK_METHOD(void, forwardSync, (), ()); + MOCK_METHOD(void, forwardAsync, (RequestList const&), ()); + MOCK_METHOD(void, terminateRequest, (std::shared_ptr<tb::LlmRequest> const& llmRequest, bool pause), ()); + MOCK_METHOD( + void, terminateRequestSync, (std::shared_ptr<tb::LlmRequest> const& llmRequest, FinishReason finishReason), ()); + MOCK_METHOD(SizeType32, getMaxNumSequences, (), (const)); + MOCK_METHOD(SizeType32, getMaxInputLen, (), (const)); + MOCK_METHOD(SizeType32, getHiddenSize, (), (const)); + MOCK_METHOD(SizeType32, getMaxSequenceLen, (), (const)); + MOCK_METHOD(SizeType32, getVocabSizePadded, (), (const)); + MOCK_METHOD(SizeType32, getMaxDraftLen, (), (const)); + MOCK_METHOD(SizeType32, getNumMicroBatches, (), (const)); + MOCK_METHOD(SizeType32, getOperatingBeamWidth, (), (const)); + MOCK_METHOD(nvinfer1::DataType, getLogitDataType, (), (const)); + MOCK_METHOD(nvinfer1::DataType, getTensorDataType, (std::string const&), (const)); + MOCK_METHOD(nvinfer1::Dims, getTensorShape, (std::string const&), (const)); + MOCK_METHOD(void, getCurrentIterationStats, (IterationStats&), (const)); + MOCK_METHOD(void, getCurrentRequestStats, (RequestStatsPerIteration&), (const)); + MOCK_METHOD(DebugTensorsPerIteration, getCurrentDebugTensors, (), (const)); + MOCK_METHOD(tr::WorldConfig const&, getWorldConfig, (), (const)); + MOCK_METHOD(tr::ModelConfig const&, getModelConfig, (), (const)); + MOCK_METHOD(tr::BufferManager const&, getBufferManager, (), (const)); + MOCK_METHOD(tr::BufferManager::CudaStreamPtr, getRuntimeStreamPtr, (), (const)); + MOCK_METHOD(IterationType, getIterCounter, (), (const, noexcept)); + MOCK_METHOD(bool, hasSpeculativeDecodingFastLogits, (), (const, noexcept)); + MOCK_METHOD(bool, getGatherGenerationLogits, (), (const)); + MOCK_METHOD(void, updatePeftCache, (LlmRequestPtr const& llmReqeust), ()); + MOCK_METHOD(void, setLogitsPostProcessorBatched, (std::optional<LogitsPostProcessorBatched>), ()); + MOCK_METHOD(void, setReplicateLogitsPostProcessor, (bool), ()); + MOCK_METHOD(bool, getReplicateLogitsPostProcessor, (), (const)); + MOCK_METHOD(bool, hasGuidedDecoder, (), (const, noexcept)); + MOCK_METHOD(void, resetIterationStats, (), ()); + MOCK_METHOD( + std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager>, getKVCacheManager, (), ()); + MOCK_METHOD(std::shared_ptr<tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager const>, + getKVCacheManager, (), (const)); + MOCK_METHOD(SizeType32, getMaxCapacityBatchSize, (SizeType32, SizeType32), (const)); +}; + +using ParamType = std::tuple<bool, bool, int>; + +std::string generateTestName(testing::TestParamInfo<ParamType> const& info) +{ + auto const streaming = std::get<0>(info.param); + auto const excludeInputFromOutput = std::get<1>(info.param); + auto const beamWidth = std::get<2>(info.param); + std::string name = "ExecutorTest"; + if (streaming) + { + name += "Streaming"; + } + if (excludeInputFromOutput) + { + name += "ExclInput"; + } + name.append("BW" + std::to_string(beamWidth)); + return name; +} + +class ParamTest : public GptExecutorTest, public ::testing::WithParamInterface<ParamType> +{ +}; + +TEST_P(ParamTest, MockedModel) +{ + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + bool const streaming = std::get<0>(GetParam()); + bool const excludeInputFromOutput = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + auto model = std::make_shared<MockedModel>(); + + EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); + EXPECT_CALL(*model, getVocabSizePadded()).Times(0); + EXPECT_CALL(*model, getLogitDataType()).Times(0); + tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); + EXPECT_CALL(*model, getModelConfig()) + .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); + SizeType32 callCount = 0; + EXPECT_CALL(*model, forwardAsync(_)) + .WillRepeatedly(Invoke( + [&](RequestList const& requestList) + { + for (auto const& llmReq : requestList) + { + // Don't add any tokens to simulate no output tokens + llmReq->addNewTokens(VecTokens(beamWidth, 1)); + llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); + if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) + { + llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); + } + } + callCount++; + })); + + EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); + EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); + tr::WorldConfig const dummyWorldConfig; + EXPECT_CALL(*model, getWorldConfig()) + .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); + + ExecutorConfig const executorConfig(beamWidth); + auto executor = Executor(model, executorConfig); + + // Create the request + constexpr SizeType32 maxNewTokens = 5; + VecTokens const inputTokens{1, 2, 3, 4}; + auto request + = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + + // Enqueue the request + auto requestId = executor.enqueueRequest(request); + + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + auto const& result = response.getResult(); + done = result.isFinal; + } + ++iter; + } + + EXPECT_LT(iter, mMaxWaitMs); + EXPECT_EQ(callCount, maxNewTokens); +} + +TEST_F(GptExecutorTest, MockedModelMaxQueueSize) +{ + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + auto model = std::make_shared<MockedModel>(); + + EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); + EXPECT_CALL(*model, terminateRequestSync(_, _)).Times(0); + EXPECT_CALL(*model, getVocabSizePadded()).Times(0); + EXPECT_CALL(*model, getLogitDataType()).Times(0); + + SizeType32 callCount = 0; + EXPECT_CALL(*model, forwardAsync(_)) + .WillRepeatedly(Invoke( + [&](RequestList const& requestList) + { + for (auto const& llmReq : requestList) + { + // Sleep to allow queue to fill up + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + // Don't add any tokens to simulate no output tokens + llmReq->addNewTokens({1}); + llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); + if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) + { + llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); + } + } + callCount++; + })); + + EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); + EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); + tr::WorldConfig const dummyWorldConfig; + EXPECT_CALL(*model, getWorldConfig()) + .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); + tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); + EXPECT_CALL(*model, getModelConfig()) + .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); + SizeType32 maxQueueSize = 6; + ExecutorConfig executorConfig; + executorConfig.setMaxQueueSize(maxQueueSize); + + auto executor = Executor(model, executorConfig); + + // Create the request + SizeType32 const maxNewTokens = 5; + VecTokens const inputTokens{1, 2, 3, 4}; + auto request = Request(inputTokens, maxNewTokens); + + // Enqueue as many requests as the queue can manage + for (int i = 0; i < maxQueueSize; i++) + { + auto requestId = executor.enqueueRequest(request); + } + try + { + auto requestId = executor.enqueueRequest(request); + + FAIL() << "Expected TllmException"; + } + catch (std::exception const& e) + { + EXPECT_THAT(e.what(), testing::HasSubstr("Maximum queue size of 6 has been reached, please try again later")); + } + + // Wait for requests to get scheduled to free up space in queue + std::this_thread::sleep_for(std::chrono::milliseconds(maxQueueSize * 200)); + auto requestId = executor.enqueueRequest(request); + + try + { + auto samplingConfig = SamplingConfig(1); + samplingConfig.setNumReturnSequences(maxQueueSize); + auto request = Request(inputTokens, maxNewTokens, false, samplingConfig); + auto requestId = executor.enqueueRequest(request); + FAIL() << "Expected TllmException"; + } + catch (std::exception const& e) + { + EXPECT_THAT(e.what(), testing::HasSubstr("Maximum queue size of 6 has been reached, please try again later")); + } +} + +TEST_F(GptExecutorTest, MockedModelReqStatsBug) +{ + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + auto model = std::make_shared<MockedModel>(); + + EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); + EXPECT_CALL(*model, getVocabSizePadded()).Times(0); + EXPECT_CALL(*model, getLogitDataType()).Times(0); + EXPECT_CALL(*model, updatePeftCache(_)).WillRepeatedly(Invoke([&]() { return; })); + + SizeType32 callCount = 0; + RequestList currentReq; + EXPECT_CALL(*model, forwardAsync(_)) + .WillRepeatedly(Invoke( + [&](RequestList const& requestList) + { + currentReq = requestList; + for (auto const& llmReq : requestList) + { + // Don't add any tokens to simulate no output tokens + llmReq->addNewTokens({1}); + llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); + } + callCount++; + })); + + EXPECT_CALL(*model, forwardSync()) + .WillRepeatedly(Invoke( + [&]() + { + for (auto const& llmReq : currentReq) + { + if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) + { + llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); + } + } + return; + })); + + EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); + EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); + tr::WorldConfig const dummyWorldConfig; + EXPECT_CALL(*model, getWorldConfig()) + .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& stats) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& stats) { return; })); + + SizeType32 beamWidth = 1; + ExecutorConfig executorConfig(beamWidth); + executorConfig.setRequestStatsMaxIterations(1000); + auto executor = Executor(model, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + int numRequests = 10000; + auto request + = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + + auto done = std::atomic<bool>{false}; + auto statsThreadDone = false; + // Spawn a thread that continuously get stats + auto statsThread = std::thread( + [&executor, &done, &statsThreadDone]() + { + while (!done) + { + auto reqStats = executor.getLatestRequestStats(); + std::this_thread::sleep_for(std::chrono::microseconds(10)); + } + statsThreadDone = true; + }); + + // Spawn a thread that enqueues the requests + std::vector<IdType> requestIds; + auto enqueueThread = std::thread( + [&executor, &requestIds, &request, &done, numRequests]() + { + for (int i = 0; i < numRequests; ++i) + { + requestIds.push_back(executor.enqueueRequest(request)); + } + done = true; + }); + enqueueThread.join(); + ASSERT_EQ(requestIds.size(), numRequests); + + // Wait for stats thread to be done, fail otherwise + int iter = 0; + while (!statsThreadDone && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + std::this_thread::sleep_for(std::chrono::milliseconds(waitTime)); + iter++; + } + ASSERT_TRUE(statsThreadDone); + statsThread.join(); +} + +TEST_F(GptExecutorTest, MockedModelEvictRestartValidityTest) +{ + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + constexpr bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + auto model = std::make_shared<MockedModel>(); + + EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); + EXPECT_CALL(*model, getVocabSizePadded()).Times(0); + EXPECT_CALL(*model, getLogitDataType()).Times(0); + EXPECT_CALL(*model, updatePeftCache(_)).WillRepeatedly(Invoke([&]() { return; })); + tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); + EXPECT_CALL(*model, getModelConfig()) + .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); + SizeType32 callCount = 0; + RequestList currentReq; + EXPECT_CALL(*model, forwardAsync(_)) + .WillRepeatedly(Invoke( + [&](RequestList const& requestList) + { + currentReq = requestList; + for (auto const& llmReq : requestList) + { + // Don't add any tokens to simulate no output tokens + llmReq->addNewTokens({1}); + llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); + } + callCount++; + })); + + EXPECT_CALL(*model, forwardSync()) + .WillRepeatedly(Invoke( + [&]() + { + for (auto const& llmReq : currentReq) + { + if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) + { + llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); + } + } + return; + })); + + EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 6; })); + EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); + EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); + tr::WorldConfig const dummyWorldConfig; + EXPECT_CALL(*model, getWorldConfig()) + .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); + + SizeType32 const beamWidth = 1; + ExecutorConfig executorConfig(beamWidth, + SchedulerConfig(CapacitySchedulerPolicy::kMAX_UTILIZATION)); // Condition 1 : MAX_UTILIZATION scheduling policy + executorConfig.setEnableChunkedContext(false); // Condition 2 : Chunked context disabled + executorConfig.setRequestStatsMaxIterations(1000); + auto executor = Executor(model, executorConfig); + + // Create the request + constexpr bool streaming = true; // Condition 3 : Streaming enabled + SizeType32 const maxNewTokens = 5; + VecTokens const tooLongInputTokens{1, 2, 3, 4, 5}; // Condition 4 : prompt input len + maxNewTokens > MaxInputLen + auto tooLongRequest = Request( + tooLongInputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + + // Enqueue the request + auto longRequestId = executor.enqueueRequest(tooLongRequest); + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(longRequestId, waitTime); + for (auto& response : responses) + { + EXPECT_EQ(response.hasError(), true); + EXPECT_THAT(response.getErrorMsg(), + testing::HasSubstr("sequence length is potentially greater than max input length")); + done = true; + } + ++iter; + } +} + +#if ENABLE_MULTI_DEVICE +// This test can be run manually to test multiGPU execution +// mpirun --allow-run-as-root -n 5 ./executorTest --gtest_filter="*MockedModelMultiGpu/ExecutorTest" +// Number of MPI ranks can be greater than tp + +TEST_P(ParamTest, MockedModelMultiGpu) +{ + auto const& world = tensorrt_llm::mpi::MpiComm::world(); + auto const worldRank = world.getRank(); + auto const worldSize = world.getSize(); + + // In this test, allow worldSize to be greater than tp = 4 + // If so, set participant ids to be the last 4 ranks + SizeType32 const tp = std::min(4, worldSize); + + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + bool const streaming = std::get<0>(GetParam()); + bool const excludeInputFromOutput = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + auto model = std::make_shared<MockedModel>(); + + // Create the request + constexpr SizeType32 maxNewTokens = 5; + VecTokens const inputTokens{1, 2, 3, 4}; + auto request + = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + + EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); + EXPECT_CALL(*model, getVocabSizePadded()).Times(0); + EXPECT_CALL(*model, getLogitDataType()).Times(0); + tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); + EXPECT_CALL(*model, getModelConfig()) + .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); + SizeType32 callCount = 0; + SizeType32 reqCallCount = 0; + EXPECT_CALL(*model, forwardAsync(_)) + .WillRepeatedly(Invoke( + [&](RequestList const& requestList) + { + for (auto const& llmReq : requestList) + { + EXPECT_EQ(llmReq->getTokens().size(), beamWidth); + // Verify that all MPI ranks get the expected request, even though only rank 0 actually gets the + // request + if (reqCallCount == 0) + { + EXPECT_EQ(llmReq->getOrigPromptLen(), request.getInputTokenIds().size()); + for (int i = 0; i < llmReq->getOrigPromptLen(); ++i) + { + EXPECT_EQ(llmReq->getTokens(beamWidth - 1).at(i), request.getInputTokenIds().at(i)); + } + } + EXPECT_EQ(llmReq->isStreaming(), request.getStreaming()); + EXPECT_EQ(llmReq->mMaxNewTokens, request.getMaxTokens()); + EXPECT_EQ( + llmReq->getTokens(beamWidth - 1).size(), request.getInputTokenIds().size() + reqCallCount); + + SizeType32 tokenId = 1; + COMM_SESSION.bcastValue(tokenId, 0); + // Don't add any tokens to simulate no output tokens + // Simulate leader rank communicating with comm session + VecTokens const newTokens(beamWidth, tokenId); + llmReq->addNewTokens(newTokens); + llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); + if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) + { + llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); + } + reqCallCount++; + } + callCount++; + })); + + EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); + + tr::WorldConfig dummyWorldConfig = tr::WorldConfig(tp, 1, 1, worldRank, tp); + EXPECT_CALL(*model, getWorldConfig()) + .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); + + ParallelConfig parallelConfig; + + // Set participant ids to be of size tp, starting at worldSize - 1 + std::vector<SizeType32> participantIds; + participantIds.reserve(tp); + for (int i = 0; i < tp; ++i) + { + participantIds.push_back(worldSize - tp + i); + } + bool const isLeader = (worldRank == participantIds.front()); + parallelConfig.setParticipantIds(participantIds); + + bool const isWorker = (std::find(participantIds.begin(), participantIds.end(), worldRank) != participantIds.end()); + + // Set device ids + std::vector<SizeType32> deviceIds(tp); + std::iota(deviceIds.begin(), deviceIds.end(), 0); + parallelConfig.setDeviceIds(deviceIds); + + ExecutorConfig executorConfig(beamWidth); + executorConfig.setParallelConfig(parallelConfig); + auto executor = Executor(model, executorConfig); + + EXPECT_EQ(isWorker, executor.isParticipant()); + + // Enqueue the request + IdType requestId = 0; + if (isLeader) + { + requestId = executor.enqueueRequest(request); + + SizeType32 numResponses{0}; + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + ++numResponses; + auto const& result = response.getResult(); + EXPECT_EQ(result.outputTokenIds.size(), beamWidth); + auto expectedSize = streaming ? (beamWidth > 1 ? numResponses : 1) + : (maxNewTokens + (excludeInputFromOutput ? 0 : inputTokens.size())); + EXPECT_EQ(result.outputTokenIds.at(beamWidth - 1).size(), expectedSize); + done = result.isFinal; + } + ++iter; + } + + EXPECT_LT(iter, mMaxWaitMs); + EXPECT_EQ(numResponses, streaming ? maxNewTokens : 1); + EXPECT_EQ(callCount, maxNewTokens); + } +} +#endif // ENABLE_MULTI_DEVICE + +TEST_F(GptExecutorTest, MockedModelWithError) +{ + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + struct MockedModelParams + { + SizeType32 maxInputLen; + SizeType32 maxSeqLen; + SizeType32 expectedTerminateCnt; + SizeType32 expectedForwardCnt; + bool computeGenLogits; + bool computeContextLogits; + std::string expectedError; + }; + + std::vector<MockedModelParams> mockedModelParams; + // Mocked error in forward call + mockedModelParams.emplace_back(MockedModelParams{10, 20, 1, 1, true, true, "mocked error"}); + // prompt longer than maxInputLen + mockedModelParams.emplace_back(MockedModelParams{1, 20, 0, 0, true, true, "exceeds maximum input length"}); + // Model doesn't support context logits output + mockedModelParams.emplace_back( + MockedModelParams{10, 20, 0, 0, false, true, "gather_generation_logits must be enabled"}); + // Model doesn't support gen logits output + mockedModelParams.emplace_back( + MockedModelParams{10, 20, 0, 0, true, false, "need to build engine with gather_context"}); + + for (auto const& mockedModelParam : mockedModelParams) + { + auto model = std::make_shared<MockedModel>(); + SizeType32 beamWidth = 1; + + // One request should be terminated + EXPECT_CALL(*model, terminateRequest(_, _)).Times(mockedModelParam.expectedTerminateCnt); + EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 1024; })); + EXPECT_CALL(*model, getLogitDataType()).WillRepeatedly(Invoke([&]() { return nvinfer1::DataType::kFLOAT; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& stats) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& stats) { return; })); + + SizeType32 callCount = 0; + EXPECT_CALL(*model, forwardAsync(_)) + .WillRepeatedly(Invoke( + [&](RequestList const&) + { + callCount++; + // There was a bug where we were missing a notify call when errors were encountered + // and this test was not catching it, probably because the error was reported + // before the first call to awaitResponses. So we add a sleep here to make sure + // the awaitResponses is called before the error is thrown + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + throw std::runtime_error("mocked error"); + })); + + EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return mockedModelParam.maxInputLen; })); + EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return mockedModelParam.maxSeqLen; })); + EXPECT_CALL(*model, getMaxDraftLen()).WillRepeatedly(Invoke([&]() { return 0; })); + tr::WorldConfig const dummyWorldConfig; + EXPECT_CALL(*model, getWorldConfig()) + .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); + tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); + dummyModelConfig.computeContextLogits(mockedModelParam.computeContextLogits); + dummyModelConfig.computeGenerationLogits(mockedModelParam.computeGenLogits); + EXPECT_CALL(*model, getModelConfig()) + .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); + EXPECT_CALL(*model, getGatherGenerationLogits()) + .WillRepeatedly(Invoke([&]() -> bool { return mockedModelParam.computeGenLogits; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& stats) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& stats) { return; })); + EXPECT_CALL(*model, getIterCounter()).WillRepeatedly(Invoke([&]() -> IterationType { return 0; })); + + ExecutorConfig executorConfig(beamWidth); + auto executor = Executor(model, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + + OutputConfig outConfig; + outConfig.returnContextLogits = true; + outConfig.returnGenerationLogits = true; + + auto streaming = false; + auto request = Request( + inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + + // Enqueue the request + auto requestId = executor.enqueueRequest(std::move(request)); + + bool done = false; + auto responses = executor.awaitResponses(requestId); + for (auto& response : responses) + { + if (!response.hasError()) + { + FAIL() << "Expecting an error to be received"; + } + else + { + auto err = response.getErrorMsg(); + EXPECT_THAT(err, testing::HasSubstr(mockedModelParam.expectedError)); + done = true; + } + } + + EXPECT_TRUE(done); + EXPECT_EQ(callCount, mockedModelParam.expectedForwardCnt); + } +} + +TEST_F(GptExecutorTest, MockedModelCancelRequest) +{ + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + constexpr bool streaming = true; + auto model = std::make_shared<MockedModel>(); + + std::unordered_map<IdType, tensorrt_llm::executor::FinishReason> reqIdsToTerminate; + // Two requests with one child request (3 in total) should be terminated + EXPECT_CALL(*model, terminateRequestSync(_, _)) + .Times(3) + .WillRepeatedly(Invoke([&](LlmRequestPtr const& llmRequest, FinishReason finishReason) + { reqIdsToTerminate.try_emplace(llmRequest->mRequestId, finishReason); })); + EXPECT_CALL(*model, terminateRequest(_, _)).Times(3); + EXPECT_CALL(*model, getVocabSizePadded()).Times(0); + EXPECT_CALL(*model, getLogitDataType()).Times(0); + tr::WorldConfig const dummyWorldConfig; + EXPECT_CALL(*model, getWorldConfig()) + .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); + tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); + EXPECT_CALL(*model, getModelConfig()) + .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); + + SizeType32 callCount = 0; + std::unordered_map<IdType, SizeType32> callCountPerSeq; + EXPECT_CALL(*model, forwardAsync(_)) + .WillRepeatedly(Invoke( + [&](RequestList const& requestList) + { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + + for (auto const& llmReq : requestList) + { + if (llmReq->isGenerationCompleteState()) + { + continue; + } + // Don't add any tokens to simulate no output tokens + llmReq->addNewTokens({1}); + llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); + if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) + { + llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); + } + if (callCountPerSeq.find(llmReq->mRequestId) != callCountPerSeq.end()) + { + callCountPerSeq[llmReq->mRequestId]++; + } + else + { + callCountPerSeq[llmReq->mRequestId] = 1; + } + + if (reqIdsToTerminate.count(llmReq->mRequestId) != 0U) + { + if (!llmReq->isGenerationToCompleteState()) + { + model->terminateRequest(llmReq, false); + llmReq->finishByReason(reqIdsToTerminate[llmReq->mRequestId]); + llmReq->clearGeneratedTokens(); + } + reqIdsToTerminate.erase(llmReq->mRequestId); + } + } + callCount++; + })); + + EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 100; })); + EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 200; })); + EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); + + SizeType32 const beamWidth = 1; + ExecutorConfig const executorConfig(beamWidth); + auto executor = Executor(model, executorConfig); + + // Create the request + SizeType32 const maxNewTokens = 150; + VecTokens const inputTokens{1, 2, 3, 4}; + auto request = Request(inputTokens, maxNewTokens, streaming); + + // Enqueue the request + auto requestId = executor.enqueueRequest(std::move(request)); + + // Cancel the request + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + executor.cancelRequest(requestId); + + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + + if (response.hasError()) + { + FAIL() << "Not expecting an error to be received"; + } + + auto const& result = response.getResult(); + done = result.isFinal; + if (done) + { + for (SizeType32 beamIdx = 0; beamIdx < beamWidth; ++beamIdx) + { + EXPECT_EQ(result.finishReasons[beamIdx], FinishReason::kCANCELLED); + } + } + } + ++iter; + } + + EXPECT_LT(iter, mMaxWaitMs); + // Expecting to receiving fewer tokens than maxNewTokens + EXPECT_LT(callCount, maxNewTokens); + + // Create the request having child requests. + auto samplingConfig2 = SamplingConfig(1); + samplingConfig2.setNumReturnSequences(2); + auto request2 = Request(inputTokens, maxNewTokens, streaming, samplingConfig2); + + // Reset call count. + callCount = 0; + callCountPerSeq.clear(); + + // Enqueue the request + auto requestId2 = executor.enqueueRequest(request2); + + // Cancel the request + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + executor.cancelRequest(requestId2); + + done = false; + iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId2, waitTime); + for (auto& response : responses) + { + + if (response.hasError()) + { + FAIL() << "Not expecting an error to be received"; + } + + auto const& result = response.getResult(); + done = result.isFinal; + if (done) + { + EXPECT_EQ(result.finishReasons[0], FinishReason::kCANCELLED); + } + } + ++iter; + } + + EXPECT_LT(iter, mMaxWaitMs); + for (auto& [reqId, count] : callCountPerSeq) + { + // Expecting to receiving fewer tokens than maxNewTokens + EXPECT_LT(count, maxNewTokens) << "Failed at request id: " << reqId; + } +} + +TEST_F(GptExecutorTest, MockedModelNumReturns) +{ + using LlmRequestPtr = std::shared_ptr<tb::LlmRequest>; + using RequestList = std::list<LlmRequestPtr>; + + SizeType32 const maxBeamWidth = 4; + OutputConfig const outConfig; + auto model = std::make_shared<MockedModel>(); + + EXPECT_CALL(*model, terminateRequest(_, _)).Times(0); + EXPECT_CALL(*model, getVocabSizePadded()).Times(0); + EXPECT_CALL(*model, getLogitDataType()).Times(0); + tr::WorldConfig const dummyWorldConfig; + EXPECT_CALL(*model, getWorldConfig()) + .WillRepeatedly(Invoke([&]() -> tr::WorldConfig const& { return dummyWorldConfig; })); + EXPECT_CALL(*model, getCurrentIterationStats(_)).WillRepeatedly(Invoke([&](IterationStats& /*stats*/) { return; })); + EXPECT_CALL(*model, getCurrentRequestStats(_)) + .WillRepeatedly(Invoke([&](RequestStatsPerIteration& /*stats*/) { return; })); + tr::ModelConfig dummyModelConfig(0, 0, 0, 0, 1, 0, nvinfer1::DataType::kHALF); + EXPECT_CALL(*model, getModelConfig()) + .WillRepeatedly(Invoke([&]() -> tr::ModelConfig const& { return dummyModelConfig; })); + SizeType32 callCount = 0; + EXPECT_CALL(*model, forwardAsync(_)) + .WillRepeatedly(Invoke( + [&](RequestList const& requestList) + { + for (auto const& llmReq : requestList) + { + // Don't add any tokens to simulate no output tokens + auto numBeams = llmReq->mSamplingConfig.getNumReturnBeams(); + llmReq->addNewTokens(VecTokens(numBeams, 1)); + llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS); + if (llmReq->getMaxNumGeneratedTokens() >= llmReq->mMaxNewTokens) + { + llmReq->setState(tb::LlmRequestState::kGENERATION_COMPLETE); + } + } + callCount++; + })); + + EXPECT_CALL(*model, getMaxNumSequences()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxInputLen()).WillRepeatedly(Invoke([&]() { return 10; })); + EXPECT_CALL(*model, getMaxSequenceLen()).WillRepeatedly(Invoke([&]() { return 20; })); + EXPECT_CALL(*model, getVocabSizePadded()).WillRepeatedly(Invoke([&]() { return 80000; })); + + ExecutorConfig const executorConfig(maxBeamWidth); + auto executor = Executor(model, executorConfig); + + // Create the request + SizeType32 const maxNewTokens = 5; + VecTokens const inputTokens{1, 2, 3, 4}; + constexpr bool streaming = false; + + auto samplingConfig1 = SamplingConfig(1); + samplingConfig1.setNumReturnSequences(3); + auto request1 = Request(inputTokens, maxNewTokens, streaming, samplingConfig1, outConfig); + auto samplingConfig2 = SamplingConfig(4); + auto request2 = Request(inputTokens, maxNewTokens, streaming, samplingConfig2, outConfig); + auto samplingConfig3 = SamplingConfig(4); + samplingConfig3.setNumReturnSequences(2); + auto request3 = Request(inputTokens, maxNewTokens, streaming, samplingConfig3, outConfig); + + // Enqueue the request + auto requestId1 = executor.enqueueRequest(request1); + auto requestId2 = executor.enqueueRequest(request2); + auto requestId3 = executor.enqueueRequest(request3); + + // Expecting one response in beam search. Instead, numReturnSequences limits the number of beams to return. + std::unordered_map<IdType, SizeType32> expectedNumResponses{{requestId1, 3}, {requestId2, 1}, {requestId3, 1}}; + std::unordered_map<IdType, SizeType32> const expectedNumBeams{{requestId1, 1}, {requestId2, 4}, {requestId3, 2}}; + + std::unordered_map<IdType, SizeType32> numResponses{{requestId1, 0}, {requestId2, 0}, {requestId3, 0}}; + std::unordered_map<IdType, SizeType32> numBeams{{requestId1, 0}, {requestId2, 0}, {requestId3, 0}}; + int numFinished = 0; + int iter = 0; + while (numFinished < 3 && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + auto const& result = response.getResult(); + auto reqId = response.getRequestId(); + numFinished += result.isFinal; + numResponses[reqId]++; + numBeams[reqId] = result.outputTokenIds.size(); + } + ++iter; + } + + EXPECT_LT(iter, mMaxWaitMs); + EXPECT_EQ(numFinished, 3); + for (auto& [reqId, numResp] : numResponses) + { + EXPECT_EQ(numResp, expectedNumResponses[reqId]); + } + for (auto& [reqId, numResp] : numResponses) + { + EXPECT_EQ(numResp, expectedNumResponses[reqId]); + } +} + +INSTANTIATE_TEST_SUITE_P(GptExecutorTest, ParamTest, + testing::Combine(testing::Values(false, true), // streaming + testing::Values(false, true), // excludeInputFromOutput + testing::Values(1, 2) // beamWidth + ), + generateTestName); diff --git a/cpp/tests/e2e_tests/executor/executorTest.cpp b/cpp/tests/e2e_tests/executor/executorTest.cpp new file mode 100644 index 000000000000..e1227970cb71 --- /dev/null +++ b/cpp/tests/e2e_tests/executor/executorTest.cpp @@ -0,0 +1,4671 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TOP_LEVEL_DIR +#error "Define TOP_LEVEL_DIR" +#endif + +#include "executorTest.h" + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/executor/dataTransceiverState.h" +#include "tensorrt_llm/executor/requestWithId.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/executor/version.h" +#include "tensorrt_llm/runtime/gptJsonConfig.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include "tensorrt_llm/runtime/utils/numpyUtils.h" +#include "tensorrt_llm/testing/modelSpec.h" +#include "tests/utils/common.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <nlohmann/json.hpp> + +#include <algorithm> +#include <chrono> +#include <cstddef> +#include <functional> +#include <memory> +#include <string> +#include <thread> +#include <vector> + +namespace tr = tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; + +using namespace tensorrt_llm::testing; +using namespace tensorrt_llm::executor; +using namespace std::chrono_literals; +namespace fs = std::filesystem; +using tensorrt_llm::testing::KVCacheType; +using tensorrt_llm::testing::ModelSpec; + +namespace +{ + +auto const LORA_DATA_PATH = DATA_PATH / "lora-test-weights-gpt2-tp1"; +auto const LORA_WEIGHTS_FILE = LORA_DATA_PATH / "source.npy"; +auto const LORA_CONFIG_FILE = LORA_DATA_PATH / "config.npy"; + +auto constexpr LLAMA_INPUT_FILE = "input_tokens_llama.npy"; +auto constexpr LLAMA_VOCAB_SIZE_PADDED = 128256; +auto constexpr LLAMA_PAD_ID = 128001; +auto constexpr LLAMA_END_ID = 128001; + +} // namespace + +void testInvalidCtor(std::filesystem::path const& enginePath, ModelType modelType, ExecutorConfig executorConfig, + std::string expectedErrMsg = "") +{ + try + { + auto executor = Executor(enginePath, modelType, executorConfig); + + FAIL() << "Expected TllmException"; + } + catch (std::exception const& e) + { + EXPECT_THAT(e.what(), testing::HasSubstr(expectedErrMsg)); + } +} + +TEST_F(GptExecutorTest, version) +{ + EXPECT_STRNE(kTensorRtLlmVersion, "@TRTLLM_VERSION@"); + EXPECT_STREQ(kTensorRtLlmVersion, version()); +} + +TEST_F(GptExecutorTest, validCtor) +{ + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); +} + +TEST_F(GptExecutorTest, invalidCtor) +{ + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + std::filesystem::path invalidPath{"Bla"}; + + // Invalid path + { + testInvalidCtor(invalidPath, ModelType::kDECODER_ONLY, executorConfig, "File does not exist"); + } +} + +TEST_F(GptExecutorTest, enqueueAfterShutdown) +{ + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + auto request = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); + auto requestId = executor.enqueueRequest(request); + + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + FAIL(); + } + else + { + done = response.getResult().isFinal; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + executor.shutdown(); + + EXPECT_FALSE(executor.canEnqueueRequests()); + + std::string expErrMsg{"Shutdown called"}; + EXPECT_THAT([&]() { auto reqId = executor.enqueueRequest(request); }, + testing::Throws<tensorrt_llm::common::TllmException>( + testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); + EXPECT_THAT([&]() { auto resp = executor.awaitResponses(); }, + testing::Throws<tensorrt_llm::common::TllmException>( + testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); + EXPECT_THAT([&]() { auto stats = executor.getLatestIterationStats(); }, + testing::Throws<tensorrt_llm::common::TllmException>( + testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); + EXPECT_THAT([&]() { auto stats = executor.getLatestRequestStats(); }, + testing::Throws<tensorrt_llm::common::TllmException>( + testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); + EXPECT_THAT([&]() { executor.cancelRequest(requestId); }, + testing::Throws<tensorrt_llm::common::TllmException>( + testing::Property(&tensorrt_llm::common::TllmException::what, testing::HasSubstr(expErrMsg)))); +} + +TEST_F(GptExecutorTest, missingPeftTask) +{ + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_LORA_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + auto request = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); + auto loraConfig = LoraConfig{10}; + request.setLoraConfig(loraConfig); + + auto requestId = executor.enqueueRequest(request); + + bool done = false; + std::chrono::milliseconds waitTime(mMaxWaitMs); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + auto err = response.getErrorMsg(); + EXPECT_EQ(err, std::string("LoRA task 10 not found in cache. Please send LoRA weights with request")); + done = true; + } + else + { + FAIL() << "Expects error due to missing Lora weights"; + } + } + EXPECT_TRUE(done); +} + +TEST_F(GptExecutorTest, ReturnAcceptedTokenLogits) +{ + SizeType32 constexpr beamWidth{1}; + SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded + + // Create executor config + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setGatherGenerationLogits(true); + + // Enable kv cache reuse of executorConfig + bool enableBlockReuse = true; + FloatType freeGpuMemoryFraction = 0.4; + auto kvCacheConfig + = KvCacheConfig(enableBlockReuse, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction); + executorConfig.setKvCacheConfig(kvCacheConfig); + + // Create executor + auto trtEnginePath + = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DRAFT_TOKENS_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4, 5, 6, 7, 8}; + + std::vector<bool> streamingOptions{false, true}; + + for (auto streaming : streamingOptions) + { + auto request = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth)); + + // Set draft tokens + auto draftTokens = VecTokens{9, 10, 11, 12, 13}; // draft tokens + auto draftLength = draftTokens.size(); + FloatType const acceptanceThreshold = 0.00001f; // Ensure the draft token can be accepted + auto externalDraftTokensConfig = ExternalDraftTokensConfig(draftTokens, std::nullopt, acceptanceThreshold); + request.setExternalDraftTokensConfig(externalDraftTokensConfig); + + // Set return accepted token logits for this request + OutputConfig outConfig; + outConfig.returnGenerationLogits = true; + request.setOutputConfig(outConfig); + + // Enqueue this request + auto requestId = executor.enqueueRequest(request); + + bool done = false; + int iter = 0; + while (!done && iter < 5000) + { + std::chrono::milliseconds waitTime(mMaxWaitMs); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + FAIL(); + } + else + { + auto result = response.getResult(); + done = result.isFinal; + auto& genLogits = result.generationLogits; + EXPECT_TRUE(genLogits.has_value()); + + // Expected shape: (1, numAcceptedDraftToken, vocabSizePadded) + auto const& acceptedTokenLogitsShape = genLogits->getShape(); + EXPECT_EQ(acceptedTokenLogitsShape.size(), 3); + EXPECT_EQ(acceptedTokenLogitsShape[0], 1); + EXPECT_LE(acceptedTokenLogitsShape[1], draftLength); // number of accepted tokens + EXPECT_EQ(acceptedTokenLogitsShape[2], vocabSizePadded); // vocabSizePadded + } + } + ++iter; + } + } +} + +TEST_F(GptExecutorTest, GenerationLogitsEarlyStop) +{ + SizeType32 constexpr beamWidth{1}; + SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded + auto constexpr streaming = false; + + ExtendedRuntimePerfKnobConfig perfKnobConfig = ExtendedRuntimePerfKnobConfig(); + + // Create executor config + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setExtendedRuntimePerfKnobConfig(perfKnobConfig); + executorConfig.setGatherGenerationLogits(true); + + // Create executor + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + auto const inputPath = DATA_PATH / "input_tokens.npy"; + ModelIds modelIds{50256, 50256}; + + auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); + auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); + auto const* const givenInputData = tr::bufferCast<TokenIdType const>(*givenInput); + + auto const& inputShape = givenInput->getShape(); + ASSERT_EQ(inputShape.nbDims, 2); + ASSERT_GT(inputShape.d[0], 0); + + BeamResult beamResult{beamWidth}; + auto const resultsPath + = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); + beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); + beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); + + // Set return generation logits for this request + OutputConfig outConfig; + outConfig.returnGenerationLogits = true; + outConfig.excludeInputFromOutput = true; + + // Load expected outputs for each beam width value + auto testData = TestData::loadTestData(beamResult, *givenInput, beamWidth, manager, outConfig, modelIds); + auto const maxSeqLen = testData.maxSeqLen; + + // Load expected outputs and inputs + std::vector<Request> requests; + std::vector<SizeType32> reqMaxNewTokens; + + auto constexpr reqIdx = 0; + SizeType32 inputLen = givenInputLengths.at(reqIdx); + auto maxNewTokens = maxSeqLen - maxInputLength; + reqMaxNewTokens.push_back(maxNewTokens); + auto const* const seqBegin = givenInputData + reqIdx * maxInputLength; + + auto request = Request(VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, modelIds.endId); + // copy request + auto request2 = request; + + auto const expectedOutputData = tr::BufferRange<TokenIdType const>(*testData.expectedOutputIds); + auto const expectedOutputLengths = testData.expectedOutputLengths; + auto const endPos = expectedOutputLengths[reqIdx] - 3; + auto const endIndex = tc::flat_index3(reqIdx, beamWidth - 1, endPos, beamWidth, maxSeqLen); + auto const endToken = expectedOutputData[endIndex]; + + // Set end id to stop early + request.setEndId(endToken); + requests.emplace_back(std::move(request)); + + // Set stop words to stop early + request2.setStopWords({{endToken}}); + requests.emplace_back(std::move(request2)); + + // Enqueue requests + auto requestIds = executor.enqueueRequests(requests); + + std::map<IdType, SizeType32> expectedNewTokens; + expectedNewTokens[requestIds.at(0)] = endPos - inputLen; + expectedNewTokens[requestIds.at(1)] = endPos - inputLen + 1; + + std::map<IdType, FinishReason> expectedFinishReason; + expectedFinishReason[requestIds.at(0)] = FinishReason::kEND_ID; + expectedFinishReason[requestIds.at(1)] = FinishReason::kSTOP_WORDS; + + std::map<IdType, bool> done; + std::for_each(requestIds.begin(), requestIds.end(), [&done](auto id) { done[id] = false; }); + int iter = 0; + while (!(std::all_of(done.begin(), done.end(), [](auto x) { return x.second; })) && iter < 5000) + { + std::chrono::milliseconds waitTime(mMaxWaitMs); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + FAIL(); + } + else + { + auto const reqId = response.getRequestId(); + auto const& result = response.getResult(); + EXPECT_TRUE(result.isFinal); + done.at(reqId) = result.isFinal; + + // only 1 beam + auto const& outputIds = result.outputTokenIds.at(0); + EXPECT_EQ(outputIds.size(), expectedNewTokens.at(reqId)) << "req " << reqId; + + auto const& finishReason = result.finishReasons.at(0); + EXPECT_EQ(finishReason, expectedFinishReason.at(reqId)) << "req " << reqId; + + auto const& genLogits = result.generationLogits; + EXPECT_TRUE(genLogits.has_value()); + + // Expected shape: (1, numAcceptedDraftToken, vocabSizePadded) + auto const& generationLogitsShape = genLogits->getShape(); + EXPECT_EQ(generationLogitsShape.size(), 3); + EXPECT_EQ(generationLogitsShape[0], 1); + EXPECT_LE(generationLogitsShape[1], maxNewTokens); + EXPECT_EQ(generationLogitsShape[2], vocabSizePadded); + + auto const genLogitsTensor = detail::toITensor(*genLogits); + genLogitsTensor->squeeze(0); // only 1 beam + + for (size_t outputIdx = 0; outputIdx < expectedNewTokens.at(reqId); ++outputIdx) + { + // logits argmax should be equal to tokenId + auto const genLogitsSlice = tr::ITensor::slice(genLogitsTensor, outputIdx, 1); + auto const genLogitsRange = tr::BufferRange<float>(*genLogitsSlice); + auto const* maxPos = std::max_element(genLogitsRange.begin(), genLogitsRange.end()); + auto const maxIdx = std::distance(genLogitsRange.begin(), maxPos); + + auto const tokenId = outputIds.at(outputIdx); + // Observed token mismatch at index 2 after building GPT engine with TRT builder optimization + // level 3. The testcase is sensitive to slight variation in kernel computation, so we skip checking + // for token id at index 2. + if (outputIdx != 2) + { + EXPECT_EQ(tokenId, maxIdx) << "req " << reqId << " outputIdx " << outputIdx; + } + } + } + } + ++iter; + } +} + +TEST_F(GptExecutorTest, GenerationChangeEndId) +{ + SizeType32 constexpr beamWidth{2}; + SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded + auto constexpr streaming = false; + + ExtendedRuntimePerfKnobConfig perfKnobConfig = ExtendedRuntimePerfKnobConfig(); + perfKnobConfig.setEnableContextFMHAFP32Acc(true); // use fmha fp32 acc for better accuracy + + // Create executor config + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setExtendedRuntimePerfKnobConfig(perfKnobConfig); + + // Create executor + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + auto const inputPath = DATA_PATH / "input_tokens.npy"; + ModelIds modelIds{50256, 50256}; + + auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); + auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); + auto const* const givenInputData = tr::bufferCast<TokenIdType const>(*givenInput); + + auto const& inputShape = givenInput->getShape(); + ASSERT_EQ(inputShape.nbDims, 2); + ASSERT_GT(inputShape.d[0], 0); + + BeamResult beamResult{beamWidth}; + auto const resultsPath + = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CONTEXTFMHAFP32ACC_RESULT_FILE(); + + // Just return tokens for check + OutputConfig outConfig; + outConfig.excludeInputFromOutput = true; + + // Load expected outputs for each beam width value + auto testData = TestData::loadTestData(beamResult, *givenInput, beamWidth, manager, outConfig, modelIds); + auto const maxSeqLen = testData.maxSeqLen; + + // Load expected outputs and inputs + std::vector<Request> requests; + std::vector<SizeType32> reqMaxNewTokens; + + // Only use the first request to test + auto constexpr reqIdx = 0; + SizeType32 inputLen = givenInputLengths.at(reqIdx); + auto maxNewTokens = maxSeqLen - maxInputLength; + reqMaxNewTokens.push_back(maxNewTokens); + auto const* const seqBegin = givenInputData + reqIdx * maxInputLength; + + // Use customized `EndId` to enqueue once + auto request = Request(VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, modelIds.endId); + + TokenIdType customizedEndId = *(seqBegin + 1); // Use a token appeared in ground-truth + request.setEndId(customizedEndId); + requests.emplace_back(std::move(request)); + + auto requestIds = executor.enqueueRequests(requests); + std::chrono::milliseconds waitTime(mMaxWaitMs); + auto responses = executor.awaitResponses(waitTime); + if (responses.at(0).hasError()) + { + FAIL(); + } + requests.clear(); + + // Change back to default `EndId` to enqueue again, and check the output + request = Request(VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, modelIds.endId); + + auto const expectedOutputData = tr::BufferRange<TokenIdType const>(*testData.expectedOutputIds); + auto const expectedOutputLengths = testData.expectedOutputLengths; + auto const endPos = expectedOutputLengths[reqIdx]; + auto const endIndex = tc::flat_index3(reqIdx, beamWidth, endPos, beamWidth, maxSeqLen); + auto const endToken = expectedOutputData[endIndex]; + + request.setEndId(endToken); + requests.emplace_back(std::move(request)); + requestIds = executor.enqueueRequests(requests); + auto const requestId = requestIds.at(0); + + std::map<IdType, SizeType32> expectedNewTokens; + expectedNewTokens[requestId] = endPos - inputLen; + + std::map<IdType, FinishReason> expectedFinishReason; + expectedFinishReason[requestId] = FinishReason::kLENGTH; + + std::map<IdType, bool> done; + std::for_each(requestIds.begin(), requestIds.end(), [&done](auto id) { done[id] = false; }); + int iter = 0; + while (!(std::all_of(done.begin(), done.end(), [](auto x) { return x.second; })) && iter < 5000) + { + std::chrono::milliseconds waitTime(mMaxWaitMs); + auto responses = executor.awaitResponses(waitTime); + auto& response = responses.at(0); + if (response.hasError()) + { + FAIL(); + } + else + { + auto const reqId = response.getRequestId(); + auto const& result = response.getResult(); + EXPECT_TRUE(result.isFinal); + done.at(reqId) = result.isFinal; + + bool anyMismatch = false; + for (int i = 0; i < result.outputTokenIds.size(); ++i) + { + auto const& outputIds = result.outputTokenIds.at(i); + EXPECT_EQ(outputIds.size(), expectedNewTokens.at(reqId)) << "req " << reqId; + anyMismatch |= outputIds.size() != expectedNewTokens.at(reqId); + + auto const& finishReason = result.finishReasons.at(i); + EXPECT_EQ(finishReason, expectedFinishReason.at(reqId)) << "req " << reqId; + anyMismatch |= finishReason != expectedFinishReason.at(reqId); + + if (anyMismatch) + { + break; + } + + for (int j = 0; j < outputIds.size(); ++j) + { + auto const resultToken = outputIds[j]; + auto const groundTruthToken = expectedOutputData[maxSeqLen * i + inputLen + j]; + EXPECT_EQ(resultToken, groundTruthToken); + anyMismatch |= resultToken != groundTruthToken; + } + } + EXPECT_FALSE(anyMismatch); + } + ++iter; + } +} + +// stream, excludeInputFromOutput, beamWidth +using ParamType = std::tuple<bool, bool, int>; +// useOrchestratorMode, beamWidth, modelName +using ParamCancelReqType = std::tuple<bool, int, std::string>; +// modelName +using LeaderApiUsageType = std::tuple<std::string>; +// iterStatsMaxIterations, useOrchestratorMode +using ParamStatsType = std::tuple<int, bool>; +// streaming, beamWidth, computeLogProbs, excludeInputInOutput, returnContextLogits, returnGenerationLogits, modelName, +// useOrchestratorMode, returnAllGeneratedTokens, numReturnSequences +using AllParamsType = std::tuple<bool, int, bool, bool, bool, bool, std::string, bool, bool, int>; +// modelName, batched, replicated +using LogitsProcParamsType = std::tuple<std::string, bool, bool>; +// modelName +using GuidedDecodingParamsType = std::tuple<std::string>; +// modelName, useOrchestratorMode, beamWidth +using TimeoutTestParamsType = std::tuple<std::string, bool, int>; + +std::string generateTestName(testing::TestParamInfo<ParamType> const& info) +{ + auto const streaming = std::get<0>(info.param); + auto const excludeInputFromOutput = std::get<1>(info.param); + auto const beamWidth = std::get<2>(info.param); + std::string name = "ExecutorTest"; + if (streaming) + { + name += "Streaming"; + } + if (excludeInputFromOutput) + { + name += "ExclInput"; + } + name.append("BW" + std::to_string(beamWidth)); + return name; +} + +std::string generateTestNameCancelReq(testing::TestParamInfo<ParamCancelReqType> const& info) +{ + auto const& useOrchestratorMode = std::get<0>(info.param); + auto const beamWidth = std::get<1>(info.param); + auto const modelName = std::get<2>(info.param); + std::string name = "ExecutorTest"; + name.append("BW" + std::to_string(beamWidth)); + name.append("_" + modelName + "_"); + + if (useOrchestratorMode) + { + name.append("OrchMode"); + } + else + { + name.append("LeaderMode"); + } + return name; +} + +std::string generateTestNameLeaderApiUsage(testing::TestParamInfo<LeaderApiUsageType> const& info) +{ + auto const modelName = std::get<0>(info.param); + std::string name = "ExecutorTest"; + name.append("_" + modelName); + return name; +} + +std::string generateTestNameLogitsProc(testing::TestParamInfo<LogitsProcParamsType> const& info) +{ + auto const modelName = std::get<0>(info.param); + bool const batched = std::get<1>(info.param); + bool const replicated = std::get<2>(info.param); + std::string name = "ExecutorTest"; + name.append("_" + modelName); + if (batched) + { + name.append("_Batched"); + } + if (replicated) + { + name.append("_Replicated"); + } + return name; +} + +std::string generateTestNameGuidedDecoding(testing::TestParamInfo<GuidedDecodingParamsType> const& info) +{ + auto const modelName = std::get<0>(info.param); + std::string name = "ExecutorTest"; + name.append("_" + modelName); + return name; +} + +std::string generateTestNameTimeoutTest(testing::TestParamInfo<TimeoutTestParamsType> const& info) +{ + auto const modelName = std::get<0>(info.param); + auto const& useOrchestratorMode = std::get<1>(info.param); + auto const beamWidth = std::get<2>(info.param); + + std::string name = "ExecutorTest"; + name.append("_" + modelName); + + if (useOrchestratorMode) + { + name.append("_OrchMode"); + } + else + { + name.append("_LeaderMode"); + } + name.append("_BW" + std::to_string(beamWidth)); + return name; +} + +std::string generateTestNameStats(testing::TestParamInfo<ParamStatsType> const& info) +{ + int iterStatsMaxIterations = std::get<0>(info.param); + auto const& useOrchestratorMode = std::get<1>(info.param); + std::string name = "ExecutorTest_"; + name.append(std::to_string(iterStatsMaxIterations) + "_"); + if (useOrchestratorMode) + { + name.append("OrchMode"); + } + else + { + name.append("LeaderMode"); + } + return name; +} + +std::string generateTestNameAllParams(testing::TestParamInfo<AllParamsType> const& info) +{ + auto const streaming = std::get<0>(info.param); + auto const& beamWidth = std::get<1>(info.param); + auto const& computeLogProbs = std::get<2>(info.param); + auto const& excludeInputInOutput = std::get<3>(info.param); + auto const& returnContextLogits = std::get<4>(info.param); + auto const& returnGenerationLogits = std::get<5>(info.param); + auto const modelName = std::get<6>(info.param); + auto const& useOrchestratorMode = std::get<7>(info.param); + auto const& returnAllGeneratedTokens = std::get<8>(info.param); + auto const& numReturnSequences = std::get<9>(info.param); + + std::string name = "ExecutorTest_"; + + if (streaming) + { + name += "Streaming"; + } + + name.append("_BW" + std::to_string(beamWidth)); + name.append("Nseq" + std::to_string(numReturnSequences)); + + if (computeLogProbs) + { + name.append("LogProbs"); + } + if (excludeInputInOutput) + { + name.append("ExcludeInput"); + } + if (returnContextLogits) + { + name.append("ContextLogits"); + } + if (returnGenerationLogits) + { + name.append("GenerationLogits"); + } + name.append("_" + modelName + "_"); + if (useOrchestratorMode) + { + name.append("OrchMode"); + } + else + { + name.append("LeaderMode"); + } + + if (returnAllGeneratedTokens) + { + name.append("returnAllGeneratedTokens"); + } + return name; +} + +class ParamTest : public GptExecutorTest, public ::testing::WithParamInterface<ParamType> +{ +}; + +class ParamStatsTest : public GptExecutorTest, public ::testing::WithParamInterface<ParamStatsType> +{ +}; + +class AllParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<AllParamsType> +{ +}; + +class ParamCancelReqTest : public GptExecutorTest, public ::testing::WithParamInterface<ParamCancelReqType> +{ +}; + +class LeaderApiUsageTest : public GptExecutorTest, public ::testing::WithParamInterface<LeaderApiUsageType> +{ +}; + +class LogitsProcParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<LogitsProcParamsType> +{ +}; + +class GuidedDecodingParamsTest : public GptExecutorTest, public ::testing::WithParamInterface<GuidedDecodingParamsType> +{ +}; + +class TimeoutTest : public GptExecutorTest, public ::testing::WithParamInterface<TimeoutTestParamsType> +{ +}; + +TEST_F(GptExecutorTest, GetLatestStats) +{ + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + auto request + = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + auto requestId = executor.enqueueRequest(std::move(request)); + + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + FAIL(); + } + else + { + done = response.getResult().isFinal; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + // Expect 6 non-empty iterations + auto stats = executor.getLatestIterationStats(); + EXPECT_EQ(stats.size(), 6); + uint64_t currentIter = 0; + for (auto const& stat : stats) + { + EXPECT_EQ(stat.timestamp.size(), 26); + EXPECT_EQ(stat.iter, currentIter); + if (currentIter != 5) + { + EXPECT_EQ(stat.numActiveRequests, 1); + } + else + { + // For the last iteration the number of active requests + // should be zero. + EXPECT_EQ(stat.numActiveRequests, 0); + } + EXPECT_EQ(stat.maxNumActiveRequests, 64); + // Very loose check to make sure the memory stats are valid + EXPECT_GT(stat.gpuMemUsage, 16); + EXPECT_GT(stat.cpuMemUsage, 16); + EXPECT_GT(stat.pinnedMemUsage, 16); + + // Stats for KV cache + EXPECT_TRUE(stat.kvCacheStats.has_value()); + KvCacheStats const& kvStats = stat.kvCacheStats.value(); + EXPECT_GT(kvStats.maxNumBlocks, 0); + EXPECT_GT(kvStats.freeNumBlocks, 0); + EXPECT_EQ(kvStats.usedNumBlocks, currentIter == maxNewTokens ? 0 : 1); + EXPECT_GT(kvStats.tokensPerBlock, 0); + EXPECT_GT(kvStats.allocTotalBlocks, 0); + EXPECT_GT(kvStats.allocNewBlocks, 0); + EXPECT_GE(kvStats.reusedBlocks, 0); + EXPECT_GE(kvStats.missedBlocks, 0); + EXPECT_GE(kvStats.cacheHitRate, 0); + + // Stats for inflight batching + EXPECT_TRUE(stat.inflightBatchingStats.has_value() && !stat.staticBatchingStats.has_value()); + InflightBatchingStats const& modelStats = stat.inflightBatchingStats.value(); + EXPECT_EQ(modelStats.numScheduledRequests, currentIter == maxNewTokens ? 0 : 1); + EXPECT_EQ(modelStats.numContextRequests, currentIter == 0 ? 1 : 0); + EXPECT_EQ(modelStats.numGenRequests, currentIter == 0 || currentIter == maxNewTokens ? 0 : 1); + EXPECT_EQ(modelStats.numPausedRequests, 0); + EXPECT_EQ(modelStats.numCtxTokens, currentIter == 0 ? inputTokens.size() : 0); + EXPECT_EQ(modelStats.microBatchId, 0); + EXPECT_NEAR( + modelStats.avgNumDecodedTokensPerIter, currentIter == 0 || currentIter == maxNewTokens ? 0.f : 1.f, 1e-9f); + + auto jsonStr = JsonSerialization::toJsonStr(stat); + EXPECT_THAT(jsonStr, testing::HasSubstr("\"iter\":" + std::to_string(currentIter))); + EXPECT_THAT(jsonStr, testing::HasSubstr("\"staticBatchingStats\":null")); + EXPECT_THAT(jsonStr, testing::HasSubstr("\"numCtxTokens\":" + std::to_string(modelStats.numCtxTokens))); + EXPECT_THAT(jsonStr, testing::HasSubstr("\"numGenRequests\":" + std::to_string(modelStats.numGenRequests))); + + ++currentIter; + } +} + +TEST_F(GptExecutorTest, GetLatestStatsWithMultipleRequests) +{ + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the requests + SizeType32 const numRequests = 2; + std::vector<SizeType32> maxNewTokens{3, 5}; + std::vector<VecTokens> inputTokens{{1, 2, 3, 4}, {5, 6, 7}}; + std::vector<IdType> reqIds; + for (SizeType32 ireq = 0; ireq < numRequests; ++ireq) + { + auto request = Request(inputTokens[ireq], maxNewTokens[ireq], streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + auto requestId = executor.enqueueRequest(std::move(request)); + reqIds.emplace_back(requestId); + // sleep for 10 ms before sending the next request + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + + for (SizeType32 ireq = 0; ireq < numRequests; ++ireq) + { + auto requestId = reqIds[ireq]; + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + FAIL(); + } + else + { + done = response.getResult().isFinal; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + } + + // NOTES: + // Expect at least max(maxNewTokens) i.e. 5 non-empty iterations + // 4th iteration should have numCompletedRequests to be 1. + // Depending on the timing, first iteration will either have: + // 2 active requests + // or + // 1 active requests and 1 queued requests + auto stats = executor.getLatestIterationStats(); + EXPECT_GT(stats.size(), 0); // make sure we have at least 1 stat before the accessing 0-th element + if (stats[0].numActiveRequests == 2) + { + // we cannot reliably check queue latency since both started in the same iteration + // there should be exactly 5 non-empty iterations + EXPECT_EQ(stats.size(), 5); + // only check numCompletedRequests in 4th iteration + EXPECT_EQ(stats[3].numCompletedRequests, 1); + // 1st iteration shall record all 2 requests queueing time; + EXPECT_EQ(stats[0].numNewActiveRequests, 2); + // all rest iterations shall not return any queueing time; + for (int i = 1; i < stats.size(); ++i) + { + EXPECT_EQ(stats[i].numNewActiveRequests, 0); + } + } + else + { + // there should be more than 5 non-empty iterations since 2nd request started after 1st iteration + EXPECT_GT(stats.size(), 5); + // 1st request's completion is at 4th iteration + EXPECT_EQ(stats[3].numCompletedRequests, 1); + // 1st iteration record 1 request's queueing time; + EXPECT_EQ(stats[0].numNewActiveRequests, 1); + // the iteration where 2nd request became active, queue latency must be > 0 + uint64_t currentIter = 0; + for (auto const& stat : stats) + { + // To check when 2nd request becomes active, we need to think about 2 cases: + // - it overlaps with first request + // => only check queue time in this case + // - it doesn't overlap with the first request (e.g. 1st request ended too fast) + // => little to no queue time, cannot check reliably + // so we only check for queue time when numActiveRequests > 1 i.e. overlap happened after first iteration + if (stat.numActiveRequests > 1) + { + EXPECT_GT(currentIter, 0); // it must be after 1st iteration + EXPECT_GT(stat.newActiveRequestsQueueLatencyMS, 0); + // 2nd request record queueing time in this iteration + EXPECT_EQ(stat.numNewActiveRequests, 1); + break; + } + ++currentIter; + } + } +} + +TEST_F(GptExecutorTest, GetLatestRequestStats) +{ + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setRequestStatsMaxIterations(1000); + executorConfig.setEnableChunkedContext(true); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the requests + std::vector<std::pair<SizeType32, VecTokens>> requestParams = { + // {maxNewTokens, inputTokens} + {5, {1, 2, 3, 4}}, {4, {1, 1, 2, 3, 5}}, {1, {1}}, + {8, VecTokens(383, 1)} // Long enough to be chunked into multiple iterations + }; + std::vector<Request> requests; + for (auto requestParam : requestParams) + { + requests.emplace_back(requestParam.second, requestParam.first, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + } + auto requestIdsVec = executor.enqueueRequests(std::move(requests)); + std::map<IdType, SizeType32> requestIdToIndex; + std::set<IdType> activeRequests; + for (SizeType32 i = 0; i < requestIdsVec.size(); ++i) + { + auto requestId = requestIdsVec[i]; + activeRequests.insert(requestId); + requestIdToIndex[requestId] = i; + } + + int iter = 0; + while (!activeRequests.empty() && iter < mMaxWaitMs) + { + for (auto i = activeRequests.begin(); i != activeRequests.end();) + { + auto requestId = *i; + bool thisDone = false; + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + else + { + thisDone = response.getResult().isFinal; + } + } + if (thisDone) + { + // Erase completed request and move to the next one + i = activeRequests.erase(i); + } + else + { + ++i; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + // Expect 5 non-empty iterations + // Note: The 6th iteration with the last finished request will be reported + // but might be unavailable when getLatestRequestStats is called since + // it could be updated after the final response has been sent. + auto stats = executor.getLatestRequestStats(); + EXPECT_GE(stats.size(), 5); + SizeType32 currentIter = 0; + auto invalidStart = std::numeric_limits<SizeType32>::max(); + std::vector<SizeType32> genStart(requestParams.size(), invalidStart); // The iteration index when generation started + std::set<IdType> completedRequests; + for (auto stat = stats.begin(); stat != stats.begin() + 5; ++stat) + { + auto jsonStrIter = JsonSerialization::toJsonStr(*stat); + EXPECT_EQ(stat->iter, currentIter); + EXPECT_THAT(jsonStrIter, testing::HasSubstr("\"iter\":" + std::to_string(currentIter))); + EXPECT_EQ(stat->requestStats.size() + completedRequests.size(), requestParams.size()); + for (auto rStat : stat->requestStats) + { + auto jsonStr = JsonSerialization::toJsonStr(rStat); + // Only a few requests here so all of them should be scheduled. A separate test + // GetLatestRequestStatsScheduling will target the scheduling stats. + if (rStat.stage != RequestStage::kGENERATION_COMPLETE) + { + EXPECT_TRUE(rStat.scheduled); + EXPECT_THAT(jsonStr, testing::HasSubstr("\"scheduled\":true")); + } + EXPECT_TRUE(!rStat.paused); + EXPECT_THAT(jsonStr, testing::HasSubstr("\"paused\":false")); + EXPECT_TRUE(requestIdToIndex.count(rStat.id)); + EXPECT_THAT(jsonStr, testing::HasSubstr("\"id\":" + std::to_string(rStat.id))); + auto requestIndex = requestIdToIndex[rStat.id]; + auto contextSize = requestParams[requestIndex].second.size(); + if (rStat.contextPrefillPosition == contextSize) // Check generation phase + { + bool firstIteration{false}; + // Context phase is done + EXPECT_TRUE(rStat.stage == RequestStage::kGENERATION_IN_PROGRESS + || rStat.stage == RequestStage::kGENERATION_COMPLETE); + EXPECT_THAT(jsonStr, testing::HasSubstr("\"stage\":\"GENERATION")); + if (genStart[requestIndex] == invalidStart) + { + // Just started generation + genStart[requestIndex] = currentIter; + firstIteration = true; + } + + // One token per iteration + EXPECT_TRUE(currentIter - genStart[requestIndex] == rStat.numGeneratedTokens); + EXPECT_NEAR(rStat.avgNumDecodedTokensPerIter, firstIteration ? 0.f : 1.0f, 1e-9); + if (rStat.stage == RequestStage::kGENERATION_COMPLETE) + { + EXPECT_TRUE(requestParams[requestIndex].first >= rStat.numGeneratedTokens); + completedRequests.insert(requestIndex); + } + else + { + EXPECT_FALSE(completedRequests.count(requestIndex)); + } + } + else if (rStat.contextPrefillPosition < contextSize) // Check context phase + { + // Must be chunked + SizeType32 const maxChunkSize = 128; + EXPECT_TRUE(rStat.contextPrefillPosition % maxChunkSize == 0); + // Context phase is on-going + EXPECT_TRUE(rStat.stage == RequestStage::kCONTEXT_IN_PROGRESS); + // No tokens are generated + EXPECT_TRUE(0 == rStat.numGeneratedTokens); + } + else + { + FAIL() << "Out-of-boundary contextPrefillPosition in stats: " << rStat.contextPrefillPosition + << " out of " << contextSize; + } + // Sanity check that disaggregated serving stats is not set in typical use case + EXPECT_FALSE(rStat.disServingStats.has_value()); + } + ++currentIter; + } + // We should have visited all requests. + // Take into consideration the last request has not been reported + EXPECT_EQ(completedRequests.size() + 1, requestParams.size()); +} + +TEST_F(GptExecutorTest, GetLatestRequestStatsScheduling) +{ + // Specifically test the case where there are too many requests to be scheduled for a iteration + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setRequestStatsMaxIterations(1000); + executorConfig.setEnableChunkedContext(true); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create 100 requests. Note the max batch size for this model is 64 so some requests won't be scheduled right away. + std::vector<std::pair<SizeType32, VecTokens>> requestParams(100, {5, {1, 2, 3, 4}}); + std::vector<Request> requests; + requests.reserve(requestParams.size()); + for (auto requestParam : requestParams) + { + requests.emplace_back(requestParam.second, requestParam.first, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + } + auto requestIdsVec = executor.enqueueRequests(std::move(requests)); + std::map<IdType, SizeType32> requestIdToIndex; + std::set<IdType> activeRequests; + for (SizeType32 i = 0; i < requestIdsVec.size(); ++i) + { + auto requestId = requestIdsVec[i]; + activeRequests.insert(requestId); + requestIdToIndex[requestId] = i; + } + + int iter = 0; + while (!activeRequests.empty() && iter < mMaxWaitMs) + { + for (auto i = activeRequests.begin(); i != activeRequests.end();) + { + auto requestId = *i; + bool thisDone = false; + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + else + { + thisDone = response.getResult().isFinal; + } + } + if (thisDone) + { + // Erase completed request and move to the next one + i = activeRequests.erase(i); + } + else + { + ++i; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + auto stats = executor.getLatestRequestStats(); + SizeType32 numFinished = 0; + SizeType32 const maxActiveSize = 64; // Decided by the model + + // The 6th iteration request stat may or may not be available when getLatestRequestStats + // is called. When there are no other active or inTransmission requests, there will be + // another request stats to properly reset all the statistics to zero. + for (auto stat = stats.begin(); stat != stats.begin() + 5; ++stat) + { + SizeType32 numReqs = 0; + SizeType32 numReqsActive = 0; + SizeType32 numReqsQueued = 0; + SizeType32 numReqsJustDone = 0; + for (auto rStat : stat->requestStats) + { + ++numReqs; + numReqsActive += rStat.scheduled ? 1 : 0; + numReqsQueued += rStat.stage == RequestStage::kQUEUED ? 1 : 0; + numReqsJustDone += rStat.stage == RequestStage::kGENERATION_COMPLETE ? 1 : 0; + } + EXPECT_EQ(numReqs, numReqsActive + numReqsQueued + numReqsJustDone); + EXPECT_EQ(numReqs + numFinished, requestParams.size()); // Should report all unfinished requests + EXPECT_TRUE(numReqsActive <= maxActiveSize); // Not all requests are active due to max active size limit. + numFinished += numReqsJustDone; + } +} + +TEST_F(GptExecutorTest, GetRequestStatsMultipleRequests) +{ + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setRequestStatsMaxIterations(1000); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + auto sendRequestWaitForResponseFn = [&]() + { + Request request({1, 2, 3}, 5); + auto requestId = executor.enqueueRequest(request); + bool isFinalResponse = false; + while (!isFinalResponse) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto response : responses) + { + if (response.getResult().isFinal) + { + isFinalResponse = true; + break; + } + } + } + return requestId; + }; + + std::unordered_map<IdType, size_t> requestIdToGenerationComplete; + auto updateStats = [&]() + { + auto stats = executor.getLatestRequestStats(); + for (auto& stat : stats) + { + for (auto const& request : stat.requestStats) + { + // only check and aggregate results when request is completed + if (request.stage == RequestStage::kGENERATION_COMPLETE) + { + requestIdToGenerationComplete[request.id] += 1; + } + } + } + }; + + auto requestId = sendRequestWaitForResponseFn(); + requestIdToGenerationComplete[requestId] = 0; + updateStats(); + + requestId = sendRequestWaitForResponseFn(); + requestIdToGenerationComplete[requestId] = 0; + updateStats(); + + for (auto [key, value] : requestIdToGenerationComplete) + { + EXPECT_EQ(value, 1); + } +} + +TEST_F(GptExecutorTest, BatchSizeTuning) +{ + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setRequestStatsMaxIterations(1000); + executorConfig.setEnableChunkedContext(true); + + DynamicBatchConfig dynamicBatchConfig(true, false, 1); // Set window size to 1 + SchedulerConfig schedulerConfig(CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT, std::nullopt, dynamicBatchConfig); + executorConfig.setSchedulerConfig(schedulerConfig); + + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + std::vector<SizeType32> tunerRecommendedBatchSizes; + + for (size_t i = 0; i <= 8; ++i) + { + auto inputLength = 1 << i; // Note that for this model max input len is 383 + Request request( + VecTokens(inputLength, 2), 5, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + auto requestId = executor.enqueueRequest(std::move(request)); + // Wait for current request to finish + while (true) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + bool done = false; + if (responses.size() != 0) + { + EXPECT_TRUE(responses.size() == 1); + auto response = responses[0]; + EXPECT_FALSE(response.hasError()); + if (response.getResult().isFinal) + { + break; + } + } + } + auto reqStats = executor.getLatestIterationStats(); + EXPECT_TRUE(reqStats.size() > 0); + auto lastStat = reqStats.back(); + tunerRecommendedBatchSizes.push_back(lastStat.maxBatchSizeTunerRecommended); + } + + EXPECT_TRUE(tunerRecommendedBatchSizes.size() > 0); + // It's supposed to be decreasing when input length increases + EXPECT_TRUE(*tunerRecommendedBatchSizes.begin() > *tunerRecommendedBatchSizes.rbegin()); +} + +TEST_F(GptExecutorTest, GetLatestDebugTensors) +{ + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + SizeType32 maxNewTokens = 5; + + tensorrt_llm::executor::DebugConfig debugConfig; + debugConfig.setDebugTensorNames({{"sequence_length"}}); + debugConfig.setDebugTensorsMaxIterations(maxNewTokens); + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setDebugConfig(debugConfig); + + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + VecTokens inputTokens{1, 2, 3, 4}; + auto request + = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + auto requestId = executor.enqueueRequest(request); + + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + FAIL(); + } + else + { + done = response.getResult().isFinal; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + auto stream = std::make_shared<tr::CudaStream>(); + + // Expect 5 non-empty iterations + auto debugTensors = executor.getLatestDebugTensors(); + EXPECT_EQ(debugTensors.size(), 5); + uint64_t currentIter = 0; + for (auto const& debugIteration : debugTensors) + { + EXPECT_EQ(debugIteration.iter, currentIter); + EXPECT_EQ(debugIteration.debugTensors.size(), 2); + + { + auto it = debugIteration.debugTensors.find("request_ids"); + EXPECT_NE(it, debugIteration.debugTensors.end()); + auto const& tensor = it->second; + auto const& shape = tensor.getShape(); + EXPECT_EQ(shape.size(), 1); + EXPECT_EQ(shape[0], 1); + EXPECT_EQ(tensor.getSize(), 1); + auto const* dataPtr = static_cast<SizeType32 const*>(tensor.getData()); + EXPECT_EQ(dataPtr[0], 1) << "currentIter " << currentIter; + } + { + auto it = debugIteration.debugTensors.find("sequence_length"); + EXPECT_NE(it, debugIteration.debugTensors.end()); + auto const& tensor = it->second; + auto const& shape = tensor.getShape(); + EXPECT_EQ(shape.size(), 1); + EXPECT_EQ(tensor.getSize(), 1); + auto tensorHost = tensor.copyToCpu(stream); + auto const* dataPtr = static_cast<SizeType32 const*>(tensorHost.getData()); + EXPECT_EQ(dataPtr[0], inputTokens.size() + currentIter); + } + + ++currentIter; + } +} + +TEST_P(ParamTest, SingleRequestDemo) +{ + bool const streaming = std::get<0>(GetParam()); + bool const excludeInputFromOutput = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + auto request + = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + + // Enqueue the request + auto requestId = executor.enqueueRequest(request); + + // Get the new tokens + VecTokens tokens; + SizeType32 numResponses{0}; + bool done = false; + int iter = 0; + std::chrono::milliseconds waitTime(1); + while (!done && iter < mMaxWaitMs) + { + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + ++numResponses; + if (response.hasError()) + { + // This request failed for some reason, get error msg + std::string errStr + = "Request id " + std::to_string(requestId) + " failed with err " + response.getErrorMsg(); + FAIL(); + } + + auto result = response.getResult(); + done = result.isFinal; + auto& newTokens = result.outputTokenIds.at(beamWidth - 1); + auto const expectedSize = streaming ? (beamWidth > 1 ? numResponses : 1) + : (maxNewTokens + (excludeInputFromOutput ? 0 : inputTokens.size())); + EXPECT_EQ(newTokens.size(), expectedSize); + + if (streaming && beamWidth > 1) + { + // replace tokens + tokens = newTokens; + } + else + { + // Append tokens + tokens.insert(tokens.end(), newTokens.begin(), newTokens.end()); + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + EXPECT_EQ(numResponses, streaming ? maxNewTokens : 1); + EXPECT_EQ( + tokens.size(), streaming ? maxNewTokens : (excludeInputFromOutput ? 0 : inputTokens.size()) + maxNewTokens); + + // Expect awaitResponse to return error message because the request is already terminated (isFinal = True) + auto response = executor.awaitResponses(requestId, waitTime).at(0); + EXPECT_TRUE(response.hasError()); + std::string err + = "ReqId " + std::to_string(response.getRequestId()) + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); +} + +TEST_P(ParamTest, MultipleRequestDemo) +{ + bool const streaming = std::get<0>(GetParam()); + bool const excludeInputFromOutput = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + SizeType32 numRequests = 20; + + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 maxPromptLen = 20; + SizeType32 maxMaxNewTokens = 20; + + SizeType32 endId = -1; + // Enqueue the requests + std::unordered_map<IdType, VecTokens> tokens; + std::unordered_map<IdType, SizeType32> expectedNumTokens; + std::unordered_map<IdType, SizeType32> expectedNumResponses; + for (SizeType32 req = 0; req < numRequests; ++req) + { + SizeType32 promptLen = rand() % maxPromptLen + 1; + SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; + + auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, endId); + auto reqId = executor.enqueueRequest(std::move(request)); + tokens[reqId] = {}; + expectedNumTokens[reqId] = ((streaming || excludeInputFromOutput) ? 0 : promptLen) + maxNewTokens; + expectedNumResponses[reqId] = streaming ? maxNewTokens : 1; + } + + // Get the new tokens for each requests + int32_t numFinished = 0; + int iter = 0; + std::unordered_map<IdType, SizeType32> numResponses; + while (numFinished < numRequests && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + auto reqId = response.getRequestId(); + ++numResponses[reqId]; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + auto& newTokens = result.outputTokenIds.at(beamWidth - 1); + auto const expectedSize + = streaming ? (beamWidth > 1 ? numResponses[reqId] : 1) : expectedNumTokens[reqId]; + EXPECT_EQ(newTokens.size(), expectedSize); + + auto& reqTokens = tokens.at(response.getRequestId()); + if (streaming && beamWidth > 1) + { + reqTokens = newTokens; + } + else + { + reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); + } + + for (SizeType32 b = 0; b < beamWidth; ++b) + { + EXPECT_EQ(result.finishReasons.at(b), + result.isFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); + } + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + // Check that number of tokens matches expectations + for (auto const& [reqId, numTokens] : expectedNumTokens) + { + EXPECT_EQ(expectedNumResponses[reqId], numResponses[reqId]) << "reqId " << reqId; + EXPECT_EQ(expectedNumTokens[reqId], tokens[reqId].size()) << "reqId " << reqId; + } +} + +TEST_P(ParamStatsTest, MultipleRequestStats) +{ + bool streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + SizeType32 numRequests = 100; + auto iterStatsMaxIterations = std::get<0>(GetParam()); + bool useOrchestratorMode = std::get<1>(GetParam()); + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setIterStatsMaxIterations(iterStatsMaxIterations); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + + std::optional<OrchestratorConfig> orchestratorConfig = std::nullopt; + if (useOrchestratorMode) + { + orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + } + auto parallelConfig = ParallelConfig(CommunicationType::kMPI, + useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, std::nullopt, + orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 maxPromptLen = 20; + SizeType32 maxMaxNewTokens = 20; + + SizeType32 endId = -1; + // Enqueue the requests + std::unordered_map<IdType, VecTokens> tokens; + std::unordered_map<IdType, SizeType32> expectedNumTokens; + for (SizeType32 req = 0; req < numRequests; ++req) + { + SizeType32 promptLen = rand() % maxPromptLen + 1; + SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; + + auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, endId); + auto reqId = executor.enqueueRequest(std::move(request)); + tokens[reqId] = {}; + expectedNumTokens[reqId] = (streaming ? 0 : (excludeInputFromOutput ? 0 : promptLen)) + maxNewTokens; + } + + std::atomic<bool> statsThreadDone = false; + std::atomic<int32_t> numFinished = 0; + std::deque<IterationStats> iterStatsReceived; + // Spawn a thread that continuously get stats + auto statsThread = std::thread( + [&executor, &numFinished, numRequests, &iterStatsReceived, &statsThreadDone]() + { + while (numFinished < numRequests) + { + auto reqStats = executor.getLatestIterationStats(); + iterStatsReceived.insert(iterStatsReceived.end(), std::make_move_iterator(reqStats.begin()), + std::make_move_iterator(reqStats.end())); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + statsThreadDone = true; + }); + + // Get the new tokens for each requests + int iter = 0; + SizeType32 numResponses = 0; + while (numFinished < numRequests && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + auto& newTokens = result.outputTokenIds.at(beamWidth - 1); + auto& reqTokens = tokens.at(response.getRequestId()); + reqTokens.insert(reqTokens.end(), std::make_move_iterator(newTokens.begin()), + std::make_move_iterator(newTokens.end())); + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + // Check that number of tokens matches expectations + for (auto const& [reqId, numTokens] : expectedNumTokens) + { + EXPECT_EQ(expectedNumTokens[reqId], tokens[reqId].size()) << "reqId " << reqId; + } + + // Wait for stats thread to be done, fail otherwise + iter = 0; + while (!statsThreadDone && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + std::this_thread::sleep_for(std::chrono::milliseconds(waitTime)); + iter++; + } + ASSERT_TRUE(statsThreadDone); + if (iterStatsMaxIterations > 0) + { + ASSERT_GT(iterStatsReceived.size(), 1); + + for (auto stats : iterStatsReceived) + { + EXPECT_GT(stats.numActiveRequests, 0); + TLLM_LOG_INFO("%d %d", stats.iter, stats.numActiveRequests); + + EXPECT_TRUE(stats.inflightBatchingStats.has_value()); + if (stats.inflightBatchingStats.has_value()) + { + EXPECT_GT(stats.inflightBatchingStats.value().numScheduledRequests, 0); + } + } + } + + statsThread.join(); +} + +TEST_P(ParamTest, MultipleRequestBatchResponses) +{ + bool const streaming = std::get<0>(GetParam()); + bool const excludeInputFromOutput = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + SizeType32 constexpr numRequests{20}; + + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 constexpr maxPromptLen{20}; + SizeType32 constexpr maxMaxNewTokens{20}; + + SizeType32 endId = -1; + // Enqueue the requests + std::unordered_map<IdType, VecTokens> tokens; + std::unordered_map<IdType, SizeType32> expectedNumTokens; + std::vector<IdType> requestIds; + for (SizeType32 req = 0; req < numRequests; ++req) + { + SizeType32 promptLen = rand() % maxPromptLen + 1; + SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; + + auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, endId); + auto reqId = executor.enqueueRequest(std::move(request)); + requestIds.push_back(reqId); + tokens[reqId] = {}; + expectedNumTokens[reqId] = (streaming ? 0 : (excludeInputFromOutput ? 0 : promptLen)) + maxNewTokens; + } + + // Get the new tokens for each requests + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + std::chrono::milliseconds waitTime(1); + while (numFinished < numRequests && iter < mMaxWaitMs) + { + auto idResponses = executor.awaitResponses(requestIds, waitTime); + for (unsigned i = 0; i < requestIds.size(); ++i) + { + auto& responses = idResponses[i]; + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + auto& newTokens = result.outputTokenIds.at(beamWidth - 1); + auto& reqTokens = tokens.at(response.getRequestId()); + if (streaming && beamWidth > 1) + { + reqTokens = newTokens; + } + else + { + reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); + } + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + // Rerun awaitResponses again and we expect to only see terminated request id error. + auto idResponses = executor.awaitResponses(requestIds, waitTime); + for (auto const& responses : idResponses) + { + for (auto& response : responses) + { + EXPECT_TRUE(response.hasError()); + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + + // Check that number of tokens matches expectations + for (auto const& [reqId, numTokens] : expectedNumTokens) + { + EXPECT_EQ(expectedNumTokens[reqId], tokens[reqId].size()) << "reqId " << reqId; + } +} + +TEST_P(ParamTest, GetNumResponsesReadyTest) +{ + bool const streaming = std::get<0>(GetParam()); + bool const excludeInputFromOutput = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 maxNumRequests = 50; + SizeType32 maxPromptLen = 20; + SizeType32 maxMaxNewTokens = 20; + + SizeType32 numRequests = rand() % maxNumRequests + 1; + SizeType32 numExpectedResponses = 0; + std::map<IdType, SizeType32> reqNumExpectedResponses; + std::vector<IdType> ids; + for (SizeType32 req = 0; req < numRequests; ++req) + { + SizeType32 promptLen = rand() % maxPromptLen + 1; + SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; + + auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + auto id = executor.enqueueRequest(std::move(request)); + ids.emplace_back(id); + reqNumExpectedResponses[id] = streaming ? maxNewTokens : 1; + numExpectedResponses += reqNumExpectedResponses.at(id); + } + + SizeType32 iter = 0; + SizeType32 numReady = 0; + while (numReady < numExpectedResponses && iter < mMaxWaitMs) + { + numReady = 0; + for (auto id : ids) + { + numReady += executor.getNumResponsesReady(id); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + // Expect one response per request + for (auto id : ids) + { + SizeType32 numReady = executor.getNumResponsesReady(id); + EXPECT_EQ(numReady, reqNumExpectedResponses.at(id)); + } + auto numResponsesReady = executor.getNumResponsesReady(); + EXPECT_EQ(numResponsesReady, numExpectedResponses); +} + +namespace +{ + +void runTest(Executor& executor, fs::path const& inputPath, ModelIds const& modelIds, + FlakyTestInfo const& flakyTestInfo, bool streaming, SizeType32 const vocabSizePadded, BeamResult const& beamResult, + OutputConfig const& outConfig, bool isSpeculativeDecoding, int maxWaitMs, bool returnAllGeneratedTokens, + SizeType32 const numReturnSequences, bool isNonGreedySampling, SizeType32 const modelParallelism) +{ + auto const beamWidth = beamResult.beamWidth; + + auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); + auto const& givenInput = tr::utils::loadNpy(manager, inputPath.string(), tr::MemoryType::kCPU); + auto [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(*givenInput, modelIds.padId); + auto const* const givenInputData = tr::bufferCast<TokenIdType const>(*givenInput); + + auto const& inputShape = givenInput->getShape(); + ASSERT_EQ(inputShape.nbDims, 2); + ASSERT_GT(inputShape.d[0], 0); + + // Load expected outputs for each beam width value + auto testData = TestData::loadTestData(beamResult, *givenInput, beamWidth, manager, outConfig, modelIds); + auto const maxSeqLen = testData.maxSeqLen; + + // Load expected outputs and inputs + SizeType32 numRequests = static_cast<SizeType32>(givenInputLengths.size()); + SizeType32 maxRequests = numRequests; + std::vector<Request> requests; + std::vector<SizeType32> reqMaxNewTokens; + + auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); + // top-k will be set by a large number to test non-identical N sequences. + if (isNonGreedySampling) + { + samplingConfig.setTopK(32); + } + samplingConfig.setNumReturnSequences(numReturnSequences); + + for (SizeType32 req = 0; req < maxRequests; ++req) + { + SizeType32 inputLen = givenInputLengths.at(req); + auto maxNewTokens = maxSeqLen - maxInputLength; + reqMaxNewTokens.push_back(maxNewTokens); + SizeType32 endId = -1; + auto const* const seqBegin = givenInputData + req * maxInputLength; + VecTokens tokens(seqBegin, seqBegin + inputLen); + auto request = Request( + VecTokens(seqBegin, seqBegin + inputLen), maxNewTokens, streaming, samplingConfig, outConfig, endId); + request.setReturnAllGeneratedTokens(returnAllGeneratedTokens); + requests.emplace_back(std::move(request)); + } + + auto& comm = tensorrt_llm::mpi::MpiComm::world(); + auto const worldRank = comm.getRank(); + + // Expected return sizes. + auto const numSequences = beamWidth > 1 ? 1 : numReturnSequences; + auto const numReturnBeams = std::min(beamWidth, numReturnSequences); + + if (worldRank == 0) + { + auto const reqIds = executor.enqueueRequests(requests); + + std::unordered_map<SizeType32, std::vector<BeamTokens>> tokens; + std::unordered_map<IdType, SizeType32> reqIdToBatchId; + + for (SizeType32 req = 0; req < reqIds.size(); ++req) + { + std::vector<BeamTokens> resultTokens(numSequences, BeamTokens(numReturnBeams)); + tokens[req] = std::move(resultTokens); + reqIdToBatchId[reqIds.at(req)] = req; + } + + // Get the new tokens for each requests + int32_t numFinished = 0; + int iter = 0; + std::unordered_map<IdType, SizeType32> numResponses; + while (numFinished < maxRequests && iter < maxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + auto batchId = reqIdToBatchId.at(response.getRequestId()); + numResponses[batchId]++; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + auto seqIdx = result.sequenceIndex; + + auto const& contextLogits = result.contextLogits; + auto const& genLogits = result.generationLogits; + auto const& outputTokenIds = result.outputTokenIds; + + EXPECT_EQ(result.finishReasons.size(), numReturnBeams); + for (SizeType32 beam = 0; beam < numReturnBeams; ++beam) + { + auto const& newTokens = outputTokenIds.at(beam); + auto& reqTokens = tokens.at(batchId).at(seqIdx).at(beam); + + if (!returnAllGeneratedTokens) + { + reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); + } + else + { + EXPECT_EQ(newTokens.size(), + (numResponses.at(batchId) + numReturnSequences - 1) / numReturnSequences); + reqTokens = newTokens; + } + // FinishReason is only supported for bw=1 and inflight batching. + if (beamWidth == 1) + { + EXPECT_EQ(result.finishReasons.at(beam), + result.isSequenceFinal ? FinishReason::kLENGTH : FinishReason::kNOT_FINISHED); + } + } + + auto const& cumLogProbs = result.cumLogProbs; + auto const& logProbs = result.logProbs; + auto const& beamTokens = tokens.at(batchId).at(seqIdx); + EXPECT_EQ(beamTokens.size(), numReturnBeams); + + if (!isNonGreedySampling) + { + float const logitsAtol = modelParallelism > 1 ? 1e-1 : 1e-2; + float const logitsRtol = modelParallelism > 1 ? 1e-2 : 1e-3; + + testData.verifyLogProbs(outConfig.returnLogProbs, streaming, outConfig.excludeInputFromOutput, + givenInputLengths.at(batchId), beamWidth, beamTokens, cumLogProbs, logProbs, batchId, + flakyTestInfo); + testData.validateContextLogits(outConfig.returnContextLogits, givenInputLengths.at(batchId), + beamWidth, contextLogits, vocabSizePadded, batchId, logitsAtol, logitsRtol); + testData.validateGenerationLogits(outConfig.returnGenerationLogits, result.isSequenceFinal, + streaming, outConfig.excludeInputFromOutput, givenInputLengths.at(batchId), + reqMaxNewTokens.at(batchId), beamWidth, beamTokens, genLogits, vocabSizePadded, batchId, + returnAllGeneratedTokens, logitsAtol, logitsRtol); + } + + // Ignore first iteration as it doesn't use draft tokens + if (outConfig.returnPerfMetrics && isSpeculativeDecoding + && result.requestPerfMetrics.value().iter > 0) + { + auto& specDecMetrics = result.requestPerfMetrics.value().speculativeDecoding; + // 4 draft tokens are used per step + EXPECT_EQ(specDecMetrics.totalDraftTokens, result.requestPerfMetrics.value().iter.value() * 4); + EXPECT_EQ(specDecMetrics.acceptanceRate, + static_cast<float>(specDecMetrics.totalAcceptedDraftTokens) + / specDecMetrics.totalDraftTokens); + } + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + ++iter; + } + EXPECT_LT(iter, maxWaitMs); + testData.verifyOutput(tokens, givenInputLengths, streaming, outConfig.excludeInputFromOutput, flakyTestInfo, + isSpeculativeDecoding, beamWidth, numSequences, isNonGreedySampling); + } +} + +void runTest(fs::path const& modelPath, ExecutorConfig const& executorConfig, fs::path const& inputPath, + ModelIds const& modelIds, FlakyTestInfo const& flakyTestInfo, bool streaming, SizeType32 const vocabSizePadded, + BeamResult const& beamResult, OutputConfig const& outConfig, bool isSpeculativeDecoding, int maxWaitMs, + bool returnAllGeneratedTokens, SizeType32 const numReturnSequences, bool isNonGreedySampling, + SizeType32 const modelParallelism) +{ + auto executor = Executor{modelPath, ModelType::kDECODER_ONLY, executorConfig}; + + runTest(executor, inputPath, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, outConfig, + isSpeculativeDecoding, maxWaitMs, returnAllGeneratedTokens, numReturnSequences, isNonGreedySampling, + modelParallelism); +} + +ExecutorConfig createExecutorConfig(SizeType32 maxBeamWidth, bool useOrchestratorMode, bool gatherGenerationLogits, + std::optional<std::vector<SizeType32>> deviceIds = std::nullopt, + std::optional<std::vector<SizeType32>> participantIds = std::nullopt) +{ + // Note: we reduce memory fraction for cases that return context/generation logits which require more free + // memory + FloatType constexpr freeGpuMemoryFraction{0.5F}; + KvCacheConfig kvCacheConfig(false, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction); + auto executorConfig = ExecutorConfig(maxBeamWidth); + executorConfig.setKvCacheConfig(kvCacheConfig); + executorConfig.setNormalizeLogProbs(false); + executorConfig.setGatherGenerationLogits(gatherGenerationLogits); + + std::optional<OrchestratorConfig> orchestratorConfig = std::nullopt; + if (useOrchestratorMode) + { + orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + } + auto parallelConfig = ParallelConfig(CommunicationType::kMPI, + useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::move(deviceIds), + std::move(participantIds), orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + return executorConfig; +} + +} // namespace + +TEST_P(AllParamsTest, TokenComparison) +{ + auto const streaming = std::get<0>(GetParam()); + auto const& beamWidth = std::get<1>(GetParam()); + OutputConfig outConfig; + outConfig.returnLogProbs = std::get<2>(GetParam()); + outConfig.excludeInputFromOutput = std::get<3>(GetParam()); + outConfig.returnContextLogits = std::get<4>(GetParam()); + outConfig.returnGenerationLogits = std::get<5>(GetParam()); + auto const modelName = std::get<6>(GetParam()); + auto const useOrchestratorMode = std::get<7>(GetParam()); + auto const returnAllGeneratedTokens = std::get<8>(GetParam()); + auto const numReturnSequences = std::get<9>(GetParam()); + if (returnAllGeneratedTokens && !streaming) + { + GTEST_SKIP() << "Test does not support returnAllGeneratedTokens without streaming"; + } + + std::optional<std::vector<SizeType32>> participantIds = std::nullopt; + + BeamResult beamResult{beamWidth}; + + ASSERT_TRUE(fs::exists(DATA_PATH)); + + fs::path modelPath; + // set defaults and adjust if needed by different models + fs::path inputPath = DATA_PATH / "input_tokens.npy"; + ModelIds modelIds{50256, 50256}; + bool isSpeculativeDecoding{false}; + + SizeType32 vocabSizePadded = 50257; + + // NOTE: This can be used to disable checks for certain prompt batch entries + FlakyTestInfo flakyTestInfo; + + if (modelName == "gpt") + { + auto const resultsPath + = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + if (outConfig.returnContextLogits || outConfig.returnGenerationLogits) + { + modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"; + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); + beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); + beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); + if (outConfig.returnLogProbs) + { + beamResult.cumLogProbsFile + = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CUM_LOG_PROBS_FILE(); + beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_LOG_PROBS_FILE(); + } + } + else + { + modelPath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); + if (outConfig.returnLogProbs) + { + beamResult.cumLogProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_FILE(); + beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_FILE(); + } + } + } + else if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1" + || modelName == "llama_tp1_pp2_cp1") + { + inputPath = DATA_PATH / LLAMA_INPUT_FILE; + modelIds.padId = LLAMA_PAD_ID; + modelIds.endId = LLAMA_END_ID; + + vocabSizePadded = LLAMA_VOCAB_SIZE_PADDED; + + auto const resultsPath + = LLAMA_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + if (modelName == "llama_tp4_pp1_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp4_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp2_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp2-cp1-gpu"; + } + else if (modelName == "llama_tp2_pp2_cp1") + { + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + } + beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_TP4_PP1_FILE(); + if (outConfig.returnLogProbs) + { + beamResult.cumLogProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_TP4_PP1_FILE(); + beamResult.logProbsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_TP4_PP1_FILE(); + } + } + else if (modelName == "medusa") + { + TLLM_CHECK_WITH_INFO(beamWidth == 1, "Medusa does not support beam search."); + auto const resultsPath = MEDUSA_DATA_PATH / "sampling"; + auto modelSpec = ModelSpec::getDefaultModelSpec() + .useMedusa() + .setInputFile("input_tokens_long.npy") + .setMaxOutputLength(128); + beamResult.resultsFile = resultsPath / modelSpec.getResultsFile(); + modelPath = MEDUSA_MODEL_PATH / modelSpec.getModelPath() / "tp1-pp1-cp1-gpu"; + + inputPath = DATA_PATH / "input_vicuna.npy"; + modelIds.padId = 2; + modelIds.endId = 2; + isSpeculativeDecoding = true; + outConfig.returnPerfMetrics = true; + } + else if (modelName == "chatglm" || modelName == "chatglm2" || modelName == "chatglm3" || modelName == "glm") + { + fs::path resultsPath; + if (modelName == "chatglm") + { + resultsPath = CHATGLM_DATA_PATH; + modelPath = CHATGLM_MODEL_PATH; + } + else if (modelName == "chatglm2") + { + resultsPath = CHATGLM2_DATA_PATH; + modelPath = CHATGLM2_MODEL_PATH; + } + else if (modelName == "chatglm3") + { + resultsPath = CHATGLM3_DATA_PATH; + modelPath = CHATGLM3_MODEL_PATH; + } + else if (modelName == "glm") + { + resultsPath = GLM_DATA_PATH; + modelPath = GLM_MODEL_PATH; + } + resultsPath /= (beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth); + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); + modelPath = modelPath / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + + char versionChatglm{0}; + if (size_t index = modelPath.string().find("chatglm"); index != std::string::npos) + { + versionChatglm = modelPath.string()[index + 7]; + std::string const vChatglmString + = (versionChatglm == '-') ? std::string("") : std::string(1, versionChatglm); + inputPath = DATA_PATH / ("input_tokens_chatglm" + vChatglmString + "-6b.npy"); + modelIds.padId = (versionChatglm == '-') ? 3 : 0; + modelIds.endId = (versionChatglm == '-') ? 130005 : 2; + } + else if (size_t index = modelPath.string().find("glm-10b"); index != std::string::npos) + { + inputPath = DATA_PATH / "input_tokens_glm-10b.npy"; + modelIds.padId = 50256; + modelIds.endId = 50258; + } + + if (versionChatglm != 0) + { + flakyTestInfo.batchIdBeams.insert(std::make_pair(1, 0)); + } + } + else + { + TLLM_THROW("Unrecognized modelName"); + } + + if (streaming && beamWidth > 1) + { + GTEST_SKIP() << "Test does not support streaming with beam search"; + } + + // Warning: This should be the last check before running the test. + // It will initialize MPI which can take significant time. + if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1" + || modelName == "llama_tp1_pp2_cp1") + { + // For llama model, only run for multiple GPUs + // This is detected by setting an env variable when running the test + char const* val = getenv("RUN_LLAMA_MULTI_GPU"); + if (val == nullptr) + { + GTEST_SKIP() << "Skipping Llama test"; + } + + if (outConfig.returnContextLogits) + { + GTEST_SKIP() << "Skipping context logits tests for mpi runs"; + } + + // Check that it was launched with right number of MPI ranks + if (!useOrchestratorMode && COMM_SESSION.getSize() != 4) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Leader mode and world size is not equal to 4"; + } + if (useOrchestratorMode && COMM_SESSION.getSize() != 1) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Orchestrator mode and World size is not equal to 1"; + } + } + auto decoderJsonConfig = tensorrt_llm::runtime::GptJsonConfig::parse(modelPath / "config.json"); + + auto const modelTP = decoderJsonConfig.getTensorParallelism(); + auto const modelPP = decoderJsonConfig.getPipelineParallelism(); + auto const modelParallelism = modelTP * modelPP; + int deviceCount = -1; + TLLM_CUDA_CHECK(cudaGetDeviceCount(&deviceCount)); + std::optional<std::vector<SizeType32>> deviceIds = std::vector<SizeType32>(modelParallelism); + for (auto i = 0; i < deviceIds->size(); i++) + { + deviceIds->at(i) = i % deviceCount; + } + if (modelName == "llama_tp1_pp2_cp1") + { + auto const& session = tensorrt_llm::mpi::MpiComm::world(); + if (session.getSize() != 4) + { + FAIL() << "Llama-tp1-pp2 is intended solely for testing coexisting engines within the same MPI world," + " which requires a session size of 4. However, the current session size is " + << session.getSize() << " ."; + } + if (session.getRank() / 2 == 0) + { + participantIds = std::vector<SizeType32>{0, 1}; + deviceIds = std::vector<SizeType32>{0, 1}; + } + else + { + participantIds = std::vector<SizeType32>{2, 3}; + deviceIds = std::vector<SizeType32>{2, 3}; + } + } + + if (modelPP > 1) + { + std::reverse(deviceIds->begin(), deviceIds->end()); + if (modelTP > 1) + { + for (SizeType32 ppRank = 0; ppRank < modelPP; ppRank++) + { + std::reverse(deviceIds->begin() + ppRank * modelTP, deviceIds->begin() + (ppRank + 1) * modelPP); + } + } + } + + // Returning logits will bring higher latency + if (streaming && (outConfig.returnContextLogits || outConfig.returnGenerationLogits)) + { + mMaxWaitMs = 20000; + } + + auto executorConfig = createExecutorConfig(beamWidth, useOrchestratorMode, outConfig.returnGenerationLogits, + std::move(deviceIds), std::move(participantIds)); + + runTest(modelPath, executorConfig, inputPath, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, + outConfig, isSpeculativeDecoding, mMaxWaitMs, returnAllGeneratedTokens, numReturnSequences, false, + modelParallelism); +} + +TEST_F(GptExecutorTest, ChangeBeamWidth) +{ + SizeType32 constexpr maxBeamWidth{2}; + auto executorConfig = ExecutorConfig(maxBeamWidth); + + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 constexpr beamWidth1{1}; + SizeType32 constexpr beamWidth2{2}; + SizeType32 constexpr maxNewTokens{2}; + VecTokens inputTokens{1, 2, 3, 4}; + + // Create requests with different beam widths + std::vector<Request> requests; + requests.emplace_back(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth1)); + requests.emplace_back(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth1)); + requests.emplace_back(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth2)); + requests.emplace_back(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth1)); + + auto requestIds = executor.enqueueRequests(requests); + + int numFinished = 0; + int iter = 0; + while (numFinished < 4 && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + auto err = response.getErrorMsg(); + std::cout << "err:" << err << std::endl; + FAIL() << "Should not get a response with error"; + } + else + { + auto result = response.getResult(); + numFinished += static_cast<int>(result.isFinal); + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + + auto stats = executor.getLatestIterationStats(); + uint64_t currentIter = 0; + for (auto const& stat : stats) + { + // TODO: enable this check when stats are cleaned + // EXPECT_EQ(stat.iter, currentIter); + if (stat.iter < 2) + { + // req 1 and 2 run with same beam width + EXPECT_EQ(stat.numActiveRequests, 2); + } + else if (stat.numActiveRequests != 0) // TODO: remove this check when stats are cleaned + { + // req 3 or 4 run width different beam width + EXPECT_EQ(stat.numActiveRequests, 1); + } + + ++currentIter; + } +} + +void doTokenComparisonChangeBeamWidth(bool enableReuse, SizeType32 maxWaitMs) +{ + SizeType32 constexpr maxBeamWidth{2}; + SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded + auto constexpr streaming = false; + + // Create executor config + auto kvCacheConfig = KvCacheConfig(enableReuse); + auto executorConfig = ExecutorConfig(maxBeamWidth, SchedulerConfig(), kvCacheConfig); + + // Create executor + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + auto const inputPath = DATA_PATH / "input_tokens.npy"; + ModelIds modelIds{50256, 50256}; + + OutputConfig outConfig; + FlakyTestInfo flakyTestInfo; + bool constexpr isSpeculativeDecoding{false}; + + for (SizeType32 beamWidth : {1, 2}) + { + TLLM_LOG_INFO("Running beam width: %d", beamWidth); + BeamResult beamResult{beamWidth}; + auto const resultsPath + = GPT_DATA_PATH / ((beamWidth == 1) ? "sampling" : "beam_search_" + std::to_string(beamWidth)); + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); + beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); + beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); + + auto const numReturnSequences = beamWidth; + + runTest(executor, inputPath, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, outConfig, + isSpeculativeDecoding, maxWaitMs, false, numReturnSequences, false, 1); + } +} + +TEST_F(GptExecutorTest, TokenComparisonChangeBeamWidth) +{ + doTokenComparisonChangeBeamWidth(false, mMaxWaitMs); +} + +TEST_F(GptExecutorTest, TokenComparisonChangeBeamWidthBlockReuse) +{ + doTokenComparisonChangeBeamWidth(true, mMaxWaitMs); +} + +TEST_F(GptExecutorTest, NReturnRandomness) +{ + SizeType32 constexpr maxBeamWidth{1}; + SizeType32 constexpr numReturnSequences{2}; + SizeType32 constexpr vocabSizePadded{50257}; // gpt vocabSizePadded + auto constexpr streaming = false; + + // Create executor config + auto executorConfig = ExecutorConfig(maxBeamWidth); + + // Create executor + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + auto const inputPath = DATA_PATH / "input_tokens.npy"; + ModelIds modelIds{50256, 50256}; + + OutputConfig outConfig; + FlakyTestInfo flakyTestInfo; + bool constexpr isSpeculativeDecoding{false}; + + BeamResult beamResult{maxBeamWidth}; + auto const resultsPath = GPT_DATA_PATH / "sampling"; + beamResult.resultsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); + beamResult.contextLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); + beamResult.genLogitsFile = resultsPath / PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); + + runTest(executor, inputPath, modelIds, flakyTestInfo, streaming, vocabSizePadded, beamResult, outConfig, + isSpeculativeDecoding, mMaxWaitMs, false, 1, true, 1); +} + +TEST_F(GptExecutorTest, TimedOut) +{ + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // No requests enqueued, expect no responses + auto numResponsesReady = executor.getNumResponsesReady(); + EXPECT_EQ(numResponsesReady, 0); + + std::chrono::milliseconds waitTime(10); + auto responses = executor.awaitResponses(waitTime); + EXPECT_EQ(responses.size(), 0); +} + +TEST_F(GptExecutorTest, MaxSeqIdleMicrosecondsError) +{ + auto executorConfig = ExecutorConfig(1); + // Request will time out + executorConfig.setMaxSeqIdleMicroseconds(1); + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 constexpr maxNewTokens{5}; + VecTokens inputTokens{1, 2, 3, 4}; + + std::vector<Request> requests; + requests.emplace_back(inputTokens, maxNewTokens, false); + + auto requestIds = executor.enqueueRequests(requests); + + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + auto err = response.getErrorMsg(); + std::cout << "err:" << err << std::endl; + EXPECT_THAT(err, testing::HasSubstr("Unable to get batch slot for request ID")); + done = true; + } + else + { + FAIL() << "Should get a response with error"; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); +} + +void logitsProcessorMixedReqsTest(std::string const& modelDir, SizeType32 worldRank, SizeType32 maxWaitMs, + bool replicated, std::optional<std::vector<SizeType32>> deviceIds); + +TEST_P(LogitsProcParamsTest, All) +{ + auto const modelName = std::get<0>(GetParam()); + auto const batched = std::get<1>(GetParam()); + auto const replicated = std::get<2>(GetParam()); + + std::string modelDir; + int tp_size = 1, pp_size = 1, cp_size = 1; + std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; + + if (modelName == "llama_tp1_pp1_cp1") + { + modelDir = "tp1-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp4_pp1_cp1") + { + modelDir = "tp4-pp1-cp1-gpu"; + tp_size = 4; + } + else if (modelName == "llama_tp1_pp4_cp1") + { + modelDir = "tp1-pp4-cp1-gpu"; + pp_size = 4; + deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; + } + else if (modelName == "llama_tp2_pp2_cp1") + { + modelDir = "tp2-pp2-cp1-gpu"; + tp_size = pp_size = 2; + deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; + } + else + { + TLLM_THROW("Unrecognized modelName"); + } + std::filesystem::path modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / modelDir; + + auto& comm = tensorrt_llm::mpi::MpiComm::world(); + auto const worldRank = comm.getRank(); + auto const worldSize = comm.getSize(); + + if (tp_size * pp_size * cp_size != 1) + { + // Run multi GPU test only when env variable is set + char const* val = getenv("RUN_LLAMA_MULTI_GPU"); + if (val == NULL) + { + GTEST_SKIP() << "Skipping multi-gpu logits post processor test"; + } + + if (worldSize != 4) + { + FAIL() << "Leader mode and world size is not equal to 4"; + } + } + else + { + // This has no effect for single-GPU tests + if (replicated) + { + GTEST_SKIP() << "Skipping single-gpu replicated logits post processor test"; + } + } + + // Configuration options + bool const streaming = false; + bool excludeInputFromOutput = false; + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + SizeType32 numRequests = 20; + IdType const kClientId = 1234; + + SizeType32 beamWidth = 1; + SizeType32 maxPromptLen = 20; + SizeType32 maxMaxNewTokens = 20; + + SizeType32 constexpr endId{2}; + SizeType32 constexpr vocabSizePadded{32000}; // llama-7b vocabSizePadded + // We just use tokenIdCalculator to generate a token_id based on request index, output position and max new tokens. + // Then LogitsPostProcessor set all other logits except the generated token_id to large negative value. + // So the output token should be the generated token by tokenIdCalculator. + auto tokenIdCalculator = [endId, vocabSizePadded](IdType req, SizeType32 pos) + { + SizeType32 tokenId = (req * 1000 + pos) % vocabSizePadded; + if (tokenId == endId) + { + tokenId = 0; + } + return tokenId; + }; + + std::unordered_map<IdType, VecTokens> tokens; + std::unordered_map<IdType, SizeType32> expectedNumTokens; + std::unordered_map<IdType, VecTokens> expectedOutputTokens; + + // Enqueue the requests + auto enqueueRequests = [&](Executor& executor, std::optional<std::string const> logitsProcessorName, + std::optional<LogitsPostProcessor> logitsProcessor = std::nullopt) + { + tokens.clear(); + expectedNumTokens.clear(); + expectedOutputTokens.clear(); + + for (SizeType32 req = 0; req < numRequests; ++req) + { + SizeType32 promptLen = rand() % maxPromptLen + 1; + SizeType32 maxNewTokens = rand() % maxMaxNewTokens + 1; + + auto request = Request(VecTokens(promptLen, 1), maxNewTokens, streaming, + tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig, endId); + request.setClientId(kClientId); + if (logitsProcessorName) + { + request.setLogitsPostProcessorName(logitsProcessorName.value()); + } + else if (logitsProcessor) + { + request.setLogitsPostProcessor(logitsProcessor.value()); + } + auto reqId = executor.enqueueRequest(std::move(request)); + tokens[reqId] = {}; + expectedNumTokens[reqId] = (streaming ? 0 : (excludeInputFromOutput ? 0 : promptLen)) + maxNewTokens; + expectedOutputTokens[reqId] = {}; + if (!streaming && !excludeInputFromOutput) + { + expectedOutputTokens[reqId].resize(promptLen, 1); + } + for (SizeType32 outputPos = 0; outputPos < maxNewTokens; ++outputPos) + { + SizeType32 outputTokenId = tokenIdCalculator(reqId, outputPos + promptLen); + expectedOutputTokens[reqId].push_back(outputTokenId); + } + } + }; + + // Get the new tokens for each requests + auto collectResponses = [&](Executor& executor) + { + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + while (numFinished < numRequests && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + EXPECT_EQ(response.getClientId().value(), kClientId); + auto result = response.getResult(); + numFinished += result.isFinal; + auto& newTokens = result.outputTokenIds.at(beamWidth - 1); + auto& reqTokens = tokens.at(response.getRequestId()); + reqTokens.insert(reqTokens.end(), std::make_move_iterator(newTokens.begin()), + std::make_move_iterator(newTokens.end())); + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + }; + + // Check that tokens matches expectations + auto checkOutput = [&]() + { + for (auto const& [reqId, numTokens] : expectedNumTokens) + { + EXPECT_EQ(expectedNumTokens[reqId], tokens[reqId].size()) << "reqId " << reqId; + for (SizeType32 tokenPos = 0; + tokenPos < std::min<SizeType32>(expectedNumTokens[reqId], tokens[reqId].size()); ++tokenPos) + { + EXPECT_EQ(expectedOutputTokens[reqId][tokenPos], tokens[reqId][tokenPos]) + << "reqId=" << reqId << ", tokenPos=" << tokenPos; + } + } + }; + + // Test non-batched logits processor + std::string const logitsProcessorName = "SelectToken"; + + auto logitsPostProcessorFn = [&](IdType reqId, Tensor& logits, BeamTokens const& tokens, StreamPtr const& streamPtr, + std::optional<IdType> clientId) + { + if (replicated) + { + EXPECT_TRUE(worldRank <= tp_size - 1); + } + else + { + EXPECT_TRUE(worldRank == 0); + } + EXPECT_TRUE(clientId.value() == kClientId); + SizeType32 numTokens = tokens.at(0).size(); + SizeType32 pos = numTokens; + SizeType32 outputTokenId = tokenIdCalculator(reqId, pos); + auto logitsDataType = logits.getDataType(); + EXPECT_TRUE(logitsDataType == DataType::kFP16 || logitsDataType == DataType::kBF16 + || logitsDataType == DataType::kFP32); + // logits has shape [draftLength + 1, reqBeamWidth, vocabSize] + auto logitsCpu = tensorrt_llm::executor::Tensor::cpu(logitsDataType, logits.getShape()); + auto* dataPtr = logitsCpu.getData(); + auto eltSize = logitsCpu.getSizeInBytes() / logitsCpu.getSize(); + EXPECT_TRUE(eltSize == 2 || eltSize == 4); + if (eltSize == 2) + { + auto* dataPtrU16 = static_cast<uint16_t*>(dataPtr); + uint16_t hugeNegValue = logitsDataType == DataType::kFP16 ? 0xFBFF : 0xFF7F; // a huge negative value + for (size_t i = 0; i < logitsCpu.getSize(); ++i) + { + dataPtrU16[i] = hugeNegValue; + } + dataPtrU16[outputTokenId] = 0; + } + else + { + auto* dataPtrFloat = static_cast<float*>(dataPtr); + for (size_t i = 0; i < logitsCpu.getSize(); ++i) + { + dataPtrFloat[i] = -HUGE_VALF; + } + dataPtrFloat[outputTokenId] = 0.0f; + } + + logits.setFrom(logitsCpu, streamPtr); + }; + + if (!batched) + { + auto executorConfig = ExecutorConfig(beamWidth); + LogitsPostProcessorConfig logitsProcConfig{ + std::unordered_map<std::string, tensorrt_llm::executor::LogitsPostProcessor>{ + {logitsProcessorName, logitsPostProcessorFn}}, + std::nullopt, replicated}; + executorConfig.setLogitsPostProcessorConfig(logitsProcConfig); + if (deviceIds.has_value()) + { + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + parallelConfig.setDeviceIds(deviceIds.value()); + executorConfig.setParallelConfig(parallelConfig); + } + auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); + + if (worldRank == 0) + { + enqueueRequests(executor, logitsProcessorName); + collectResponses(executor); + checkOutput(); + + if (!replicated || tp_size == 1) + { + // Dynamic logits postprocessor must be used with replicate=false or no tensor parallelism. + enqueueRequests(executor, std::nullopt, logitsPostProcessorFn); + collectResponses(executor); + checkOutput(); + } + } + } + + // Test batched logits processor + auto logitsPostProcessorBatchedFn + = [logitsPostProcessorFn](std::vector<IdType> const& reqIdBatch, std::vector<Tensor>& logitsBatch, + std::vector<std::reference_wrapper<BeamTokens const>> const& tokensBatch, StreamPtr const& streamPtr, + std::vector<std::optional<IdType>> const& clientIdBatch) + { + for (int sample = 0; sample < reqIdBatch.size(); sample++) + { + logitsPostProcessorFn( + reqIdBatch[sample], logitsBatch[sample], tokensBatch[sample], streamPtr, clientIdBatch[sample]); + } + }; + + if (batched) + { + auto batchedExecutorConfig = ExecutorConfig(beamWidth); + if (deviceIds.has_value()) + { + auto parallelConfig = batchedExecutorConfig.getParallelConfig().value_or(ParallelConfig()); + + parallelConfig.setDeviceIds(deviceIds.value()); + batchedExecutorConfig.setParallelConfig(parallelConfig); + } + LogitsPostProcessorConfig logitsProcConfig{std::nullopt, logitsPostProcessorBatchedFn, replicated}; + batchedExecutorConfig.setLogitsPostProcessorConfig(logitsProcConfig); + + auto batchedExecutor = Executor(modelPath, ModelType::kDECODER_ONLY, batchedExecutorConfig); + + if (worldRank == 0) + { + enqueueRequests(batchedExecutor, Request::kBatchedPostProcessorName); + collectResponses(batchedExecutor); + checkOutput(); + } + } + + if (!batched) + { + logitsProcessorMixedReqsTest(modelDir, worldRank, mMaxWaitMs, replicated, std::move(deviceIds)); + } +} + +// Test for mixing requests with and without logits processor. +void logitsProcessorMixedReqsTest(std::string const& modelDir, SizeType32 worldRank, SizeType32 maxWaitMs, + bool replicated, std::optional<std::vector<SizeType32>> deviceIds) +{ + std::string const logitsProcessorName = "dummy"; + auto logitsPostProcessorFn = [&](IdType reqId, Tensor& logits, BeamTokens const& tokens, StreamPtr const& streamPtr, + std::optional<IdType> clientId) + { + // Dummy callback that does not modify logits + assert(!clientId.has_value()); + }; + + LogitsPostProcessorConfig logitsProcConfig{ + std::unordered_map<std::string, tensorrt_llm::executor::LogitsPostProcessor>{ + {logitsProcessorName, logitsPostProcessorFn}}, + std::nullopt, replicated}; + + // Create executor + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + executorConfig.setLogitsPostProcessorConfig(logitsProcConfig); + if (deviceIds.has_value()) + { + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + + parallelConfig.setDeviceIds(deviceIds.value()); + executorConfig.setParallelConfig(parallelConfig); + } + std::filesystem::path modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / modelDir; + auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); + + if (worldRank == 0) + { + SizeType32 numRequests = 2; + SizeType32 promptLen = 5; + + // First request with no LP and many output tokens + auto request1 = Request(VecTokens(promptLen, 1), 25); + // Second request with LP and few output tokens + auto request2 = Request(VecTokens(promptLen, 1), 5); + request2.setLogitsPostProcessorName(logitsProcessorName); + + // Enqueue requests + auto reqId1 = executor.enqueueRequest(request1); + auto reqId2 = executor.enqueueRequest(request2); + + // Wait for responses + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + while (numFinished < numRequests && iter < maxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + EXPECT_EQ(response.getErrorMsg(), err); + } + } + ++iter; + } + EXPECT_LT(iter, maxWaitMs); + } +} + +TEST_F(GptExecutorTest, LogitsPostProcessorThrow) +{ + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + std::string const logitsProcessorName = "UnExistProcessor"; + + auto request + = Request(VecTokens(10, 1), 10, false, tensorrt_llm::executor::SamplingConfig(beamWidth), OutputConfig()); + request.setLogitsPostProcessorName(logitsProcessorName); + EXPECT_THROW({ auto reqId = executor.enqueueRequest(std::move(request)); }, tensorrt_llm::common::TllmException); +} + +static Response executeDraftRequest(Executor& executor) +{ + OutputConfig outputConfig; + outputConfig.returnGenerationLogits = true; + + // Create the request + SizeType32 maxNewTokens = 4; + VecTokens inputTokens{1, 2, 3, 4}; + + Request request{std::move(inputTokens), maxNewTokens}; + request.setOutputConfig(outputConfig); + + // Enqueue the request + auto requestId = executor.enqueueRequest(std::move(request)); + + // Wait for the response + auto responses = executor.awaitResponses(requestId); + + return responses.at(0); +} + +static Response executeTargetRequest(Executor& executor, Result const& draftResult) +{ + // Create the request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + + Request request{std::move(inputTokens), maxNewTokens}; + + VecTokens const& outputTokenIds = draftResult.outputTokenIds.at(0); + VecTokens draftTokens(outputTokenIds.end() - 4, outputTokenIds.end()); + + auto const& logitsInfo = draftResult.specDecFastLogitsInfo.value(); + auto logitsTensor = logitsInfo.toTensor(); + + ExternalDraftTokensConfig draftTokensConfig( + std::move(draftTokens), logitsTensor, std::nullopt /* acceptance threshold */, true /* fastLogits */); + request.setExternalDraftTokensConfig(draftTokensConfig); + + // Enqueue the request + auto requestId = executor.enqueueRequest(std::move(request)); + + // Wait for the response + auto responses = executor.awaitResponses(requestId); + + return responses.at(0); +} + +class SpeculativeDecodingTest : public GptExecutorTest +{ +}; + +TEST_F(SpeculativeDecodingTest, SpecDecFastLogits) +{ + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtDraftEnginePath + = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() / "tp1-pp1-cp1-gpu"; + auto trtEnginePath + = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DRAFT_TOKENS_DIR() / "tp1-pp1-cp1-gpu"; + + FloatType freeGpuMemoryFraction = 0.3; + auto kvCacheConfig + = KvCacheConfig(true /* enableBlockReuse */, std::nullopt, std::nullopt, std::nullopt, freeGpuMemoryFraction); + executorConfig.setKvCacheConfig(kvCacheConfig); + + tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE); + int const worldSize = tensorrt_llm::mpi::MpiComm::world().getSize(); + ASSERT_EQ(worldSize, 3); + int const myRank = tensorrt_llm::mpi::MpiComm::world().getRank(); + bool const isOrchestrator = (myRank == 0); + + auto orchestratorConfig + = OrchestratorConfig(isOrchestrator, "" /* workerExecutablePath */, nullptr, false /* spawnPrcesses */); + auto parallelConfig = ParallelConfig( + CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR, std::nullopt, std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + auto specDecConfig = SpeculativeDecodingConfig(true /* fastLogits */); + executorConfig.setSpecDecConfig(specDecConfig); + + std::unique_ptr<Executor> draftExecutor; + std::unique_ptr<Executor> targetExecutor; + + if (isOrchestrator) + { + auto executorConfigDraft = executorConfig; + parallelConfig.setParticipantIds({1}); + executorConfigDraft.setParallelConfig(parallelConfig); + + draftExecutor = std::make_unique<Executor>(trtDraftEnginePath, ModelType::kDECODER_ONLY, executorConfigDraft); + + parallelConfig.setParticipantIds({2}); + executorConfig.setParallelConfig(parallelConfig); + + targetExecutor = std::make_unique<Executor>(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + } + else if (myRank == 1) // draft model process + { + parallelConfig.setParticipantIds({1}); + parallelConfig.setDeviceIds({0}); + executorConfig.setParallelConfig(parallelConfig); + executorConfig.setGatherGenerationLogits(true); + draftExecutor = std::make_unique<Executor>(trtDraftEnginePath, ModelType::kDECODER_ONLY, executorConfig); + } + else if (myRank == 2) // target model process + { + parallelConfig.setParticipantIds({2}); + parallelConfig.setDeviceIds({0}); + executorConfig.setParallelConfig(parallelConfig); + draftExecutor = std::make_unique<Executor>(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + } + + if (isOrchestrator) + { + auto response = executeDraftRequest(*draftExecutor); + ASSERT_FALSE(response.hasError()); + response = executeTargetRequest(*targetExecutor, response.getResult()); + ASSERT_FALSE(response.hasError()); + } +} + +TEST_F(GptExecutorTest, OrchestratorMaxQueueSize) +{ + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + SizeType32 maxQueueSize = 6; + ExecutorConfig executorConfig; + executorConfig.setMaxQueueSize(maxQueueSize); + auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + auto parallelConfig = ParallelConfig( + CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR, std::nullopt, std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 100; + VecTokens inputTokens{1, 2, 3, 4}; + auto request = Request(inputTokens, maxNewTokens); + std::vector<IdType> requestIds; + auto numberOfRequests = maxQueueSize * 5; + requestIds.reserve(numberOfRequests); + + // Enqueue more requests than the queue can manage + for (int i = 0; i < numberOfRequests; i++) + { + auto requestId = executor.enqueueRequest(request); + requestIds.emplace_back(requestId); + } + + auto responseVectors = executor.awaitResponses(std::move(requestIds)); + bool failedWithFullQueue = false; + for (auto& responseVector : responseVectors) + { + for (auto& response : responseVector) + { + if (response.hasError()) + { + EXPECT_THAT(response.getErrorMsg(), + testing::HasSubstr("Maximum queue size of 6 has been reached, please try again later")); + failedWithFullQueue = true; + } + } + } + EXPECT_TRUE(failedWithFullQueue) << "Expected requests to fail due to maximum queue size reached"; + + // Wait for requests to get scheduled to free up space in queue + std::this_thread::sleep_for(std::chrono::milliseconds(maxQueueSize * 200)); + auto requestId = executor.enqueueRequest(std::move(request)); + auto responses = executor.awaitResponses(requestId); + for (auto& response : responses) + { + EXPECT_FALSE(response.hasError()); + } +} + +TEST_F(GptExecutorTest, SingleRequestInvalidInputs) +{ + bool streaming = true; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + + std::vector<std::string> expectedErrMsgs; + std::vector<Request> requests; + + // Invalid embedding bias shape + { + requests.emplace_back(inputTokens, maxNewTokens, streaming); + auto embeddingBias = Tensor::cpu(DataType::kFP32, {1}); + requests.back().setEmbeddingBias(embeddingBias); + expectedErrMsgs.emplace_back("embedding bias shape is not as expected"); + } + + for (auto req = 0; req < requests.size(); ++req) + { + auto& request = requests.at(req); + auto const& expectedErrMsg = expectedErrMsgs.at(req); + + auto requestId = executor.enqueueRequest(std::move(request)); + + // Try to get the new tokens + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + + auto err = response.getErrorMsg(); + EXPECT_THAT(err, testing::HasSubstr(expectedErrMsg)); + done = true; + } + else + { + FAIL() << "Expected an err: " << expectedErrMsg; + } + } + ++iter; + } + EXPECT_EQ(done, true); + } +} + +TEST_F(GptExecutorTest, ExecutorKVCacheManager) +{ + + bool streaming = true; + int numRequests = 3; + + SizeType32 beamWidth = 1; + SizeType32 maxNewTokens = 5; + auto executorConfig = ExecutorConfig(beamWidth); + auto kvCacheConfig = KvCacheConfig(true, 128); + kvCacheConfig.setEventBufferMaxSize(1024); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + auto kvCacheManager = *executor.getKVCacheEventManager(); + + // Created event should be available before any requests. + auto events = kvCacheManager->getLatestEvents(std::chrono::seconds(1)); + EXPECT_EQ(events.size(), 1); + EXPECT_TRUE(std::holds_alternative<KVCacheCreatedData>(events.front().data)); + + // Create requests + std::vector<Request> requests; + for (int request = 0; request < 3; request++) + { + VecTokens inputTokens; + for (int i = 0; i < 63; i++) + { + inputTokens.emplace_back(i + request); + } + requests.emplace_back(inputTokens, maxNewTokens, streaming); + } + + for (auto req = 0; req < requests.size(); ++req) + { + auto& request = requests.at(req); + + auto requestId = executor.enqueueRequest(std::move(request)); + + // Get the new tokens + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + // This request failed for some reason, get error msg + std::string errStr + = "Request id " + std::to_string(requestId) + " failed with err " + response.getErrorMsg(); + FAIL(); + } + else + { + auto result = response.getResult(); + done = result.isFinal; + if (done) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + auto events = kvCacheManager->getLatestEvents(std::chrono::milliseconds(100)); + if (req == 0) + { + EXPECT_EQ(events.size(), 3); + + // Store the first context block + EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).parentHash, std::nullopt); + EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); + events.pop_front(); + // Store the second (now completed) context block and the partial decode block. + EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); + EXPECT_EQ(std::get<KVCacheStoredData>(events.back().data).blocks.size(), 1); + EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks[0].blockHash, + std::get<KVCacheStoredData>(events.back().data).parentHash); + } + else + { + EXPECT_EQ(events.size(), 5); + + // Remove a block to make room for the second context block. On the second request, we need + // to remove 2 blocks. + EXPECT_EQ(std::get<KVCacheRemovedData>(events.front().data).blockHashes.size(), req); + events.pop_front(); + // Store the first filled context block + EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); + events.pop_front(); + // Remove a block for the decode phase + EXPECT_EQ(std::get<KVCacheRemovedData>(events.front().data).blockHashes.size(), 1); + events.pop_front(); + // Store the final context block and the decode block + EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); + events.pop_front(); + EXPECT_EQ(std::get<KVCacheStoredData>(events.front().data).blocks.size(), 1); + } + } + } + } + iter++; + } + EXPECT_EQ(done, true); + } +} + +TEST_F(GptExecutorTest, SingleRequestLora) +{ + bool streaming = true; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Load lora weights, config + auto manager = tr::BufferManager(std::make_shared<tr::CudaStream>()); + auto loraWeightsTensor + = std::shared_ptr(tr::utils::loadNpy(manager, LORA_WEIGHTS_FILE.string(), tr::MemoryType::kCPU)); + auto loraConfigTensor + = std::shared_ptr(tr::utils::loadNpy(manager, LORA_CONFIG_FILE.string(), tr::MemoryType::kCPU)); + + // Create the request + SizeType32 maxNewTokens = 5; + VecTokens inputTokens{1, 2, 3, 4}; + auto request = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig()); + auto loraConfig = LoraConfig(0, detail::ofITensor(loraWeightsTensor), detail::ofITensor(loraConfigTensor)); + request.setLoraConfig(loraConfig); + + // Enqueue the request + auto requestId = executor.enqueueRequest(std::move(request)); + + // Get the new tokens + VecTokens tokens; + bool done = false; + int iter = 0; + std::chrono::milliseconds waitTime(1); + while (!done && iter < mMaxWaitMs) + { + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + // This request failed for some reason, get error msg + std::string errStr + = "Request id " + std::to_string(requestId) + " failed with err " + response.getErrorMsg(); + FAIL(); + } + else + { + auto result = response.getResult(); + done = result.isFinal; + // Append tokens + auto& newTokens = result.outputTokenIds.at(beamWidth - 1); + tokens.insert( + tokens.end(), std::make_move_iterator(newTokens.begin()), std::make_move_iterator(newTokens.end())); + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); + EXPECT_EQ(tokens.size(), maxNewTokens); +} + +TEST_P(GuidedDecodingParamsTest, All) +{ + auto const modelName = std::get<0>(GetParam()); + std::filesystem::path enginePath; + std::filesystem::path tokenizerInfoPath; + int tp_size = 1, pp_size = 1, cp_size = 1; + std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; + + if (modelName == "gpt") + { + enginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + tokenizerInfoPath = GPT_XGRAMMAR_TOKENIZER_INFO_PATH; + } + else if (modelName == "llama_tp1_pp1_cp1") + { + enginePath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + tokenizerInfoPath = LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH; + } + else if (modelName == "llama_tp4_pp1_cp1") + { + enginePath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + tokenizerInfoPath = LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH; + tp_size = 4; + } + else if (modelName == "llama_tp1_pp4_cp1") + { + enginePath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + tokenizerInfoPath = LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH; + pp_size = 4; + deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; + } + else if (modelName == "llama_tp2_pp2_cp1") + { + enginePath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + tokenizerInfoPath = LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH; + tp_size = 2; + pp_size = 2; + deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; + } + else + { + TLLM_THROW("Unrecognized modelName"); + } + + auto& comm = tensorrt_llm::mpi::MpiComm::world(); + auto const worldRank = comm.getRank(); + auto const worldSize = comm.getSize(); + + if (tp_size * pp_size * cp_size > 1) + { + // Run multi GPU test only when env variable is set + char const* val = getenv("RUN_LLAMA_MULTI_GPU"); + if (val == NULL) + { + GTEST_SKIP() << "Skipping multi-gpu guided decoding test"; + } + else + { + if (worldSize != 4) + { + FAIL() << "Leader mode and world size is not equal to 4"; + } + } + } + + bool streaming = false; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + + auto const tokenizerInfo = nlohmann::json::parse(std::ifstream{tokenizerInfoPath}); + auto const encodedVocab = tokenizerInfo["encoded_vocab"].template get<std::vector<std::string>>(); + auto const tokenizerStr = tokenizerInfo["tokenizer_str"].template get<std::string>(); + auto const stopTokenIds = tokenizerInfo["stop_token_ids"].template get<std::vector<TokenIdType>>(); + GuidedDecodingConfig guidedDecodingConfig( + GuidedDecodingConfig::GuidedDecodingBackend::kXGRAMMAR, encodedVocab, tokenizerStr, stopTokenIds); + executorConfig.setGuidedDecodingConfig(guidedDecodingConfig); + + if (deviceIds.has_value()) + { + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + + parallelConfig.setDeviceIds(deviceIds.value()); + executorConfig.setParallelConfig(parallelConfig); + } + auto executor = Executor(enginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the requests + VecTokens inputTokens; + if (modelName == "gpt") + { + inputTokens = {2061, 318, 352, 10, 16, 30, 23998, 39559, 287, 257, 8633, 287, 33918, 5794, 25, 220}; + } + else // llama + { + inputTokens = { + 128000, 62, 3923, 7037, 62, 16, 10, 16, 30, 62, 16533, 87710, 1265, 4404, 5356, 1265, 9643, 9132, 25, 62}; + } + SizeType32 maxNewTokens = 10; + SamplingConfig samplingConfig{}; + OutputConfig outputConfig{false, false, false, true}; + + std::vector<Request> requests; + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); + + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); + requests.back().setGuidedDecodingParams(GuidedDecodingParams(GuidedDecodingParams::GuideType::kJSON)); + + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); + std::string jsonSchema{ + R"({"properties": {"answer": {"title": "Answer", "type": "integer"}}, "required": ["answer"], "title": "Answer", "type": "object"})"}; + requests.back().setGuidedDecodingParams( + GuidedDecodingParams(GuidedDecodingParams::GuideType::kJSON_SCHEMA, jsonSchema)); + + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); + std::string regex{R"(\d+)"}; + requests.back().setGuidedDecodingParams(GuidedDecodingParams(GuidedDecodingParams::GuideType::kREGEX, regex)); + + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); + std::string ebnfGrammar{R"(root ::= [0-9]+)"}; + requests.back().setGuidedDecodingParams( + GuidedDecodingParams(GuidedDecodingParams::GuideType::kEBNF_GRAMMAR, ebnfGrammar)); + + std::vector<VecTokens> expectedOutputTokens; + if (modelName == "gpt") + { + expectedOutputTokens.push_back({1849, 7, 16, 10, 16, 8, 198, 16, 10, 16}); + expectedOutputTokens.push_back({90, 366, 3672, 1298, 366, 7554, 31780, 1600, 366, 12888}); + expectedOutputTokens.push_back({90, 366, 64, 77, 2032, 68, 81, 1, 1058, 352}); + expectedOutputTokens.push_back({25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645}); + expectedOutputTokens.push_back({25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645, 25645}); + } + else // llama + { + expectedOutputTokens.push_back({16, 10, 16, 28, 17, 198, 62, 3923, 7037, 62}); + expectedOutputTokens.push_back({5018, 16, 794, 330, 16, 498, 330, 17, 794, 330}); + expectedOutputTokens.push_back({5018, 9399, 794, 16, 92}); + expectedOutputTokens.push_back({16}); + expectedOutputTokens.push_back({16}); + } + + if (executor.canEnqueueRequests()) + { + // Enqueue the requests + auto reqIds = executor.enqueueRequests(std::move(requests)); + + // Get the responses + int numFinished = 0; + int iter = 0; + while (numFinished < 5 && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + auto reqId = response.getRequestId(); + if (response.hasError()) + { + // This request failed for some reason, get error msg + std::string errStr + = "Request id " + std::to_string(reqId) + " failed with err " + response.getErrorMsg(); + FAIL(); + } + else + { + auto result = response.getResult(); + auto& newTokens = result.outputTokenIds.at(0); + + int reqIdx = std::find(reqIds.begin(), reqIds.end(), reqId) - reqIds.begin(); + EXPECT_THAT(newTokens, ::testing::ElementsAreArray(expectedOutputTokens[reqIdx])); + } + numFinished++; + } + } + EXPECT_LT(iter, mMaxWaitMs); + EXPECT_EQ(numFinished, 5); + } +} + +TEST_F(GptExecutorTest, GuidedDecodingFailure) +{ + bool streaming = false; + + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + + std::vector<int> stopTokenIds{50256}; + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the requests + SizeType32 maxNewTokens = 10; + SamplingConfig samplingConfig{}; + OutputConfig outputConfig{false, false, false, true}; + VecTokens inputTokens{2061, 318, 352, 10, 16, 30, 23998, 39559, 287, 257, 8633, 287, 33918, 5794, 25, 220}; + + std::vector<Request> requests; + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outputConfig, stopTokenIds[0]); + requests.back().setGuidedDecodingParams(GuidedDecodingParams(GuidedDecodingParams::GuideType::kJSON)); + + // Enqueue the requests + auto reqIds = executor.enqueueRequests(std::move(requests)); + + // Get the responses + int numFinished = 0; + int iter = 0; + while (numFinished < 2 && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + auto reqId = response.getRequestId(); + int reqIdx = std::find(reqIds.begin(), reqIds.end(), reqId) - reqIds.begin(); + if (reqIdx == 0) + { + EXPECT_FALSE(response.hasError()); + } + else + { + EXPECT_TRUE(response.hasError()); + } + numFinished++; + } + } + EXPECT_LT(iter, mMaxWaitMs); + EXPECT_EQ(numFinished, 2); +} + +TEST_P(ParamTest, SingleRequestCancelRequest) +{ + bool const streaming = std::get<0>(GetParam()); + bool const excludeInputFromOutput = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + OutputConfig outConfig; + outConfig.excludeInputFromOutput = excludeInputFromOutput; + + auto executorConfig = ExecutorConfig(beamWidth); + auto trtEnginePath = GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 300; + VecTokens inputTokens{1, 2, 3, 4}; + auto request + = Request(inputTokens, maxNewTokens, streaming, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + + auto requestId = executor.enqueueRequest(std::move(request)); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + executor.cancelRequest(requestId); + + // Try to get the new tokens + bool done = false; + int iter = 0; + VecTokens tokens; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(requestId, waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + FAIL() << "Did not expect errors"; + } + else + { + auto result = response.getResult(); + done = result.isFinal; + // Append tokens + auto& newTokens = result.outputTokenIds.at(beamWidth - 1); + if (done) + { + for (SizeType32 beamIdx = 0; beamIdx < beamWidth; ++beamIdx) + { + EXPECT_EQ(result.finishReasons[beamIdx], FinishReason::kCANCELLED); + } + } + + if (streaming && beamWidth > 1) + { + tokens = newTokens; + } + else + { + tokens.insert(tokens.end(), newTokens.begin(), newTokens.end()); + } + } + } + ++iter; + } + EXPECT_EQ(done, true); + EXPECT_LT(iter, mMaxWaitMs); + auto expectedNumTokens + = streaming ? maxNewTokens : (excludeInputFromOutput ? 0 : inputTokens.size()) + maxNewTokens; + TLLM_LOG_INFO("num tokens: %d, expected %d", tokens.size(), expectedNumTokens); + EXPECT_LT(tokens.size(), expectedNumTokens); +} + +TEST_F(GptExecutorTest, orchModeFetchNewReqErr) +{ + SizeType32 beamWidth = 1; + auto executorConfig = ExecutorConfig(beamWidth); + + auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + auto parallelConfig = ParallelConfig( + CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR, std::nullopt, std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Create a req with invalid parameters + SizeType32 maxNewTokens = 5; + // Create very long prompt which should result in error during request validate + VecTokens inputTokens(10000000); + + auto request = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); + auto requestId = executor.enqueueRequest(request); + auto requestId2 = executor.enqueueRequest(request); + + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + auto err = response.getErrorMsg(); + EXPECT_THAT(err, testing::HasSubstr("exceeds maximum input length")); + EXPECT_THAT(err, testing::HasSubstr("Encountered an error when fetching new request:")); + done = true; + } + else + { + FAIL() << "Should get a response with error"; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); +} + +TEST_F(GptExecutorTest, orchModeForwardError) +{ + SizeType32 constexpr maxBeamWidth{1}; + auto executorConfig = ExecutorConfig(maxBeamWidth); + + auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + auto parallelConfig = ParallelConfig( + CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR, std::nullopt, std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + + // Setting request beam width to 2 which should cause failure + SizeType32 constexpr beamWidth{2}; + SizeType32 constexpr maxNewTokens{5}; + VecTokens inputTokens{1, 2, 3, 4}; + + auto request = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); + auto requestId = executor.enqueueRequest(request); + auto requestId2 = executor.enqueueRequest(request); + + bool done = false; + int iter = 0; + while (!done && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + if (response.hasError()) + { + auto err = response.getErrorMsg(); + std::cout << "err:" << err << std::endl; + EXPECT_THAT( + err, testing::HasSubstr("Requested beam width 2 is larger than configured max beam width 1")); + done = true; + } + else + { + FAIL() << "Should get a response with error"; + } + } + ++iter; + } + EXPECT_LT(iter, mMaxWaitMs); +} + +TEST_P(ParamCancelReqTest, MultipleRequestsMultiGpuCancelRequest) +{ + auto const useOrchestratorMode = std::get<0>(GetParam()); + auto const beamWidth = std::get<1>(GetParam()); + auto const modelName = std::get<2>(GetParam()); + + std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; + + OutputConfig outConfig; + + auto executorConfig = ExecutorConfig(beamWidth); + std::filesystem::path modelPath; + if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1") + { + if (modelName == "llama_tp4_pp1_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp4_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; + } + else if (modelName == "llama_tp2_pp2_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; + } + } + + // For llama model, only run for multiple GPUs + // This is detected by setting an env variable when running the test + char const* val = getenv("RUN_LLAMA_MULTI_GPU"); + if (val == NULL) + { + GTEST_SKIP() << "Skipping Llama test"; + } + else + { + // Check that it was launched with right number of MPI ranks + if (!useOrchestratorMode && COMM_SESSION.getSize() != 4) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Leader mode and world size is not equal to 4"; + } + else if (useOrchestratorMode && COMM_SESSION.getSize() != 1) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Orchestrator mode and World size is not equal to 1"; + } + } + + if (useOrchestratorMode) + { + auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + auto parallelConfig = ParallelConfig(CommunicationType::kMPI, + useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, + std::nullopt, orchestratorConfig); + if (deviceIds.has_value()) + { + parallelConfig.setDeviceIds(deviceIds.value()); + } + executorConfig.setParallelConfig(parallelConfig); + } + else + { + if (deviceIds.has_value()) + { + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + parallelConfig.setDeviceIds(deviceIds.value()); + executorConfig.setParallelConfig(parallelConfig); + } + } + + auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + SizeType32 maxNewTokens = 50; + VecTokens inputTokens{1, 2, 3, 4}; + + std::vector<Request> requests; + for (auto streaming : {false, true}) + { + // Add two requests with numReturnSequences = 1 + auto samplingConfig = tensorrt_llm::executor::SamplingConfig(beamWidth); + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outConfig); + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig, outConfig); + // Add a request with numReturnSequences > 1 + auto samplingConfig2 = tensorrt_llm::executor::SamplingConfig(beamWidth); + auto constexpr numReturnSequences = 2; + samplingConfig2.setNumReturnSequences(numReturnSequences); + requests.emplace_back(inputTokens, maxNewTokens, streaming, samplingConfig2, outConfig); + } + std::vector<bool> cancelRequests{true, false, true, true, false, true}; + + if (executor.canEnqueueRequests()) + { + auto const requestIds = executor.enqueueRequests(requests); + + // Cancel the first and third requests + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + for (SizeType32 i = 0; i < requests.size(); i++) + { + if (cancelRequests.at(i)) + { + executor.cancelRequest(requestIds.at(i)); + } + } + + std::unordered_map<IdType, bool> isStreaming; + std::unordered_map<IdType, SizeType32> expectedNumTokens; + SizeType32 expectedNumResponses = 0; + for (SizeType32 i = 0; i < requests.size(); i++) + { + auto const& request = requests.at(i); + auto requestId = requestIds.at(i); + isStreaming[requestId] = request.getStreaming(); + expectedNumTokens[requestId] = (request.getStreaming() ? 0 : inputTokens.size()) + maxNewTokens; + auto const numResponses = request.getStreaming() ? expectedNumTokens[requestId] : 1; + auto const numReturnSequences = request.getSamplingConfig().getBeamWidth() > 1 + ? 1 + : request.getSamplingConfig().getNumReturnSequences().value_or(1); + expectedNumResponses += numResponses * numReturnSequences; + } + + std::unordered_map<IdType, std::unordered_map<SizeType32, VecTokens>> tokens; + + // Get the new tokens for each requests + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + while (numFinished < requests.size() && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + auto requestId = response.getRequestId(); + auto result = response.getResult(); + numFinished += result.isFinal; + auto seqIdx = result.sequenceIndex; + auto numSequences = result.outputTokenIds.size(); + auto& newTokens = result.outputTokenIds.at(numSequences - 1); + auto& reqResults = tokens[response.getRequestId()]; + auto& reqTokens = reqResults[seqIdx]; + if (isStreaming.at(requestId) && beamWidth > 1) + { + reqTokens = newTokens; + } + else + { + reqTokens.insert(reqTokens.end(), newTokens.begin(), newTokens.end()); + } + } + else + { + FAIL() << "Did not expect errors"; + } + } + ++iter; + } + + EXPECT_LE(numResponses, expectedNumResponses); + EXPECT_EQ(numFinished, requests.size()); + EXPECT_LT(iter, mMaxWaitMs); + + for (auto requestIdx = 0; requestIdx < requests.size(); requestIdx++) + { + auto const requestId = requestIds.at(requestIdx); + for (auto seqIdx = 0; seqIdx < tokens.at(requestId).size(); seqIdx++) + { + auto const& seqTokens = tokens.at(requestId).at(seqIdx); + if (cancelRequests.at(requestIdx)) + { + EXPECT_LT(seqTokens.size(), expectedNumTokens.at(requestId)); + } + else + { + EXPECT_EQ(seqTokens.size(), expectedNumTokens.at(requestId)); + } + } + } + } +} + +TEST_P(LeaderApiUsageTest, LeaderModeTest) +{ + auto const modelName = std::get<0>(GetParam()); + + SizeType32 beamWidth = 2; + OutputConfig outConfig; + std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; + + auto executorConfig = ExecutorConfig(beamWidth); + std::filesystem::path modelPath; + if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1") + { + if (modelName == "llama_tp4_pp1_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp4_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; + } + else if (modelName == "llama_tp2_pp2_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; + } + } + + // For llama model, only run for multiple GPUs + // This is detected by setting an env variable when running the test + char const* val = getenv("RUN_LLAMA_MULTI_GPU"); + if (val == NULL) + { + GTEST_SKIP() << "Skipping Llama test"; + } + else + { + // Check that it was launched with right number of MPI ranks + if (COMM_SESSION.getSize() != 4) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Leader mode and world size is not equal to 4"; + } + } + + if (deviceIds.has_value()) + { + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + + parallelConfig.setDeviceIds(deviceIds.value()); + executorConfig.setParallelConfig(parallelConfig); + } + auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); + + // Since this is leader mode, all ranks should participate + EXPECT_TRUE(executor.isParticipant()); + + // Create the request + SizeType32 maxNewTokens = 50; + VecTokens inputTokens{1, 2, 3, 4}; + auto request + = Request(inputTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + auto requestStreaming + = Request(inputTokens, maxNewTokens, true, tensorrt_llm::executor::SamplingConfig(beamWidth), outConfig); + + // Leader enqueues requests and wait for responses + if (executor.canEnqueueRequests()) + { + auto requestId = executor.enqueueRequest(request); + auto requestId2 = executor.enqueueRequest(request); + auto requestId3 = executor.enqueueRequest(requestStreaming); + auto requestId4 = executor.enqueueRequest(requestStreaming); + + int32_t numFinished = 0; + int iter = 0; + SizeType32 numResponses = 0; + while (numFinished < 4 && iter < mMaxWaitMs) + { + std::chrono::milliseconds waitTime(1); + auto responses = executor.awaitResponses(waitTime); + for (auto& response : responses) + { + numResponses++; + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + } + else + { + FAIL() << "Did not expect errors"; + } + } + ++iter; + } + EXPECT_EQ(numFinished, 4); + EXPECT_LT(iter, mMaxWaitMs); + } + else + { + // Check that non-leader cannot enqueue requests + EXPECT_THROW({ auto reqId = executor.enqueueRequest(request); }, tensorrt_llm::common::TllmException); + EXPECT_THROW({ auto responses = executor.awaitResponses(); }, tensorrt_llm::common::TllmException); + EXPECT_THROW({ auto numResp = executor.getNumResponsesReady(); }, tensorrt_llm::common::TllmException); + EXPECT_THROW({ executor.cancelRequest(1); }, tensorrt_llm::common::TllmException); + EXPECT_THROW({ auto stats = executor.getLatestIterationStats(); }, tensorrt_llm::common::TllmException); + EXPECT_THROW({ auto stats = executor.getLatestRequestStats(); }, tensorrt_llm::common::TllmException); + } +} + +TEST_F(GptExecutorTest, validateParallelConfig) +{ + + auto trtEnginePath = (GPT_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"); + { + auto executorConfig = ExecutorConfig(); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + } + + { + std::string expectedErrMsg = "OrchestratorConfig must be set"; + try + { + auto executorConfig = ExecutorConfig(); + auto parallelConfig = ParallelConfig(CommunicationType::kMPI, CommunicationMode::kORCHESTRATOR); + executorConfig.setParallelConfig(parallelConfig); + auto executor = Executor(trtEnginePath, ModelType::kDECODER_ONLY, executorConfig); + FAIL() << "Expected TllmException"; + } + catch (tc::TllmException& e) + { + EXPECT_THAT(e.what(), testing::HasSubstr(expectedErrMsg)); + } + catch (std::exception const& e) + { + FAIL() << "Expected TllmException"; + } + } +} + +TEST_P(TimeoutTest, TimeoutStreamingTest) +{ + auto const modelName = std::get<0>(GetParam()); + auto const useOrchestratorMode = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + + auto executorConfig = ExecutorConfig(beamWidth); + std::filesystem::path modelPath; + bool isMultiGpu{false}; + std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; + + if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1") + { + isMultiGpu = true; + if (modelName == "llama_tp4_pp1_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp4_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; + } + else if (modelName == "llama_tp2_pp2_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; + } + } + if (modelName == "llama_tp1_pp1_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + } + // For llama model, only run for multiple GPUs + // This is detected by setting an env variable when running the test + char const* val = getenv("RUN_LLAMA_MULTI_GPU"); + if (val == NULL && isMultiGpu) + { + GTEST_SKIP() << "Skipping MultiGpu tests"; + } + if (val != NULL && !isMultiGpu) + { + GTEST_SKIP() << "Skipping SingleGpu tests"; + } + if (val != NULL && isMultiGpu) + { + // Check that it was launched with right number of MPI ranks + if (!useOrchestratorMode && COMM_SESSION.getSize() != 4) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Leader mode and world size is not equal to 4"; + } + if (useOrchestratorMode && COMM_SESSION.getSize() != 1) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Orchestrator mode and World size is not equal to 1"; + } + } + + if (useOrchestratorMode) + { + auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + auto parallelConfig = ParallelConfig(CommunicationType::kMPI, + useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, + std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + if (deviceIds.has_value()) + { + parallelConfig.setDeviceIds(deviceIds.value()); + } + executorConfig.setParallelConfig(parallelConfig); + } + else + { + if (deviceIds.has_value()) + { + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + parallelConfig.setDeviceIds(deviceIds.value()); + executorConfig.setParallelConfig(parallelConfig); + } + } + auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 constexpr maxNewTokens = 10; + // create 1 request that times out immediately + // momentarily we don't cancel requests before forwardAsync so it will get scheduled for at least 1 forward + VecTokens immediateCancelTokens{1, 2, 3, 4}; + auto immediateCancelRequest + = Request(immediateCancelTokens, maxNewTokens, true, tensorrt_llm::executor::SamplingConfig(beamWidth)); + immediateCancelRequest.setReturnAllGeneratedTokens(true); + immediateCancelRequest.setAllottedTimeMs(std::chrono::milliseconds(0)); + SizeType32 constexpr immediateCancelMinLength = 0; + SizeType32 constexpr immediateCancelMaxLength = 1; + + // create 1 request that times out during the first forward + VecTokens oneForwardTokens{11, 12, 13, 14}; + auto oneForwardRequest + = Request(oneForwardTokens, maxNewTokens, true, tensorrt_llm::executor::SamplingConfig(beamWidth)); + oneForwardRequest.setReturnAllGeneratedTokens(true); + oneForwardRequest.setAllottedTimeMs(std::chrono::milliseconds(1)); + SizeType32 constexpr oneForwardlMinLength = 0; + SizeType32 constexpr oneForwardlMaxLength = 1; + + // Create the request that finishes by the number of tokens + VecTokens finishedTokens{101, 102, 103, 104}; + auto finishedRequest + = Request(finishedTokens, maxNewTokens, true, tensorrt_llm::executor::SamplingConfig(beamWidth)); + finishedRequest.setReturnAllGeneratedTokens(true); + finishedRequest.setAllottedTimeMs(std::chrono::milliseconds(5000)); + SizeType32 constexpr finishedMinLength = 5; + SizeType32 constexpr finishedMaxLength = maxNewTokens; + + std::vector<FinishReason> referenceFinishReasons + = {FinishReason::kTIMED_OUT, FinishReason::kTIMED_OUT, FinishReason::kLENGTH}; + std::vector<SizeType32> minLengths = {immediateCancelMinLength, oneForwardlMinLength, finishedMinLength}; + std::vector<SizeType32> maxLengths = {immediateCancelMaxLength, oneForwardlMaxLength, finishedMaxLength}; + // workaround because the last response will be empty, but we want to have at least *some* responses surpass the + // minLength + std::vector<SizeType32> achievedLength = {0, 0, 0}; + SizeType32 itNr{0}; + + if (executor.canEnqueueRequests()) + { + + std::vector<Request> requests = {immediateCancelRequest, oneForwardRequest, finishedRequest}; + auto requestIds = executor.enqueueRequests(requests); + + auto numFinished = 0; + + while (numFinished < static_cast<SizeType32>(requests.size())) + { + itNr++; + std::chrono::milliseconds waitTime(mMaxWaitMs); + auto responses = executor.awaitResponses(requestIds, waitTime); + for (auto const& response : responses) + { + for (auto const& responseIt : response) + { + auto const reqId = responseIt.getRequestId(); + if (responseIt.hasError()) + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err + = "ReqId " + std::to_string(reqId) + " has already been processed and was terminated."; + if (responseIt.getErrorMsg() != err) + { + TLLM_THROW("Request id %lu encountered error: %s", reqId, responseIt.getErrorMsg().c_str()); + } + continue; + } + + auto const& result = responseIt.getResult(); + if (result.isFinal) + { + requestIds.erase(std::remove(requestIds.begin(), requestIds.end(), reqId), requestIds.end()); + numFinished++; + } + + auto const finishReason = result.finishReasons; + auto const actualResponse = result.outputTokenIds; + TLLM_LOG_DEBUG("reqId %d finished %d", reqId, result.isFinal); + TLLM_LOG_DEBUG("actual response:"); + + for (auto const& beam : actualResponse) + { + std::string tokenStr; + for (auto tok : beam) + { + tokenStr += std::to_string(tok) + " "; + } + TLLM_LOG_DEBUG("%s", tokenStr.c_str()); + } + + TLLM_LOG_DEBUG( + "beams' length must be in range [%d, %d]", minLengths[reqId - 1], maxLengths[reqId - 1]); + + if (result.isFinal) + { + TLLM_LOG_DEBUG("finishReason"); + std::string reasonStr; + for (auto const reason : finishReason) + { + // cast for easier visibility during debugging + EXPECT_EQ(static_cast<int>(reason), static_cast<int>(referenceFinishReasons[reqId - 1])); + reasonStr += std::to_string(static_cast<int>(reason)) + " "; + } + TLLM_LOG_DEBUG("%s", reasonStr.c_str()); + } + + EXPECT_EQ(beamWidth, actualResponse.size()); + for (int beam = 0; beam < beamWidth; beam++) + { + EXPECT_LE(actualResponse.at(beam).size(), maxLengths[reqId - 1]) << "for request " << reqId; + achievedLength[reqId - 1] = std::max( + achievedLength[reqId - 1], static_cast<SizeType32>(actualResponse.at(beam).size())); + } + } + } + } + + for (int reqIt = 0; reqIt < achievedLength.size(); ++reqIt) + { + EXPECT_GE(achievedLength[reqIt], minLengths[reqIt]) + << "request " << reqIt + 1 << " has not achieved min lengths"; + } + } +} + +TEST_P(TimeoutTest, TimeoutNonstreamingTest) +{ + auto const modelName = std::get<0>(GetParam()); + auto const useOrchestratorMode = std::get<1>(GetParam()); + auto const beamWidth = std::get<2>(GetParam()); + + std::optional<std::vector<SizeType32>> deviceIds = std::nullopt; + + auto executorConfig = ExecutorConfig(beamWidth); + std::filesystem::path modelPath; + bool isMultiGpu{false}; + if (modelName == "llama_tp4_pp1_cp1" || modelName == "llama_tp1_pp4_cp1" || modelName == "llama_tp2_pp2_cp1") + { + isMultiGpu = true; + if (modelName == "llama_tp4_pp1_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp4-pp1-cp1-gpu"; + } + else if (modelName == "llama_tp1_pp4_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp4-cp1-gpu"; + deviceIds = std::vector<SizeType32>{3, 2, 1, 0}; + } + else if (modelName == "llama_tp2_pp2_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp2-pp2-cp1-gpu"; + deviceIds = std::vector<SizeType32>{2, 3, 0, 1}; + } + } + if (modelName == "llama_tp1_pp1_cp1") + { + modelPath = LLAMA_MODEL_PATH / PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() / "tp1-pp1-cp1-gpu"; + } + // For llama model, only run for multiple GPUs + // This is detected by setting an env variable when running the test + char const* val = getenv("RUN_LLAMA_MULTI_GPU"); + if (val == NULL && isMultiGpu) + { + GTEST_SKIP() << "Skipping MultiGpu tests"; + } + if (val != NULL && !isMultiGpu) + { + GTEST_SKIP() << "Skipping SingleGpu tests"; + } + if (val != NULL && isMultiGpu) + { + // Check that it was launched with right number of MPI ranks + if (!useOrchestratorMode && COMM_SESSION.getSize() != 4) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Leader mode and world size is not equal to 4"; + } + if (useOrchestratorMode && COMM_SESSION.getSize() != 1) + { + // No orchestrator, need worldSize to match TP*PP + FAIL() << "Orchestrator mode and World size is not equal to 1"; + } + } + + if (useOrchestratorMode) + { + auto orchestratorConfig = OrchestratorConfig(true, PathUtil::EXECUTOR_WORKER_PATH()); + auto parallelConfig = ParallelConfig(CommunicationType::kMPI, + useOrchestratorMode ? CommunicationMode::kORCHESTRATOR : CommunicationMode::kLEADER, std::nullopt, + std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + if (deviceIds.has_value()) + { + parallelConfig.setDeviceIds(deviceIds.value()); + } + executorConfig.setParallelConfig(parallelConfig); + } + else + { + if (deviceIds.has_value()) + { + auto parallelConfig = executorConfig.getParallelConfig().value_or(ParallelConfig()); + parallelConfig.setDeviceIds(deviceIds.value()); + executorConfig.setParallelConfig(parallelConfig); + } + } + auto executor = Executor(modelPath, ModelType::kDECODER_ONLY, executorConfig); + + SizeType32 constexpr maxNewTokens = 5; + // create 1 request that times out immediately + // momentarily we don't cancel requests before forwardAsync so it will get scheduled for at least 1 forward + VecTokens immediateCancelTokens{1, 2, 3, 4}; + auto immediateCancelRequest + = Request(immediateCancelTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); + immediateCancelRequest.setAllottedTimeMs(std::chrono::milliseconds(0)); + std::vector<std::vector<int>> immediateCancelResponse = {immediateCancelTokens, immediateCancelTokens}; + + // create 1 request that times out during the first forward + VecTokens oneForwardTokens{11, 12, 13, 14}; + auto oneForwardRequest + = Request(oneForwardTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); + oneForwardRequest.setAllottedTimeMs(std::chrono::milliseconds(1)); + std::vector<std::vector<int>> oneForwardResponse = {oneForwardTokens, oneForwardTokens}; + + // Create the request that finishes by the number of tokens + VecTokens finishedTokens{101, 102, 103, 104}; + auto finishedRequest + = Request(finishedTokens, maxNewTokens, false, tensorrt_llm::executor::SamplingConfig(beamWidth)); + finishedRequest.setAllottedTimeMs(std::chrono::milliseconds(6000)); + std::vector<std::vector<int>> finishedReponse + = {{101, 102, 103, 104, 49849, 225, 49849, 232, 55742}, {101, 102, 103, 104, 49849, 225, 49849, 232, 29082}}; + + // assume responses will come in FIFO order + std::vector<BeamTokens> refResponses = {immediateCancelResponse, oneForwardResponse, finishedReponse}; + std::vector<FinishReason> referenceFinishReasons + = {FinishReason::kTIMED_OUT, FinishReason::kTIMED_OUT, FinishReason::kLENGTH}; + if (executor.canEnqueueRequests()) + { + + std::vector<Request> requests = {immediateCancelRequest, oneForwardRequest, finishedRequest}; + auto requestIds = executor.enqueueRequests(requests); + + std::chrono::milliseconds waitTime(mMaxWaitMs); + auto responses = executor.awaitResponses(requestIds, waitTime); + for (auto const& response : responses) + { + for (auto const& responseIt : response) + { + auto const reqId = responseIt.getRequestId(); + if (responseIt.hasError()) + { + TLLM_THROW("Request id %lu encountered error: %s", reqId, responseIt.getErrorMsg().c_str()); + } + + auto const& result = responseIt.getResult(); + + auto const finishReason = result.finishReasons; + auto const actualResponse = result.outputTokenIds; + TLLM_LOG_DEBUG("reqId %d finished %d", reqId, result.isFinal); + TLLM_LOG_DEBUG("actual response:"); + + for (auto const& beam : actualResponse) + { + std::string tokenStr; + for (auto tok : beam) + { + tokenStr += std::to_string(tok) + " "; + } + TLLM_LOG_DEBUG("%s", tokenStr.c_str()); + } + + TLLM_LOG_DEBUG("reference:"); + auto referenceResponse = refResponses[reqId - 1]; + for (auto const& beam : referenceResponse) + { + std::string tokenStr; + for (auto tok : beam) + { + tokenStr += std::to_string(tok) + " "; + } + TLLM_LOG_DEBUG("%s", tokenStr.c_str()); + } + + if (result.isFinal) + { + TLLM_LOG_DEBUG("finishReason"); + std::string reasonStr; + for (auto const reason : finishReason) + { + // cast for easier visibility during debugging + EXPECT_EQ(static_cast<int>(reason), static_cast<int>(referenceFinishReasons[reqId - 1])); + reasonStr += std::to_string(static_cast<int>(reason)) + " "; + } + TLLM_LOG_DEBUG("%s", reasonStr.c_str()); + } + + EXPECT_EQ(beamWidth, actualResponse.size()); + for (int beam = 0; beam < beamWidth; beam++) + { + EXPECT_EQ(referenceResponse.at(beam).size(), actualResponse.at(beam).size()); + EXPECT_THAT(actualResponse.at(beam), testing::ElementsAreArray(referenceResponse.at(beam))); + } + } + } + } +} + +INSTANTIATE_TEST_SUITE_P(GptExecutorTest, ParamTest, + testing::Combine( // + testing::Values(false, true), // streaming + testing::Values(false, true), // excludeInputFromOutput + testing::Values(1, 2) // beamWidth + ), + generateTestName); + +INSTANTIATE_TEST_SUITE_P(GptExecutorTest, ParamStatsTest, + testing::Combine( // + testing::Values(0, 1000), // iterStatsMaxIterations + testing::Values(false, true) // useOrchestratorMode + ), + generateTestNameStats); + +INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, ParamCancelReqTest, + testing::Combine( // + testing::Values(false, true), // useOrchestratorMode + testing::Values(1, 2), // beamWidth + testing::Values("llama_tp1_pp4_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1") // modelName + ), + generateTestNameCancelReq); + +INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, TimeoutTest, + testing::Combine( // + testing::Values("llama_tp1_pp4_cp1", "llama_tp4_pp1_cp1", "llama_tp1_pp1_cp1"), // modelName + testing::Values(false, true), // useOrchestratorMode + testing::Values(2) // beamWidth + ), + generateTestNameTimeoutTest); + +INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, LeaderApiUsageTest, + testing::Combine( // + testing::Values("llama_tp1_pp4_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1") // modelName + ), + generateTestNameLeaderApiUsage); + +INSTANTIATE_TEST_SUITE_P(GptExecutorTest, AllParamsTest, + testing::Combine( // + testing::Values(false, true), // streaming + testing::Values(1, 2), // beamWidth + testing::Values(true), // computeLogProbs + testing::Values(false, true), // excludeInputInOutput + testing::Values(true), // returnContextLogits + testing::Values(true), // returnGenerationLogits + testing::Values("gpt"), // modelName + testing::Values(false, true), // useOrchestratorMode + testing::Values(false, true), // returnAllGeneratedTokens + testing::Values(1, 2) // numReturnSequences + ), + generateTestNameAllParams); + +INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, AllParamsTest, + testing::Combine( // + testing::Values(false, true), // streaming + testing::Values(1, 2), // beamWidth + testing::Values(true), // computeLogProbs + testing::Values(false, true), // excludeInputInOutput + testing::Values(false), // returnContextLogits + testing::Values(true), // returnGenerationLogits + testing::Values("llama_tp1_pp4_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1"), // modelName + testing::Values(false, true), // useOrchestratorMode + testing::Values(false), // returnAllGeneratedTokens + testing::Values(1) // numReturnSequences + ), + generateTestNameAllParams); + +INSTANTIATE_TEST_SUITE_P(LlamaMultiExecutorTest, AllParamsTest, + testing::Combine( // + testing::Values(false, true), // streaming + testing::Values(1, 2), // beamWidth + testing::Values(false), // computeLogProbs + testing::Values(false, true), // excludeInputInOutput + testing::Values(false), // returnContextLogits + testing::Values(false), // returnGenerationLogits + testing::Values("llama_tp1_pp2_cp1"), // modelName + testing::Values(false), // useOrchestratorMode + testing::Values(false), // returnAllGeneratedTokens + testing::Values(1) // numReturnSequences + ), + generateTestNameAllParams); + +INSTANTIATE_TEST_SUITE_P(MedusaExecutorTest, AllParamsTest, + testing::Combine( // + testing::Values(false, true), // streaming + testing::Values(1), // beamWidth + testing::Values(false), // computeLogProbs + testing::Values(false, true), // excludeInputInOutput + testing::Values(false), // returnContextLogits + testing::Values(false), // returnGenerationLogits + testing::Values("medusa"), // modelName + testing::Values(false, true), // useOrchestratorMode + testing::Values(false), // returnAllGeneratedTokens + testing::Values(1) // numReturnSequences + ), + generateTestNameAllParams); + +// Disable some of ChatGLM's tests since they are the same as gpt's. +INSTANTIATE_TEST_SUITE_P(ChatGlmExecutorTest, AllParamsTest, + testing::Combine( // + testing::Values(false), // streaming + testing::Values(1, 2), // beamWidth + testing::Values(false), // computeLogProbs + testing::Values(false), // excludeInputInOutput + testing::Values(false), // returnContextLogits + testing::Values(false), // returnGenerationLogits + testing::Values("chatglm"), // modelName + testing::Values(false), // useOrchestratorMode + testing::Values(false), // returnAllGeneratedTokens + testing::Values(1, 2) // numReturnSequences + ), + generateTestNameAllParams); + +// ChatGlm0 Test is for glm-10b. +INSTANTIATE_TEST_SUITE_P(ChatGlm0ExecutorTest, AllParamsTest, + testing::Combine( // + testing::Values(false), // streaming + testing::Values(1), // beamWidth + testing::Values(false), // computeLogProbs + testing::Values(false), // excludeInputInOutput + testing::Values(false), // returnContextLogits + testing::Values(false), // returnGenerationLogits + testing::Values("glm"), // modelName + testing::Values(false), // useOrchestratorMode + testing::Values(false), // returnAllGeneratedTokens + testing::Values(1) // numReturnSequences + ), + generateTestNameAllParams); + +INSTANTIATE_TEST_SUITE_P(ChatGlm2ExecutorTest, AllParamsTest, + testing::Combine( // + testing::Values(false), // streaming + testing::Values(1), // beamWidth + testing::Values(false), // computeLogProbs + testing::Values(false), // excludeInputInOutput + testing::Values(false), // returnContextLogits + testing::Values(false), // returnGenerationLogits + testing::Values("chatglm2"), // modelName + testing::Values(false), // useOrchestratorMode + testing::Values(false), // returnAllGeneratedTokens + testing::Values(1) // numReturnSequences + ), + generateTestNameAllParams); + +INSTANTIATE_TEST_SUITE_P(ChatGlm3ExecutorTest, AllParamsTest, + testing::Combine( // + testing::Values(false), // streaming + testing::Values(1), // beamWidth + testing::Values(false), // computeLogProbs + testing::Values(false), // excludeInputInOutput + testing::Values(false), // returnContextLogits + testing::Values(false), // returnGenerationLogits + testing::Values("chatglm3"), // modelName + testing::Values(false), // useOrchestratorMode + testing::Values(false), // returnAllGeneratedTokens + testing::Values(1) // numReturnSequences + ), + generateTestNameAllParams); + +INSTANTIATE_TEST_SUITE_P(LlamaExecutorTest, LogitsProcParamsTest, + testing::Combine( // + testing::Values( + "llama_tp1_pp1_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1", "llama_tp1_pp4_cp1"), // modelName + testing::Values(false, true), // batched + testing::Values(false, true) // replicated + ), + generateTestNameLogitsProc); + +INSTANTIATE_TEST_SUITE_P(GptExecutorGuidedDecodingTest, GuidedDecodingParamsTest, + testing::Combine(testing::Values("gpt")), generateTestNameGuidedDecoding); + +INSTANTIATE_TEST_SUITE_P(LlamaExecutorGuidedDecodingTest, GuidedDecodingParamsTest, + testing::Combine( + testing::Values("llama_tp1_pp1_cp1", "llama_tp4_pp1_cp1", "llama_tp2_pp2_cp1", "llama_tp1_pp4_cp1")), + generateTestNameGuidedDecoding); diff --git a/cpp/tests/e2e_tests/executor/executorTest.h b/cpp/tests/e2e_tests/executor/executorTest.h new file mode 100644 index 000000000000..7866a6992266 --- /dev/null +++ b/cpp/tests/e2e_tests/executor/executorTest.h @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tests/utils/common.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +#include <memory> + +namespace tensorrt_llm::testing +{ + +class GptExecutorTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) +{ +public: + using SizeType32 = tensorrt_llm::testing::SizeType32; + +protected: + void SetUp() override + { + mDeviceCount = tensorrt_llm::common::getDeviceCount(); + if (mDeviceCount == 0) + { + GTEST_SKIP() << "No GPUs found"; + } + + mLogger = std::make_shared<tensorrt_llm::runtime::TllmLogger>(); + initTrtLlmPlugins(mLogger.get()); + } + + void TearDown() override {} + + int mDeviceCount{}; + std::shared_ptr<nvinfer1::ILogger> mLogger{}; + SizeType32 mMaxWaitMs = 300000; + SizeType32 mTrigWarnMs = 10000; +}; + +} // namespace tensorrt_llm::testing diff --git a/cpp/tests/resources/scripts/build_chatglm_engines.py b/cpp/tests/resources/scripts/build_chatglm_engines.py new file mode 100644 index 000000000000..abe187307604 --- /dev/null +++ b/cpp/tests/resources/scripts/build_chatglm_engines.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import platform +import shutil +import sys +import typing +from pathlib import Path +from typing import Optional + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + +resources_dir = Path(__file__).parent.resolve().parent +model_dir = resources_dir / "models" +chatglm_example_dir = Path("examples/chatglm") +bCopyModel = True # "False" to remove redundant copy of model from model_cache + + +def convert_ckpt(model_dir: str, output_dir: str, world_size: int): + if os.path.exists(output_dir): + print('Skip ckpt convert - output already exists') + return + + convert_cmd = [ + sys.executable, + str(chatglm_example_dir / "convert_checkpoint.py"), "--dtype=float16", + f"--model_dir={model_dir}", f"--output_dir={output_dir}", + f"--tp_size={world_size}" + ] + run_command(convert_cmd) + + +def build_engine(ckpt_dir: str, + engine_dir: str, + is_ifb: bool = False, + is_chatglm_6b_or_glm_10b: bool = False): + if os.path.exists(engine_dir): + print('Skip engine build - output already exists') + return + + build_cmd = [ + "trtllm-build", + f"--checkpoint_dir={ckpt_dir}", + f"--output_dir={engine_dir}", + "--log_level=error", + "--max_batch_size=8", + "--max_beam_width=2", + "--max_input_len=256", + "--max_seq_len=384", + "--gpt_attention_plugin=float16", + "--gemm_plugin=float16", + ] + if is_ifb: + build_cmd.extend([ + "--remove_input_padding=enable", + "--paged_kv_cache=enable", + "--context_fmha=enable", + "--use_paged_context_fmha=enable", + ]) + else: + build_cmd.extend([ + "--remove_input_padding=disable", + "--paged_kv_cache=disable", + ]) + + if is_chatglm_6b_or_glm_10b: + print("Disable Context FMHA for ChatGLM-6B and GLM-10B") + build_cmd.extend(["--context_fmha=disable"]) + + run_command(build_cmd) + + +def build_engines(model_cache: typing.Optional[str] = None, + world_size: int = 1, + clean: Optional[bool] = False): + + for model_name in [ + "chatglm-6b", "chatglm2-6b", "chatglm3-6b", "glm-10b", "glm-4-9b", + "chatglm3-6b-32k" + ]: + is_chatglm_6b_or_glm_10b = model_name in ["chatglm-6b", "glm-10b"] + if model_cache and (Path(model_cache) / model_name).is_dir(): + model_cache_dir = Path(model_cache) / model_name + if bCopyModel or model_name == "chatglm-6b": + print("Copy model from model_cache") + hf_dir = model_dir / model_name + if platform.system() == "Windows": + wincopy(source=str(model_cache_dir), + dest=model_name, + isdir=True, + cwd=model_dir) + else: + run_command(["rsync", "-rlptD", + str(model_cache_dir), "."], + cwd=model_dir) + else: + print("Use model from model_cache directly except ChatGLM-6B") + hf_dir = Path(model_cache) + + else: + hf_dir = model_dir / model_name + if not hf_dir.is_dir(): + print("Clone model from HF") + run_command( + [ + "git", "clone", + f"https://huggingface.co/THUDM/{model_name}", model_name + ], + cwd=model_dir, + ) + + # Build engines + print(f"Building {model_name}") + ckpt_dir = Path(model_dir) / "c-model" / model_name + if clean: + print('clean up ckpt folder ', ckpt_dir) + if ckpt_dir.is_dir(): + shutil.rmtree(ckpt_dir, ignore_errors=True) + + # Fix HF error for ChatGLM-6B / GLM-4-9B / ChatGLM2-6B / ChatGLM3-6B-32K, hope to remove this in the future + if model_name in [ + "chatglm-6b", "glm-4-9b", "chatglm2-6b", "chatglm3-6b-32k" + ]: + shutil.copy( + chatglm_example_dir / f"{model_name}/tokenization_chatglm.py", + hf_dir, + ) + + convert_ckpt(hf_dir, ckpt_dir, world_size) + + model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) + model_spec_obj.use_gpt_plugin() + engine_dir = Path( + model_dir + ) / "rt_engine" / model_name / model_spec_obj.get_model_path( + ) / "tp1-pp1-cp1-gpu" + if clean: + print('clean up engine folder ', engine_dir) + if engine_dir.is_dir(): + shutil.rmtree(engine_dir, ignore_errors=True) + build_engine(ckpt_dir, engine_dir, False, is_chatglm_6b_or_glm_10b) + + model_spec_obj.use_packed_input() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + engine_dir = Path( + model_dir + ) / "rt_engine" / model_name / model_spec_obj.get_model_path( + ) / "tp1-pp1-cp1-gpu" + if clean: + print('clean up engine folder ', engine_dir) + if engine_dir.is_dir(): + shutil.rmtree(engine_dir, ignore_errors=True) + build_engine(ckpt_dir, engine_dir, True, is_chatglm_6b_or_glm_10b) + + print("Done") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + + parser.add_argument('--world_size', + type=int, + default=1, + help='world size, only support tensor parallelism now') + + parser.add_argument('--clean', + action='store_true', + default=False, + help='Clean target folders before building engines') + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_eagle_engines.py b/cpp/tests/resources/scripts/build_eagle_engines.py new file mode 100755 index 000000000000..8b10698a603b --- /dev/null +++ b/cpp/tests/resources/scripts/build_eagle_engines.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import pathlib as _pl +import platform as _pf +import sys as _sys + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def build_engine(base_model_dir: _pl.Path, eagle_model_dir: _pl.Path, + engine_dir: _pl.Path, build_base_model: bool, *args): + + if build_base_model: + checkpoint_path = "examples/models/core/llama/convert_checkpoint.py" + else: + checkpoint_path = "examples/eagle/convert_checkpoint.py" + + covert_cmd = [_sys.executable, checkpoint_path] + ( + ['--model_dir', str(base_model_dir)] if base_model_dir else []) + [ + '--output_dir', str(engine_dir), '--dtype=float16' + ] + list(args) + + if not build_base_model: + covert_cmd += [ + '--eagle_model_dir', + str(eagle_model_dir), '--num_eagle_layers=4', '--max_draft_len=63' + ] + + run_command(covert_cmd) + + build_args = ["trtllm-build"] + ( + ['--checkpoint_dir', str(engine_dir)] if engine_dir else []) + [ + '--output_dir', + str(engine_dir), + '--gemm_plugin=float16', + '--max_batch_size=8', + '--max_input_len=12', + '--max_seq_len=140', + '--log_level=error', + '--paged_kv_cache=enable', + '--remove_input_padding=enable', + '--use_paged_context_fmha=enable', + ] + + if not build_base_model: + build_args += ['--speculative_decoding_mode=eagle'] + + run_command(build_args) + + +def build_engines(model_cache: str): + resources_dir = _pl.Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_name = 'vicuna-7b-eagle' + base_model_name = 'vicuna-7b-v1.3' + eagle_model_name = 'EAGLE-Vicuna-7B-v1.3' + + if model_cache: + print(f"Copy model from {model_cache}") + base_model_cache_dir = _pl.Path(model_cache) / base_model_name + eagle_cache_dir = _pl.Path(model_cache) / eagle_model_name + assert base_model_cache_dir.is_dir(), base_model_cache_dir + assert eagle_cache_dir.is_dir(), eagle_cache_dir + + if _pf.system() == "Windows": + wincopy(source=str(base_model_cache_dir), + dest=base_model_name, + isdir=True, + cwd=models_dir) + wincopy(source=str(eagle_cache_dir), + dest=eagle_model_name, + isdir=True, + cwd=models_dir) + else: + run_command(["rsync", "-rlptD", + str(base_model_cache_dir), "."], + cwd=models_dir) + run_command(["rsync", "-rlptD", + str(eagle_cache_dir), "."], + cwd=models_dir) + + base_model_dir = models_dir / base_model_name + eagle_model_dir = models_dir / eagle_model_name + assert base_model_dir.is_dir() + assert eagle_model_dir.is_dir() + + eagle_engine_dir = models_dir / 'rt_engine' / model_name + base_engine_dir = models_dir / 'rt_engine' / base_model_name + + model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + + base_full_engine_path = base_engine_dir / model_spec_obj.get_model_path( + ) / 'tp1-pp1-cp1-gpu' + print(f"\nBuilding fp16 engine at {str(base_full_engine_path)}") + build_engine(base_model_dir, + eagle_model_dir, + base_full_engine_path, + build_base_model=True) + + model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + model_spec_obj.use_eagle() + eagle_full_engine_path = eagle_engine_dir / model_spec_obj.get_model_path( + ) / 'tp1-pp1-cp1-gpu' + print(f"\nBuilding fp16 engine at {str(eagle_full_engine_path)}") + build_engine(base_model_dir, + eagle_model_dir, + eagle_full_engine_path, + build_base_model=False) + + print("Done.") + + +if __name__ == "__main__": + parser = _arg.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_enc_dec_engines.py b/cpp/tests/resources/scripts/build_enc_dec_engines.py new file mode 100644 index 000000000000..7079916267d9 --- /dev/null +++ b/cpp/tests/resources/scripts/build_enc_dec_engines.py @@ -0,0 +1,187 @@ +import os.path +from argparse import ArgumentParser +from dataclasses import dataclass, fields +from subprocess import run +from sys import stderr, stdout +from typing import List, Literal, Union + +split = os.path.split +join = os.path.join +dirname = os.path.dirname + + +@dataclass +class Arguments: + download: bool = False + dtype: Literal['float16', 'float32', 'bfloat16'] = 'float16' + + hf_repo_name: Literal[ + 'facebook/bart-large-cnn', 't5-small', + 'language_adapter-enc_dec_language_adapter'] = 'facebook/bart-large-cnn' + + model_cache: str = '/llm-models' + + tp: int = 1 + pp: int = 1 + + beams: str = '1' + gpus_per_node: int = 4 + debug: bool = False + + rm_pad: bool = True + gemm: bool = True + + max_new_tokens: int = 64 + + @property + def beams_tuple(self): + return eval(f'tuple([{self.beams}])') + + @property + def max_beam(self): + return max(self.beams_tuple) + + @property + def ckpt(self): + return self.hf_repo_name.split('/')[-1] + + @property + def base_dir(self): + return dirname(dirname(__file__)) + + @property + def data_dir(self): + return join(self.base_dir, 'data/enc_dec') + + @property + def models_dir(self): + return join(self.base_dir, 'models/enc_dec') + + @property + def hf_models_dir(self): + return join(self.model_cache, self.ckpt) + + @property + def trt_models_dir(self): + return join(self.models_dir, 'trt_models', self.ckpt) + + @property + def engines_dir(self): + return join(self.models_dir, 'trt_engines', self.ckpt, + f'{self.tp * self.pp}-gpu', self.dtype) + + @property + def model_type(self): + return self.ckpt.split('-')[0] + + def __post_init__(self): + parser = ArgumentParser() + for k in fields(self): + k = k.name + v = getattr(self, k) + if isinstance(v, bool): + parser.add_argument(f'--{k}', action='store_true') + else: + parser.add_argument(f'--{k}', default=v, type=type(v)) + + args = parser.parse_args() + for k, v in args._get_kwargs(): + setattr(self, k, v) + + +@dataclass +class RunCMDMixin: + args: Arguments + + def command(self) -> Union[str, List[str]]: + raise NotImplementedError + + def run(self): + cmd = self.command() + if cmd: + cmd = ' '.join(cmd) if isinstance(cmd, list) else cmd + print('+ ' + cmd) + run(cmd, shell='bash', stdout=stdout, stderr=stderr, check=True) + + +class DownloadHF(RunCMDMixin): + + def command(self): + args = self.args + return [ + 'git', 'clone', f'https://huggingface.co/{args.hf_repo_name}', + args.hf_models_dir + ] if args.download and args.model_type != 'language_adapter' else '' + + +class Convert(RunCMDMixin): + + def command(self): + args = self.args + return [ + f'python examples/models/core/enc_dec/convert_checkpoint.py', + f'--model_type {args.model_type}', + f'--model_dir {args.hf_models_dir}', + f'--output_dir {args.trt_models_dir}', + f'--tp_size {args.tp} --pp_size {args.pp}' + ] + + +class Build(RunCMDMixin): + + def command(self): + args = self.args + engine_dir = args.engines_dir + weight_dir = args.trt_models_dir + encoder_build = [ + f"trtllm-build --checkpoint_dir {join(weight_dir, 'encoder')}", + f"--output_dir {join(engine_dir, 'encoder')}", + f'--paged_kv_cache disable', + f'--max_beam_width {args.max_beam}', + f'--max_batch_size 8', + f'--max_input_len 512', + f'--gemm_plugin {args.dtype}', + f'--bert_attention_plugin {args.dtype}', + f'--gpt_attention_plugin {args.dtype}', + f'--remove_input_padding enable', + ] + + decoder_build = [ + f"trtllm-build --checkpoint_dir {join(weight_dir, 'decoder')}", + f"--output_dir {join(engine_dir, 'decoder')}", + f'--paged_kv_cache enable', + f'--max_beam_width {args.max_beam}', + f'--max_batch_size 8', + f'--max_seq_len 201', + f'--max_encoder_input_len 512', + f'--gemm_plugin {args.dtype}', + f'--bert_attention_plugin {args.dtype}', + f'--gpt_attention_plugin {args.dtype}', + f'--remove_input_padding enable', + '--max_input_len 1', + ] + + # t5 model with relative attention cannot use context_fmha + encoder_build.append(f'--context_fmha disable') + decoder_build.append(f'--context_fmha disable') + + # language adapter plugin leverages MOE plugin for static expert selection + if args.model_type == 'language_adapter': + encoder_build.append(f'--moe_plugin auto') + decoder_build.append(f'--moe_plugin auto') + else: + encoder_build.append(f'--moe_plugin disable') + decoder_build.append(f'--moe_plugin disable') + + encoder_build = ' '.join(encoder_build) + decoder_build = ' '.join(decoder_build) + ret = ' && '.join((encoder_build, decoder_build)) + return ret + + +if __name__ == "__main__": + # TODO: add support for more models / setup + args = Arguments() + DownloadHF(args).run() + Convert(args).run() + Build(args).run() diff --git a/cpp/tests/resources/scripts/build_engines_utils.py b/cpp/tests/resources/scripts/build_engines_utils.py new file mode 100644 index 000000000000..ad8525217e3a --- /dev/null +++ b/cpp/tests/resources/scripts/build_engines_utils.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging as _log +import os as _os +import pathlib as _pl +import subprocess as _sp +import typing as _tp + + +def run_command(command: _tp.Sequence[str], + *, + cwd=None, + timeout=None, + **kwargs) -> None: + _log.info("Running: cd %s && %s", str(cwd), " ".join(command)) + override_timeout = int(_os.environ.get("CPP_TEST_TIMEOUT_OVERRIDDEN", "-1")) + if override_timeout > 0 and (timeout is None or override_timeout > timeout): + _log.info("Overriding the command timeout: %s (before) and %s (after)", + timeout, override_timeout) + timeout = override_timeout + _sp.check_call(command, cwd=cwd, timeout=timeout, **kwargs) + + +# We can't use run_command() because robocopy (Robust Copy, rsync equivalent on Windows) +# for some reason uses nonzero return codes even on *successful* copies, so we need to check it manually. +# Also, robocopy only accepts dirs, not individual files, so we need a separate command for the +# single-file case. +def wincopy(source: str, dest: str, isdir: bool, cwd=None) -> None: + if not isdir: # Single-file copy + run_command(["cmd", "/c", "copy", + str(_pl.Path(source)), f".\\{dest}"], + cwd=cwd) + else: # Directory sync + copy_cmd = ["robocopy", source, f"./{dest}", "/mir", "/e"] + print(f"Running: cd %s && %s" % + (str(cwd or _pl.Path.cwd()), " ".join(copy_cmd))) + + # Run the command from the specified directory + result = _sp.run(copy_cmd, cwd=cwd) + + # Check for valid exit code + if result.returncode < 8: + print("ROBOCOPY completed successfully.") + else: + print( + "ROBOCOPY failure. Displaying error. See https://ss64.com/nt/robocopy-exit.html for exit code info." + ) + raise _sp.CalledProcessError(returncode=result.returncode, + cmd=copy_cmd, + output=result.stderr) diff --git a/cpp/tests/resources/scripts/build_gpt_engines.py b/cpp/tests/resources/scripts/build_gpt_engines.py new file mode 100755 index 000000000000..fa089d773dc0 --- /dev/null +++ b/cpp/tests/resources/scripts/build_gpt_engines.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import platform +import shutil +import sys +from pathlib import Path +from typing import Optional + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec, QuantMethod + + +def convert_ckpt(model_dir: str, + output_dir: str, + *args, + world_size: int = 1, + dtype: str = 'float16'): + convert_cmd = [ + sys.executable, "examples/models/core/gpt/convert_checkpoint.py", + f"--model_dir={model_dir}", f"--output_dir={output_dir}", + f"--dtype={dtype}", f"--tp_size={world_size}" + ] + list(args) + run_command(convert_cmd) + + +def build_engine( + checkpoint_dir: str, + engine_dir: str, + *args, + max_input_len: int = 256, + max_seq_len: int = 384, +): + + build_cmd = [ + "trtllm-build", + '--log_level=error', + f'--checkpoint_dir={checkpoint_dir}', + f'--output_dir={engine_dir}', + '--max_batch_size=64', + f'--max_input_len={max_input_len}', + f'--max_seq_len={max_seq_len}', + '--max_beam_width=2', + '--kv_cache_type=continuous', + ] + legacy_args = [ + "--gpt_attention_plugin=disable", + "--context_fmha=disable", + "--remove_input_padding=disable", + ] + build_cmd = build_cmd + legacy_args + list(args) + run_command(build_cmd) + + +def build_engines(model_cache: Optional[str] = None, + world_size: int = 1, + clean: Optional[bool] = False): + # TODO add support of Pipeline parallelism to GPT + tp_size = world_size + pp_size = 1 + cp_size = 1 + + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_name = 'gpt2' + + # Clone or update the model directory without lfs + hf_dir = models_dir / model_name + if hf_dir.exists(): + assert hf_dir.is_dir() + run_command(["git", "pull"], cwd=hf_dir) + else: + if platform.system() == "Windows": + url_prefix = "" + else: + url_prefix = "file://" + + model_url = url_prefix + str( + Path(model_cache) / + model_name) if model_cache else "https://huggingface.co/gpt2" + run_command([ + "git", "clone", model_url, "--single-branch", "--no-local", + model_name + ], + cwd=hf_dir.parent, + env={ + **os.environ, "GIT_LFS_SKIP_SMUDGE": "1" + }) + + assert hf_dir.is_dir() + + # Download the model file + model_file_name = "pytorch_model.bin" + if model_cache: + if platform.system() == "Windows": + wincopy(source=str( + Path(model_cache) / model_name / model_file_name), + dest=model_file_name, + isdir=False, + cwd=hf_dir) + else: + run_command([ + "rsync", "-rlptD", + str(Path(model_cache) / model_name / model_file_name), "." + ], + cwd=hf_dir) + else: + run_command(["git", "lfs", "pull", "--include", model_file_name], + cwd=hf_dir) + + safetensor_file = hf_dir / "model.safetensors" + has_safetensor = safetensor_file.exists() + if has_safetensor: + safetensor_file.rename(str(safetensor_file) + ".bak") + + assert (hf_dir / model_file_name).is_file() + + ckpt_dir = models_dir / 'c-model' / model_name + engine_dir = models_dir / 'rt_engine' / model_name + + if clean: + target_dir = Path(engine_dir) + print('clean up target folder ', target_dir) + if target_dir.is_dir(): + shutil.rmtree(target_dir, ignore_errors=True) + + tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" + tp_dir = f"{world_size}-gpu" + + print("\nConverting to fp16") + fp16_ckpt_dir = ckpt_dir / 'fp16' / tp_dir + convert_ckpt(str(hf_dir), + str(fp16_ckpt_dir), + world_size=tp_size, + dtype='float16') + + print("\nBuilding fp16 engines") + + input_file = 'input_tokens.npy' + # this engine can be use for in-flight batching + ifb_base_args = [ + '--gpt_attention_plugin=float16', + '--remove_input_padding=enable', + '--context_fmha=enable', + '--max_num_tokens=10000', + '--use_paged_context_fmha=enable', + ] + + paged_kv_cache_args = ['--kv_cache_type=paged'] + + no_kv_cache_args = ['--kv_cache_type=disabled'] + + def get_ifb_args(kv_cache_type): + if kv_cache_type == _tb.KVCacheType.DISABLED: + return ifb_base_args + no_kv_cache_args + elif kv_cache_type == _tb.KVCacheType.PAGED: + return ifb_base_args + paged_kv_cache_args + else: + assert False, f"Unsupported kv_cache_type: {kv_cache_type}" + + model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + + model_spec_current = model_spec_obj.__copy__() + + for kv_cache_type in [_tb.KVCacheType.DISABLED, _tb.KVCacheType.PAGED]: + model_spec_current.set_kv_cache_type(kv_cache_type) + build_engine( + str(fp16_ckpt_dir), + str(engine_dir / model_spec_current.get_model_path() / + tp_pp_cp_dir), *get_ifb_args(kv_cache_type)) + + model_spec_current = model_spec_obj.__copy__() + max_draft_tokens = 5 + model_spec_current.use_draft_tokens_external_decoding() + model_spec_current.set_draft_tokens(max_draft_tokens) + + build_engine( + str(fp16_ckpt_dir), + str(engine_dir / model_spec_current.get_model_path() / tp_pp_cp_dir), + f'--max_draft_len={max_draft_tokens}', + '--speculative_decoding_mode=draft_tokens_external', + *get_ifb_args(_tb.KVCacheType.PAGED)) + + model_spec_current = model_spec_obj.__copy__() + model_spec_current.use_multiple_profiles() + + build_engine( + str(fp16_ckpt_dir), + str(engine_dir / model_spec_current.get_model_path() / tp_pp_cp_dir), + '--multiple_profiles=enable', *get_ifb_args(_tb.KVCacheType.PAGED)) + + model_spec_current = model_spec_obj.__copy__() + max_input_len = 128 + model_spec_current.set_max_input_length(max_input_len) + + build_engine(str(fp16_ckpt_dir), + str(engine_dir / model_spec_current.get_model_path() / + tp_pp_cp_dir), + *get_ifb_args(_tb.KVCacheType.PAGED), + max_input_len=max_input_len) + + # We build almost the same engine twice. But this engine has gather_context_logits + # to extract logits from python runtime and uses context FMHA for generation to match draft model executions, + # which uses context FMHA for draft tokens prediction. + # Currently the gather_context_logits is not supported with target model of speculative decoding + model_spec_current = model_spec_obj.__copy__() + model_spec_current.gather_logits() + + build_engine( + str(fp16_ckpt_dir), + str(engine_dir / model_spec_current.get_model_path() / tp_pp_cp_dir), + '--gather_context_logits', *get_ifb_args(_tb.KVCacheType.PAGED)) + + # build engine with lora enabled + model_spec_current = model_spec_obj.__copy__() + model_spec_current.use_lora_plugin() + build_engine( + str(fp16_ckpt_dir), + str(engine_dir / model_spec_current.get_model_path() / tp_pp_cp_dir), + "--lora_target_modules=attn_qkv", '--lora_plugin=float16', + *get_ifb_args(_tb.KVCacheType.PAGED)) + + if model_cache: + llm_datasets_root = Path(model_cache) / "datasets" + calib_dataset = llm_datasets_root / "cimec/lambada/" + else: + calib_dataset = "lambada" + print("\nConverting to fp16 SQ") + fp16_sq_ckpt_dir = ckpt_dir / 'fp16-sq' / tp_dir + convert_ckpt(str(hf_dir), + str(fp16_sq_ckpt_dir), + "--smoothquant=0.5", + f"--calib_dataset={calib_dataset}", + world_size=tp_size, + dtype='float16') + + print("\nBuilding fp16 SQ engines") + model_spec_current = ModelSpec(input_file, _tb.DataType.HALF) + model_spec_current.use_gpt_plugin() + model_spec_current.use_packed_input() + model_spec_current.set_quant_method(QuantMethod.SMOOTH_QUANT) + + for kv_cache_type in [_tb.KVCacheType.DISABLED, _tb.KVCacheType.PAGED]: + model_spec_current.set_kv_cache_type(kv_cache_type) + build_engine( + str(fp16_sq_ckpt_dir), + str(engine_dir / model_spec_current.get_model_path() / + tp_pp_cp_dir), *get_ifb_args(kv_cache_type)) + + if has_safetensor: + Path(str(safetensor_file) + ".bak").rename(safetensor_file) + + print("Done.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + + parser.add_argument('--world_size', + type=int, + default=1, + help='World size, only support tensor parallelism now') + + parser.add_argument('--clean', + action='store_true', + default=False, + help='Clean target folders before building engines') + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_gptj_engines.py b/cpp/tests/resources/scripts/build_gptj_engines.py new file mode 100755 index 000000000000..bfab97e0ec11 --- /dev/null +++ b/cpp/tests/resources/scripts/build_gptj_engines.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import os as _os +import pathlib as _pl +import platform as _pf +import sys as _sys +import typing as _tp + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def get_ckpt_without_quatization(model_dir, output_dir): + build_args = [ + _sys.executable, "examples/models/contrib/gpt/convert_checkpoint.py" + ] + [ + '--model_dir={}'.format(model_dir), + '--output_dir={}'.format(output_dir), + ] + run_command(build_args) + + +def get_ckpt_with_modelopt_quant(model_dir, output_dir, model_cache): + build_args = [_sys.executable, "examples/quantization/quantize.py"] + [ + '--model_dir={}'.format(model_dir), + '--output_dir={}'.format(output_dir), '--qformat=fp8', + '--kv_cache_dtype=fp8', + f'--calib_dataset={model_cache}/datasets/cnn_dailymail' + ] + run_command(build_args) + + +def build_engine(checkpoint_dir: _pl.Path, engine_dir: _pl.Path, *args): + build_args = ["trtllm-build"] + ( + ['--checkpoint_dir', str(checkpoint_dir)] if checkpoint_dir else []) + [ + '--output_dir', + str(engine_dir), + '--logits_dtype=float16', + '--gemm_plugin=float16', + '--max_batch_size=32', + '--max_input_len=40', + '--max_seq_len=60', + '--max_beam_width=2', + '--log_level=error', + ] + list(args) + run_command(build_args) + + +def build_engines(model_cache: _tp.Optional[str] = None, only_fp8=False): + resources_dir = _pl.Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_name = 'gpt-j-6b' + + # Clone or update the model directory without lfs + hf_dir = models_dir / model_name + if hf_dir.exists(): + assert hf_dir.is_dir() + run_command(["git", "pull"], cwd=hf_dir) + else: + if _pf.system() == "Windows": + url_prefix = "" + else: + url_prefix = "file://" + model_url = url_prefix + str( + _pl.Path(model_cache) / model_name + ) if model_cache else "https://huggingface.co/EleutherAI/gpt-j-6b" + run_command([ + "git", "clone", model_url, "--single-branch", "--no-local", + model_name + ], + cwd=hf_dir.parent, + env={ + **_os.environ, "GIT_LFS_SKIP_SMUDGE": "1" + }) + + assert (hf_dir.is_dir()) + + # Download the model file + model_file_name = "pytorch_model.bin" + if model_cache: + if _pf.system() == "Windows": + wincopy(source=str( + _pl.Path(model_cache) / model_name / model_file_name), + dest=model_file_name, + isdir=False, + cwd=hf_dir) + else: + run_command([ + "rsync", "-rlptD", + str(_pl.Path(model_cache) / model_name / model_file_name), "." + ], + cwd=hf_dir) + else: + run_command(["git", "lfs", "pull", "--include", model_file_name], + cwd=hf_dir) + + assert ((hf_dir / model_file_name).is_file()) + + engine_dir = models_dir / 'rt_engine' / model_name + + # TODO add Tensor and Pipeline parallelism to GPT-J + tp_size = 1 + pp_size = 1 + cp_size = 1 + tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" + input_file = 'input_tokens.npy' + + if only_fp8: + # with ifb, new plugin + print( + "\nBuilding fp8-plugin engine using gpt_attention_plugin with inflight-batching, packed" + ) + # TODO: use dummy scales atm; to re-enable when data is uploaded to the model cache + # quantized_fp8_model_arg = '--quantized_fp8_model_path=' + \ + # str(_pl.Path(model_cache) / 'fp8-quantized-modelopt' / 'gptj_tp1_rank0.npz') + fp8_ckpt_path = engine_dir / 'fp8' / tp_pp_cp_dir + get_ckpt_with_modelopt_quant(hf_dir, fp8_ckpt_path, model_cache) + model_spec_obj = ModelSpec(input_file, _tb.DataType.FP8) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + build_engine( + fp8_ckpt_path, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--gpt_attention_plugin=float16', + '--paged_kv_cache=enable', + '--remove_input_padding=enable', + '--use_paged_context_fmha=enable', + ) + else: + fp16_ckpt_path = engine_dir / 'fp16' / tp_pp_cp_dir + get_ckpt_without_quatization(hf_dir, fp16_ckpt_path) + print("\nBuilding fp16-plugin engine") + model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) + + build_engine( + fp16_ckpt_path, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--gpt_attention_plugin=float16', '--paged_kv_cache=disable', + '--remove_input_padding=disable', "--context_fmha=disable") + + print("\nBuilding fp16-plugin-packed engine") + model_spec_obj.use_packed_input() + build_engine( + fp16_ckpt_path, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--gpt_attention_plugin=float16', '--paged_kv_cache=disable', + '--remove_input_padding=enable', "--context_fmha=disable") + + print("\nBuilding fp16-plugin-packed-paged engine") + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + build_engine( + fp16_ckpt_path, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--gpt_attention_plugin=float16', '--paged_kv_cache=enable', + '--remove_input_padding=enable', "--context_fmha=disable") + print("Done.") + + +if __name__ == "__main__": + parser = _arg.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + parser.add_argument( + "--only_fp8", + action="store_true", + help="Build engines for only FP8 tests. Implemented for H100 runners.") + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_llama_engines.py b/cpp/tests/resources/scripts/build_llama_engines.py new file mode 100644 index 000000000000..dbac12621c73 --- /dev/null +++ b/cpp/tests/resources/scripts/build_llama_engines.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import pathlib as _pl +import platform as _pf +import sys as _sys +import time + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def build_engine(weight_dir: _pl.Path, engine_dir: _pl.Path, convert_extra_args, + build_extra_args): + + ckpt_dir = engine_dir / 'ckpt' + + convert_cmd = [ + _sys.executable, "examples/models/core/llama/convert_checkpoint.py" + ] + ([f'--model_dir={weight_dir}'] if weight_dir else []) + [ + f'--output_dir={ckpt_dir}', + '--dtype=float16', + ] + convert_extra_args + + run_command(convert_cmd) + + build_args = [ + 'trtllm-build', + f'--checkpoint_dir={ckpt_dir}', + f'--output_dir={engine_dir}', + '--gpt_attention_plugin=float16', + '--gemm_plugin=float16', + '--max_batch_size=32', + '--max_input_len=40', + '--max_seq_len=60', + '--max_beam_width=2', + '--log_level=error', + '--paged_kv_cache=enable', + '--remove_input_padding=enable', + ] + build_extra_args + + run_command(build_args) + + +def build_engines(model_cache: str, only_multi_gpu: bool): + resources_dir = _pl.Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_name = 'Llama-3.2-1B' + + if model_cache: + print("Copy model from model_cache") + model_cache_dir = _pl.Path( + model_cache) / 'llama-3.2-models' / model_name + assert (model_cache_dir.is_dir()), model_cache_dir + + if _pf.system() == "Windows": + wincopy(source=str(model_cache_dir), + dest=model_name, + isdir=True, + cwd=models_dir) + else: + run_command(["rsync", "-rlptD", + str(model_cache_dir), "."], + cwd=models_dir) + + hf_dir = models_dir / model_name + assert hf_dir.is_dir(), f"testing {hf_dir}" + + engine_dir = models_dir / 'rt_engine' / model_name + + model_spec_obj = ModelSpec('input_tokens_llama.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + + tp_pp_cp_sizes = [(1, 1, 1)] + if only_multi_gpu: + tp_pp_cp_sizes = [(1, 4, 1), (4, 1, 1), (1, 2, 1), (2, 2, 1), (2, 1, 1), + (1, 1, 2), (2, 1, 2)] + for tp_size, pp_size, cp_size in tp_pp_cp_sizes: + print(f"\nBuilding fp16 tp{tp_size} pp{pp_size} cp{cp_size} engine") + start_time = time.time() + + tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" + model_spec_obj.use_tensor_parallelism(tp_size) + model_spec_obj.use_pipeline_parallelism(pp_size) + model_spec_obj.use_context_parallelism(cp_size) + + build_engine( + hf_dir, engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + [ + f'--tp_size={tp_size}', f'--pp_size={pp_size}', + f'--cp_size={cp_size}' + ], ['--use_paged_context_fmha=disable']) + + duration = time.time() - start_time + print( + f"Building fp16 tp{tp_size} pp{pp_size} cp{cp_size} engine took {duration} seconds" + ) + + if not only_multi_gpu: + print(f"\nBuilding lookahead engine") + start_time = time.time() + + model_spec_obj.use_tensor_parallelism(1) + model_spec_obj.use_pipeline_parallelism(1) + model_spec_obj.use_context_parallelism(1) + model_spec_obj.use_lookahead_decoding() + build_engine( + hf_dir, + engine_dir / model_spec_obj.get_model_path() / 'tp1-pp1-cp1-gpu', + [], [ + '--max_draft_len=39', + '--speculative_decoding_mode=lookahead_decoding' + ]) + + duration = time.time() - start_time + print(f"Building lookahead engine took {duration} seconds") + + print("Done.") + + +if __name__ == "__main__": + parser = _arg.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + parser.add_argument( + "--only_multi_gpu", + action="store_true", + help="Flag to build only for Tensor and Pipeline parallelism") + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_mamba_engines.py b/cpp/tests/resources/scripts/build_mamba_engines.py new file mode 100644 index 000000000000..6b10a5b03531 --- /dev/null +++ b/cpp/tests/resources/scripts/build_mamba_engines.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import os as _os +import pathlib as _pl +import platform as _pf +import sys as _sys +import typing as _tp + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def build_engine(weight_dir: _pl.Path, ckpt_dir: _pl.Path, engine_dir: _pl.Path, + *args): + convert_args = [ + _sys.executable, "examples/models/core/mamba/convert_checkpoint.py" + ] + (['--model_dir', str(weight_dir)] if weight_dir else []) + [ + '--output_dir', + str(ckpt_dir), + '--dtype=float16', + ] + run_command(convert_args) + build_args = ["trtllm-build"] + ['--checkpoint_dir', + str(ckpt_dir)] + [ + '--output_dir', + str(engine_dir), + '--gpt_attention_plugin=disable', + '--paged_kv_cache=disable', + '--gemm_plugin=disable', + '--max_batch_size=8', + '--max_input_len=924', + '--max_seq_len=1024', + '--max_beam_width=1', + ] + list(args) + run_command(build_args) + + +def build_engines(model_cache: _tp.Optional[str] = None): + resources_dir = _pl.Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_name = 'mamba-2.8b-hf' + + if model_cache: + print("Copy model from model_cache") + model_cache_dir = _pl.Path(model_cache) / 'mamba' / model_name + if _pf.system() == "Windows": + wincopy(source=str(model_cache_dir), + dest=model_name, + isdir=True, + cwd=models_dir) + else: + run_command(["rsync", "-rlptD", + str(model_cache_dir), "."], + cwd=models_dir) + else: + print("Clone model from HF") + hf_dir = _pl.Path(models_dir) / model_name + run_command( + [ + "git", "clone", + "https://huggingface.co/state-spaces/mamba-2.8b-hf", model_name + ], + cwd=models_dir, + ) + hf_dir = models_dir / model_name + assert (hf_dir.is_dir()) + + # Clone or update the tokenizer directory without lfs + tokenizer_name = 'gpt-neox-20b' + tokenizer_hf_dir = models_dir / tokenizer_name + if tokenizer_hf_dir.exists(): + assert tokenizer_hf_dir.is_dir() + run_command(["git", "pull"], cwd=tokenizer_hf_dir) + else: + if _pf.system() == "Windows": + url_prefix = "" + else: + url_prefix = "file://" + tokenizer_url = url_prefix + str( + _pl.Path(model_cache) / tokenizer_name + ) if model_cache else "https://huggingface.co/EleutherAI/gpt-neox-20b" + run_command([ + "git", "clone", tokenizer_url, "--single-branch", "--no-local", + tokenizer_name + ], + cwd=tokenizer_hf_dir.parent, + env={ + **_os.environ, "GIT_LFS_SKIP_SMUDGE": "1" + }) + + tp_size = 1 + pp_size = 1 + cp_size = 1 + tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" + + ckpt_dir = models_dir / 'rt_ckpt' / model_name + engine_dir = models_dir / 'rt_engine' / model_name + model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) + model_spec_obj.use_tensor_parallelism(tp_size) + model_spec_obj.use_pipeline_parallelism(pp_size) + model_spec_obj.use_context_parallelism(cp_size) + + print("\nBuilding fp16 engine") + build_engine(hf_dir, + ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--remove_input_padding=disable', '--paged_state=disable', + '--mamba_conv1d_plugin=disable') + print("\nBuilding fp16-plugin engine") + model_spec_obj.use_mamba_plugin() + build_engine(hf_dir, + ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--remove_input_padding=disable', '--paged_state=disable') + print("\nBuilding fp16-plugin-packed engine") + model_spec_obj.use_packed_input() + build_engine(hf_dir, + ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--remove_input_padding=enable', '--paged_state=disable') + print("\nBuilding fp16-plugin-packed-paged engine") + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + build_engine(hf_dir, + ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--remove_input_padding=enable', '--paged_state=enable') + print("Done.") + + +if __name__ == "__main__": + parser = _arg.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_medusa_engines.py b/cpp/tests/resources/scripts/build_medusa_engines.py new file mode 100755 index 000000000000..cf9c74f8779f --- /dev/null +++ b/cpp/tests/resources/scripts/build_medusa_engines.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import pathlib as _pl +import platform as _pf +import sys as _sys + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def build_engine(base_model_dir: _pl.Path, medusa_model_dir: _pl.Path, + engine_dir: _pl.Path, *args): + + covert_cmd = [_sys.executable, "examples/medusa/convert_checkpoint.py"] + ( + ['--model_dir', str(base_model_dir)] if base_model_dir else []) + [ + '--medusa_model_dir', str(medusa_model_dir), \ + '--output_dir', str(engine_dir), '--dtype=float16', '--num_medusa_heads=4' + ] + list(args) + + run_command(covert_cmd) + + build_args = ["trtllm-build"] + ( + ['--checkpoint_dir', str(engine_dir)] if engine_dir else []) + [ + '--output_dir', + str(engine_dir), + '--gemm_plugin=float16', + '--max_batch_size=8', + '--max_input_len=12', + '--max_seq_len=140', + '--log_level=error', + '--paged_kv_cache=enable', + '--use_paged_context_fmha=enable', + '--remove_input_padding=enable', + '--speculative_decoding_mode=medusa', + ] + + run_command(build_args) + + +def build_engines(model_cache: str): + resources_dir = _pl.Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_name = 'vicuna-7b-medusa' + base_model_name = 'vicuna-7b-v1.3' + medusa_model_name = 'medusa-vicuna-7b-v1.3' + + if model_cache: + print(f"Copy model from {model_cache}") + base_model_cache_dir = _pl.Path(model_cache) / base_model_name + medusa_head_cache_dir = _pl.Path(model_cache) / medusa_model_name + assert base_model_cache_dir.is_dir(), base_model_cache_dir + assert medusa_head_cache_dir.is_dir(), medusa_head_cache_dir + + if _pf.system() == "Windows": + wincopy(source=str(base_model_cache_dir), + dest=base_model_name, + isdir=True, + cwd=models_dir) + wincopy(source=str(medusa_head_cache_dir), + dest=medusa_model_name, + isdir=True, + cwd=models_dir) + else: + run_command(["rsync", "-rlptD", + str(base_model_cache_dir), "."], + cwd=models_dir) + run_command(["rsync", "-rlptD", + str(medusa_head_cache_dir), "."], + cwd=models_dir) + + base_model_dir = models_dir / base_model_name + medusa_model_dir = models_dir / medusa_model_name + assert base_model_dir.is_dir() + assert medusa_model_dir.is_dir() + + engine_dir = models_dir / 'rt_engine' / model_name + + model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + model_spec_obj.use_medusa() + + full_engine_path = engine_dir / model_spec_obj.get_model_path( + ) / 'tp1-pp1-cp1-gpu' + print(f"\nBuilding fp16 engine at {str(full_engine_path)}") + build_engine(base_model_dir, medusa_model_dir, full_engine_path) + + print("Done.") + + +if __name__ == "__main__": + parser = _arg.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_recurrentgemma_engines.py b/cpp/tests/resources/scripts/build_recurrentgemma_engines.py new file mode 100644 index 000000000000..293aab101d38 --- /dev/null +++ b/cpp/tests/resources/scripts/build_recurrentgemma_engines.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import os as _os +import pathlib as _pl +import platform as _pf +import sys as _sys +import typing as _tp + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def build_engine(weight_dir: _pl.Path, ckpt_dir: _pl.Path, engine_dir: _pl.Path, + *args): + convert_args = [ + _sys.executable, + "examples/models/core/recurrentgemma/convert_checkpoint.py" + ] + (['--model_dir', str(weight_dir)] if weight_dir else []) + [ + '--output_dir', + str(ckpt_dir), + '--ckpt_type=hf', + '--dtype=float16', + ] + run_command(convert_args) + build_args = ["trtllm-build"] + ['--checkpoint_dir', + str(ckpt_dir)] + [ + '--output_dir', + str(engine_dir), + '--gpt_attention_plugin=float16', + '--paged_kv_cache=enable', + '--gemm_plugin=float16', + '--max_batch_size=8', + '--max_input_len=924', + '--max_seq_len=1024', + '--max_beam_width=1', + ] + list(args) + run_command(build_args) + + +def build_engines(model_cache: _tp.Optional[str] = None): + resources_dir = _pl.Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_name = 'recurrentgemma-2b' + hf_dir = models_dir / model_name + + # Clone or update the model directory without lfs + if model_cache: + print("Copy model from model_cache") + model_cache_dir = _pl.Path(model_cache) / 'recurrentgemma' / model_name + print(model_cache_dir) + assert (model_cache_dir.is_dir()) + if _pf.system() == "Windows": + wincopy(source=str(model_cache_dir), + dest=model_name, + isdir=True, + cwd=models_dir) + else: + run_command(["rsync", "-rlptD", + str(model_cache_dir), "."], + cwd=models_dir) + else: + if not hf_dir.is_dir(): + if _pf.system() == "Windows": + url_prefix = "" + else: + url_prefix = "file://" + model_url = "https://huggingface.co/google/recurrentgemma-2b" + run_command([ + "git", "clone", model_url, "--single-branch", "--no-local", + model_name + ], + cwd=models_dir, + env={ + **_os.environ, "GIT_LFS_SKIP_SMUDGE": "1" + }) + + assert (hf_dir.is_dir()) + + # Download the model file + model_file_name = "*" + if not model_cache: + run_command(["git", "lfs", "pull", "--include", model_file_name], + cwd=hf_dir) + + tp_size = 1 + pp_size = 1 + cp_size = 1 + tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu" + + ckpt_dir = models_dir / 'rt_ckpt' / model_name + engine_dir = models_dir / 'rt_engine' / model_name + + python_exe = _sys.executable + run_command([python_exe, "-m", "pip", "install", "transformers>=4.40.0"], + env=_os.environ, + timeout=300) + input_file = 'input_tokens.npy' + model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.use_packed_input() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + + print("\nBuilding fp16-plugin-packed-paged engine") + build_engine(hf_dir, + ckpt_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + engine_dir / model_spec_obj.get_model_path() / tp_pp_cp_dir, + '--remove_input_padding=enable', '--paged_state=enable') + + print("Done.") + + +if __name__ == "__main__": + parser = _arg.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/build_redrafter_engines.py b/cpp/tests/resources/scripts/build_redrafter_engines.py new file mode 100755 index 000000000000..cdf3e889ac35 --- /dev/null +++ b/cpp/tests/resources/scripts/build_redrafter_engines.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import pathlib as _pl +import platform as _pf +import sys as _sys + +from build_engines_utils import run_command, wincopy + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def build_engine(base_model_dir: _pl.Path, drafter_model_dir: _pl.Path, + engine_dir: _pl.Path, *args): + + base_ckpt_dir = f'{base_model_dir}-ckpt' + covert_cmd_base = [ + _sys.executable, "examples/models/core/llama/convert_checkpoint.py" + ] + (['--model_dir', str(base_model_dir)] if base_model_dir else []) + [ + '--output_dir', str(base_ckpt_dir), '--dtype=float16' + ] + list(args) + + run_command(covert_cmd_base) + + covert_cmd = [ + _sys.executable, "examples/redrafter/convert_checkpoint.py"] + ( + ['--base_model_checkpoint_dir', str(base_ckpt_dir)] if base_model_dir else []) + [ + '--drafter_model_dir', str(drafter_model_dir), \ + '--output_dir', str(engine_dir), '--dtype=float16', + '--redrafter_num_beams=5', '--redrafter_draft_len_per_beam=5' + ] + list(args) + + run_command(covert_cmd) + + build_args = ["trtllm-build"] + ( + ['--checkpoint_dir', str(engine_dir)] if engine_dir else []) + [ + '--output_dir', + str(engine_dir), + '--gemm_plugin=float16', + '--max_batch_size=8', + '--max_input_len=64', + '--max_seq_len=1024', + '--log_level=error', + '--paged_kv_cache=enable', + '--use_paged_context_fmha=enable', + '--remove_input_padding=enable', + '--speculative_decoding_mode=explicit_draft_tokens', + ] + + run_command(build_args) + + +def build_engines(model_cache: str): + resources_dir = _pl.Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_name = 'vicuna-7b-redrafter' + base_model_name = 'vicuna-7b-v1.3' + drafter_model_name = 'redrafter-vicuna-7b-v1.3' + + if model_cache: + print(f"Copy model from {model_cache}") + base_model_cache_dir = _pl.Path(model_cache) / base_model_name + drafter_cache_dir = _pl.Path(model_cache) / drafter_model_name + assert base_model_cache_dir.is_dir(), base_model_cache_dir + assert drafter_cache_dir.is_dir(), drafter_cache_dir + + if _pf.system() == "Windows": + wincopy(source=str(base_model_cache_dir), + dest=base_model_name, + isdir=True, + cwd=models_dir) + wincopy(source=str(drafter_cache_dir), + dest=drafter_model_name, + isdir=True, + cwd=models_dir) + else: + run_command(["rsync", "-rlptD", + str(base_model_cache_dir), "."], + cwd=models_dir) + run_command(["rsync", "-rlptD", + str(drafter_cache_dir), "."], + cwd=models_dir) + + base_model_dir = models_dir / base_model_name + drafter_model_dir = models_dir / drafter_model_name + assert base_model_dir.is_dir() + assert drafter_model_dir.is_dir() + + engine_dir = models_dir / 'rt_engine' / model_name + + model_spec_obj = ModelSpec('input_tokens.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + model_spec_obj.use_explicit_draft_tokens_decoding() + + full_engine_path = engine_dir / model_spec_obj.get_model_path( + ) / 'tp1-pp1-cp1-gpu' + print(f"\nBuilding fp16 engine at {str(full_engine_path)}") + build_engine(base_model_dir, drafter_model_dir, full_engine_path) + + print("Done.") + + +if __name__ == "__main__": + parser = _arg.ArgumentParser() + parser.add_argument("--model_cache", + type=str, + help="Directory where models are stored") + + build_engines(**vars(parser.parse_args())) diff --git a/cpp/tests/resources/scripts/generate_expected_chatglm_output.py b/cpp/tests/resources/scripts/generate_expected_chatglm_output.py new file mode 100755 index 000000000000..416f76938700 --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_chatglm_output.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path + +import numpy as np + +# isort: off +import run +# isort: on + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + +resources_dir = Path(__file__).parent.resolve().parent +model_path = resources_dir / "models" + + +def generate_output( + model_name: str = "", + num_beams: int = 1, + max_output_len: int = 8, + output_logits: bool = False, + output_cum_log_probs: bool = False, + output_log_probs: bool = False, +): + hf_path = model_path / model_name + tp_size = 1 + pp_size = 1 + cp_size = 1 + tp_pp_cp_dir = f"tp{tp_size}-pp{pp_size}-cp{cp_size}-gpu/" + input_file = f"input_tokens_{model_name}.npy" + + data_input_file_name = resources_dir / "data" / input_file + if num_beams == 1: + output_dir = resources_dir / "data" / model_name / "sampling" + else: + output_dir = resources_dir / "data" / model_name / f"beam_search_{num_beams}" + output_dir.mkdir(exist_ok=True, parents=True) + + model_spec_obj_list = [ + ModelSpec(input_file, + _tb.DataType.HALF).use_gpt_plugin().set_kv_cache_type( + _tb.KVCacheType.CONTINUOUS), + ModelSpec(input_file, _tb.DataType.HALF).use_gpt_plugin(). + use_packed_input().set_kv_cache_type(_tb.KVCacheType.PAGED), + ] + + for model_spec_obj in model_spec_obj_list: + engine_dir = model_path / 'rt_engine' / model_name / model_spec_obj.get_model_path( + ) / tp_pp_cp_dir + base_output_name = os.path.splitext( + model_spec_obj.get_results_file())[0] + output_npy_file_name = output_dir / f'{base_output_name}.npy' + output_csv_file_name = output_dir / f'{base_output_name}.csv' + + args_list = [ + '--engine_dir', + str(engine_dir), + '--tokenizer_dir', + str(hf_path), + '--input_file', + str(data_input_file_name), + '--output_npy', + str(output_npy_file_name), + '--output_csv', + str(output_csv_file_name), + '--max_output_len', + str(max_output_len), + '--num_beams', + str(num_beams), + '--use_py_session', + ] + + if output_logits: + file_name = str(output_npy_file_name)[:-4] + "_logits.npy" + args_list.extend(['--output_logits_npy', file_name]) + + if output_cum_log_probs: + file_name = str(output_npy_file_name)[:-4] + "_cum_log_probs.npy" + args_list.extend(['--output_cum_log_probs_npy', file_name]) + + if output_log_probs: + file_name = str(output_npy_file_name)[:-4] + "_log_probs.npy" + args_list.extend(['--output_log_probs_npy', file_name]) + + args = run.parse_arguments(args_list) + run.main(args) + + # Convert pad_id to end_id in .npy out put file + data = np.load(str(output_npy_file_name)) + if model_name == 'chatglm-6b': + data[data == 3] = 130005 + elif model_name == 'chatglm2-6b' or model_name == 'chatglm3-6b': + data[data == 0] = 2 + elif model_name == 'glm-10b': + data[data == 50256] = 50258 + else: + raise NameError('bad model name') + + np.save(str(output_npy_file_name), data) + + +if __name__ == '__main__': + generate_output(model_name='chatglm-6b', num_beams=1) + generate_output(model_name='chatglm-6b', num_beams=2) + generate_output(model_name='chatglm2-6b', num_beams=1) + generate_output(model_name='chatglm2-6b', num_beams=2) + generate_output(model_name='chatglm3-6b', num_beams=1) + generate_output(model_name='chatglm3-6b', num_beams=2) + generate_output(model_name='glm-10b', num_beams=1) + print("Done") diff --git a/cpp/tests/resources/scripts/generate_expected_eagle_output.py b/cpp/tests/resources/scripts/generate_expected_eagle_output.py new file mode 100755 index 000000000000..253a98beaf4e --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_eagle_output.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import os +from pathlib import Path + +# isort: off +import run +# isort: on + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def generate_output(engine: str, + model_spec_obj: ModelSpec, + max_output_len: int = 8): + + model = 'vicuna-7b-v1.3' + model_eagle = 'vicuna-7b-eagle' + hf_model = 'vicuna-7b-v1.3' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + hf_dir = models_dir / hf_model + tp_pp_cp_dir = 'tp1-pp1-cp1-gpu/' + engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir + + data_dir = resources_dir / 'data' + input_file = data_dir / 'input_vicuna.npy' + model_data_dir = data_dir / model_eagle + output_dir = model_data_dir / 'sampling' + + base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] + + args = run.parse_arguments([ + '--engine_dir', + str(engine_dir), '--input_file', + str(input_file), '--tokenizer_dir', + str(hf_dir), '--output_npy', + str(output_dir / (base_output_name + '.npy')), '--output_csv', + str(output_dir / (base_output_name + '.csv')), '--max_output_len', + str(max_output_len), '--use_py_session', '--temperature', '1.0' + ]) + run.main(args) + print(f"Output saved at {str(output_dir / base_output_name)}.[npy|csv]") + + +def generate_outputs(): + print(f'Generating outputs for Vicuna 7B v1.3 FP16') + max_output_len = 128 + model_spec_obj = ModelSpec('input_tokens_long.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_max_output_length(max_output_len) + model_spec_obj.use_packed_input() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + + generate_output(engine=model_spec_obj.get_model_path(), + model_spec_obj=model_spec_obj, + max_output_len=max_output_len) + + +if __name__ == '__main__': + parser = _arg.ArgumentParser() + parser.add_argument( + "--only_multi_gpu", + action="store_true", + help="Generate data with Pipeline and Tensor Parallelism") + + args = parser.parse_args() + + generate_outputs() + print("Done") diff --git a/cpp/tests/resources/scripts/generate_expected_enc_dec_output.py b/cpp/tests/resources/scripts/generate_expected_enc_dec_output.py new file mode 100644 index 000000000000..fc3dc615c918 --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_enc_dec_output.py @@ -0,0 +1,30 @@ +from build_enc_dec_engines import Arguments, RunCMDMixin + + +class Run(RunCMDMixin): + + def command(self): + args = self.args + world_size = args.tp * args.pp + mpi_run = f'mpirun --allow-run-as-root -np {world_size}' if world_size > 1 else '' + ret = [] + for beam in args.beams_tuple: + ret.append(( + mpi_run, + f'python3 examples/models/core/enc_dec/run.py --engine_dir {args.engines_dir}', + f'--engine_name {args.ckpt}', + f'--model_name "{args.hf_models_dir}"', + f'--max_new_tokens={args.max_new_tokens}', + f'--num_beams={beam}', + f'--compare_hf_fp32', + f'--output_npy={args.data_dir}', + "--debug_mode" if args.debug else "", + )) + ret = [' '.join(x) for x in ret] + ret = ' && '.join(ret) + return ret + + +if __name__ == '__main__': + args = Arguments() + Run(args).run() diff --git a/cpp/tests/resources/scripts/generate_expected_gpt_output.py b/cpp/tests/resources/scripts/generate_expected_gpt_output.py new file mode 100755 index 000000000000..16fa5cc8db64 --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_gpt_output.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from pathlib import Path + +# isort: off +import run +# isort: on + +import os +import shutil + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec, QuantMethod + + +def get_model_data_dir(): + resources_dir = Path(__file__).parent.resolve().parent + data_dir = resources_dir / 'data' + return data_dir / 'gpt2' + + +def generate_output(engine: str, + num_beams: int, + input_name: str, + model_spec_obj: ModelSpec, + max_output_len: int = 8, + output_logits: bool = False, + output_cum_log_probs: bool = False, + output_log_probs: bool = False): + tp_size = 1 + pp_size = 1 + cp_size = 1 + model = 'gpt2' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( + cp_size) + '-gpu/' + engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir + + data_dir = resources_dir / 'data' + input_file = data_dir / input_name + model_data_dir = get_model_data_dir() + if num_beams <= 1: + output_dir = model_data_dir / 'sampling' + else: + output_dir = model_data_dir / ('beam_search_' + str(num_beams)) + + model_spec_obj.use_tensor_parallelism(tp_size).use_pipeline_parallelism( + pp_size).use_context_parallelism(cp_size) + + base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] + + args_list = [ + f'--engine_dir={engine_dir}', + f'--input_file={input_file}', + f'--tokenizer_dir={models_dir / model}', + f'--output_npy={output_dir / (base_output_name + ".npy")}', + f'--output_csv={output_dir / (base_output_name + ".csv")}', + f'--max_output_len={max_output_len}', + f'--num_beams={num_beams}', + '--use_py_session', + ] + + if output_logits: + args_list.extend([ + f'--output_logits_npy={output_dir / (base_output_name + "_logits.npy")}', + '--output_generation_logits', + ]) + + # Generate context_fmha_fp32_acc enabled results for GptExecutorTest.GenerationLogitsEarlyStop + if model_spec_obj.get_enable_context_fmha_fp32_acc(): + args_list.extend(["--enable_context_fmha_fp32_acc"]) + + if output_cum_log_probs: + args_list.extend([ + f'--output_cum_log_probs_npy={output_dir / model_spec_obj.get_cum_log_probs_file()}' + ]) + + if output_log_probs: + args_list.extend([ + f'--output_log_probs_npy={output_dir / model_spec_obj.get_log_probs_file()}' + ]) + + args = run.parse_arguments(args_list) + run.main(args) + + +def generate_outputs(num_beams): + input_name = 'input_tokens.npy' + input_name_long = 'input_tokens_long.npy' + + print('Generating GPT2 FP16 outputs') + model_spec_obj = ModelSpec(input_name, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.use_packed_input() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.gather_logits() + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj, + output_logits=True, + output_log_probs=True, + output_cum_log_probs=True) + # GptExecutorTest.GenerationLogitsEarlyStop and several tests require to use context_fmha_fp32_acc flag in runtime + model_spec_obj.enable_context_fmha_fp32_acc() + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj, + output_logits=True, + output_log_probs=True, + output_cum_log_probs=True) + + model_spec_obj = ModelSpec(input_name, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj, + output_logits=False, + output_log_probs=True, + output_cum_log_probs=True) + model_spec_obj.enable_context_fmha_fp32_acc() + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj, + output_logits=False, + output_log_probs=True, + output_cum_log_probs=True) + model_spec_obj.set_max_output_length(128) + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj, + output_logits=False, + max_output_len=128) + + model_spec_obj = ModelSpec(input_name_long, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.use_packed_input() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name_long, + model_spec_obj=model_spec_obj, + output_logits=False) + + model_spec_obj = ModelSpec(input_name, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.use_packed_input() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.set_quant_method(QuantMethod.SMOOTH_QUANT) + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj, + output_logits=False) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--clean', + action='store_true', + default=False, + help='Clean target folders before building engines') + args = parser.parse_args() + if args.clean: + model_data_dir = get_model_data_dir() + print(f'Cleaning target folder {model_data_dir}') + shutil.rmtree(model_data_dir, ignore_errors=True) + generate_outputs(num_beams=1) + generate_outputs(num_beams=2) diff --git a/cpp/tests/resources/scripts/generate_expected_gptj_output.py b/cpp/tests/resources/scripts/generate_expected_gptj_output.py new file mode 100755 index 000000000000..8d650d6bfc31 --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_gptj_output.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import os +from pathlib import Path + +# isort: off +import run +# isort: on + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def generate_output(engine: str, + num_beams: int, + model_spec_obj: ModelSpec, + max_output_len: int = 4): + + tp_size = 1 + pp_size = 1 + cp_size = 1 + model = 'gpt-j-6b' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + hf_dir = models_dir / model + tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( + cp_size) + '-gpu/' + engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir + + data_dir = resources_dir / 'data' + input_file = data_dir / 'input_tokens.npy' + model_data_dir = data_dir / model + if num_beams <= 1: + output_dir = model_data_dir / 'sampling' + else: + output_dir = model_data_dir / ('beam_search_' + str(num_beams)) + + base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] + + args = run.parse_arguments([ + '--engine_dir', + str(engine_dir), '--input_file', + str(input_file), '--tokenizer_dir', + str(hf_dir), '--output_npy', + str(output_dir / (base_output_name + '.npy')), '--output_csv', + str(output_dir / (base_output_name + '.csv')), '--max_output_len', + str(max_output_len), '--num_beams', + str(num_beams), '--use_py_session' + ]) + run.main(args) + + +def generate_outputs(only_fp8, num_beams): + input_file = 'input_tokens.npy' + if only_fp8 and num_beams == 1: + model_spec_obj = ModelSpec(input_file, _tb.DataType.FP8) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + + print('Generating GPT-J FP8-kv-cache outputs') + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + model_spec_obj=model_spec_obj) + elif not only_fp8: + print('Generating GPT-J FP16 outputs') + model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + model_spec_obj=model_spec_obj) + + model_spec_obj.use_packed_input() + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + model_spec_obj=model_spec_obj) + + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + model_spec_obj=model_spec_obj) + + +if __name__ == '__main__': + parser = _arg.ArgumentParser() + parser.add_argument( + "--only_fp8", + action="store_true", + help="Generate data for only FP8 tests. Implemented for H100 runners.") + + generate_outputs(**vars(parser.parse_args()), num_beams=1) + generate_outputs(**vars(parser.parse_args()), num_beams=2) diff --git a/cpp/tests/resources/scripts/generate_expected_llama_output.py b/cpp/tests/resources/scripts/generate_expected_llama_output.py new file mode 100644 index 000000000000..74916e77d053 --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_llama_output.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import os +import time +from pathlib import Path + +from mpi4py.MPI import COMM_WORLD + +# isort: off +import run +# isort: on + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def generate_output(engine: str, + num_beams: int, + model_spec_obj: ModelSpec, + tp_size: int = 1, + pp_size: int = 1, + cp_size: int = 1, + max_output_len: int = 8, + output_logits: bool = False, + output_cum_log_probs: bool = False, + output_log_probs: bool = False): + + model = 'Llama-3.2-1B' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( + cp_size) + '-gpu/' + engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir + + data_dir = resources_dir / 'data' + input_file = data_dir / 'input_tokens_llama.npy' + model_data_dir = data_dir / model + if num_beams <= 1: + output_dir = model_data_dir / 'sampling' + else: + output_dir = model_data_dir / ('beam_search_' + str(num_beams)) + + base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] + + args_list = [ + f'--engine_dir={engine_dir}', + f'--input_file={input_file}', + f'--tokenizer_dir={models_dir / model}', + f'--output_npy={output_dir / (base_output_name + ".npy")}', + f'--output_csv={output_dir / (base_output_name + ".csv")}', + f'--max_output_len={max_output_len}', + f'--num_beams={num_beams}', + '--use_py_session', + ] + + if output_logits: + args_list.extend([ + f'--output_logits_npy={output_dir / (base_output_name + "_logits.npy")}', + '--output_generation_logits', + ]) + + if output_cum_log_probs: + args_list.extend([ + f'--output_cum_log_probs_npy={output_dir / model_spec_obj.get_cum_log_probs_file()}' + ]) + + if output_log_probs: + args_list.extend([ + f'--output_log_probs_npy={output_dir / model_spec_obj.get_log_probs_file()}' + ]) + + args = run.parse_arguments(args_list) + run.main(args) + + +def generate_outputs(num_beams, only_multi_gpu=False): + if not only_multi_gpu: + tp_pp_cp_sizes = [(1, 1, 1)] + elif COMM_WORLD.size == 4: + tp_pp_cp_sizes = [(4, 1, 1), (2, 2, 1), (1, 4, 1)] + elif COMM_WORLD.size == 2: + tp_pp_cp_sizes = [(1, 2, 1), (2, 1, 1)] + else: + raise RuntimeError( + f"The world size of MPI {COMM_WORLD.size} is not equal to 1, 2, or 4." + ) + model_spec_obj = ModelSpec('input_tokens_llama.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + + for tp_size, pp_size, cp_size in tp_pp_cp_sizes: + print( + f'Generating outputs for Llama FP16 with TP={tp_size}, PP={pp_size}, CP={cp_size}, BW={num_beams}' + ) + start_time = time.time() + + output_logits = False + output_log_probs = False + output_cum_log_probs = False + if tp_size == 4 and pp_size == 1: + output_logits = True + output_log_probs = True + output_cum_log_probs = True + + model_spec_obj.use_tensor_parallelism(tp_size) + model_spec_obj.use_pipeline_parallelism(pp_size) + model_spec_obj.use_context_parallelism(cp_size) + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + tp_size=tp_size, + pp_size=pp_size, + cp_size=cp_size, + model_spec_obj=model_spec_obj, + output_logits=output_logits, + output_log_probs=output_log_probs, + output_cum_log_probs=output_cum_log_probs) + + duration = time.time() - start_time + print( + f"Generating outputs for Llama FP16 with TP={tp_size}, PP={pp_size}, CP={cp_size}, BW={num_beams} took {duration} seconds" + ) + + +if __name__ == '__main__': + parser = _arg.ArgumentParser() + parser.add_argument( + "--only_multi_gpu", + action="store_true", + help="Generate data with Pipeline and Tensor Parallelism") + + args = parser.parse_args() + + generate_outputs(num_beams=1, only_multi_gpu=args.only_multi_gpu) + generate_outputs(num_beams=2, only_multi_gpu=args.only_multi_gpu) + print("Done") diff --git a/cpp/tests/resources/scripts/generate_expected_mamba_output.py b/cpp/tests/resources/scripts/generate_expected_mamba_output.py new file mode 100644 index 000000000000..16779c434775 --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_mamba_output.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path + +# isort: off +import run +# isort: on + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def generate_output(engine: str, + num_beams: int, + input_name: str, + model_spec_obj: ModelSpec, + max_output_len: int = 8, + output_logits: bool = False): + tp_size = 1 + pp_size = 1 + cp_size = 1 + model = 'mamba-2.8b-hf' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( + cp_size) + '-gpu/' + engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir + + data_dir = resources_dir / 'data' + input_file = data_dir / input_name + model_data_dir = data_dir / model + if num_beams <= 1: + output_dir = model_data_dir / 'sampling' + else: + output_dir = model_data_dir / ('beam_search_' + str(num_beams)) + + base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] + + output_logits_npy = None + if output_logits: + output_logits_npy = str(output_dir / + (base_output_name + '_logits' + '.npy')) + + args = run.parse_arguments([ + '--engine_dir', + str(engine_dir), '--input_file', + str(input_file), '--tokenizer_dir', + str(models_dir / 'gpt-neox-20b'), '--output_npy', + str(output_dir / (base_output_name + '.npy')), '--output_csv', + str(output_dir / (base_output_name + '.csv')), '--max_output_len', + str(max_output_len), '--num_beams', + str(num_beams), '--output_logits_npy', + str(output_logits_npy), '--use_py_session' + ]) + run.main(args) + + +def generate_outputs(num_beams): + print('Generating Mamba FP16 outputs') + input_name = 'input_tokens.npy' + model_spec_obj = ModelSpec(input_name, _tb.DataType.HALF) + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.CONTINUOUS) + + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj) + + print('Generating Mamba FP16-plugin outputs') + model_spec_obj.use_gpt_plugin() + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj) + + print('Generating Mamba FP16-plugin-packed outputs') + model_spec_obj.use_packed_input() + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj) + + print('Generating Mamba FP16-plugin-packed-paged outputs') + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_name, + model_spec_obj=model_spec_obj) + + +if __name__ == '__main__': + generate_outputs(num_beams=1) diff --git a/cpp/tests/resources/scripts/generate_expected_medusa_output.py b/cpp/tests/resources/scripts/generate_expected_medusa_output.py new file mode 100755 index 000000000000..e1cbc20c051b --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_medusa_output.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import os +from pathlib import Path + +# isort: off +import run +# isort: on + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def generate_output(engine: str, + model_spec_obj: ModelSpec, + max_output_len: int = 8): + + model = 'vicuna-7b-medusa' + hf_model = 'vicuna-7b-v1.3' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + hf_dir = models_dir / hf_model + tp_pp_dir = 'tp1-pp1-cp1-gpu/' + engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_dir + + data_dir = resources_dir / 'data' + input_file = data_dir / 'input_vicuna.npy' + model_data_dir = data_dir / model + output_dir = model_data_dir / 'sampling' + + base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] + + args = run.parse_arguments([ + '--engine_dir', + str(engine_dir), '--input_file', + str(input_file), '--tokenizer_dir', + str(hf_dir), '--output_npy', + str(output_dir / (base_output_name + '.npy')), '--output_csv', + str(output_dir / (base_output_name + '.csv')), '--max_output_len', + str(max_output_len), '--use_py_session', + '--medusa_choices=[[0], [0, 0], [1], [0, 1], [2], [0, 0, 0], [1, 0], [0, 2], [3], [0, 3], [4], [0, 4], [2, 0], [0, 5], [0, 0, 1], [5], [0, 6], [6], [0, 7], [0, 1, 0], [1, 1], [7], [0, 8], [0, 0, 2], [3, 0], [0, 9], [8], [9], [1, 0, 0], [0, 2, 0], [1, 2], [0, 0, 3], [4, 0], [2, 1], [0, 0, 4], [0, 0, 5], [0, 0, 0, 0], [0, 1, 1], [0, 0, 6], [0, 3, 0], [5, 0], [1, 3], [0, 0, 7], [0, 0, 8], [0, 0, 9], [6, 0], [0, 4, 0], [1, 4], [7, 0], [0, 1, 2], [2, 0, 0], [3, 1], [2, 2], [8, 0], [0, 5, 0], [1, 5], [1, 0, 1], [0, 2, 1], [9, 0], [0, 6, 0], [0, 0, 0, 1], [1, 6], [0, 7, 0]]', + '--temperature', '1.0' + ]) + run.main(args) + print(f"Output saved at {str(output_dir / base_output_name)}.[npy|csv]") + + +def generate_outputs(): + print(f'Generating outputs for Medusa FP16') + max_output_len = 128 + model_spec_obj = ModelSpec('input_tokens_long.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_max_output_length(max_output_len) + model_spec_obj.use_packed_input() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_medusa() + + generate_output(engine=model_spec_obj.get_model_path(), + model_spec_obj=model_spec_obj, + max_output_len=max_output_len) + + +if __name__ == '__main__': + parser = _arg.ArgumentParser() + parser.add_argument( + "--only_multi_gpu", + action="store_true", + help="Generate data with Pipeline and Tensor Parallelism") + + args = parser.parse_args() + + generate_outputs() + print("Done") diff --git a/cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py b/cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py new file mode 100644 index 000000000000..0ef4cc4509fd --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path + +# isort: off +import run +# isort: on + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def generate_output(engine: str, + num_beams: int, + input_name: str, + model_spec_obj: ModelSpec, + max_output_len: int = 8, + output_logits: bool = False): + tp_size = 1 + pp_size = 1 + cp_size = 1 + model = 'recurrentgemma-2b' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + tp_pp_cp_dir = 'tp' + str(tp_size) + '-pp' + str(pp_size) + '-cp' + str( + cp_size) + '-gpu/' + engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_cp_dir + + data_dir = resources_dir / 'data' + input_file = data_dir / input_name + model_data_dir = data_dir / model + if num_beams <= 1: + output_dir = model_data_dir / 'sampling' + else: + output_dir = model_data_dir / ('beam_search_' + str(num_beams)) + + base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] + + output_logits_npy = None + if output_logits: + output_logits_npy = str(output_dir / + (base_output_name + '_logits' + '.npy')) + + args = run.parse_arguments([ + '--engine_dir', + str(engine_dir), '--input_file', + str(input_file), '--tokenizer_dir', + str(models_dir / model), '--output_npy', + str(output_dir / (base_output_name + '.npy')), '--output_csv', + str(output_dir / (base_output_name + '.csv')), '--max_output_len', + str(max_output_len), '--num_beams', + str(num_beams), '--output_logits_npy', + str(output_logits_npy), '--use_py_session' + ]) + run.main(args) + + +def generate_outputs(num_beams): + input_file = 'input_tokens.npy' + model_spec_obj = ModelSpec(input_file, _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_packed_input() + + print('Generating RecurrentGemma FP16-plugin-packed-paged outputs') + generate_output(engine=model_spec_obj.get_model_path(), + num_beams=num_beams, + input_name=input_file, + model_spec_obj=model_spec_obj) + + +if __name__ == '__main__': + generate_outputs(num_beams=1) diff --git a/cpp/tests/resources/scripts/generate_expected_redrafter_output.py b/cpp/tests/resources/scripts/generate_expected_redrafter_output.py new file mode 100644 index 000000000000..989e029a5ab1 --- /dev/null +++ b/cpp/tests/resources/scripts/generate_expected_redrafter_output.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse as _arg +import os +from pathlib import Path + +# isort: off +import run +# isort: on + +import tensorrt_llm.bindings as _tb +from tensorrt_llm.bindings.internal.testing import ModelSpec + + +def generate_output(engine: str, + model_spec_obj: ModelSpec, + max_output_len: int = 8): + + model = 'vicuna-7b-redrafter' + hf_model = 'vicuna-7b-v1.3' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + hf_dir = models_dir / hf_model + tp_pp_dir = 'tp1-pp1-cp1-gpu/' + engine_dir = models_dir / 'rt_engine' / model / engine / tp_pp_dir + + data_dir = resources_dir / 'data' + input_filename = model_spec_obj.get_input_file() + input_file = data_dir / input_filename + model_data_dir = data_dir / model + output_dir = model_data_dir / 'sampling' + + base_output_name = os.path.splitext(model_spec_obj.get_results_file())[0] + + args = run.parse_arguments([ + '--engine_dir', + str(engine_dir), + '--input_file', + str(input_file), + '--tokenizer_dir', + str(hf_dir), + '--output_npy', + str(output_dir / (base_output_name + '.npy')), + '--output_csv', + str(output_dir / (base_output_name + '.csv')), + '--max_output_len', + str(max_output_len), + '--use_py_session', + ]) + run.main(args) + print(f"Output saved at {str(output_dir / base_output_name)}.[npy|csv]") + + +def generate_outputs(): + print(f'Generating outputs for ReDrafter FP16') + max_output_len = 128 + model_spec_obj = ModelSpec('input_vicuna.npy', _tb.DataType.HALF) + model_spec_obj.use_gpt_plugin() + model_spec_obj.set_max_output_length(max_output_len) + model_spec_obj.use_packed_input() + model_spec_obj.set_kv_cache_type(_tb.KVCacheType.PAGED) + model_spec_obj.use_explicit_draft_tokens_decoding() + + generate_output(engine=model_spec_obj.get_model_path(), + model_spec_obj=model_spec_obj, + max_output_len=max_output_len) + + +if __name__ == '__main__': + parser = _arg.ArgumentParser() + args = parser.parse_args() + + generate_outputs() + print("Done") diff --git a/cpp/tests/resources/scripts/generate_hf_gpt_output.py b/cpp/tests/resources/scripts/generate_hf_gpt_output.py new file mode 100755 index 000000000000..a40ada8cb455 --- /dev/null +++ b/cpp/tests/resources/scripts/generate_hf_gpt_output.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import run_hf + + +def generate_hf_output(data_type: str, + output_name: str, + max_output_len: int = 8): + + model = 'gpt2' + resources_dir = Path(__file__).parent.resolve().parent + models_dir = resources_dir / 'models' + model_dir = models_dir / model + + data_dir = resources_dir / 'data' + input_file = data_dir / 'input_tokens.npy' + output_dir = data_dir / model / 'huggingface' + + run_hf.generate(model_dir=str(model_dir), + data_type=data_type, + input_file=str(input_file), + output_npy=str(output_dir / (output_name + '.npy')), + output_csv=str(output_dir / (output_name + '.csv')), + max_output_len=max_output_len) + + +def generate_hf_outputs(): + generate_hf_output(data_type='fp32', + output_name='output_tokens_fp32_huggingface') + generate_hf_output(data_type='fp16', + output_name='output_tokens_fp16_huggingface') + + +if __name__ == '__main__': + generate_hf_outputs() diff --git a/cpp/tests/resources/scripts/io_converter.py b/cpp/tests/resources/scripts/io_converter.py new file mode 100755 index 000000000000..0ed6413c5ac4 --- /dev/null +++ b/cpp/tests/resources/scripts/io_converter.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import csv +import os + +import numpy as np + + +def csv_to_npy(input_file, output_file, pad_id, verbose): + data = [] + with open(input_file, newline='') as csvfile: + csv_reader = csv.reader(csvfile, delimiter=',') + for line in csv_reader: + data.append([int(e) for e in line]) + max_input_length = max([len(x) for x in data]) + data = [row + [pad_id] * (max_input_length - len(row)) for row in data] + data = np.array(data, dtype='int32') + if (verbose): + print(data, data.dtype) + np.save(output_file, data) + + +def npy_to_csv(input_file, output_file, verbose): + data = np.load(input_file) + if (verbose): + print(data, data.dtype) + np.savetxt(output_file, data, delimiter=",", fmt='%i') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + 'input_file', + type=str, + help='Read token ids from this file. Must be csv or npy.') + parser.add_argument('output_file', + type=str, + help='Write token ids this file. Must be csv or npy.') + parser.add_argument( + '-p', + '--pad_id', + type=int, + help= + 'Token id used for padding csv input with different sequence lengths.', + default=-1) + parser.add_argument('-v', '--verbose', action="store_true") + args = parser.parse_args() + + _, input_ext = os.path.splitext(args.input_file) + _, output_ext = os.path.splitext(args.output_file) + + if (input_ext == '.csv' and output_ext == '.npy'): + print('Converting csv to npy') + csv_to_npy(args.input_file, args.output_file, args.pad_id, args.verbose) + elif (input_ext == '.npy' and output_ext == '.csv'): + print('Converting npy to csv') + npy_to_csv(args.input_file, args.output_file, args.verbose) + else: + print('unknown file extensions') diff --git a/cpp/tests/unit_tests/CMakeLists.txt b/cpp/tests/unit_tests/CMakeLists.txt index 9d22bd03b52a..034de457bb78 100644 --- a/cpp/tests/unit_tests/CMakeLists.txt +++ b/cpp/tests/unit_tests/CMakeLists.txt @@ -27,3 +27,4 @@ add_subdirectory(multi_gpu) add_subdirectory(layers) add_subdirectory(runtime) add_subdirectory(thop) +add_subdirectory(utils) diff --git a/cpp/tests/unit_tests/batch_manager/CMakeLists.txt b/cpp/tests/unit_tests/batch_manager/CMakeLists.txt index 75329d192a8c..9425d5d13c2a 100644 --- a/cpp/tests/unit_tests/batch_manager/CMakeLists.txt +++ b/cpp/tests/unit_tests/batch_manager/CMakeLists.txt @@ -30,5 +30,7 @@ add_gtest(microBatchSchedulerTest microBatchSchedulerTest.cpp) add_gtest(peftCacheManagerTest peftCacheManagerTest.cpp) add_gtest(staticThreadPoolTest staticThreadPoolTest.cpp) add_gtest(rnnCacheFormatterTest rnnCacheFormatterTest.cpp) +add_gtest(cudaGraphExecutorCacheTest cudaGraphExecutorCacheTest.cpp) add_gtest(agentTreeTest agentTreeTest.cpp) add_gtest(truncateBlocksTest truncateBlocksTest.cpp) +add_gtest(encDecBeamSearchTest encDecBeamSearchTest.cpp) diff --git a/cpp/tests/unit_tests/batch_manager/bufferIndexHolderTest.cpp b/cpp/tests/unit_tests/batch_manager/bufferIndexHolderTest.cpp index 31400831a5bb..daa4f284b95d 100644 --- a/cpp/tests/unit_tests/batch_manager/bufferIndexHolderTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/bufferIndexHolderTest.cpp @@ -18,7 +18,7 @@ #include "tensorrt_llm/batch_manager/baseTransBuffer.h" #include "tensorrt_llm/batch_manager/cacheTransBuffer.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <gtest/gtest.h> #include <memory> #include <optional> @@ -48,16 +48,6 @@ class ObservableTransBufferManager : public CacheTransBufferManager { return mConcurrenceRecvResource.mConcurrence.load(); } - - void configureIndexPoolsForTest(size_t count) - { - ASSERT_EQ(mConcurrenceSendResource.mConcurrence.load(), 0); - ASSERT_EQ(mConcurrenceRecvResource.mConcurrence.load(), 0); - mSendBufferCount = count; - mRecvBufferCount = count; - mConcurrenceSendResource.mBufferIndexFlag.assign(count, 0); - mConcurrenceRecvResource.mBufferIndexFlag.assign(count, 0); - } }; } // namespace @@ -79,6 +69,12 @@ class BufferIndexHolderLifecycleTest : public ::testing::TestWithParam<HolderCas protected: void SetUp() override { + setenv("TRTLLM_USE_UCX_KVCACHE", "1", 1); + // Move-assignment coverage requires two simultaneously held indices from either pool. + setenv("TRTLLM_REQUEST_KV_CACHE_CONCURRENT", "1", 1); + setenv("TRTLLM_KVCACHE_RECV_BUFFER_COUNT", "2", 1); + setenv("TRTLLM_KVCACHE_SEND_MAX_CONCURRENCY_NUM", "2", 1); + int constexpr numLayers = 2; int constexpr numHeads = 2; int constexpr sizePerHead = 8; @@ -95,13 +91,10 @@ class BufferIndexHolderLifecycleTest : public ::testing::TestWithParam<HolderCas mKv = std::make_unique<KVCacheManager>(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, std::vector<BlockManager::SizeType32>{kvMaxNumTokens}, - tensorrt_llm::DataType::kFLOAT, sinkTokenLength, stream, kvMaxNumTokens, kvMaxNumTokens, + nvinfer1::DataType::kFLOAT, sinkTokenLength, stream, kvMaxNumTokens, kvMaxNumTokens, /*enableBlockReuse=*/true, CacheType::kSELF, std::nullopt, nullptr, true); mKv->allocatePools(false); mTrans = std::make_unique<ObservableTransBufferManager>(mKv.get(), std::optional<size_t>{kvMaxNumTokens}); - // envUtils caches process-wide settings, so set up the index-only test - // pools directly instead of relying on test-order-sensitive setenv calls. - mTrans->configureIndexPoolsForTest(2); } void TearDown() override @@ -168,44 +161,6 @@ TEST_P(BufferIndexHolderLifecycleTest, NulloptIndexReleasesNothing) EXPECT_EQ(inUse(), before); } -// A nullopt holder owns no concrete slot but retains the manager binding needed -// to poison a dynamic buffer after an unquiesced transfer exit. -TEST_P(BufferIndexHolderLifecycleTest, NulloptIndexCanPoisonManager) -{ - int const before = inUse(); - BufferIndexHolder holder{mgr(), std::nullopt, isRecv()}; - EXPECT_FALSE(holder.held()); - - holder.poison(); - - EXPECT_FALSE(holder.held()); - EXPECT_TRUE(mgr().hasPoisonedBuffer()); - EXPECT_EQ(inUse(), before); -} - -// Poisoning a concrete slot fails the entire direction closed: the slot stays -// reserved and no later request can acquire another buffer from that pool. -TEST_P(BufferIndexHolderLifecycleTest, ValidIndexPoisonPreventsReuse) -{ - int const before = inUse(); - auto idx = acquire(); - ASSERT_TRUE(idx.has_value()); - BufferIndexHolder holder{mgr(), idx, isRecv()}; - EXPECT_EQ(inUse(), before + 1); - - holder.poison(); - - EXPECT_FALSE(holder.held()); - EXPECT_TRUE(mgr().hasPoisonedBuffer()); - EXPECT_EQ(inUse(), before + 1); - EXPECT_THROW((void) acquire(), std::exception); - - // poison() disarms the holder; neither an explicit release nor its - // destructor may return the quarantined slot to the pool. - holder.release(); - EXPECT_EQ(inUse(), before + 1); -} - // RAII: a held slot is released when the holder goes out of scope. TEST_P(BufferIndexHolderLifecycleTest, ValidIndexReleasedOnDestruction) { diff --git a/cpp/tests/unit_tests/batch_manager/cacheTransBufferTest.cpp b/cpp/tests/unit_tests/batch_manager/cacheTransBufferTest.cpp index 2fa0477d2352..8150c6fa5406 100644 --- a/cpp/tests/unit_tests/batch_manager/cacheTransBufferTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/cacheTransBufferTest.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/batch_manager/cacheTransBuffer.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -52,7 +51,7 @@ class CacheTransBufferTest : public ::testing::Test auto constexpr blocksInSecondaryPool = 0; auto constexpr enableBlockReuse = true; - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {totalNumBlocks, blocksInSecondaryPool}}}; diff --git a/cpp/tests/unit_tests/batch_manager/capacitySchedulerTest.cpp b/cpp/tests/unit_tests/batch_manager/capacitySchedulerTest.cpp index 2c9757b847b2..7bb87c91e361 100644 --- a/cpp/tests/unit_tests/batch_manager/capacitySchedulerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/capacitySchedulerTest.cpp @@ -31,7 +31,7 @@ #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferPlugin.h> #include <cstdlib> #include <functional> @@ -132,7 +132,7 @@ class CapacitySchedulerTest : public ::testing::Test // NOLINT(cppcoreguidelines auto const nbKvHeads = 10; auto constexpr sizePerHead = 1; auto const maxNumBlocks = tc::divUp(maxNumTokens, tokensPerBlock); - auto const kvDtype = tensorrt_llm::DataType::kHALF; + auto const kvDtype = nvinfer1::DataType::kHALF; CudaStreamPtr streamPtr = std::make_shared<tensorrt_llm::runtime::CudaStream>(); using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; diff --git a/cpp/tests/unit_tests/batch_manager/cudaGraphExecutorCacheTest.cpp b/cpp/tests/unit_tests/batch_manager/cudaGraphExecutorCacheTest.cpp new file mode 100644 index 000000000000..f8ae0a9db64f --- /dev/null +++ b/cpp/tests/unit_tests/batch_manager/cudaGraphExecutorCacheTest.cpp @@ -0,0 +1,161 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h" + +#include <gtest/gtest.h> + +#include <memory> + +namespace tb = tensorrt_llm::batch_manager; +namespace tbu = tensorrt_llm::batch_manager::utils; +using SizeType32 = tensorrt_llm::runtime::SizeType32; + +namespace +{ +// A default-constructed CudaGraphExecutor holds mInstance == nullptr, so its destructor +// is a no-op and these tests do not require an active CUDA context. +std::shared_ptr<tbu::CudaGraphExecutor> makeDummyExecutor() +{ + return std::make_shared<tbu::CudaGraphExecutor>(); +} + +tb::BatchState makeBatchState(SizeType32 numTokens) +{ + return tb::BatchState{/*numCtxRequests=*/0, /*numGenRequests=*/1, numTokens, /*maxKvCacheLength=*/256}; +} +} // namespace + +class CudaGraphExecutorCacheTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) +{ +}; + +TEST_F(CudaGraphExecutorCacheTest, EmptyByDefault) +{ + tbu::CudaGraphExecutorCache cache(/*capacity=*/4); + EXPECT_EQ(cache.size(), 0); + EXPECT_FALSE(cache.get(makeBatchState(1)).has_value()); +} + +TEST_F(CudaGraphExecutorCacheTest, PutAndGetReturnsSameInstance) +{ + tbu::CudaGraphExecutorCache cache(/*capacity=*/4); + + auto bs = makeBatchState(1); + auto exec = makeDummyExecutor(); + cache.put(bs, exec); + + ASSERT_EQ(cache.size(), 1); + auto got = cache.get(bs); + ASSERT_TRUE(got.has_value()); + EXPECT_EQ(got->get(), exec.get()); +} + +TEST_F(CudaGraphExecutorCacheTest, PutWithExistingKeyReplaces) +{ + tbu::CudaGraphExecutorCache cache(/*capacity=*/4); + + auto bs = makeBatchState(1); + auto first = makeDummyExecutor(); + auto second = makeDummyExecutor(); + + cache.put(bs, first); + cache.put(bs, second); + + EXPECT_EQ(cache.size(), 1); + auto got = cache.get(bs); + ASSERT_TRUE(got.has_value()); + EXPECT_EQ(got->get(), second.get()); +} + +TEST_F(CudaGraphExecutorCacheTest, EvictsLeastRecentlyUsedAtCapacity) +{ + tbu::CudaGraphExecutorCache cache(/*capacity=*/2); + + auto bsA = makeBatchState(1); + auto bsB = makeBatchState(2); + auto bsC = makeBatchState(3); + + auto execA = makeDummyExecutor(); + auto execB = makeDummyExecutor(); + auto execC = makeDummyExecutor(); + + cache.put(bsA, execA); + cache.put(bsB, execB); + ASSERT_EQ(cache.size(), 2); + + // Access A so that B becomes the LRU entry. + EXPECT_TRUE(cache.get(bsA).has_value()); + + // Inserting C must evict B (the LRU), not A (just touched). + cache.put(bsC, execC); + EXPECT_EQ(cache.size(), 2); + EXPECT_TRUE(cache.get(bsA).has_value()); + EXPECT_FALSE(cache.get(bsB).has_value()); + EXPECT_TRUE(cache.get(bsC).has_value()); +} + +TEST_F(CudaGraphExecutorCacheTest, ClearDropsAllEntries) +{ + tbu::CudaGraphExecutorCache cache(/*capacity=*/4); + + auto bsA = makeBatchState(1); + auto bsB = makeBatchState(2); + auto bsC = makeBatchState(3); + + cache.put(bsA, makeDummyExecutor()); + cache.put(bsB, makeDummyExecutor()); + cache.put(bsC, makeDummyExecutor()); + ASSERT_EQ(cache.size(), 3); + + cache.clear(); + + EXPECT_EQ(cache.size(), 0); + EXPECT_FALSE(cache.get(bsA).has_value()); + EXPECT_FALSE(cache.get(bsB).has_value()); + EXPECT_FALSE(cache.get(bsC).has_value()); + + // After clearing, the cache must remain functional (i.e. clear() must not + // leave it in a broken state). + auto execA2 = makeDummyExecutor(); + cache.put(bsA, execA2); + EXPECT_EQ(cache.size(), 1); + auto got = cache.get(bsA); + ASSERT_TRUE(got.has_value()); + EXPECT_EQ(got->get(), execA2.get()); +} + +TEST_F(CudaGraphExecutorCacheTest, ClearReleasesExecutorOwnership) +{ + tbu::CudaGraphExecutorCache cache(/*capacity=*/4); + + auto exec = makeDummyExecutor(); + std::weak_ptr<tbu::CudaGraphExecutor> weak = exec; + + cache.put(makeBatchState(1), exec); + exec.reset(); + // The cache still owns one strong reference at this point. + ASSERT_FALSE(weak.expired()); + + cache.clear(); + + // After clear(), no strong references should remain. This guarantees that + // ~CudaGraphExecutor (which calls cudaGraphExecDestroy) actually runs for + // every cached entry - exactly what changeBeamWidth() relies on. + EXPECT_TRUE(weak.expired()); +} diff --git a/cpp/tests/unit_tests/batch_manager/encDecBeamSearchTest.cpp b/cpp/tests/unit_tests/batch_manager/encDecBeamSearchTest.cpp new file mode 100644 index 000000000000..8e96c6a52ee3 --- /dev/null +++ b/cpp/tests/unit_tests/batch_manager/encDecBeamSearchTest.cpp @@ -0,0 +1,154 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/batch_manager/kvCacheManager.h" +#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/runtimeBuffers.h" +#include "tensorrt_llm/batch_manager/utils/inflightBatchingUtils.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/kernels/kvCacheIndex.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/cudaStream.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/samplingConfig.h" +#include "gtest/gtest.h" +#include <memory> + +using namespace tensorrt_llm::batch_manager; +using namespace tensorrt_llm::batch_manager::kv_cache_manager; +namespace tr = tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; +namespace tk = tensorrt_llm::kernels; +using SizeType32 = tr::SizeType32; + +// Verify that copyGenerationLogits correctly assembles the host logits buffer +// using the real kernel merge path, and that two back-to-back calls (simulating +// two requests flushing in the same batch) use distinct fragmentPointerDevice +// slots so their pointer arrays do not clobber each other. +TEST(CopyGenerationLogitsTest, KernelMergePathProducesCorrectHostLayoutAndSlotsAreIsolated) +{ + SizeType32 constexpr beamWidth = 2; + SizeType32 constexpr numSteps = RuntimeBuffers::GenerationLogitsCache::kCACHE_LENGTH; // full flush + SizeType32 constexpr vocabSize = 8; + SizeType32 constexpr promptLen = 1; + SizeType32 constexpr maxBatchSize = 4; // must be >= 2 to test slot isolation + + auto stream = std::make_shared<tr::CudaStream>(); + tr::BufferManager bufferMgr{stream}; + + // Build a real GenerationLogitsCache so that transposedLogits, + // fragmentPointerDevice and fragmentPointerHost are all properly allocated. + // cache.logits uses pinned memory so the test can fill it from the CPU while + // the GPU kernel can still read from it via DMA. + RuntimeBuffers::GenerationLogitsCache cache; + cache.logits = tr::BufferManager::pinnedPool( + tr::ITensor::makeShape({numSteps, maxBatchSize * beamWidth, vocabSize}), nvinfer1::DataType::kFLOAT); + cache.transposedLogits + = bufferMgr.gpu(tr::ITensor::makeShape({beamWidth, numSteps, vocabSize}), nvinfer1::DataType::kFLOAT); + cache.fragmentPointerDevice + = bufferMgr.gpu(tr::ITensor::makeShape({maxBatchSize, numSteps}), nvinfer1::DataType::kINT64); + cache.fragmentPointerHost + = tr::BufferManager::pinnedPool(tr::ITensor::makeShape({maxBatchSize, numSteps}), nvinfer1::DataType::kINT64); + + // Helper: build one LlmRequest that has numSteps fragments pointing into + // cache.logits[0..numSteps-1][logitsIndex:logitsIndex+beamWidth]. + // Each fragment is filled with sentinel value (step*100 + beam + reqOffset). + auto makeRequest = [&](RequestIdType reqId, SizeType32 logitsIndex, float reqOffset) -> std::shared_ptr<LlmRequest> + { + auto tokens = std::make_shared<VecTokens>(promptLen, 0); + tr::SamplingConfig sc{beamWidth}; + auto req = std::make_shared<LlmRequest>(reqId, numSteps, tokens, sc, false); + + LlmRequest::BeamTokens gen(beamWidth, VecTokens(numSteps, 1)); + req->setGeneratedTokens(gen); + req->allocGenerationLogitsHost(vocabSize, nvinfer1::DataType::kFLOAT); + + // Write known values into the logits cache slots for this request and + // create matching fragment slice views. + for (SizeType32 step = 0; step < numSteps; ++step) + { + // cache.logits shape: [numSteps, maxBatchSize*beamWidth, vocabSize] + // Slice to [1, maxBS*bw, vocab], squeeze to [maxBS*bw, vocab]. + tr::ITensor::SharedPtr slot = tr::ITensor::slice(cache.logits, step, 1); + slot->squeeze(0); // [maxBS*bw, vocab] + auto* slotPtr = tr::bufferCast<float>(*slot); + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + float const val = reqOffset + static_cast<float>(step * 100 + beam); + for (SizeType32 v = 0; v < vocabSize; ++v) + { + slotPtr[(logitsIndex + beam) * vocabSize + v] = val; + } + } + + // Fragment matches HandleGenerationLogits: slice [logitsIndex:logitsIndex+beamWidth] + // from the step slot, then unsqueeze(0) → [1, beamWidth, vocab]. + tr::ITensor::SharedPtr fragView = tr::ITensor::slice(slot, logitsIndex, beamWidth); + fragView->unsqueeze(0); // [1, beamWidth, vocab] + req->addGenerationLogitsFragment(fragView); + } + return req; + }; + + // Request 0 occupies logitsIndex=0 in the batch slot. + auto req0 = makeRequest(1, /*logitsIndex=*/0, /*reqOffset=*/0.0f); + // Request 1 occupies logitsIndex=beamWidth in the batch slot. + auto req1 = makeRequest(2, /*logitsIndex=*/beamWidth, /*reqOffset=*/1000.0f); + + // Flush request 0 — uses workIdx=0. + utils::copyGenerationLogits(cache, bufferMgr, *req0, /*beforeDecoder=*/false, {}); + // Flush request 1 — uses workIdx=1 (different slot → no pointer clobbering). + utils::copyGenerationLogits(cache, bufferMgr, *req1, /*beforeDecoder=*/false, {}); + + ASSERT_EQ(cudaStreamSynchronize(stream->get()), cudaSuccess); + + // Verify req0 host buffer: host[beam, step, v] == step*100 + beam + auto const* host0 = tr::bufferCast<float>(*req0->getGenerationLogitsHost()); + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + for (SizeType32 step = 0; step < numSteps; ++step) + { + float const expected = static_cast<float>(step * 100 + beam); + for (SizeType32 v = 0; v < vocabSize; ++v) + { + SizeType32 const idx = (beam * numSteps + step) * vocabSize + v; + EXPECT_FLOAT_EQ(host0[idx], expected) << "req0 host[beam=" << beam << ",step=" << step << ",v=" << v + << "]=" << host0[idx] << " expected " << expected; + } + } + } + + // Verify req1 host buffer: host[beam, step, v] == 1000 + step*100 + beam + auto const* host1 = tr::bufferCast<float>(*req1->getGenerationLogitsHost()); + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + for (SizeType32 step = 0; step < numSteps; ++step) + { + float const expected = 1000.0f + static_cast<float>(step * 100 + beam); + for (SizeType32 v = 0; v < vocabSize; ++v) + { + SizeType32 const idx = (beam * numSteps + step) * vocabSize + v; + EXPECT_FLOAT_EQ(host1[idx], expected) << "req1 host[beam=" << beam << ",step=" << step << ",v=" << v + << "]=" << host1[idx] << " expected " << expected; + } + } + } + + // Both requests must have had their fragments cleared. + EXPECT_EQ(req0->getGenerationLogitsFragmentsSize(), 0); + EXPECT_EQ(req1->getGenerationLogitsFragmentsSize(), 0); +} diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerFabricMemoryTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerFabricMemoryTest.cpp index 4d41d751476c..d23cbabe4144 100644 --- a/cpp/tests/unit_tests/batch_manager/kvCacheManagerFabricMemoryTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerFabricMemoryTest.cpp @@ -20,7 +20,6 @@ #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/samplingConfig.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" @@ -109,7 +108,7 @@ TEST_F(KVCacheManagerFabricMemoryTest, AllocatePoolsFallbackWhenFabricUnsupporte BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, 0); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, 0); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -148,7 +147,7 @@ TEST_F(KVCacheManagerFabricMemoryTest, AllocatePoolsWithFabricMemory) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, 0); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, 0); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getMaxNumBlocks(), blocksInPrimaryPool); @@ -192,7 +191,7 @@ TEST_F(KVCacheManagerFabricMemoryTest, OffloadOnboardRoundTripWithFabricPrimary) BlockManager blockManager(std::vector<BlockManager::SizeType32>(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, 0); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, 0); blockManager.allocatePools(false); auto primaryPoolPtr = blockManager.getPrimaryPool(0); @@ -293,7 +292,7 @@ TEST_F(KVCacheManagerFabricMemoryTest, ReleasePoolsClearsFabricMemory) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, 0); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, 0); size_t freeBefore = 0; size_t freeAfterAlloc = 0; diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp index 98c2232b4062..1d21b94a45ad 100644 --- a/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/kvCacheManagerTest.cpp @@ -26,7 +26,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/transferAgent.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/kernels/kvCacheIndex.h" @@ -175,8 +174,7 @@ TEST_F(KVCacheManagerTest, BlockManagerTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, - maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -312,7 +310,7 @@ void writePatternToOffloadedBlocksGDS( } } -template <typename T, tensorrt_llm::DataType type, int mask, KvCacheTransferMode transferMode> +template <typename T, nvinfer1::DataType type, int mask, KvCacheTransferMode transferMode> void runPartialCopyTest() { auto constexpr numLayers = 12; @@ -523,59 +521,59 @@ void runPartialCopyTest() TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyINT64) { - runPartialCopyTest<std::uint64_t, tensorrt_llm::DataType::kINT64, -1, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint64_t, tensorrt_llm::DataType::kINT64, -1, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint64_t, nvinfer1::DataType::kINT64, -1, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint64_t, nvinfer1::DataType::kINT64, -1, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyINT32) { - runPartialCopyTest<std::uint32_t, tensorrt_llm::DataType::kINT32, -1, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint32_t, tensorrt_llm::DataType::kINT32, -1, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint32_t, nvinfer1::DataType::kINT32, -1, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint32_t, nvinfer1::DataType::kINT32, -1, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyFLOAT) { - runPartialCopyTest<std::uint32_t, tensorrt_llm::DataType::kFLOAT, -1, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint32_t, tensorrt_llm::DataType::kFLOAT, -1, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint32_t, nvinfer1::DataType::kFLOAT, -1, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint32_t, nvinfer1::DataType::kFLOAT, -1, KvCacheTransferMode::GDS>(); } #ifdef ENABLE_BF16 TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyBF16) { - runPartialCopyTest<std::uint16_t, tensorrt_llm::DataType::kBF16, 65535, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint16_t, tensorrt_llm::DataType::kBF16, 65535, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint16_t, nvinfer1::DataType::kBF16, 65535, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint16_t, nvinfer1::DataType::kBF16, 65535, KvCacheTransferMode::GDS>(); } #endif TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyHALF) { - runPartialCopyTest<std::uint16_t, tensorrt_llm::DataType::kHALF, 65535, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint16_t, tensorrt_llm::DataType::kHALF, 65535, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint16_t, nvinfer1::DataType::kHALF, 65535, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint16_t, nvinfer1::DataType::kHALF, 65535, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyBOOL) { - runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kBOOL, 255, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kBOOL, 255, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kBOOL, 255, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kBOOL, 255, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyUINT8) { - runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kUINT8, 255, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kUINT8, 255, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kUINT8, 255, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kUINT8, 255, KvCacheTransferMode::GDS>(); } TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyINT8) { - runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kINT8, 255, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kINT8, 255, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kINT8, 255, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kINT8, 255, KvCacheTransferMode::GDS>(); } #ifdef ENABLE_FP8 TEST_F(KVCacheManagerTest, BlockManagerTestPartialCopyFP8) { - runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kFP8, 255, KvCacheTransferMode::DRAM>(); - runPartialCopyTest<std::uint8_t, tensorrt_llm::DataType::kFP8, 255, KvCacheTransferMode::GDS>(); + runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kFP8, 255, KvCacheTransferMode::DRAM>(); + runPartialCopyTest<std::uint8_t, nvinfer1::DataType::kFP8, 255, KvCacheTransferMode::GDS>(); } #endif @@ -733,8 +731,8 @@ TEST_F(KVCacheManagerTest, FindBlocksInReuseTreeByBlockKeysTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, false, - stream, maxAttentionWindow, maxAttentionWindow, true); + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, false, stream, + maxAttentionWindow, maxAttentionWindow, true); // Add sequence [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16] (17 tokens, three blocks) auto inputTokens = std::make_shared<VecTokens>(VecTokens{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}); @@ -785,8 +783,8 @@ TEST_F(KVCacheManagerTest, FP4BlockScaleManagementTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kFP4, false, - stream, maxAttentionWindow, maxAttentionWindow, true); + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kFP4, false, stream, + maxAttentionWindow, maxAttentionWindow, true); kvCacheManager.allocatePools(/*useUvm=*/false); @@ -801,90 +799,6 @@ TEST_F(KVCacheManagerTest, FP4BlockScaleManagementTest) // The expected block size of pool 1 should be the number of FP4 elements / vectorSize. EXPECT_EQ(blockManager.getBlockSize(0) * numFp4EltsPerContainer / vectorSize, blockManager.getBlockSize(1)); } - -TEST_F(KVCacheManagerTest, FP4AttentionWithHalfRecurrentStatesPoolTest) -{ - auto constexpr numKvHeads = 2; - auto constexpr sizePerHead = 16; - auto constexpr tokensPerBlock = 4; - auto constexpr blocksInPrimaryPool = 4; - auto constexpr blocksInSecondaryPool = 0; - auto constexpr maxNumSequences = 2; - auto constexpr maxBeamWidth = 1; - auto constexpr maxAttentionWindow = 16; - auto constexpr recurrentStatesBytes = 64; - SizeType32 constexpr recurrentStatesWindow = LinearAttentionMetadata::LinearCacheType::kRecurrentStates; - - LinearAttentionMetadata const linearAttentionMetadata{ - .linearLayerIndices = {0}, - .cacheType = recurrentStatesWindow, - .allRecurrentStatesBytes = recurrentStatesBytes, - }; - auto const blocksPerWindow = BlocksPerWindow{ - {recurrentStatesWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}, - {maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}, - }; - auto const poolConfigurations = std::vector<PoolConfiguration>{ - {recurrentStatesWindow, sizePerHead, tensorrt_llm::DataType::kHALF}, - {maxAttentionWindow, sizePerHead, tensorrt_llm::DataType::kFP4}, - }; - auto const stream = std::make_shared<tr::CudaStream>(); - - KVCacheManager kvCacheManager(std::vector<SizeType32>{0, numKvHeads}, sizePerHead, tokensPerBlock, blocksPerWindow, - maxNumSequences, maxBeamWidth, std::vector<SizeType32>{recurrentStatesWindow, maxAttentionWindow}, - tensorrt_llm::DataType::kFP4, - /*sinkTokenLength=*/0, stream, maxAttentionWindow, /*chunkSize=*/0, /*enableBlockReuse=*/false, - CacheType::kSELF, std::nullopt, nullptr, /*enablePartialReuse=*/false, /*copyOnPartialReuse=*/true, nullptr, - /*enableIndexerKCache=*/false, /*indexerKCacheQuantBlockSize=*/128, /*indexerKCacheIndexHeadDim=*/0, - /*indexerKCacheUseFp4=*/false, linearAttentionMetadata, poolConfigurations); - kvCacheManager.allocatePools(/*useUvm=*/false); - auto const& blockManager = kvCacheManager.getBlockManager(); - - EXPECT_EQ(blockManager.getDataTypeForWindow(recurrentStatesWindow), tensorrt_llm::DataType::kHALF); - EXPECT_EQ(blockManager.getDataTypeForWindow(maxAttentionWindow), tensorrt_llm::DataType::kFP4); - - auto const& recurrentStatesPool = blockManager.getRecurrentStatesPool(); - ASSERT_NE(recurrentStatesPool.primaryPtr, nullptr); - EXPECT_EQ(recurrentStatesPool.primaryPtr->getDataType(), tensorrt_llm::DataType::kHALF); - auto const recurrentStatesElementsPerBlock = recurrentStatesBytes / tc::getDTypeSize(tensorrt_llm::DataType::kHALF); - EXPECT_EQ(recurrentStatesPool.blockSize, recurrentStatesElementsPerBlock); - - SizeType32 numRecurrentScalePools = 0; - SizeType32 numAttentionScalePools = 0; - for (SizeType32 poolIdx = 0; poolIdx < blockManager.getNumPools(); ++poolIdx) - { - if (!blockManager.containsBlockScales(poolIdx)) - { - continue; - } - if (blockManager.getPoolWindowSize(poolIdx) == recurrentStatesWindow) - { - ++numRecurrentScalePools; - } - else if (blockManager.getPoolWindowSize(poolIdx) == maxAttentionWindow) - { - ++numAttentionScalePools; - } - } - EXPECT_EQ(numRecurrentScalePools, 0); - EXPECT_EQ(numAttentionScalePools, 1); - - auto const blockPoolPointers = kvCacheManager.getBlockPoolPointers(); - auto const blockScalePoolPointers = kvCacheManager.getBlockScalePoolPointers(); - ASSERT_NE(blockPoolPointers, nullptr); - ASSERT_NE(blockScalePoolPointers, nullptr); - EXPECT_EQ(blockPoolPointers->getShape().d[0], 2); - EXPECT_EQ(blockScalePoolPointers->getShape().d[0], 1); - auto const blockScalePoolPointersRange = tr::BufferRange<void*>(*blockScalePoolPointers); - EXPECT_NE(blockScalePoolPointersRange[0], nullptr); - EXPECT_EQ(blockScalePoolPointersRange[1], nullptr); - - auto const layerToPoolMapping = kvCacheManager.getLayerToPoolMapping(); - ASSERT_NE(layerToPoolMapping, nullptr); - auto const layerToPoolMappingRange = tr::BufferRange<SizeType32>(*layerToPoolMapping); - EXPECT_EQ(layerToPoolMappingRange[0], 0); - EXPECT_EQ(layerToPoolMappingRange[2], 1); -} #endif TEST_F(KVCacheManagerTest, BlockManagerReuseTest) @@ -907,8 +821,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, - maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -1243,8 +1156,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, - maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -1479,8 +1391,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithMultimodalHashTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, - maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -1684,8 +1595,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithLoraTaskIdTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, - maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -1978,8 +1888,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithExtraIdAndLoraTaskIdTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, - maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -2248,8 +2157,7 @@ TEST_F(KVCacheManagerTest, BlockManagerReuseWithCacheSaltTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, - maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -2481,7 +2389,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerPerRequestStatsTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -2538,8 +2446,7 @@ TEST_F(KVCacheManagerTest, BlockManagerBlockPriorityTest) BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, - maxAttentionWindow); + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, maxAttentionWindow); blockManager.allocatePools(false); EXPECT_EQ(blockManager.getTokensPerBlock(), tokensPerBlock); @@ -2677,7 +2584,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerDecodeBlockPriorityTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -2784,7 +2691,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerTimedEvictionTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -2856,7 +2763,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerDecodeTimedEvictionTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); { @@ -2949,7 +2856,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerSecondaryBlockPrimaryChildTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -3046,7 +2953,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerStoreContextBlocksUsesMaterializedConte auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, /*chunkSize*/ 0, true); kvCacheManager.allocatePools(false); @@ -3090,7 +2997,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerReleaseBlocksUsesMaterializedContextExt auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, /*chunkSize*/ 0, true); kvCacheManager.allocatePools(false); @@ -3132,7 +3039,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerLeafBlockTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -3217,7 +3124,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerLeafBlockWithDependentTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true); kvCacheManager.allocatePools(false); @@ -3321,7 +3228,7 @@ TEST_P(KVCacheManagerTest, DISABLED_KVCacheManagerAllocationTest) auto constexpr maxNumSequences = 8; auto constexpr maxBeamWidth = 4; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr maxSequenceLength = tokensPerBlock * maxBlocksPerSeq; @@ -3344,12 +3251,11 @@ TEST_P(KVCacheManagerTest, DISABLED_KVCacheManagerAllocationTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(std::vector<KVCacheManager::SizeType32>(numLayers, numHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, - enableBlockReuse); + nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); auto const& blockManager = kvCacheManager.getBlockManager(); auto const& bufferManager = blockManager.getBufferManager(theOnlyWindowSize(kvCacheManager)); @@ -3420,10 +3326,10 @@ TEST_P(KVCacheManagerTest, KVCacheManagerTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -3590,12 +3496,11 @@ TEST_P(KVCacheManagerTest, KVCacheManagerRewindTokensTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(std::vector<KVCacheManager::SizeType32>(numLayers, numHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, - enableBlockReuse); + nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); EXPECT_EQ(kvCacheManager.getTokensPerBlock(), tokensPerBlock); @@ -3699,10 +3604,10 @@ TEST_P(KVCacheManagerTest, KVCacheManagerMaxAttentionWindowTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -3823,7 +3728,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerMaxAttentionWindowSmallerThanBlockSizeT auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -3917,7 +3822,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStream) auto constexpr blocksInPrimaryPool = 8; auto constexpr blocksInSecondaryPool = 2; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -4101,7 +4006,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerMaxAttentionWindowWithReuseTest) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/tokensPerBlock, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -4228,7 +4133,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerSWAInvalidateReuseTest) auto constexpr maxNumSequences = 8; auto constexpr maxBeamWidth = 1; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr maxSequenceLength = 128; SizeType32 constexpr maxNewTokens = 40; @@ -4311,7 +4216,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerVariableWindowAttentionWithReuseTest) auto constexpr maxNumSequences = 8; auto constexpr maxBeamWidth = 1; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr maxSequenceLength = 128; @@ -4437,7 +4342,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamOverflow) auto constexpr blocksInPrimaryPool = 8; auto constexpr blocksInSecondaryPool = 2; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -4497,7 +4402,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamPriority) auto constexpr blocksInPrimaryPool = 8; auto constexpr blocksInSecondaryPool = 2; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -4574,7 +4479,7 @@ TEST_F(KVCacheManagerTest, GetPriorityByBlockId) auto constexpr maxAttentionWindow = 32; auto constexpr maxNumSequences = 4; auto constexpr beamWidth = 1; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens = 4; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -4640,7 +4545,7 @@ TEST_F(KVCacheManagerTest, CommitAndGetBlockHashesForRequest) auto constexpr maxNumSequences = 4; auto constexpr beamWidth = 1; auto constexpr beamIdx = 0; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens = 8; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -4766,7 +4671,7 @@ TEST_F(KVCacheManagerTest, CommitAndGetBlockHashesFrontRunsTrailingFullBlock) auto constexpr maxNumSequences = 4; auto constexpr beamWidth = 1; auto constexpr beamIdx = 0; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens = 8; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -4882,7 +4787,7 @@ TEST_F(KVCacheManagerTest, PinAndUnpinBlocksById) BlocksPerWindow const blocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxAttentionWindow, maxAttentionWindow, true); kvCacheManager.allocatePools(false); @@ -4934,7 +4839,7 @@ TEST_F(KVCacheManagerTest, StoreBlocksForReuseWithPinDoesNotCreateGhostFreeBlock BlocksPerWindow const blocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxAttentionWindow, maxAttentionWindow, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -5006,7 +4911,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamBlocking) auto constexpr blocksInPrimaryPool = 8; auto constexpr blocksInSecondaryPool = 2; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -5026,7 +4931,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamBlocking) EXPECT_EQ(getEvents(kvCacheManagerTest).size(), 0); KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength, true, CacheType::kSELF, std::nullopt, std::make_unique<tlk::KVCacheEventManager>(1024)); @@ -5061,7 +4966,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStreamWindowSize) auto blocksInPool = std::vector<SizeType32>{8, 2}; auto blocksInSlidingWindowPool = std::vector<SizeType32>{4, 2}; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr beamWidth = 1; @@ -5120,11 +5025,10 @@ TEST_F(KVCacheManagerTest, KVCacheTransferManagerConcurrencyTest) auto pool = KVCacheBlockPool(0, 2, 0, 0, 0); - pool.primaryPtr = bufferManager.gpu(tr::ITensor::makeShape({1, blockSize}), tensorrt_llm::DataType::kFLOAT); + pool.primaryPtr = bufferManager.gpu(tr::ITensor::makeShape({1, blockSize}), nvinfer1::DataType::kFLOAT); bufferManager.setZero(*pool.primaryPtr); - pool.secondaryPtr - = tr::BufferManager::pinned(tr::ITensor::makeShape({1, blockSize}), tensorrt_llm::DataType::kFLOAT); + pool.secondaryPtr = tr::BufferManager::pinned(tr::ITensor::makeShape({1, blockSize}), nvinfer1::DataType::kFLOAT); // Write some specific data into the cpu blocks. for (int i = 0; i < blockSize; i++) @@ -5161,11 +5065,11 @@ TEST_F(KVCacheManagerTest, KVCacheTransferManagerPendingTransfersDistinguishPrim auto pool = KVCacheBlockPool(0, 2, 0, 0, 0); pool.primaryPtr - = bufferManager.gpu(tr::ITensor::makeShape({kNumSlotsPerPool, kBlockSize}), tensorrt_llm::DataType::kFLOAT); + = bufferManager.gpu(tr::ITensor::makeShape({kNumSlotsPerPool, kBlockSize}), nvinfer1::DataType::kFLOAT); bufferManager.setZero(*pool.primaryPtr); - pool.secondaryPtr = tr::BufferManager::pinned( - tr::ITensor::makeShape({kNumSlotsPerPool, kBlockSize}), tensorrt_llm::DataType::kFLOAT); + pool.secondaryPtr + = tr::BufferManager::pinned(tr::ITensor::makeShape({kNumSlotsPerPool, kBlockSize}), nvinfer1::DataType::kFLOAT); auto primarySlot0 = std::make_shared<KVCacheBlock>(0, tk::KVCacheIndex(0, false)); auto primarySlot1 = std::make_shared<KVCacheBlock>(1, tk::KVCacheIndex(1, false)); @@ -5252,10 +5156,10 @@ TEST_P(KVCacheManagerTest, DISABLED_KVCacheManagerSinkTokenLengthTest) auto const maxSequenceLength = tokensPerBlock * maxBlocksPerSeq; KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -5414,10 +5318,10 @@ TEST_P(KVCacheManagerTest, KVCacheManagerBatchTest) KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, maxBeamWidth, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, sinkTokenLength, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, maxSequenceLength, enableBlockReuse); kvCacheManager.allocatePools(false); @@ -5554,12 +5458,12 @@ void testNeededBlocksOneStep(bool kv_cache_block_reuse, int beamWidth, int draft KVCacheManager kvCacheManager = homogeneousLayers ? KVCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/tokensPerBlock, kv_cache_block_reuse) : KVCacheManager(numHeadsPerLayer, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - tensorrt_llm::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, + maxBeamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, + sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/tokensPerBlock, kv_cache_block_reuse); kvCacheManager.allocatePools(false); @@ -5762,7 +5666,7 @@ struct KvCacheManagerInstantiationParameters SizeType32 maxNumTokens; bool kvCacheBlockReuse; std::vector<SizeType32> maxAttentionWindowVec = {maxAttentionWindow}; - tensorrt_llm::DataType dtype = tensorrt_llm::DataType::kFLOAT; + nvinfer1::DataType dtype = nvinfer1::DataType::kFLOAT; }; BlocksPerWindow blocksAndWindow(SizeType32 numPrimaryBlocks, SizeType32 windowSize) @@ -7258,7 +7162,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventRemovedBatchedWithinWindow) auto constexpr maxNumSequences = 4; auto constexpr maxAttentionWindow = 32; auto constexpr beamWidth = 1; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens{0}; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -7336,7 +7240,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventRemovedOrderedBeforeStore) auto constexpr maxNumSequences = 4; auto constexpr maxAttentionWindow = 32; auto constexpr beamWidth = 1; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens{0}; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -7430,7 +7334,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerEventStoreForDifferentWindowDoesNotFlus auto constexpr blocksInSecondaryPool = 0; auto constexpr maxNumSequences = 4; auto constexpr beamWidth = 1; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); SizeType32 constexpr maxNewTokens{0}; tr::SamplingConfig const samplingConfig{beamWidth}; @@ -7543,8 +7447,7 @@ void testBlockManagerLinearAttention_ContextNoReuse(int beamWidth, int numTokens BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, tensorrt_llm::DataType::kHALF, - 0, + std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, /*chunkSize*/ 0, CacheType::kSELF, std::nullopt, nullptr, false, true, nullptr, std::nullopt, false, 128, 0, false, linearAttentionMetadata); blockManager.allocatePools(false); @@ -7689,8 +7592,7 @@ void testBlockManagerLinearAttention_ContextReuse(int beamWidth, int numTokens0, BlockManager blockManager(std::vector(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, maxAttentionWindow, beamWidth, - std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, tensorrt_llm::DataType::kHALF, - 0, + std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, /*chunkSize*/ 0, CacheType::kSELF, std::nullopt, nullptr, false, true, nullptr, std::nullopt, false, 128, 0, false, linearAttentionMetadata); blockManager.allocatePools(false); @@ -7914,7 +7816,7 @@ void testKVCacheManagerLinearAttention_DecodingBlockGrowth( {linearWindowSizeCode, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, beamWidth, std::vector<BlockManager::SizeType32>{linearWindowSizeCode}, - /*dtype*/ tensorrt_llm::DataType::kHALF, + /*dtype*/ nvinfer1::DataType::kHALF, /*sinkTokenLen*/ sinkTokenLen, /*stream*/ stream, /*maxSequenceLength*/ maxAttentionWindow, @@ -8026,7 +7928,7 @@ void testKVCacheManagerLinearAttention_BlockCopying( {linearWindowSizeCode, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, beamWidth, std::vector<BlockManager::SizeType32>{linearWindowSizeCode, maxAttentionWindow}, - tensorrt_llm::DataType::kHALF, sinkTokenLen, stream, maxAttentionWindow, /*chunkSize*/ 0, enableContextReuse, + nvinfer1::DataType::kHALF, sinkTokenLen, stream, maxAttentionWindow, /*chunkSize*/ 0, enableContextReuse, CacheType::kSELF, std::nullopt, nullptr, false, true, nullptr, false, 128, 0, false, linearAttentionMetadata); kvCacheManager.allocatePools(false); @@ -8352,7 +8254,7 @@ TEST_F(KVCacheManagerTest, StaticLinearHybridAllocationTest) // Static-hybrid path requires block reuse to be disabled. tle::KvCacheConfig const kvCacheConfigDisabledReuse{/*enableBlockReuse=*/false}; auto const blocksPerWindow - = KVCacheManager::calculateMaxNumBlocks(kvCacheConfigDisabledReuse, tensorrt_llm::DataType::kHALF, + = KVCacheManager::calculateMaxNumBlocks(kvCacheConfigDisabledReuse, nvinfer1::DataType::kHALF, numKvHeadsPerLayer, sizePerHead, tokensPerBlock, worldConfig, windowSizeToLayers, allottedPrimaryMemBytes, allottedSecondaryMemBytes, extraCostMemory, kvFactor, maxBatchSize, linearAttentionMetadata); @@ -8371,7 +8273,7 @@ TEST_F(KVCacheManagerTest, StaticLinearHybridAllocationTest) // so the linear pool falls back to memory-budget-based sizing rather than maxBatchSize. tle::KvCacheConfig const kvCacheConfigEnabledReuse{/*enableBlockReuse=*/true}; auto const dynamicBlocksPerWindow - = KVCacheManager::calculateMaxNumBlocks(kvCacheConfigEnabledReuse, tensorrt_llm::DataType::kHALF, + = KVCacheManager::calculateMaxNumBlocks(kvCacheConfigEnabledReuse, nvinfer1::DataType::kHALF, numKvHeadsPerLayer, sizePerHead, tokensPerBlock, worldConfig, windowSizeToLayers, allottedPrimaryMemBytes, allottedSecondaryMemBytes, extraCostMemory, kvFactor, maxBatchSize, linearAttentionMetadata); EXPECT_NE(std::get<0>(dynamicBlocksPerWindow.at(linearWindowSizeCode)), maxBatchSize); @@ -8404,7 +8306,7 @@ static auto makeBatchTestKVCacheManager(std::shared_ptr<tensorrt_llm::runtime::C auto mgr = std::make_unique<KVCacheManager>(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - tensorrt_llm::DataType::kHALF, 0, stream, maxAttentionWindow, /*chunkSize=*/maxAttentionWindow, + nvinfer1::DataType::kHALF, 0, stream, maxAttentionWindow, /*chunkSize=*/maxAttentionWindow, /*enableBlockReuse=*/true, CacheType::kSELF, /*secondaryOffloadMinPriority=*/std::nullopt, /*eventManager=*/nullptr, @@ -8733,7 +8635,7 @@ TEST_F(KVCacheManagerTest, BatchAddSequence_NonLeafCopySourceTightPool) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numKvHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxAttentionWindow, /*chunkSize=*/maxAttentionWindow, /*enableBlockReuse=*/true, CacheType::kSELF, /*secondaryOffloadMinPriority=*/std::nullopt, /*eventManager=*/nullptr, @@ -8890,7 +8792,7 @@ std::unique_ptr<KVCacheManager> makePriorityEvictionManager( auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, 0}}}; auto mgr = std::make_unique<KVCacheManager>(kPE_NUM_LAYERS, kPE_NUM_HEADS, kPE_SIZE_PER_HEAD, kPE_TOKENS_PER_BLOCK, blocksPerWindow, kPE_MAX_NUM_SEQUENCES, kPE_BEAM_WIDTH, - std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxAttentionWindow, /*chunkSize=*/maxAttentionWindow, /*enableBlockReuse=*/true); mgr->allocatePools(false); return mgr; @@ -9224,7 +9126,7 @@ std::unique_ptr<KVCacheManager> makeVSWAManager( { auto const blocksPerWindow = BlocksPerWindow{{kVSWA_ATTENTION_WINDOW, {blocksInPrimaryPool, 0}}}; auto mgr = std::make_unique<KVCacheManager>(2, 2, 64, kVSWA_TOKENS_PER_BLOCK, blocksPerWindow, 8, kVSWA_BEAM_WIDTH, - std::vector<SizeType32>{kVSWA_ATTENTION_WINDOW}, tensorrt_llm::DataType::kHALF, 0, stream, + std::vector<SizeType32>{kVSWA_ATTENTION_WINDOW}, nvinfer1::DataType::kHALF, 0, stream, kVSWA_MAX_SEQUENCE_LENGTH, /*chunkSize=*/kVSWA_MAX_SEQUENCE_LENGTH, enableBlockReuse); mgr->allocatePools(false); return mgr; @@ -9243,7 +9145,7 @@ std::unique_ptr<KVCacheManager> makeSmallWindowManager( SizeType32 constexpr kSmallMaxSeqLen = 128; auto const blocksPerWindow = BlocksPerWindow{{kSmallWindow, {blocksInPrimaryPool, 0}}}; auto mgr = std::make_unique<KVCacheManager>(2, 2, 64, kSmallTpb, blocksPerWindow, 8, kVSWA_BEAM_WIDTH, - std::vector<SizeType32>{kSmallWindow}, tensorrt_llm::DataType::kHALF, 0, stream, kSmallMaxSeqLen, + std::vector<SizeType32>{kSmallWindow}, nvinfer1::DataType::kHALF, 0, stream, kSmallMaxSeqLen, /*chunkSize=*/kSmallMaxSeqLen, /*enableBlockReuse=*/true); mgr->allocatePools(false); return mgr; @@ -9966,7 +9868,7 @@ TEST_F(KVCacheManagerTest, VSWAEvictedPlaceholderAnchorAllowsTrailingReuse) auto const blocksPerWindow = BlocksPerWindow{{window, {blocksInPrimaryPool, 0}}}; KVCacheManager kvCacheManager(2, 2, 64, tpb, blocksPerWindow, 8, kVSWA_BEAM_WIDTH, std::vector<SizeType32>{window}, - tensorrt_llm::DataType::kHALF, 0, stream, + nvinfer1::DataType::kHALF, 0, stream, /*maxSequenceLength=*/128, /*chunkSize=*/128, /*enableBlockReuse=*/true); kvCacheManager.allocatePools(false); auto const& blockManager = kvCacheManager.getBlockManager(); @@ -10102,7 +10004,7 @@ std::unique_ptr<KVCacheManager> makeConnectorTestKVCacheManager( auto mgr = std::make_unique<KVCacheManager>(std::vector<SizeType32>(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, - /*dtype*/ tensorrt_llm::DataType::kHALF, + /*dtype*/ nvinfer1::DataType::kHALF, /*sinkTokenLength*/ 0, stream, /*maxSequenceLength*/ maxAttentionWindow, /*chunkSize*/ maxAttentionWindow, @@ -10296,7 +10198,7 @@ TEST_F(KVCacheManagerTest, BlockManagerTestPerWindowFallback) auto constexpr maxBeamWidth = 1; auto constexpr smallWindow = 1024; auto constexpr largeWindow = 4096; - auto constexpr scalarDtype = tensorrt_llm::DataType::kHALF; + auto constexpr scalarDtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto const maxAttentionWindowVec = std::vector<SizeType32>{smallWindow, largeWindow}; auto const blocksPerWindow = BlocksPerWindow{ @@ -10344,7 +10246,7 @@ TEST(BaseKVCacheManagerCalculateMaxNumBlocks, PerWindowOverrideDivergesByteBudge uint64_t const allottedPrimaryMemBytes = static_cast<uint64_t>(1) << 30; // 1 GiB uint64_t const allottedSecondaryMemBytes = static_cast<uint64_t>(1) << 30; size_t const extraCostMemory = 0; - auto const dtype = tensorrt_llm::DataType::kHALF; + auto const dtype = nvinfer1::DataType::kHALF; tensorrt_llm::executor::KvCacheConfig const config{}; tensorrt_llm::runtime::WorldConfig const worldConfig{}; @@ -10398,7 +10300,7 @@ TEST_F(KVCacheManagerTest, KVCacheManagerSWAEvictionCountPerWindow) auto constexpr maxNumSequences = 4; auto constexpr maxBeamWidth = 1; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto const stream = std::make_shared<tr::CudaStream>(); auto constexpr maxSequenceLength = 128; auto constexpr maxNewTokens = 40; @@ -10488,7 +10390,7 @@ TEST_F(KVCacheManagerTest, GenerationRequestClearCacheBlocksPerWindowResetsOnlyT BlockManager blockManager(std::vector<BlockManager::SizeType32>(numLayers, numKvHeads), sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, stream, /*maxSequenceLength=*/fullWindow, maxBeamWidth, maxAttentionWindowVec, - tensorrt_llm::DataType::kHALF, /*sinkBubbleLength=*/0, /*chunkSize=*/0); + nvinfer1::DataType::kHALF, /*sinkBubbleLength=*/0, /*chunkSize=*/0); blockManager.allocatePools(/*useUvm=*/false); auto constexpr requestId = 7; @@ -10541,7 +10443,7 @@ TEST_F(KVCacheManagerTest, VswaMixedHeadDimReuseSmoke) auto constexpr smallSizePerHead = 256; auto constexpr largeSizePerHead = 512; auto constexpr sinkTokenLength = 0; - auto constexpr dtype = tensorrt_llm::DataType::kHALF; + auto constexpr dtype = nvinfer1::DataType::kHALF; auto constexpr maxSequenceLength = 64; auto constexpr maxNewTokens = 0; auto const stream = std::make_shared<tr::CudaStream>(); @@ -10647,12 +10549,11 @@ TEST_F(KVCacheManagerTest, VswaDisaggDtypeMismatchTriggersGuard) auto const maxAttentionWindowVec = std::vector<SizeType32>{smallWindow, largeWindow}; auto const blocksPerWindow = BlocksPerWindow{ {smallWindow, {blocksInPrimary, blocksInSecondary}}, {largeWindow, {blocksInPrimary, blocksInSecondary}}}; - auto const poolConfigurations - = std::vector<PoolConfiguration>{{smallWindow, sizePerHead, tensorrt_llm::DataType::kHALF}, - {largeWindow, sizePerHead, tensorrt_llm::DataType::kBF16}}; + auto const poolConfigurations = std::vector<PoolConfiguration>{ + {smallWindow, sizePerHead, nvinfer1::DataType::kHALF}, {largeWindow, sizePerHead, nvinfer1::DataType::kBF16}}; auto kvCacheManager = std::make_unique<KVCacheManager>(numLayers, numKvHeads, sizePerHead, tokensPerBlock, - blocksPerWindow, maxNumSequences, maxBeamWidth, maxAttentionWindowVec, /*dtype=*/tensorrt_llm::DataType::kHALF, + blocksPerWindow, maxNumSequences, maxBeamWidth, maxAttentionWindowVec, /*dtype=*/nvinfer1::DataType::kHALF, sinkTokenLength, stream, maxSequenceLength, /*chunkSize=*/0, /*enableBlockReuse=*/false, CacheType::kSELF, /*secondaryOffloadMinPriority=*/std::nullopt, /*eventManager=*/nullptr, diff --git a/cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp b/cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp index a7b15e5ae8e0..2047a885d98c 100644 --- a/cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/kvCacheUtilsTest.cpp @@ -21,7 +21,6 @@ #include <gtest/gtest.h> #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" namespace tc = tensorrt_llm::common; namespace tr = tensorrt_llm::runtime; @@ -49,7 +48,7 @@ TEST_F(BlockIteratorTest, BasicTest) auto constexpr mNumLayers = 5; auto constexpr mBlockSize = 32; auto const cacheShape = tr::ITensor::makeShape({mNumPrimaryBlocks, mNumLayers, 2, mBlockSize}); - constexpr tensorrt_llm::DataType dtype{tr::TRTDataType<DataType>::value}; + constexpr nvinfer1::DataType dtype{tr::TRTDataType<DataType>::value}; tr::ITensor::SharedPtr pool = tr::BufferManager::cpu(cacheShape, dtype); std::vector<SizeType32> blockIds(mNumPrimaryBlocks); std::iota(blockIds.begin(), blockIds.end(), 0); @@ -76,7 +75,7 @@ TEST_F(BlockIteratorTest, BasicTest) TEST_F(BlockIteratorTest, CacheManagerTest) { - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; auto constexpr numLayers = 12; auto constexpr numKvHeads = 6; auto constexpr sizePerHead = 16; diff --git a/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp b/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp index 418178ff5023..120263b01af1 100644 --- a/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp @@ -16,7 +16,6 @@ */ #include "tensorrt_llm/batch_manager/llmRequest.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/executor/types.h" @@ -98,7 +97,7 @@ TEST_F(LlmRequestTest, fromExecutorRequest) EXPECT_TRUE(llmReq.getStopWordsList().has_value()); { auto badWordsTensor = llmReq.getBadWordsList().value(); - EXPECT_EQ(badWordsTensor->getDataType(), tensorrt_llm::DataType::kINT32); + EXPECT_EQ(badWordsTensor->getDataType(), nvinfer1::DataType::kINT32); EXPECT_EQ(badWordsTensor->getShape().nbDims, 3); EXPECT_EQ(badWordsTensor->getShape().d[0], 1); EXPECT_EQ(badWordsTensor->getShape().d[1], 2); @@ -120,7 +119,7 @@ TEST_F(LlmRequestTest, fromExecutorRequest) { auto stopWordsTensor = llmReq.getStopWordsList().value(); - EXPECT_EQ(stopWordsTensor->getDataType(), tensorrt_llm::DataType::kINT32); + EXPECT_EQ(stopWordsTensor->getDataType(), nvinfer1::DataType::kINT32); EXPECT_EQ(stopWordsTensor->getShape().nbDims, 3); EXPECT_EQ(stopWordsTensor->getShape().d[0], 1); EXPECT_EQ(stopWordsTensor->getShape().d[1], 2); @@ -152,7 +151,7 @@ TEST_F(LlmRequestTest, fromExecutorRequest) EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[0], 1); EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[1], vocabSize); EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getShape().d[2], hiddenSize); - EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getDataType(), tensorrt_llm::DataType::kFLOAT); + EXPECT_EQ(llmReq.getPromptEmbeddingTable().value()->getDataType(), nvinfer1::DataType::kFLOAT); EXPECT_EQ(llmReq.getPromptVocabSize().value(), vocabSize); VecUniqueTokens uniqueTokens; for (size_t i = 0; i < inputTokens.size(); ++i) @@ -374,7 +373,7 @@ TEST_F(LlmRequestTest, testAllocateLogitsBuffer) EXPECT_EQ(llmReq.mPromptLen, 5); SizeType32 vocabSizePadded = 32000; - tensorrt_llm::DataType logitsDataType = tensorrt_llm::DataType::kFLOAT; + nvinfer1::DataType logitsDataType = nvinfer1::DataType::kFLOAT; // Test the allocation of context logits EXPECT_EQ(llmReq.getContextLogitsHost(), nullptr); @@ -463,7 +462,7 @@ TEST_F(LlmRequestTest, testCreateRequests) SizeType32 maxNewTokens{60}; tb::LlmRequest::RequestIdType requestId{77}; SizeType32 vocabSize{32}; - tensorrt_llm::DataType dtype{tensorrt_llm::DataType::kHALF}; + nvinfer1::DataType dtype{nvinfer1::DataType::kHALF}; tr::SamplingConfig samplingConfig(1); samplingConfig.randomSeed = std::vector<texec::RandomSeedType>{7}; diff --git a/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp b/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp index c392b05dccf7..c2c308b37dd9 100644 --- a/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/microBatchSchedulerTest.cpp @@ -23,7 +23,6 @@ #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/batch_manager/microBatchScheduler.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" #include <numeric> @@ -60,7 +59,7 @@ class MicroBatchSchedulerTest : public ::testing::Test // NOLINT(cppcoreguidelin { draftTokens = std::make_shared<std::vector<int32_t>>(draftTokensLen, 2); draftLogits = BufferManager::cpu( - ITensor::makeShape({draftTokensLen, /* vocabSizePadded*/ 42}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({draftTokensLen, /* vocabSizePadded*/ 42}), nvinfer1::DataType::kFLOAT); } return std::make_shared<LlmRequest>(reqId, maxNewTokens, inputTokens, samplingConfig, /*isStreaming=*/false, @@ -1247,7 +1246,7 @@ class CombinedSchedulerTest : public ::testing::Test return std::make_shared<kv_cache_manager::KVCacheManager>( /*numLayers=*/10, /*nbKvHeads=*/10, /*sizePerHead=*/1, tokensPerBlock, blocksPerWindow, maxNumRequests, - /*maxBeamWidth=*/1, std::vector<SizeType32>{maxNumTokensPerSeq}, tensorrt_llm::DataType::kHALF, + /*maxBeamWidth=*/1, std::vector<SizeType32>{maxNumTokensPerSeq}, nvinfer1::DataType::kHALF, /*sinkTokenLength=*/0, stream, maxNumTokensPerSeq, /*chunkSize=*/maxNumTokensPerSeq, enableReuse); } @@ -2054,19 +2053,19 @@ class ForceChunkTest : public MicroBatchSchedulerTest } }; -TEST_F(ForceChunkTest, NoSnapshotPointsUsesRemainingContext) +TEST_F(ForceChunkTest, Basic) { - // A request without snapshot points is not split at chunkUnitSize. + // A single request with prompt_len > chunk_unit_size is chunked to unit_size. auto reqs = initRequests({30}); MicroBatchScheduler::setCtxRequestsChunkSize(reqs, Policy::kFORCE_CHUNK, /*ctxTokensCapacity=*/std::nullopt, /*chunkUnitSize=*/10, /*maxContextLength=*/std::nullopt); - EXPECT_EQ(reqs[0]->getContextChunkSize(), 30); + EXPECT_EQ(reqs[0]->getContextChunkSize(), 10); } TEST_F(ForceChunkTest, PromptSmallerThanUnit) { - // Without snapshot points, a short prompt is consumed in full. + // When prompt_len < chunk_unit_size, chunk_size = prompt_len (min). auto reqs = initRequests({8}); MicroBatchScheduler::setCtxRequestsChunkSize(reqs, Policy::kFORCE_CHUNK, std::nullopt, 20, std::nullopt); @@ -2075,7 +2074,7 @@ TEST_F(ForceChunkTest, PromptSmallerThanUnit) TEST_F(ForceChunkTest, ExactUnitSize) { - // Without snapshot points, an exact-unit prompt is consumed in full. + // When prompt_len == chunk_unit_size, chunk_size = prompt_len. auto reqs = initRequests({10}); MicroBatchScheduler::setCtxRequestsChunkSize(reqs, Policy::kFORCE_CHUNK, std::nullopt, 10, std::nullopt); @@ -2084,36 +2083,31 @@ TEST_F(ForceChunkTest, ExactUnitSize) TEST_F(ForceChunkTest, MultipleRequests) { - // Requests without snapshot points independently consume their remaining contexts. + // Each request independently gets min(remaining, unit_size). auto reqs = initRequests({25, 15, 5}); MicroBatchScheduler::setCtxRequestsChunkSize(reqs, Policy::kFORCE_CHUNK, std::nullopt, 10, std::nullopt); - EXPECT_EQ(reqs[0]->getContextChunkSize(), 25); - EXPECT_EQ(reqs[1]->getContextChunkSize(), 15); - EXPECT_EQ(reqs[2]->getContextChunkSize(), 5); + EXPECT_EQ(reqs[0]->getContextChunkSize(), 10); + EXPECT_EQ(reqs[1]->getContextChunkSize(), 10); + EXPECT_EQ(reqs[2]->getContextChunkSize(), 5); // min(5, 10) = 5 } TEST_F(ForceChunkTest, CapacityLimits) { - // Budget truncation is unit-aligned; later requests with less than one - // unit available are delayed. + // When capacity is limited, later requests get chunk_size=0. auto reqs = initRequests({30, 30}); MicroBatchScheduler::setCtxRequestsChunkSize( reqs, Policy::kFORCE_CHUNK, /*ctxTokensCapacity=*/15, /*chunkUnitSize=*/10, std::nullopt); - // req0 is budget-truncated to 10; only 5 remain, so req1 gets 0. + // req0 gets 10, req1 would push total to 20 > 15 → 0 EXPECT_EQ(reqs[0]->getContextChunkSize(), 10); EXPECT_EQ(reqs[1]->getContextChunkSize(), 0); } TEST_F(ForceChunkTest, CapacityExactFit) { - // Capacity exactly accommodates both requested snapshot chunks. + // When capacity exactly accommodates all chunks. auto reqs = initRequests({30, 30}); - for (auto const& req : reqs) - { - req->setExpectedSnapshotPoints({10}); - } MicroBatchScheduler::setCtxRequestsChunkSize( reqs, Policy::kFORCE_CHUNK, /*ctxTokensCapacity=*/20, /*chunkUnitSize=*/10, std::nullopt); @@ -2121,38 +2115,11 @@ TEST_F(ForceChunkTest, CapacityExactFit) EXPECT_EQ(reqs[1]->getContextChunkSize(), 10); } -TEST_F(ForceChunkTest, ExpectedChunkingPoints) -{ - // Expected snapshot points are absolute context positions. - auto reqs = initRequests({30}); - reqs[0]->setExpectedSnapshotPoints({12, 25}); - - chunkIteration(reqs, 10); - expectPositions(reqs, {12}, "iter 1"); - - chunkIteration(reqs, 10); - expectPositions(reqs, {25}, "iter 2"); - - chunkIteration(reqs, 10); - expectPositions(reqs, {30}, "iter 3"); -} - -TEST_F(ForceChunkTest, CapacityRoundsExpectedChunkDownToUnit) -{ - auto reqs = initRequests({50}); - reqs[0]->setExpectedSnapshotPoints({30}); - - MicroBatchScheduler::setCtxRequestsChunkSize( - reqs, Policy::kFORCE_CHUNK, /*ctxTokensCapacity=*/25, /*chunkUnitSize=*/10, std::nullopt); - - EXPECT_EQ(reqs[0]->getContextChunkSize(), 20); -} - TEST_F(ForceChunkTest, MultiIteration) { - // Snapshot points at 10 and 20 split a 25-token prompt into three iterations. + // A request with prompt_len=25 and chunk_unit_size=10 processes in 3 iterations: + // chunk 1: 10, chunk 2: 10, chunk 3: 5. auto reqs = initRequests({25}); - reqs[0]->setExpectedSnapshotPoints({10, 20}); // Iteration 1 chunkIteration(reqs, 10); @@ -2170,10 +2137,8 @@ TEST_F(ForceChunkTest, MultiIteration) TEST_F(ForceChunkTest, MultiRequestMultiIteration) { // Two requests with different lengths processed over multiple iterations. - // Their expected snapshot points determine each boundary. + // prompt_len={25, 12}, chunk_unit_size=10. auto reqs = initRequests({25, 12}); - reqs[0]->setExpectedSnapshotPoints({10, 20}); - reqs[1]->setExpectedSnapshotPoints({10}); // Iteration 1: both get 10 chunkIteration(reqs, 10); @@ -2191,12 +2156,8 @@ TEST_F(ForceChunkTest, MultiRequestMultiIteration) TEST_F(ForceChunkTest, CapacityAcrossIterations) { // With limited capacity, some requests may be delayed to later iterations. - // Both requests have snapshot points at 10 and 20; capacity is 15. + // prompt_len={25, 25}, chunk_unit_size=10, capacity=15. auto reqs = initRequests({25, 25}); - for (auto const& req : reqs) - { - req->setExpectedSnapshotPoints({10, 20}); - } // Iteration 1: req0=10, req1=0 (10+10=20 > 15) chunkIteration(reqs, 10, /*ctxTokensCapacity=*/15); @@ -2219,10 +2180,10 @@ TEST_F(ForceChunkTest, CapacityAcrossIterations) expectPositions(reqs, {25, 25}, "iter 5"); } -TEST_F(ForceChunkTest, FullSchedulerWithoutSnapshotPoints) +TEST_F(ForceChunkTest, FullSchedulerPath) { - // Test via MicroBatchScheduler::operator(): without snapshot points, a - // context that fits is not split at chunkUnitSize. + // Test via MicroBatchScheduler::operator() — FORCE_CHUNK always re-chunks + // even when all contexts fit within the token budget. batch_scheduler::ContextChunkingConfig chunkConfig; chunkConfig.chunkingPolicy = Policy::kFORCE_CHUNK; chunkConfig.chunkUnitSize = 10; @@ -2239,33 +2200,9 @@ TEST_F(ForceChunkTest, FullSchedulerWithoutSnapshotPoints) auto const [contextRequests, genRequests] = (*scheduler)(activeRequests, inflightReqIds, maxBatchSize, maxNumTokens); + // Despite budget=100 >> prompt=30, FORCE_CHUNK limits chunk to unit_size=10. ASSERT_EQ(contextRequests.size(), 1); - EXPECT_EQ(contextRequests[0]->getContextChunkSize(), 30); - EXPECT_EQ(genRequests.size(), 0); -} - -TEST_F(ForceChunkTest, FullSchedulerUsesExpectedChunkingPoints) -{ - batch_scheduler::ContextChunkingConfig chunkConfig; - chunkConfig.chunkingPolicy = Policy::kFORCE_CHUNK; - chunkConfig.chunkUnitSize = 10; - - auto scheduler = std::make_shared<MicroBatchScheduler>(chunkConfig); - - constexpr SizeType32 maxBatchSize = 4; - constexpr SizeType32 maxNumTokens = 100; - - RequestVector activeRequests; - auto request = createRequest(/*promptLen=*/30, /*maxNewTokens=*/1, /*reqId=*/0); - request->setExpectedSnapshotPoints({12, 25}); - activeRequests.push_back(request); - - ReqIdsSet inflightReqIds; - auto const [contextRequests, genRequests] - = (*scheduler)(activeRequests, inflightReqIds, maxBatchSize, maxNumTokens); - - ASSERT_EQ(contextRequests.size(), 1); - EXPECT_EQ(contextRequests[0]->getContextChunkSize(), 12); + EXPECT_EQ(contextRequests[0]->getContextChunkSize(), 10); EXPECT_EQ(genRequests.size(), 0); } @@ -2297,8 +2234,8 @@ TEST_F(ForceChunkTest, FullSchedulerMultipleRequests) { chunks[req->mRequestId] = req->getContextChunkSize(); } - EXPECT_EQ(chunks[0], 25); - EXPECT_EQ(chunks[1], 15); + EXPECT_EQ(chunks[0], 10); + EXPECT_EQ(chunks[1], 10); EXPECT_EQ(chunks[2], 5); } @@ -2330,7 +2267,6 @@ TEST_F(ForceChunkTest, FullSchedulerWithGeneration) EXPECT_EQ(genRequests.size(), 1); ASSERT_EQ(contextRequests.size(), 1); - // Budget remaining is 14, so the context is rounded down to one - // 10-token chunk-unit boundary. + // Budget remaining = 15 - 1 (gen) = 14; chunk = min(30, 10) = 10 EXPECT_EQ(contextRequests[0]->getContextChunkSize(), 10); } diff --git a/cpp/tests/unit_tests/batch_manager/peftCacheManagerTest.cpp b/cpp/tests/unit_tests/batch_manager/peftCacheManagerTest.cpp index ef513894bfb6..49adfe5a6cb6 100644 --- a/cpp/tests/unit_tests/batch_manager/peftCacheManagerTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/peftCacheManagerTest.cpp @@ -32,7 +32,7 @@ #include "tensorrt_llm/runtime/utils/numpyUtils.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <cuda_runtime.h> #include <gmock/gmock-matchers.h> @@ -78,7 +78,7 @@ class PeftCacheManagerTest : public ::testing::Test // NOLINT(cppcoreguidelines- void SetUp() override { - mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); mModelConfig->setMlpHiddenSize(32); mWorldConfig = std::make_unique<WorldConfig>(2, 1, 1, 0); std::vector<LoraModule> modules{ @@ -285,7 +285,7 @@ TEST_F(PeftCacheManagerTest, gptManagerSim) auto peftManager = std::make_unique<PeftCacheManager>(config, *mModelConfig, *mWorldConfig, *mManager); auto pageConfig = LoraCachePageManagerConfig( - runtime::MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 128, 128, 2 * 8 * 64, 4 * 16, 1); + runtime::MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 128, 128, 2 * 8 * 64, 4 * 16, 1); auto loraCache = std::make_unique<LoraCache>(pageConfig, *mModelConfig, *mWorldConfig, *mManager); std::map<uint64_t, std::pair<TensorPtr, TensorPtr>> loras; @@ -505,7 +505,7 @@ TEST_F(PeftCacheManagerTest, getMaxNumSlots) config.numHostModuleLayer = 8192 * 8; config.numDeviceModuleLayer = 8292 * 2; auto [hostSlots, deviceSlots] - = PeftCacheManager::getMaxNumSlots(config, tensorrt_llm::DataType::kHALF, 256, 4 * 256, *mManager); + = PeftCacheManager::getMaxNumSlots(config, nvinfer1::DataType::kHALF, 256, 4 * 256, *mManager); EXPECT_EQ(262144, hostSlots); EXPECT_EQ(66336, deviceSlots); @@ -516,13 +516,13 @@ TEST_F(PeftCacheManagerTest, getMaxNumSlots) config.maxPagesPerBlockDevice = 8; std::tie(hostSlots, deviceSlots) - = PeftCacheManager::getMaxNumSlots(config, tensorrt_llm::DataType::kHALF, 256, 4 * 256, *mManager); + = PeftCacheManager::getMaxNumSlots(config, nvinfer1::DataType::kHALF, 256, 4 * 256, *mManager); EXPECT_EQ(195, hostSlots); EXPECT_EQ(66336, deviceSlots); std::tie(hostSlots, deviceSlots) - = PeftCacheManager::getMaxNumSlots(config, tensorrt_llm::DataType::kFLOAT, 384, 4 * 1024, *mManager); + = PeftCacheManager::getMaxNumSlots(config, nvinfer1::DataType::kFLOAT, 384, 4 * 1024, *mManager); config.hostCacheSize = 100000000; config.numHostModuleLayer = 8291 * 2; @@ -539,7 +539,7 @@ TEST_F(PeftCacheManagerTest, getPageManagerConfig) auto [hostCfg, deviceCfg] = PeftCacheManager::getPageManagerConfig(config, *mModelConfig, *mWorldConfig, *mManager); EXPECT_EQ(runtime::MemoryType::kCPU, hostCfg.getMemoryType()); - EXPECT_EQ(tensorrt_llm::DataType::kFLOAT, hostCfg.getDataType()); + EXPECT_EQ(nvinfer1::DataType::kFLOAT, hostCfg.getDataType()); EXPECT_EQ(456, hostCfg.getTotalNumPages()); EXPECT_EQ(24, hostCfg.getMaxPagesPerBlock()); EXPECT_EQ(288, hostCfg.getSlotsPerPage()); @@ -547,7 +547,7 @@ TEST_F(PeftCacheManagerTest, getPageManagerConfig) EXPECT_FALSE(hostCfg.getInitToZero()); EXPECT_EQ(runtime::MemoryType::kGPU, deviceCfg.getMemoryType()); - EXPECT_EQ(tensorrt_llm::DataType::kFLOAT, deviceCfg.getDataType()); + EXPECT_EQ(nvinfer1::DataType::kFLOAT, deviceCfg.getDataType()); EXPECT_EQ(116, deviceCfg.getTotalNumPages()); EXPECT_EQ(8, deviceCfg.getMaxPagesPerBlock()); EXPECT_EQ(288, deviceCfg.getSlotsPerPage()); @@ -563,7 +563,7 @@ TEST_F(PeftCacheManagerTest, getPageManagerConfig) = PeftCacheManager::getPageManagerConfig(config, *mModelConfig, *mWorldConfig, *mManager); EXPECT_EQ(runtime::MemoryType::kCPU, hostCfg.getMemoryType()); - EXPECT_EQ(tensorrt_llm::DataType::kFLOAT, hostCfg.getDataType()); + EXPECT_EQ(nvinfer1::DataType::kFLOAT, hostCfg.getDataType()); EXPECT_EQ(3617, hostCfg.getTotalNumPages()); EXPECT_EQ(4, hostCfg.getMaxPagesPerBlock()); EXPECT_EQ(288, hostCfg.getSlotsPerPage()); @@ -571,7 +571,7 @@ TEST_F(PeftCacheManagerTest, getPageManagerConfig) EXPECT_FALSE(hostCfg.getInitToZero()); EXPECT_EQ(runtime::MemoryType::kGPU, deviceCfg.getMemoryType()); - EXPECT_EQ(tensorrt_llm::DataType::kFLOAT, deviceCfg.getDataType()); + EXPECT_EQ(nvinfer1::DataType::kFLOAT, deviceCfg.getDataType()); EXPECT_EQ(116, deviceCfg.getTotalNumPages()); EXPECT_EQ(8, deviceCfg.getMaxPagesPerBlock()); EXPECT_EQ(288, deviceCfg.getSlotsPerPage()); @@ -586,7 +586,7 @@ class PeftCacheManagerPrefetchTest : public ::testing::Test // NOLINT(cppcoregui void SetUp() override { - mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); mModelConfig->setMlpHiddenSize(32); mWorldConfig = std::make_unique<WorldConfig>(2, 1, 1, 0); std::vector<LoraModule> modules{ diff --git a/cpp/tests/unit_tests/batch_manager/rnnCacheFormatterTest.cpp b/cpp/tests/unit_tests/batch_manager/rnnCacheFormatterTest.cpp index 7dc4cea6e9af..8840130df1ea 100644 --- a/cpp/tests/unit_tests/batch_manager/rnnCacheFormatterTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/rnnCacheFormatterTest.cpp @@ -7,7 +7,6 @@ #include "tensorrt_llm/batch_manager/cacheFormatter.h" #include "tensorrt_llm/batch_manager/rnnCacheFormatter.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include <random> @@ -28,12 +27,12 @@ class RnnTargetIRanksTest : public ::testing::Test std::vector<SizeType32> kvLayersPerPP(pp, 0); // No attention layers auto state = texec::kv_cache::CacheState( /*nbAttentionLayers=*/0, /*nbKvHeads=*/1, /*sizePerHead=*/64, /*tokensPerBlock=*/32, tp, pp, - /*contextParallelism=*/1, kvLayersPerPP, tensorrt_llm::DataType::kFLOAT); + /*contextParallelism=*/1, kvLayersPerPP, nvinfer1::DataType::kFLOAT); texec::kv_cache::CacheState::RnnModelConfig rnnModelConfig{/*mDState=*/16, /*mDConv=*/4, /*mHiddenSize=*/256, /*mHeadDim=*/64, /*mConvDimSize=*/128, /*mNGroups=*/1, /*mNumLayers=*/numLayers, /*mNumHeads=*/4}; - state.setRnnConfig(rnnModelConfig, layersPerPP, tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kFLOAT); + state.setRnnConfig(rnnModelConfig, layersPerPP, nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kFLOAT); return state; } }; @@ -217,7 +216,7 @@ class HybridModelCounterpartsTest : public ::testing::Test SizeType32 tokensPerBlock = 32) { return texec::kv_cache::CacheState(numLayers, numHeads, sizePerHead, tokensPerBlock, tp, pp, - /*contextParallelism=*/1, layersPerPP, tensorrt_llm::DataType::kFLOAT, + /*contextParallelism=*/1, layersPerPP, nvinfer1::DataType::kFLOAT, texec::kv_cache::CacheState::AttentionType::kDEFAULT, /*kvFactor=*/2, /*enableAttentionDP=*/false, /*DPrank=*/0, /*DPsize=*/1); } @@ -229,8 +228,7 @@ class HybridModelCounterpartsTest : public ::testing::Test { auto state = makeKvCacheState(kvNumLayers, tp, pp, kvLayersPerPP, numHeads, sizePerHead, tokensPerBlock); texec::kv_cache::CacheState::RnnModelConfig rnnModelConfig{16, 4, 256, 64, 128, 1, rnnNumLayers, 4}; - state.setRnnConfig( - rnnModelConfig, rnnLayersPerPP, tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kFLOAT); + state.setRnnConfig(rnnModelConfig, rnnLayersPerPP, nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kFLOAT); return state; } @@ -473,7 +471,7 @@ class AttentionOnlyModelTest : public ::testing::Test SizeType32 tokensPerBlock = 32) { return texec::kv_cache::CacheState(numLayers, numHeads, sizePerHead, tokensPerBlock, tp, pp, - /*contextParallelism=*/1, layersPerPP, tensorrt_llm::DataType::kFLOAT, + /*contextParallelism=*/1, layersPerPP, nvinfer1::DataType::kFLOAT, texec::kv_cache::CacheState::AttentionType::kDEFAULT, /*kvFactor=*/2, /*enableAttentionDP=*/false, /*DPrank=*/0, /*DPsize=*/1); } diff --git a/cpp/tests/unit_tests/batch_manager/truncateBlocksTest.cpp b/cpp/tests/unit_tests/batch_manager/truncateBlocksTest.cpp index 85d4ac112244..d0dce8eb71c5 100644 --- a/cpp/tests/unit_tests/batch_manager/truncateBlocksTest.cpp +++ b/cpp/tests/unit_tests/batch_manager/truncateBlocksTest.cpp @@ -13,7 +13,6 @@ #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/batch_manager/llmRequest.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/samplingConfig.h" #include "tensorrt_llm/testing/kvCacheManagerTestUtil.h" @@ -90,7 +89,7 @@ TEST_F(TruncateBlocksTest, MultiTurnConversationTruncation) // Create KVCacheManager with block reuse enabled KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -220,7 +219,7 @@ TEST_F(TruncateBlocksTest, SharedPrefixTruncation) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -318,7 +317,7 @@ TEST_F(TruncateBlocksTest, CompleteTruncation) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -378,7 +377,7 @@ TEST_F(TruncateBlocksTest, NonExistentTokensTruncation) auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {blocksInPrimaryPool, blocksInSecondaryPool}}}; KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); @@ -454,7 +453,7 @@ TEST_F(TruncateBlocksTest, ComplexMultiTurnConversationTruncation) // Create KVCacheManager with block reuse enabled KVCacheManager kvCacheManager(numLayers, numHeads, sizePerHead, tokensPerBlock, blocksPerWindow, maxNumSequences, - beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, tensorrt_llm::DataType::kHALF, 0, stream, + beamWidth, std::vector<BlockManager::SizeType32>{maxAttentionWindow}, nvinfer1::DataType::kHALF, 0, stream, maxSequenceLength, maxSequenceLength /* chunkSize */, true /* enableBlockReuse */); kvCacheManager.allocatePools(false); diff --git a/cpp/tests/unit_tests/common/loggerTest.cpp b/cpp/tests/unit_tests/common/loggerTest.cpp index 6fa28a69b5f8..8aaea6863fb9 100644 --- a/cpp/tests/unit_tests/common/loggerTest.cpp +++ b/cpp/tests/unit_tests/common/loggerTest.cpp @@ -26,7 +26,7 @@ using namespace tensorrt_llm::common; TEST(LoggerModuleTest, FormatModuleNoTrailingSpaces) { for (auto const* raw : {"batch_manager", "common", "cutlass_extensions", "deep_ep", "deep_gemm", "executor", - "flash_mla", "kernels", "layers", "nanobind", "runtime", "testing", "thop"}) + "executor_worker", "flash_mla", "kernels", "layers", "nanobind", "plugins", "runtime", "testing", "thop"}) { auto const fmt = formatModule(raw); EXPECT_FALSE(fmt.empty()); diff --git a/cpp/tests/unit_tests/executor/CMakeLists.txt b/cpp/tests/unit_tests/executor/CMakeLists.txt index 2275683879a4..a51baa6ed00f 100644 --- a/cpp/tests/unit_tests/executor/CMakeLists.txt +++ b/cpp/tests/unit_tests/executor/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -19,12 +19,21 @@ add_gtest(decodingConfigTest decodingConfigTest.cpp) add_gtest(requestTest requestTest.cpp) add_gtest(responseTest responseTest.cpp) +add_gtest(executorTestSmall executorTestSmall.cpp) +target_link_libraries(executorTestSmall PRIVATE testingUtils) + +add_gtest(executorTestSmallArbitraryOutputTensors + executorTestSmallArbitraryOutputTensors.cpp) +target_link_libraries(executorTestSmallArbitraryOutputTensors + PRIVATE testingUtils) + add_gtest(executorConfigTest executorConfigTest.cpp) add_gtest(executorTensorTest tensorTest.cpp) add_gtest(serializeUtilsTest serializeUtilsTest.cpp) add_gtest(requestWithIdTest requestWithIdTest.cpp) add_gtest(loraConfigTest loraConfigTest.cpp) -add_gtest(coalesceTest coalesceTest.cpp) +add_gtest(intervalSetTest intervalSetTest.cpp) +add_gtest(dynamicBatchTunerTest dynamicBatchTunerTest.cpp) add_gtest(genUniqueAgentNameTest genUniqueAgentNameTest.cpp) target_link_libraries(genUniqueAgentNameTest PRIVATE ${Python3_LIBRARIES}) add_gtest(ucxCommTest ucxCommTest.cpp) @@ -47,6 +56,10 @@ if(NIXL_ROOT OR (MOONCAKE_ROOT AND NOT IS_ROCKY8)) ${Python3_LIBRARIES}) target_compile_definitions(transferAgentTest PRIVATE TEST_NIXL_BACKEND=1) target_compile_definitions(agentCommTest PRIVATE TEST_NIXL_BACKEND=1) + + add_gtest(coalesceTest coalesceTest.cpp) + target_link_libraries(coalesceTest PRIVATE tensorrt_llm_nixl_wrapper + NIXL::nixl) endif() if(MOONCAKE_ROOT) diff --git a/cpp/tests/unit_tests/executor/agentCommTest.cpp b/cpp/tests/unit_tests/executor/agentCommTest.cpp index 89488ba373e8..194d5267c6b3 100644 --- a/cpp/tests/unit_tests/executor/agentCommTest.cpp +++ b/cpp/tests/unit_tests/executor/agentCommTest.cpp @@ -15,7 +15,6 @@ * limitations under the License. */ -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include <gtest/gtest.h> @@ -121,7 +120,7 @@ class AgentCommTest : public ::testing::TestWithParam<std::string> auto constexpr blocksInSecondaryPool = 0; auto constexpr enableBlockReuse = true; - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; BlocksPerWindow const blocksPerWindow diff --git a/cpp/tests/unit_tests/executor/coalesceTest.cpp b/cpp/tests/unit_tests/executor/coalesceTest.cpp index f115d0fd06db..8675d93fc3b4 100644 --- a/cpp/tests/unit_tests/executor/coalesceTest.cpp +++ b/cpp/tests/unit_tests/executor/coalesceTest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,60 +15,154 @@ * limitations under the License. */ -#include "tensorrt_llm/executor/transferAgent.h" +#include "tensorrt_llm/executor/cache_transmission/nixl_utils/transferAgent.h" #include <gtest/gtest.h> using namespace tensorrt_llm::executor::kv_cache; -namespace +// ==================== coalesceMemoryDescs tests ==================== + +TEST(CoalesceMemoryDescsTest, EmptyInput) +{ + MemoryDescs descs{MemoryType::kVRAM, {}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + EXPECT_EQ(result.getDescs().size(), 0); +} + +TEST(CoalesceMemoryDescsTest, SingleEntry) +{ + MemoryDescs descs{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 1); + EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); + EXPECT_EQ(result.getDescs()[0].getLen(), 256); + EXPECT_EQ(result.getDescs()[0].getDeviceId(), 0); +} + +TEST(CoalesceMemoryDescsTest, TwoContiguous) +{ + // [0x1000, 256) then [0x1100, 256) — adjacent, should merge into one + MemoryDescs descs{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 1); + EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); + EXPECT_EQ(result.getDescs()[0].getLen(), 512); + EXPECT_EQ(result.getDescs()[0].getDeviceId(), 0); +} + +TEST(CoalesceMemoryDescsTest, TwoNonContiguous) { -VramRegionMap const kEmptyMap; + // Gap between blocks — should stay as two + MemoryDescs descs{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x2000, 256, 0}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 2); + EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); + EXPECT_EQ(result.getDescs()[0].getLen(), 256); + EXPECT_EQ(result.getDescs()[1].getAddr(), 0x2000); + EXPECT_EQ(result.getDescs()[1].getLen(), 256); +} -// Merging requires region metadata: an address that misses its region map is never coalesced. -// Tests exercising merge behavior therefore provide maps covering their addresses; a flat -// region (chunkSize=0) imposes no chunk boundaries within it. -VramRegionMap flatRegion(uintptr_t base, size_t len) +TEST(CoalesceMemoryDescsTest, ThreeContiguous) { - VramRegionMap map; - map[base] = {len, 0}; - return map; + MemoryDescs descs{ + MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x1200, 256, 0}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 1); + EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); + EXPECT_EQ(result.getDescs()[0].getLen(), 768); } -std::pair<MemoryDescs, MemoryDescs> run(TransferDescs const& src, TransferDescs const& dst, - VramRegionMap const& localMap = kEmptyMap, VramRegionMap const& remoteMap = kEmptyMap) +TEST(CoalesceMemoryDescsTest, UnsortedInput) { - return VmmDescSplitter::splitAndCoalesceTransferDescs(src, dst, localMap, remoteMap); + // Same three contiguous blocks but in reverse order — sorting should fix it + MemoryDescs descs{ + MemoryType::kVRAM, {MemoryDesc{0x1200, 256, 0}, MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 1); + EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); + EXPECT_EQ(result.getDescs()[0].getLen(), 768); } -} // namespace -// ==================== coalescing within flat regions (chunkSize=0) ==================== +TEST(CoalesceMemoryDescsTest, DifferentDevices) +{ + // Contiguous addresses but different devices — should NOT merge + MemoryDescs descs{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 1}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 2); +} -TEST(SplitAndCoalesceTest, EmptyInput) +TEST(CoalesceMemoryDescsTest, MixedContiguousAndGaps) +{ + // First two are contiguous, then a gap before the third + MemoryDescs descs{ + MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x3000, 128, 0}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 2); + EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); + EXPECT_EQ(result.getDescs()[0].getLen(), 512); + EXPECT_EQ(result.getDescs()[1].getAddr(), 0x3000); + EXPECT_EQ(result.getDescs()[1].getLen(), 128); +} + +TEST(CoalesceMemoryDescsTest, MultipleDevicesEachContiguous) +{ + // Two contiguous on device 0, two contiguous on device 1 + MemoryDescs descs{MemoryType::kVRAM, + {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x2000, 128, 1}, + MemoryDesc{0x2080, 128, 1}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 2); + EXPECT_EQ(result.getDescs()[0].getAddr(), 0x1000); + EXPECT_EQ(result.getDescs()[0].getLen(), 512); + EXPECT_EQ(result.getDescs()[0].getDeviceId(), 0); + EXPECT_EQ(result.getDescs()[1].getAddr(), 0x2000); + EXPECT_EQ(result.getDescs()[1].getLen(), 256); + EXPECT_EQ(result.getDescs()[1].getDeviceId(), 1); +} + +TEST(CoalesceMemoryDescsTest, PreservesMemoryType) +{ + MemoryDescs descs{MemoryType::kDRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + EXPECT_EQ(result.getType(), MemoryType::kDRAM); +} + +TEST(CoalesceMemoryDescsTest, AllSeparate) +{ + // Nothing can be merged — all have gaps + MemoryDescs descs{MemoryType::kVRAM, + {MemoryDesc{0x1000, 100, 0}, MemoryDesc{0x2000, 100, 0}, MemoryDesc{0x3000, 100, 0}, + MemoryDesc{0x4000, 100, 0}}}; + auto result = NixlHelper::coalesceMemoryDescs(descs); + ASSERT_EQ(result.getDescs().size(), 4); +} + +// ==================== coalesceTransferDescs tests ==================== + +TEST(CoalesceTransferDescsTest, EmptyInput) { TransferDescs src{MemoryType::kVRAM, {}}; TransferDescs dst{MemoryType::kVRAM, {}}; - auto [resSrc, resDst] = run(src, dst); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); EXPECT_EQ(resSrc.getDescs().size(), 0); EXPECT_EQ(resDst.getDescs().size(), 0); } -TEST(SplitAndCoalesceTest, SinglePair) +TEST(CoalesceTransferDescsTest, SinglePair) { TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 1); ASSERT_EQ(resDst.getDescs().size(), 1); - EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x1000); - EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x5000); } -TEST(SplitAndCoalesceTest, BothSidesContiguous) +TEST(CoalesceTransferDescsTest, BothSidesContiguous) { - // src contiguous AND dst contiguous within known regions — should merge into one transfer + // src contiguous AND dst contiguous — should merge into one transfer TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 1); ASSERT_EQ(resDst.getDescs().size(), 1); EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x1000); @@ -77,79 +171,98 @@ TEST(SplitAndCoalesceTest, BothSidesContiguous) EXPECT_EQ(resDst.getDescs()[0].getLen(), 512); } -TEST(SplitAndCoalesceTest, SrcContiguousDstNot) +TEST(CoalesceTransferDescsTest, SrcContiguousDstNot) { + // src is contiguous but dst has a gap — can't merge TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x6000, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x2000)); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); } -TEST(SplitAndCoalesceTest, DstContiguousSrcNot) +TEST(CoalesceTransferDescsTest, DstContiguousSrcNot) { + // dst is contiguous but src has a gap — can't merge TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x2000, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x2000), flatRegion(0x5000, 0x1000)); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); } -TEST(SplitAndCoalesceTest, DifferentDevicesOnSrc) +TEST(CoalesceTransferDescsTest, DifferentDevicesOnSrc) { + // src addresses look contiguous but are on different devices — can't merge TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 1}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 0}, MemoryDesc{0x5100, 256, 0}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); } -TEST(SplitAndCoalesceTest, DifferentDevicesOnDst) +TEST(CoalesceTransferDescsTest, DifferentDevicesOnDst) { + // dst addresses look contiguous but are on different devices — can't merge TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 0}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); } -TEST(SplitAndCoalesceTest, ThreePairsAllContiguous) +TEST(CoalesceTransferDescsTest, MismatchedSizes) +{ + // src has 2 entries, dst has 1 — sizes don't match, return as-is + TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}}}; + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); + ASSERT_EQ(resSrc.getDescs().size(), 2); + ASSERT_EQ(resDst.getDescs().size(), 1); +} + +TEST(CoalesceTransferDescsTest, ThreePairsAllContiguous) { TransferDescs src{ MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x1200, 256, 0}}}; TransferDescs dst{ MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}, MemoryDesc{0x5200, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 1); ASSERT_EQ(resDst.getDescs().size(), 1); EXPECT_EQ(resSrc.getDescs()[0].getLen(), 768); EXPECT_EQ(resDst.getDescs()[0].getLen(), 768); } -TEST(SplitAndCoalesceTest, PartialMerge) +TEST(CoalesceTransferDescsTest, PartialMerge) { - // First two pairs merge; third pair's dst has a gap — stays separate + // First two pairs: both sides contiguous — merge + // Third pair: src contiguous but dst has gap — stays separate TransferDescs src{ MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x1200, 256, 0}}}; TransferDescs dst{ MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}, MemoryDesc{0x9000, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x8000)); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 2); ASSERT_EQ(resDst.getDescs().size(), 2); + // Merged pair EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x1000); EXPECT_EQ(resSrc.getDescs()[0].getLen(), 512); EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x5000); EXPECT_EQ(resDst.getDescs()[0].getLen(), 512); + // Separate pair EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x1200); + EXPECT_EQ(resSrc.getDescs()[1].getLen(), 256); EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0x9000); + EXPECT_EQ(resDst.getDescs()[1].getLen(), 256); } -TEST(SplitAndCoalesceTest, UnsortedInput) +TEST(CoalesceTransferDescsTest, UnsortedInput) { - // Same as BothSidesContiguous but in reverse order — sorting by src addr should fix it + // Same as BothSidesContiguous but in reverse order — sorting should fix it TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1100, 256, 0}, MemoryDesc{0x1000, 256, 0}}}; TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5100, 256, 1}, MemoryDesc{0x5000, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000), flatRegion(0x5000, 0x1000)); + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); ASSERT_EQ(resSrc.getDescs().size(), 1); ASSERT_EQ(resDst.getDescs().size(), 1); EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x1000); @@ -158,259 +271,11 @@ TEST(SplitAndCoalesceTest, UnsortedInput) EXPECT_EQ(resDst.getDescs()[0].getLen(), 512); } -TEST(SplitAndCoalesceTest, NonVramPassthrough) +TEST(CoalesceTransferDescsTest, PreservesMemoryType) { - // Non-kVRAM descs pass through unchanged: no region info exists to bound a merge - TransferDescs src{MemoryType::kDRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; - TransferDescs dst{MemoryType::kDRAM, {MemoryDesc{0x5000, 256, 0}, MemoryDesc{0x5100, 256, 0}}}; - auto [resSrc, resDst] = run(src, dst); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); + TransferDescs src{MemoryType::kDRAM, {MemoryDesc{0x1000, 256, 0}}}; + TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 0}}}; + auto [resSrc, resDst] = NixlHelper::coalesceTransferDescs(src, dst); EXPECT_EQ(resSrc.getType(), MemoryType::kDRAM); -} - -// ==================== unknown regions never merge ==================== - -TEST(SplitAndCoalesceTest, NoMergeWithoutRegionMetadata) -{ - // Contiguous on both sides, but neither map covers the addresses: two unknown regions are - // indistinguishable (both look up as a miss), so merging is disabled and pairs stay separate. - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); -} - -TEST(SplitAndCoalesceTest, NoMergeWhenRemoteRegionUnknown) -{ - // Local map covers src, but the remote side sent no region info (e.g. an older peer): - // dst lookups miss, so nothing merges even though both sides are contiguous. - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, flatRegion(0x1000, 0x1000)); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); -} - -TEST(SplitAndCoalesceTest, NoMergeWhenLocalRegionUnknown) -{ - // Remote map covers dst, but src addresses miss the local map: no merging. - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 256, 0}, MemoryDesc{0x1100, 256, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x5000, 256, 1}, MemoryDesc{0x5100, 256, 1}}}; - auto [resSrc, resDst] = run(src, dst, kEmptyMap, flatRegion(0x5000, 0x1000)); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); -} - -// ==================== chunk-boundary-constrained coalescing ==================== - -TEST(SplitAndCoalesceTest, MergeStopsAtSrcChunkBoundary) -{ - // Local VMM region: base=0x100000, 4MB total, 2MB chunks → boundary at 0x300000. - VramRegionMap localMap; - localMap[0x100000] = {0x400000, 0x200000}; - - // Four contiguous 1MB pairs covering 4MB on both sides; dst is one flat remote region. - std::vector<MemoryDesc> srcVec, dstVec; - for (size_t i = 0; i < 4; ++i) - { - srcVec.emplace_back(0x100000 + i * 0x100000, 0x100000, 0); - dstVec.emplace_back(0x900000 + i * 0x100000, 0x100000, 1); - } - TransferDescs src{MemoryType::kVRAM, srcVec}; - TransferDescs dst{MemoryType::kVRAM, dstVec}; - - auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x400000)); - // Merged per src chunk: two 2MB transfers, split exactly at the chunk boundary. - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); - EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x100000); - EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x200000); - EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x300000); - EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x200000); - EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x900000); - EXPECT_EQ(resDst.getDescs()[0].getLen(), 0x200000); - EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0xB00000); - EXPECT_EQ(resDst.getDescs()[1].getLen(), 0x200000); -} - -TEST(SplitAndCoalesceTest, MergeStopsAtDstChunkBoundary) -{ - // Remote VMM region: base=0x800000, 1MB chunks → boundary at 0x900000. - VramRegionMap remoteMap; - remoteMap[0x800000] = {0x400000, 0x100000}; - - // Two contiguous pairs whose dst junction sits exactly on the remote chunk boundary. - auto localMap = flatRegion(0x1000, 0x100000); - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x1000, 0x80000, 0}, MemoryDesc{0x81000, 0x80000, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x880000, 0x80000, 1}, MemoryDesc{0x900000, 0x80000, 1}}}; - - // With a flat remote region they would merge into one transfer... - { - auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x800000, 0x400000)); - ASSERT_EQ(resSrc.getDescs().size(), 1); - ASSERT_EQ(resDst.getDescs().size(), 1); - } - // ...but the dst chunk boundary must block the merge. - auto [resSrc, resDst] = run(src, dst, localMap, remoteMap); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); - EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x880000); - EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0x900000); -} - -TEST(SplitAndCoalesceTest, SplitPiecesAreNotRemerged) -{ - // A single pair spanning two src chunks stays split even though the pieces are contiguous. - VramRegionMap localMap; - localMap[0x100000] = {0x400000, 0x100000}; - - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x100000, 0x200000, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x900000, 0x200000, 1}}}; - - auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x400000)); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); - EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x100000); - EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x100000); -} - -TEST(SplitAndCoalesceTest, UnalignedRegionBaseBoundary) -{ - // Chunk boundaries are relative to the region base, not absolute alignment. - // base=0x180000, 1MB chunks → boundaries at 0x280000, 0x380000, ... - VramRegionMap localMap; - localMap[0x180000] = {0x300000, 0x100000}; - - // Three contiguous 512KB pairs: first two share the first chunk, third starts a new chunk. - std::vector<MemoryDesc> srcVec, dstVec; - for (size_t i = 0; i < 3; ++i) - { - srcVec.emplace_back(0x180000 + i * 0x80000, 0x80000, 0); - dstVec.emplace_back(0x900000 + i * 0x80000, 0x80000, 1); - } - TransferDescs src{MemoryType::kVRAM, srcVec}; - TransferDescs dst{MemoryType::kVRAM, dstVec}; - - auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x300000)); - ASSERT_EQ(resSrc.getDescs().size(), 2); - EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x180000); - EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x100000); - EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x280000); - EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x80000); -} - -TEST(SplitAndCoalesceTest, NoMergeAcrossRegions) -{ - // Two VA-adjacent but distinct local regions (cudaMalloc-style, chunkSize=0): - // contiguous descs must not merge across the region boundary. - VramRegionMap localMap; - localMap[0x100000] = {0x100000, 0}; - localMap[0x200000] = {0x100000, 0}; - - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x100000, 0x100000, 0}, MemoryDesc{0x200000, 0x100000, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x900000, 0x100000, 1}, MemoryDesc{0xA00000, 0x100000, 1}}}; - - auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x200000)); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); -} - -TEST(SplitAndCoalesceTest, MergeWithinSingleRegion) -{ - // Control for NoMergeAcrossRegions: same layout as one region (chunkSize=0) merges freely. - VramRegionMap localMap; - localMap[0x100000] = {0x200000, 0}; - - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x100000, 0x100000, 0}, MemoryDesc{0x200000, 0x100000, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x900000, 0x100000, 1}, MemoryDesc{0xA00000, 0x100000, 1}}}; - - auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x200000)); - ASSERT_EQ(resSrc.getDescs().size(), 1); - EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x200000); - EXPECT_EQ(resDst.getDescs()[0].getLen(), 0x200000); -} - -TEST(SplitAndCoalesceTest, NoMergeAcrossRemoteRegions) -{ - // The remote side registered two discrete but VA-adjacent buffers (chunkSize=0 each): - // contiguous dst descs must not merge across the remote registration boundary. - VramRegionMap remoteMap; - remoteMap[0x900000] = {0x100000, 0}; - remoteMap[0xA00000] = {0x100000, 0}; - - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x100000, 0x100000, 0}, MemoryDesc{0x200000, 0x100000, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x900000, 0x100000, 1}, MemoryDesc{0xA00000, 0x100000, 1}}}; - - auto [resSrc, resDst] = run(src, dst, flatRegion(0x100000, 0x200000), remoteMap); - ASSERT_EQ(resSrc.getDescs().size(), 2); - ASSERT_EQ(resDst.getDescs().size(), 2); -} - -TEST(SplitAndCoalesceTest, CoalesceDisabledSplitsOnly) -{ - // With coalescing disabled, contiguous pairs stay separate and chunk splitting still applies. - VramRegionMap localMap; - localMap[0x100000] = {0x400000, 0x100000}; - - // Two contiguous 512KB pairs within one chunk plus one pair spanning a chunk boundary. - TransferDescs src{MemoryType::kVRAM, - {MemoryDesc{0x100000, 0x80000, 0}, MemoryDesc{0x180000, 0x80000, 0}, MemoryDesc{0x200000, 0x200000, 0}}}; - TransferDescs dst{MemoryType::kVRAM, - {MemoryDesc{0x900000, 0x80000, 1}, MemoryDesc{0x980000, 0x80000, 1}, MemoryDesc{0xA00000, 0x200000, 1}}}; - - auto [resSrc, resDst] - = VmmDescSplitter::splitAndCoalesceTransferDescs(src, dst, localMap, kEmptyMap, /*enableCoalesce=*/false); - // No merging: pair 1, pair 2, and pair 3 split into two chunk pieces → 4 descs. - ASSERT_EQ(resSrc.getDescs().size(), 4); - ASSERT_EQ(resDst.getDescs().size(), 4); - EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x80000); - EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x80000); - EXPECT_EQ(resSrc.getDescs()[2].getLen(), 0x100000); - EXPECT_EQ(resSrc.getDescs()[3].getLen(), 0x100000); -} - -TEST(SplitAndCoalesceTest, CoalesceDisabledPreservesInputOrder) -{ - // With coalescing disabled there is no sorting either: descs come out in input order, - // matching the historical split-only behavior. - TransferDescs src{MemoryType::kVRAM, {MemoryDesc{0x2000, 256, 0}, MemoryDesc{0x1000, 256, 0}}}; - TransferDescs dst{MemoryType::kVRAM, {MemoryDesc{0x6000, 256, 1}, MemoryDesc{0x5000, 256, 1}}}; - - auto [resSrc, resDst] - = VmmDescSplitter::splitAndCoalesceTransferDescs(src, dst, kEmptyMap, kEmptyMap, /*enableCoalesce=*/false); - ASSERT_EQ(resSrc.getDescs().size(), 2); - EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x2000); - EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x1000); - EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x6000); - EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0x5000); -} - -TEST(SplitAndCoalesceTest, SplitAndMergeScatteredBlocks) -{ - // Scattered input order + chunked src region: blocks are sorted, merged per chunk. - // base=0x100000, 1MB chunks; four 512KB blocks given out of order. - VramRegionMap localMap; - localMap[0x100000] = {0x400000, 0x100000}; - - std::vector<size_t> perm{2, 0, 3, 1}; - std::vector<MemoryDesc> srcVec, dstVec; - for (size_t i : perm) - { - srcVec.emplace_back(0x100000 + i * 0x80000, 0x80000, 0); - dstVec.emplace_back(0x900000 + i * 0x80000, 0x80000, 1); - } - TransferDescs src{MemoryType::kVRAM, srcVec}; - TransferDescs dst{MemoryType::kVRAM, dstVec}; - - auto [resSrc, resDst] = run(src, dst, localMap, flatRegion(0x900000, 0x400000)); - // 2MB of contiguous data over two 1MB chunks → one transfer per chunk. - ASSERT_EQ(resSrc.getDescs().size(), 2); - EXPECT_EQ(resSrc.getDescs()[0].getAddr(), 0x100000); - EXPECT_EQ(resSrc.getDescs()[0].getLen(), 0x100000); - EXPECT_EQ(resSrc.getDescs()[1].getAddr(), 0x200000); - EXPECT_EQ(resSrc.getDescs()[1].getLen(), 0x100000); - EXPECT_EQ(resDst.getDescs()[0].getAddr(), 0x900000); - EXPECT_EQ(resDst.getDescs()[1].getAddr(), 0xA00000); + EXPECT_EQ(resDst.getType(), MemoryType::kVRAM); } diff --git a/cpp/tests/unit_tests/executor/dynamicBatchTunerTest.cpp b/cpp/tests/unit_tests/executor/dynamicBatchTunerTest.cpp new file mode 100644 index 000000000000..04ce10393b67 --- /dev/null +++ b/cpp/tests/unit_tests/executor/dynamicBatchTunerTest.cpp @@ -0,0 +1,99 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/executor/dynamicBatchTuner.h" +#include "tensorrt_llm/common/tllmException.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +using ::testing::_; +using ::testing::Invoke; + +using namespace tensorrt_llm::executor; +using namespace tensorrt_llm::common; + +TEST(DynamicBatchTunerTest, Stats) +{ + // moving average window size is 3 + DynamicBatchConfig dynamicBatchConfig(true, true, 3); + DynamicBatchTuner dynamicBatchTuner(dynamicBatchConfig); + + // check no division by zero issue + EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 0); + EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 0); + + dynamicBatchTuner.updateStats(1, 2); + EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 1); + EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 2); + + dynamicBatchTuner.updateStats(2, 3); + EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 1.5); + EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 2.5); + + dynamicBatchTuner.updateStats(3, 4); + EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 2); + EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 3); + + // check that the first element is removed from the moving average window + dynamicBatchTuner.updateStats(4, 5); + EXPECT_EQ(dynamicBatchTuner.getAverageInputLength(), 3); + EXPECT_EQ(dynamicBatchTuner.getAverageOutputLength(), 4); +} + +TEST(DynamicBatchConfig, RuntimeBatchSize) +{ + // moving average window size is 3 + DynamicBatchConfig dynamicBatchConfig(true, true, 3); + DynamicBatchTuner dynamicBatchTuner(dynamicBatchConfig); + // check runtime batch size computation + EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(143), 128); + EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(335), 256); + EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(671), 512); + EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(831), 768); + EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(1279), 1024); + EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(1663), 1536); + // fall back + EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(2049), 2048); + EXPECT_EQ(dynamicBatchTuner.getRuntimeBatchSize(1665), 1665); +} + +TEST(DynamicBatchConfig, RuntimeMaxNumTokens) +{ + // moving average window size is 1 + DynamicBatchConfig dynamicBatchConfig(true, true, 1); + DynamicBatchTuner dynamicBatchTuner(dynamicBatchConfig); + + // context heavy + dynamicBatchTuner.updateStats(100, 2); + EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(1), 8192); + // context heavy fall back + EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(256), 16384); + + // balanced + dynamicBatchTuner.updateStats(100, 100); + EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(1), 4096); + // balanced fall back + EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(4000), 8192); + + // gen heavy + dynamicBatchTuner.updateStats(2, 256); + EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(1), 2048); + // gen heavy fall back + EXPECT_EQ(dynamicBatchTuner.getRuntimeMaxNumTokens(4000), 4096); +} diff --git a/cpp/tests/unit_tests/executor/executorTestSmall.cpp b/cpp/tests/unit_tests/executor/executorTestSmall.cpp new file mode 100644 index 000000000000..2987509f16ac --- /dev/null +++ b/cpp/tests/unit_tests/executor/executorTestSmall.cpp @@ -0,0 +1,289 @@ +#include "tensorrt_llm/batch_manager/trtGptModelInflightBatching.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tests/utils/common.h" +#include "tests/utils/engines.h" +#include "tests/utils/executorUtils.h" + +#include "gtest/gtest.h" + +#include <random> +#include <tuple> +#include <unordered_map> + +namespace tensorrt_llm::testing +{ + +struct TrivialConstantDecoderTestParameters +{ + using TupleT = std::tuple<runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, + runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32>; + runtime::SizeType32 randomSeed; + runtime::SizeType32 vocabSize; + runtime::SizeType32 maxNumTokens; + runtime::SizeType32 maxBeamWidth; + runtime::SizeType32 maxBatchSize; + runtime::SizeType32 numRequests; + runtime::SizeType32 promptLength; + runtime::SizeType32 maxOutputLength; + + // Constructor that takes a tuple + TrivialConstantDecoderTestParameters( // NOLINT: implicit to allow gtest to convert from tuple generated by + // 'combine' + TupleT t) + : randomSeed(std::get<0>(t)) + , vocabSize(std::get<1>(t)) + , maxNumTokens(std::get<2>(t)) + , maxBeamWidth(std::get<3>(t)) + , maxBatchSize(std::get<4>(t)) + , numRequests(std::get<5>(t)) + , promptLength(std::get<6>(t)) + , maxOutputLength(std::get<7>(t)) + { + } +}; + +template <typename TLogits> +struct DecoderTestShared +{ + static constexpr runtime::SizeType32 kNumTokensPerBlock = 64; + static constexpr runtime::SizeType32 kKvCacheMaxTokens = 2048 * 8; + + DecoderTestShared(std::shared_ptr<runtime::TllmLogger> logger, std::mt19937 rng, + std::shared_ptr<executor::Executor> executor, std::vector<TLogits> randomLogits) + : logger(std::move(logger)) + , rng(rng) + , executor(std::move(executor)) + , randomLogits(std::move(randomLogits)){}; + std::shared_ptr<runtime::TllmLogger> logger; + std::mt19937 rng; + std::shared_ptr<executor::Executor> executor; + std::vector<TLogits> randomLogits; +}; + +template <typename TLogits> +std::unique_ptr<DecoderTestShared<TLogits>> SetupDecoderTest(TrivialConstantDecoderTestParameters const& params) +{ + auto logger = std::make_shared<runtime::TllmLogger>(); + auto rng = std::mt19937(params.randomSeed); + auto randomLogits = tensorrt_llm::testing::randomLogits<std::mt19937, TLogits>(params.vocabSize, &rng); + auto const decoderParameters = tensorrt_llm::testing::utils::engines::ConstantTrivialDecoderParameters<TLogits>{ + tensorrt_llm::testing::utils::engines::TrivialDecoderParameters{params.vocabSize, params.maxBatchSize, + params.maxNumTokens, DecoderTestShared<TLogits>::kNumTokensPerBlock, params.maxBeamWidth, false}, + randomLogits}; + auto engineHostMemory + = tensorrt_llm::testing::utils::engines::createConstantTrivialDecoder<TLogits>(decoderParameters, logger); + auto const engine = runtime::RawEngine(engineHostMemory.release()); + auto const dtype = runtime::TRTDataType<TLogits>::value; + auto modelConfig = runtime::ModelConfig(params.vocabSize, 1, 1, 0, 1, 1, dtype); + modelConfig.useGptAttentionPlugin(true); + modelConfig.setModelVariant(runtime::ModelConfig::ModelVariant::kGpt); + modelConfig.usePackedInput(true); + modelConfig.setKVCacheType(runtime::ModelConfig::KVCacheType::kPAGED); + modelConfig.setMaxNumTokens(params.maxNumTokens); + modelConfig.setMaxBatchSize(params.maxBatchSize); + modelConfig.setMaxBeamWidth(params.maxBeamWidth); + modelConfig.setMaxSequenceLen(params.maxNumTokens); + modelConfig.setMaxInputLen(params.maxNumTokens); + modelConfig.setLayerTypes({runtime::ModelConfig::LayerType::kATTENTION}); + modelConfig.setTokensPerBlock(DecoderTestShared<TLogits>::kNumTokensPerBlock); + modelConfig.setPagedContextFMHA(true); + + auto const worldConfig = runtime::WorldConfig(); + auto kvCacheConfig = executor::KvCacheConfig{}; + kvCacheConfig.setMaxTokens(DecoderTestShared<TLogits>::kKvCacheMaxTokens); + + auto const executorConfig + = tensorrt_llm::executor::ExecutorConfig(params.maxBeamWidth, executor::SchedulerConfig(), kvCacheConfig, true, + true, 1, 1, executor::BatchingType::kINFLIGHT, params.maxBatchSize, params.maxNumTokens, std::nullopt, + std::nullopt, std::nullopt, std::nullopt, false, 1, std::nullopt, executor::ExtendedRuntimePerfKnobConfig(), + std::nullopt, 0, executor::ExecutorConfig::kDefaultMaxSeqIdleMicroseconds, std::nullopt, std::nullopt); + + auto model = std::make_shared<batch_manager::TrtGptModelInflightBatching>( + logger, modelConfig, worldConfig, engine, false, executorConfig, false); + + return std::make_unique<DecoderTestShared<TLogits>>( + logger, rng, std::make_shared<executor::Executor>(model, executorConfig), randomLogits); +} + +template <typename TLogits> +class DecoderTest : public ::testing::Test, public ::testing::WithParamInterface<TrivialConstantDecoderTestParameters> +{ +protected: + std::unique_ptr<DecoderTestShared<TLogits>> state; + + DecoderTest() + { + auto const params = GetParam(); + state = SetupDecoderTest<TLogits>(params); + } + + void runDecoderTest(TrivialConstantDecoderTestParameters const& parameters) + { + auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, 0); + auto requests = std::vector<executor::Request>{}; + requests.reserve(static_cast<std::size_t>(parameters.numRequests)); + for (auto i = 0; i < parameters.numRequests; i++) + { + requests.emplace_back(requestTokens, parameters.maxOutputLength, false, executor::SamplingConfig{}, + executor::OutputConfig{false, false, false, true, false, false}); + } + auto const accumulatedResponses + = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(3600000)); + ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); + + std::sort(state->randomLogits.begin(), state->randomLogits.end()); + std::reverse(state->randomLogits.begin(), state->randomLogits.end()); + for (auto const& [requestId, responses] : accumulatedResponses) + { + for (auto const& response : responses) + { + ASSERT_FALSE(response.hasError()); + auto const& tokensByBeam = response.getResult().outputTokenIds; + ASSERT_EQ(tokensByBeam.size(), 1); + for (auto const& tokensForBeam : tokensByBeam) + { + ASSERT_EQ(tokensForBeam.size(), parameters.maxOutputLength); + } + } + } + } +}; + +namespace +{ +constexpr runtime::SizeType32 kRandomSeed1 = 45; +auto const randomSeeds = ::testing::Values(kRandomSeed1); + +constexpr runtime::SizeType32 kMinVocabSize = 16; +auto const vocabSizes = ::testing::Values(kMinVocabSize); + +constexpr runtime::SizeType32 kMinMaxNumTokens = 2048; +auto const maxNumTokenses = ::testing::Values(kMinMaxNumTokens); + +constexpr runtime::SizeType32 kMinBeamWidth = 1; +auto const beamWidths = ::testing::Values(kMinBeamWidth); + +constexpr runtime::SizeType32 kMinMaxBatchSize = 2048; +auto const maxBatchSizes = ::testing::Values(kMinMaxBatchSize); + +constexpr runtime::SizeType32 kMinNumRequests = 64; +auto const numRequestses = ::testing::Values(kMinNumRequests); + +constexpr runtime::SizeType32 kMinPromptLength = 32; +auto const promptLengths = ::testing::Values(kMinPromptLength); + +constexpr runtime::SizeType32 kMinMaxOutputLength = 16; +auto const maxOutputLengths = ::testing::Values(kMinMaxOutputLength); + +auto const paramGenerator + = ::testing::ConvertGenerator<TrivialConstantDecoderTestParameters::TupleT>(::testing::Combine(randomSeeds, + vocabSizes, maxNumTokenses, beamWidths, maxBatchSizes, numRequestses, promptLengths, maxOutputLengths)); +} // namespace + +using DecoderFloatTest = DecoderTest<float>; + +TEST_P(DecoderFloatTest, TestSizeAndValues) +{ + runDecoderTest(GetParam()); +} + +INSTANTIATE_TEST_SUITE_P(Float, DecoderFloatTest, paramGenerator, + [](::testing::TestParamInfo<TrivialConstantDecoderTestParameters> const& info) -> std::string + { + std::stringstream nameStringStream; + nameStringStream << "_maxBatchSize_" << info.param.maxBatchSize << "_vocabSize_" << info.param.vocabSize + << "_maxBeamWidth_" << info.param.maxBeamWidth << "_maxNumTokens_" << info.param.maxNumTokens + << "_maxOutputLength_" << info.param.maxOutputLength << "_numRequests_" + << info.param.numRequests << "_promptLength_" << info.param.promptLength << "_randomSeed_" + << info.param.randomSeed; + return nameStringStream.str(); + }); + +// Helper function to test calculateCacheSizePerToken with given parameters. +std::map<runtime::SizeType32, runtime::SizeType32> calculateCacheSizePerTokenHelper( + std::vector<runtime::SizeType32> const& maxAttentionWindowVec, runtime::SizeType32 kvFactor = 2, + runtime::SizeType32 vocabSize = 32, runtime::SizeType32 nbLayers = 4, runtime::SizeType32 nbAttentionLayers = 4, + runtime::SizeType32 nbRnnLayers = 0, runtime::SizeType32 nbHeads = 8, runtime::SizeType32 hiddenSize = 512, + bool isCrossAttention = false) +{ + // Create minimal ModelConfig for testing. + auto modelConfig = runtime::ModelConfig( + vocabSize, nbLayers, nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, nvinfer1::DataType::kFLOAT); + modelConfig.useGptAttentionPlugin(true); + modelConfig.setModelVariant(runtime::ModelConfig::ModelVariant::kGpt); + modelConfig.setKVCacheType(runtime::ModelConfig::KVCacheType::kPAGED); + + auto const worldConfig = runtime::WorldConfig(); + + return batch_manager::TrtGptModelInflightBatching::calculateCacheSizePerTokenForDisagg( + modelConfig, worldConfig, maxAttentionWindowVec, isCrossAttention, kvFactor); +} + +// Test for TrtGptModelInflightBatching::calculateCacheSizePerToken function with different layer types. +TEST(TrtInflightBatchingTest, CalculateCacheSizePerTokenForDisagg) +{ + // Common parameters. + constexpr runtime::SizeType32 nbLayers = 5; + constexpr runtime::SizeType32 hiddenSize = 512; + constexpr runtime::SizeType32 kvFactor = 2; + constexpr runtime::SizeType32 vocabSize = 32; + constexpr runtime::SizeType32 nbHeads = 8; + // Test case 1: Single attention window size - attention layers only. + { + std::vector<runtime::SizeType32> maxAttentionWindowVec = {128}; + constexpr runtime::SizeType32 nbAttentionLayers = 5; + constexpr runtime::SizeType32 numBytesPerFloatElement = 4; + constexpr runtime::SizeType32 nbRnnLayers = 0; + auto result = calculateCacheSizePerTokenHelper(maxAttentionWindowVec, kvFactor, vocabSize, nbLayers, + nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, false); + EXPECT_EQ(result.size(), 1); + EXPECT_EQ(result.at(128), nbAttentionLayers * kvFactor * hiddenSize * numBytesPerFloatElement); + } + + // Test case 2: Multiple attention window sizes - attention layers only. + { + std::vector<runtime::SizeType32> maxAttentionWindowVec = {128, 256}; + constexpr runtime::SizeType32 nbAttentionLayers = 5; + constexpr runtime::SizeType32 numBytesPerFloatElement = 4; + constexpr runtime::SizeType32 nbRnnLayers = 0; + auto result = calculateCacheSizePerTokenHelper(maxAttentionWindowVec, kvFactor, vocabSize, nbLayers, + nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, false); + EXPECT_EQ(result.size(), 2); + auto const nbAttentionLayersIn128Window = 3; + auto const nbAttentionLayersIn256Window = 2; + EXPECT_EQ(result.at(128), nbAttentionLayersIn128Window * kvFactor * hiddenSize * numBytesPerFloatElement); + EXPECT_EQ(result.at(256), nbAttentionLayersIn256Window * kvFactor * hiddenSize * numBytesPerFloatElement); + } + + // Test case 3: Single attention window size - attention and rnn layers. + { + std::vector<runtime::SizeType32> maxAttentionWindowVec = {128}; + constexpr runtime::SizeType32 nbAttentionLayers = 3; + constexpr runtime::SizeType32 numBytesPerFloatElement = 4; + constexpr runtime::SizeType32 nbRnnLayers = 2; + auto result = calculateCacheSizePerTokenHelper(maxAttentionWindowVec, kvFactor, vocabSize, nbLayers, + nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, false); + EXPECT_EQ(result.size(), 1); + EXPECT_EQ(result.at(128), nbAttentionLayers * kvFactor * hiddenSize * numBytesPerFloatElement); + } + + // Test case 4: Multiple attention window sizes - attention and rnn layers. + { + std::vector<runtime::SizeType32> maxAttentionWindowVec = {128, 256}; + constexpr runtime::SizeType32 nbAttentionLayers = 3; + constexpr runtime::SizeType32 numBytesPerFloatElement = 4; + constexpr runtime::SizeType32 nbRnnLayers = 2; + auto result = calculateCacheSizePerTokenHelper(maxAttentionWindowVec, kvFactor, vocabSize, nbLayers, + nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, false); + EXPECT_EQ(result.size(), 2); + auto const nbAttentionLayersIn128Window = 2; + auto const nbAttentionLayersIn256Window = 1; + EXPECT_EQ(result.at(128), nbAttentionLayersIn128Window * kvFactor * hiddenSize * numBytesPerFloatElement); + EXPECT_EQ(result.at(256), nbAttentionLayersIn256Window * kvFactor * hiddenSize * numBytesPerFloatElement); + } +} + +} // namespace tensorrt_llm::testing diff --git a/cpp/tests/unit_tests/executor/executorTestSmallArbitraryOutputTensors.cpp b/cpp/tests/unit_tests/executor/executorTestSmallArbitraryOutputTensors.cpp new file mode 100644 index 000000000000..b64bd775fe30 --- /dev/null +++ b/cpp/tests/unit_tests/executor/executorTestSmallArbitraryOutputTensors.cpp @@ -0,0 +1,491 @@ +#include "include/tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/batch_manager/trtGptModelInflightBatching.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/modelConfig.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/worldConfig.h" +#include "tests/utils/common.h" +#include "tests/utils/engines.h" +#include "tests/utils/executorUtils.h" + +#include "gtest/gtest.h" +#include <NvInfer.h> +#include <NvInferRuntime.h> +#include <NvInferRuntimeBase.h> +#include <gmock/gmock.h> + +#include <algorithm> +#include <chrono> +#include <cstddef> +#include <memory> +#include <optional> +#include <random> +#include <ratio> +#include <utility> +#include <vector> + +namespace tensorrt_llm::testing +{ + +struct TrivialConstantDecoderWithTopKLogitsTestParameters +{ + using TupleT = std::tuple<runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, + runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, runtime::SizeType32, bool>; + runtime::SizeType32 randomSeed; + runtime::SizeType32 vocabSize; + runtime::SizeType32 maxNumTokens; + runtime::SizeType32 maxBeamWidth; + runtime::SizeType32 maxBatchSize; + runtime::SizeType32 numTopKLogits; + runtime::SizeType32 numRequests; + runtime::SizeType32 promptLength; + runtime::SizeType32 maxOutputLength; + bool gatherContext; + + // Constructor that takes a tuple + TrivialConstantDecoderWithTopKLogitsTestParameters( // NOLINT: implicit to allow gtest to convert from tuple + // generated by 'combine' + TupleT t) + : randomSeed(std::get<0>(t)) + , vocabSize(std::get<1>(t)) + , maxNumTokens(std::get<2>(t)) + , maxBeamWidth(std::get<3>(t)) + , maxBatchSize(std::get<4>(t)) + , numTopKLogits(std::get<5>(t)) + , numRequests(std::get<6>(t)) + , promptLength(std::get<7>(t)) + , maxOutputLength(std::get<8>(t)) + , gatherContext(std::get<9>(t)) + { + } +}; + +template <typename TLogits> +struct DecoderTestShared +{ + static constexpr runtime::SizeType32 kNumTokensPerBlock = 64; + static constexpr runtime::SizeType32 kKvCacheMaxTokens = 2048 * 8; + static constexpr auto kTopKTensorName = "topKLogits"; + + DecoderTestShared(std::shared_ptr<runtime::TllmLogger> logger, std::mt19937 rng, + std::shared_ptr<executor::Executor> executor, std::vector<TLogits> randomLogits) + : logger(std::move(logger)) + , rng(rng) + , executor(std::move(executor)) + , randomLogits(std::move(randomLogits)){}; + std::shared_ptr<runtime::TllmLogger> logger; + std::mt19937 rng; + std::shared_ptr<executor::Executor> executor; + std::vector<TLogits> randomLogits; +}; + +template <typename TLogits> +std::unique_ptr<DecoderTestShared<TLogits>> SetupDecoderTest( + TrivialConstantDecoderWithTopKLogitsTestParameters const& params) +{ + auto logger = std::make_shared<runtime::TllmLogger>(); + auto rng = std::mt19937(params.randomSeed); + auto randomLogits = tensorrt_llm::testing::randomLogits<std::mt19937, TLogits>(params.vocabSize, &rng); + auto const decoderParameters = tensorrt_llm::testing::utils::engines::ConstantTrivialDecoderParameters<TLogits>{ + tensorrt_llm::testing::utils::engines::TrivialDecoderParameters{params.vocabSize, params.maxBatchSize, + params.maxNumTokens, DecoderTestShared<TLogits>::kNumTokensPerBlock, params.maxBeamWidth, + params.gatherContext}, + randomLogits}; + auto engineHostMemory = tensorrt_llm::testing::utils::engines::createConstantTrivialDecoderWithTopKLogits<TLogits>( + decoderParameters, params.numTopKLogits, DecoderTestShared<TLogits>::kTopKTensorName, logger); + auto const engine = runtime::RawEngine(engineHostMemory.release()); + + auto const dtype = runtime::TRTDataType<TLogits>::value; + auto modelConfig = runtime::ModelConfig(params.vocabSize, 1, 1, 0, 1, 1, dtype); + modelConfig.useGptAttentionPlugin(true); + modelConfig.setModelVariant(runtime::ModelConfig::ModelVariant::kGpt); + modelConfig.usePackedInput(true); + modelConfig.setKVCacheType(runtime::ModelConfig::KVCacheType::kPAGED); + modelConfig.setMaxNumTokens(params.maxNumTokens); + modelConfig.setMaxBatchSize(params.maxBatchSize); + modelConfig.setMaxBeamWidth(params.maxBeamWidth); + modelConfig.setMaxSequenceLen(params.maxNumTokens); + modelConfig.setMaxInputLen(params.maxNumTokens); + modelConfig.setLayerTypes({runtime::ModelConfig::LayerType::kATTENTION}); + modelConfig.setTokensPerBlock(DecoderTestShared<TLogits>::kNumTokensPerBlock); + modelConfig.setPagedContextFMHA(true); + modelConfig.computeContextLogits(params.gatherContext); + + auto const worldConfig = runtime::WorldConfig(); + + auto kvCacheConfig = executor::KvCacheConfig{}; + kvCacheConfig.setMaxTokens(DecoderTestShared<TLogits>::kKvCacheMaxTokens); + + auto const executorConfig + = executor::ExecutorConfig(params.maxBeamWidth, executor::SchedulerConfig(), kvCacheConfig, true, true, 1, 1, + executor::BatchingType::kINFLIGHT, params.maxBatchSize, params.maxNumTokens, std::nullopt, std::nullopt, + std::nullopt, std::nullopt, false, 1, std::nullopt, executor::ExtendedRuntimePerfKnobConfig(), std::nullopt, + 0, executor::ExecutorConfig::kDefaultMaxSeqIdleMicroseconds, std::nullopt, std::nullopt, + std::vector<executor::AdditionalModelOutput>{ + executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName, params.gatherContext}}); + + auto model = std::make_shared<batch_manager::TrtGptModelInflightBatching>( + logger, modelConfig, worldConfig, engine, false, executorConfig, false); + + return std::make_unique<DecoderTestShared<TLogits>>( + logger, rng, std::make_shared<executor::Executor>(model, executorConfig), randomLogits); +} + +template <typename TLogits> +class DecoderTopKGenerationLogitsTest + : public ::testing::Test, + public ::testing::WithParamInterface<TrivialConstantDecoderWithTopKLogitsTestParameters> +{ +protected: + std::unique_ptr<DecoderTestShared<TLogits>> state; + + DecoderTopKGenerationLogitsTest() + { + auto const params = GetParam(); + state = SetupDecoderTest<TLogits>(params); + } + + void runTopKGenerationLogitsTest(TrivialConstantDecoderWithTopKLogitsTestParameters const& parameters) + { + auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, 0); + auto requests = std::vector<executor::Request>{}; + requests.reserve(static_cast<std::size_t>(parameters.numRequests)); + for (auto i = 0; i < parameters.numRequests; i++) + { + std::vector<executor::AdditionalModelOutput> additionalOutputs{ + executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName}}; + requests.emplace_back(requestTokens, parameters.maxOutputLength, false, executor::SamplingConfig{}, + executor::OutputConfig{false, false, false, true, false, false, additionalOutputs}); + } + auto const accumulatedResponses + = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(100000)); + ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); + + std::sort(state->randomLogits.begin(), state->randomLogits.end()); + std::reverse(state->randomLogits.begin(), state->randomLogits.end()); + for (auto const& [requestId, responses] : accumulatedResponses) + { + for (auto const& response : responses) + { + ASSERT_FALSE(response.hasError()); + auto const& tokensByBeam = response.getResult().outputTokenIds; + auto const& additionalOutputs = response.getResult().additionalOutputs; + ASSERT_EQ(additionalOutputs.size(), 1); + auto const& topKLogits = additionalOutputs.front(); + auto const expectedOutputSize = parameters.maxOutputLength * parameters.numTopKLogits; + ASSERT_EQ(topKLogits.output.getSize(), expectedOutputSize); + auto const* topKLogitsData = reinterpret_cast<TLogits const*>(topKLogits.output.getData()); + for (auto i = 0; i < parameters.numTopKLogits; i++) + { + EXPECT_TRUE(almostEqual(topKLogitsData[i], state->randomLogits[i], 1e-5)) + << "requestId " << requestId << " i " << i << ": " << topKLogitsData[i] + << " != " << state->randomLogits[i]; + } + ASSERT_EQ(tokensByBeam.size(), 1); + for (auto const& tokensForBeam : tokensByBeam) + { + ASSERT_EQ(tokensForBeam.size(), parameters.maxOutputLength); + } + } + } + } +}; + +template <typename TLogits> +class DecoderTopKGenerationLogitsStreamingTest + : public ::testing::Test, + public ::testing::WithParamInterface<TrivialConstantDecoderWithTopKLogitsTestParameters> +{ +protected: + std::unique_ptr<DecoderTestShared<TLogits>> state; + + DecoderTopKGenerationLogitsStreamingTest() + { + auto const params = GetParam(); + state = SetupDecoderTest<TLogits>(params); + } + + void runTopKGenerationLogitsStreamingTest(TrivialConstantDecoderWithTopKLogitsTestParameters const& parameters) + { + auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, 0); + auto requests = std::vector<executor::Request>{}; + requests.reserve(static_cast<std::size_t>(parameters.numRequests)); + for (auto i = 0; i < parameters.numRequests; i++) + { + std::vector<executor::AdditionalModelOutput> additionalOutputs{ + executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName}}; + requests.emplace_back(requestTokens, parameters.maxOutputLength, true, executor::SamplingConfig{}, + executor::OutputConfig{false, false, false, true, false, false, additionalOutputs}); + } + auto const accumulatedResponses + = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(100000)); + ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); + + std::sort(state->randomLogits.begin(), state->randomLogits.end()); + std::reverse(state->randomLogits.begin(), state->randomLogits.end()); + for (auto const& idResponsesKvp : accumulatedResponses) + { + auto const& [requestId, responses] = idResponsesKvp; + auto numTokensForRequest = 0; + for (auto const& response : responses) + { + ASSERT_FALSE(response.hasError()); + auto const& tokensByBeam = response.getResult().outputTokenIds; + auto const& additionalOutputs = response.getResult().additionalOutputs; + ASSERT_EQ(additionalOutputs.size(), 1); + auto const& topKLogits = additionalOutputs.front(); + auto const expectedOutputSize = parameters.maxOutputLength * parameters.numTopKLogits; + ASSERT_EQ(topKLogits.output.getSize(), expectedOutputSize); + auto const* topKLogitsData = reinterpret_cast<TLogits const*>(topKLogits.output.getData()); + for (auto i = 0; i < parameters.numTopKLogits; i++) + { + EXPECT_TRUE(almostEqual(topKLogitsData[i], state->randomLogits[i], 1e-5)) + << "requestId " << requestId << " i " << i << ": " << topKLogitsData[i] + << " != " << state->randomLogits[i]; + } + ASSERT_EQ(tokensByBeam.size(), 1); + for (auto const& tokensForBeam : tokensByBeam) + { + numTokensForRequest += tokensForBeam.size(); + } + } + ASSERT_EQ(numTokensForRequest, parameters.maxOutputLength); + } + } +}; + +template <typename TLogits> +class DecoderTopKContextLogitsStreamingTest + : public ::testing::Test, + public ::testing::WithParamInterface<TrivialConstantDecoderWithTopKLogitsTestParameters> +{ +protected: + std::unique_ptr<DecoderTestShared<TLogits>> state; + + DecoderTopKContextLogitsStreamingTest() + { + auto const params = GetParam(); + state = SetupDecoderTest<TLogits>(params); + } + + void runTopKContextLogitsTest(TrivialConstantDecoderWithTopKLogitsTestParameters const& parameters) + { + auto requests = std::vector<executor::Request>{}; + requests.reserve(static_cast<std::size_t>(parameters.numRequests)); + for (auto i = 0; i < parameters.numRequests; i++) + { + // create different sequence for each request to avoid KV cache reuse + auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, i); + std::vector<executor::AdditionalModelOutput> additionalOutputs{ + executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName, true}}; + requests.emplace_back(requestTokens, parameters.maxOutputLength, true, executor::SamplingConfig{}, + executor::OutputConfig{false, false, false, true, false, false, additionalOutputs}); + } + auto const& accumulatedResponses + = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(100000)); + ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); + + std::sort(state->randomLogits.begin(), state->randomLogits.end()); + std::reverse(state->randomLogits.begin(), state->randomLogits.end()); + std::string const expectedAdditionalOutputName + = std::string("context_") + DecoderTestShared<TLogits>::kTopKTensorName; + for (auto const& idResponsesKvp : accumulatedResponses) + { + auto const& [requestId, responses] = idResponsesKvp; + std::size_t numTokensForRequest{0}; + for (auto const& response : responses) + { + ASSERT_FALSE(response.hasError()); + auto const& tokensByBeam = response.getResult().outputTokenIds; + auto const& additionalOutputs = response.getResult().additionalOutputs; + ASSERT_EQ(additionalOutputs.size(), 2); + auto const contextTopKLogitsPtr = std::find_if(additionalOutputs.cbegin(), additionalOutputs.cend(), + [&expectedAdditionalOutputName](auto const& ao) + { return ao.name == expectedAdditionalOutputName; }); + auto const expectedOutputSize = parameters.promptLength * parameters.numTopKLogits; + ASSERT_EQ(contextTopKLogitsPtr->output.getSize(), expectedOutputSize); + auto const* topKLogitsData = reinterpret_cast<TLogits const*>(contextTopKLogitsPtr->output.getData()); + for (auto i = 0; i < parameters.numTopKLogits; i++) + { + EXPECT_TRUE(almostEqual(topKLogitsData[i], state->randomLogits[i], 1e-5)) + << "requestId " << requestId << " i " << i << ": " << topKLogitsData[i] + << " != " << state->randomLogits[i]; + } + ASSERT_EQ(tokensByBeam.size(), 1); + for (auto const& tokensForBeam : tokensByBeam) + { + numTokensForRequest += static_cast<std::size_t>(tokensForBeam.size()); + } + } + ASSERT_EQ(numTokensForRequest, parameters.maxOutputLength); + } + } +}; + +template <typename TLogits> +class DecoderTopKContextLogitsTest + : public ::testing::Test, + public ::testing::WithParamInterface<TrivialConstantDecoderWithTopKLogitsTestParameters> +{ +protected: + std::unique_ptr<DecoderTestShared<TLogits>> state; + + DecoderTopKContextLogitsTest() + { + auto const params = GetParam(); + state = SetupDecoderTest<TLogits>(params); + } + + void runTopKContextLogitsTest(TrivialConstantDecoderWithTopKLogitsTestParameters const& parameters) + { + auto requests = std::vector<executor::Request>{}; + requests.reserve(static_cast<std::size_t>(parameters.numRequests)); + for (auto i = 0; i < parameters.numRequests; i++) + { + // create different sequence for each request to avoid KV cache reuse + auto const requestTokens = createConsecutiveTokenSequence(parameters.promptLength, parameters.vocabSize, i); + std::vector<executor::AdditionalModelOutput> additionalOutputs{ + executor::AdditionalModelOutput{DecoderTestShared<TLogits>::kTopKTensorName, true}}; + requests.emplace_back(requestTokens, parameters.maxOutputLength, false, executor::SamplingConfig{}, + executor::OutputConfig{false, false, false, true, false, false, additionalOutputs}); + } + auto const accumulatedResponses + = runThroughRequests(*state->executor, requests, std::chrono::duration<float, std::milli>(100000)); + ASSERT_EQ(accumulatedResponses.size(), parameters.numRequests); + + std::sort(state->randomLogits.begin(), state->randomLogits.end()); + std::reverse(state->randomLogits.begin(), state->randomLogits.end()); + std::string const expectedAdditionalOutputName + = std::string("context_") + DecoderTestShared<TLogits>::kTopKTensorName; + for (auto const& idResponsesKvp : accumulatedResponses) + { + auto const& [requestId, responses] = idResponsesKvp; + for (auto const& response : responses) + { + ASSERT_FALSE(response.hasError()); + auto const& tokensByBeam = response.getResult().outputTokenIds; + auto const& additionalOutputs = response.getResult().additionalOutputs; + ASSERT_EQ(additionalOutputs.size(), 2); + auto const contextTopKLogitsPtr = std::find_if(additionalOutputs.cbegin(), additionalOutputs.cend(), + [&expectedAdditionalOutputName](auto const& ao) + { return ao.name == expectedAdditionalOutputName; }); + auto const expectedOutputSize = parameters.promptLength * parameters.numTopKLogits; + ASSERT_EQ(contextTopKLogitsPtr->output.getSize(), expectedOutputSize); + auto const* topKLogitsData = reinterpret_cast<TLogits const*>(contextTopKLogitsPtr->output.getData()); + for (auto i = 0; i < parameters.numTopKLogits; i++) + { + EXPECT_TRUE(almostEqual(topKLogitsData[i], state->randomLogits[i], 1e-5)) + << "requestId " << requestId << " i " << i << ": " << topKLogitsData[i] + << " != " << state->randomLogits[i]; + } + ASSERT_EQ(tokensByBeam.size(), 1); + for (auto const& tokensForBeam : tokensByBeam) + { + ASSERT_EQ(tokensForBeam.size(), parameters.maxOutputLength); + } + } + } + } +}; + +namespace +{ +constexpr runtime::SizeType32 kRandomSeed1 = 45; +auto const randomSeeds = ::testing::Values(kRandomSeed1); + +constexpr runtime::SizeType32 kMinVocabSize = 64; +constexpr runtime::SizeType32 kMaxVocabSize = 2048; +auto const vocabSizes = ::testing::Values(kMinVocabSize); + +constexpr runtime::SizeType32 kMinMaxNumTokens = 2048; +auto const maxNumTokenses = ::testing::Values(kMinMaxNumTokens); + +constexpr runtime::SizeType32 kMinBeamWidth = 1; +auto const beamWidths = ::testing::Values(kMinBeamWidth); + +constexpr runtime::SizeType32 kMinMaxBatchSize = 2048; +auto const batchSizes = ::testing::Values(kMinMaxBatchSize); + +constexpr runtime::SizeType32 kMinNumTopKLogits = 4; +constexpr runtime::SizeType32 kMaxNumTopKLogits = 32; +auto const numTopKLogitses = ::testing::Values(kMinNumTopKLogits, kMaxNumTopKLogits); + +constexpr runtime::SizeType32 kMinNumRequests = 16; +constexpr runtime::SizeType32 kMaxNumRequests = 2048; +auto const numRequestses = ::testing::Values(kMinNumRequests); + +constexpr runtime::SizeType32 kMinPromptLength = 4; +constexpr runtime::SizeType32 kMaxPromptLength = 512; +auto const promptLengths = ::testing::Values(kMinPromptLength, kMaxPromptLength); + +constexpr runtime::SizeType32 kMinMaxOutputLength = 4; +constexpr runtime::SizeType32 kMaxMaxOutputLength = 256; +auto const maxOutputLengths = ::testing::Values(kMinMaxOutputLength, kMaxMaxOutputLength); + +auto const gatherContext = ::testing::Values(false, true); +auto const alwaysGatherContext = ::testing::Values(true); + +auto const paramGenerator = ::testing::ConvertGenerator<TrivialConstantDecoderWithTopKLogitsTestParameters::TupleT>( + ::testing::Combine(randomSeeds, vocabSizes, maxNumTokenses, beamWidths, batchSizes, numTopKLogitses, numRequestses, + promptLengths, maxOutputLengths, gatherContext)); + +auto const paramGeneratorGatherContext + = ::testing::ConvertGenerator<TrivialConstantDecoderWithTopKLogitsTestParameters::TupleT>( + ::testing::Combine(randomSeeds, vocabSizes, maxNumTokenses, beamWidths, batchSizes, numTopKLogitses, + numRequestses, promptLengths, maxOutputLengths, alwaysGatherContext)); + +auto const nameSuffixGenerator + = [](::testing::TestParamInfo<TrivialConstantDecoderWithTopKLogitsTestParameters> const& info) -> std::string +{ + std::stringstream nameStringStream; + nameStringStream << "gatherContext_" << info.param.gatherContext << "_maxBatchSize_" << info.param.maxBatchSize + << "_vocabSize_" << info.param.vocabSize << "_maxBeamWidth_" << info.param.maxBeamWidth + << "_maxNumTokens_" << info.param.maxNumTokens << "_maxOutputLength_" << info.param.maxOutputLength + << "_numRequests_" << info.param.numRequests << "_numTopKLogits_" << info.param.numTopKLogits + << "_promptLength_" << info.param.promptLength << "_randomSeed_" << info.param.randomSeed; + return nameStringStream.str(); +}; + +} // namespace + +using DecoderTopKGenerationLogitsFloatTest = DecoderTopKGenerationLogitsTest<float>; + +TEST_P(DecoderTopKGenerationLogitsFloatTest, TestSizeAndValues) +{ + runTopKGenerationLogitsTest(GetParam()); +} + +INSTANTIATE_TEST_SUITE_P(Float, DecoderTopKGenerationLogitsFloatTest, paramGenerator, nameSuffixGenerator); + +using DecoderTopKGenerationLogitsStreamingFloatTest = DecoderTopKGenerationLogitsStreamingTest<float>; + +TEST_P(DecoderTopKGenerationLogitsStreamingFloatTest, TestSizeAndValues) +{ + runTopKGenerationLogitsStreamingTest(GetParam()); +} + +INSTANTIATE_TEST_SUITE_P(Float, DecoderTopKGenerationLogitsStreamingFloatTest, paramGenerator, nameSuffixGenerator); + +using DecoderTopKContextLogitsStreamingFloatTest = DecoderTopKContextLogitsStreamingTest<float>; + +TEST_P(DecoderTopKContextLogitsStreamingFloatTest, TestSizeAndValues) +{ + runTopKContextLogitsTest(GetParam()); +} + +INSTANTIATE_TEST_SUITE_P( + Float, DecoderTopKContextLogitsStreamingFloatTest, paramGeneratorGatherContext, nameSuffixGenerator); + +using DecoderTopKContextLogitsFloatTest = DecoderTopKContextLogitsTest<float>; + +TEST_P(DecoderTopKContextLogitsFloatTest, TestSizeAndValues) +{ + runTopKContextLogitsTest(GetParam()); +} + +INSTANTIATE_TEST_SUITE_P(Float, DecoderTopKContextLogitsFloatTest, paramGeneratorGatherContext, nameSuffixGenerator); + +} // namespace tensorrt_llm::testing diff --git a/cpp/tests/unit_tests/executor/intervalSetTest.cpp b/cpp/tests/unit_tests/executor/intervalSetTest.cpp new file mode 100644 index 000000000000..a2bb0a8f7532 --- /dev/null +++ b/cpp/tests/unit_tests/executor/intervalSetTest.cpp @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/executor/intervalSet.h" + +#include <gtest/gtest.h> + +using tensorrt_llm::executor::IntervalSet; +using tensorrt_llm::executor::IdType; + +class IntervalSetTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) +{ +protected: + void SetUp() override {} + + void TearDown() override {} + + IntervalSet<IdType> mIntervalSet; +}; + +namespace +{ + +TEST_F(IntervalSetTest, testPublicAPI) +{ + EXPECT_FALSE(mIntervalSet.contains(0)); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); + mIntervalSet.insert(0); + mIntervalSet.insert(1); + mIntervalSet.insert(4); + mIntervalSet.insert(6); + EXPECT_TRUE(mIntervalSet.contains(0)); + EXPECT_TRUE(mIntervalSet.contains(4)); + EXPECT_FALSE(mIntervalSet.contains(2125)); + EXPECT_EQ(mIntervalSet.getNumElements(), 4); + mIntervalSet.insert(6); + mIntervalSet.insert(4); + mIntervalSet.insert(1); + mIntervalSet.insert(0); + EXPECT_EQ(mIntervalSet.getNumElements(), 4); + EXPECT_TRUE(mIntervalSet.contains(0)); + EXPECT_TRUE(mIntervalSet.contains(4)); + EXPECT_FALSE(mIntervalSet.contains(9)); + EXPECT_FALSE(mIntervalSet.contains(3)); + EXPECT_FALSE(mIntervalSet.contains(11)); + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); +} + +TEST_F(IntervalSetTest, testClear) +{ + for (int i = 0; i < 100; i++) + { + if (i % 2 == 0) + { + EXPECT_FALSE(mIntervalSet.contains(i)); + mIntervalSet.insert(i); + EXPECT_TRUE(mIntervalSet.contains(i)); + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); + } + EXPECT_FALSE(mIntervalSet.contains(i)); + } + for (int i = 0; i < 100; i++) + { + EXPECT_FALSE(mIntervalSet.contains(i)); + } + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); + + for (int i = 19; i >= 10; i--) + { + if (i % 2 == 0) + { + EXPECT_FALSE(mIntervalSet.contains(i)); + mIntervalSet.insert(i); + EXPECT_TRUE(mIntervalSet.contains(i)); + } + else + { + EXPECT_FALSE(mIntervalSet.contains(i)); + } + } + EXPECT_EQ(mIntervalSet.getNumElements(), 5); + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); +} + +TEST_F(IntervalSetTest, testRandomInsert) +{ + mIntervalSet.insert(4); + mIntervalSet.insert(8); + EXPECT_EQ(mIntervalSet.getNumElements(), 2); + std::vector<int> idToAdd{9, 7, 5, 1, 6, 0, 2}; + for (auto id : idToAdd) + { + mIntervalSet.insert(id); + } + for (int i = 0; i < 10; i++) + { + if (i != 3) + { + EXPECT_TRUE(mIntervalSet.contains(i)); + } + else + { + EXPECT_FALSE(mIntervalSet.contains(i)); + } + } + EXPECT_EQ(mIntervalSet.getNumElements(), 9); + mIntervalSet.insert(3); + for (int i = 0; i < 10; i++) + { + EXPECT_TRUE(mIntervalSet.contains(i)); + } + EXPECT_EQ(mIntervalSet.getNumElements(), 10); + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); +} + +TEST_F(IntervalSetTest, testTerminatedReqIdIntervals) +{ + mIntervalSet.insert(4); + mIntervalSet.insert(8); + EXPECT_EQ(mIntervalSet.getNumElements(), 2); + // terminatedReqIdIntervals is [[4, 4], [8, 8]] + EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); + mIntervalSet.insert(3); + mIntervalSet.insert(5); + // terminatedReqIdIntervals is [[3, 5], [8, 8]] + EXPECT_EQ(mIntervalSet.getNumElements(), 4); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); + mIntervalSet.insert(9); + mIntervalSet.insert(7); + // terminatedReqIdIntervals is [[3, 5], [7, 9]] + EXPECT_EQ(mIntervalSet.getNumElements(), 6); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); + mIntervalSet.insert(6); + // terminatedReqIdIntervals is [[3, 9]] + EXPECT_EQ(mIntervalSet.getNumElements(), 7); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); + mIntervalSet.insert(1); + // terminatedReqIdIntervals is [[1, 1], [3, 9]] + EXPECT_EQ(mIntervalSet.getNumElements(), 8); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); + mIntervalSet.insert(0); + // terminatedReqIdIntervals is [[0, 1], [3, 9]] + EXPECT_EQ(mIntervalSet.getNumElements(), 9); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 2); + mIntervalSet.insert(2); + // terminatedReqIdIntervals is [[0, 9]] + EXPECT_EQ(mIntervalSet.getNumElements(), 10); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); + for (int i = 0; i < 10; i++) + { + mIntervalSet.insert(i); + // terminatedReqIdIntervals is always [[0, 9]] + EXPECT_EQ(mIntervalSet.getNumElements(), 10); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); + } + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 0); + + // Insert continuous decreasing numbers. Interval size is always one. + for (int i = 19; i >= 10; i--) + { + mIntervalSet.insert(i); + EXPECT_TRUE(mIntervalSet.contains(i)); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); + } + mIntervalSet.clear(); + EXPECT_EQ(mIntervalSet.getNumElements(), 0); + EXPECT_EQ(mIntervalSet.getIntervals().size(), 0); + + // Insert 50 disjoint even numbers + for (int i = 0; i < 100; i++) + { + if (i % 2 == 0) + { + mIntervalSet.insert(i); + EXPECT_EQ(mIntervalSet.getNumElements(), (i / 2) + 1); + EXPECT_EQ(mIntervalSet.getIntervals().size(), (i / 2) + 1); + } + } + + // Insert 50 disjoint odd numbers. Interval size should go down as the intervals are merged. + for (int i = 0; i < 100; i++) + { + if (i % 2 != 0) + { + mIntervalSet.insert(i); + EXPECT_EQ(mIntervalSet.getNumElements(), 50 + (i + 1) / 2); + if (i != 99) + { + EXPECT_EQ(mIntervalSet.getIntervals().size(), 50 - (i + 1) / 2); + } + else + { + EXPECT_EQ(mIntervalSet.getIntervals().size(), 1); + } + } + } +} + +} // namespace diff --git a/cpp/tests/unit_tests/executor/requestTest.cpp b/cpp/tests/unit_tests/executor/requestTest.cpp index ec43d088657f..f44f55b5141d 100644 --- a/cpp/tests/unit_tests/executor/requestTest.cpp +++ b/cpp/tests/unit_tests/executor/requestTest.cpp @@ -26,9 +26,7 @@ using ::testing::_; using ::testing::Invoke; using namespace tensorrt_llm::executor; -// Not a namespace-wide import: common also exports DataType/Dims, which would -// make the unqualified DataType (= executor::DataType) below ambiguous. -using tensorrt_llm::common::TllmException; +using namespace tensorrt_llm::common; TEST(RequestTest, validInputs) { diff --git a/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp b/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp index 3c78659c87c4..a39756bf7243 100644 --- a/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp +++ b/cpp/tests/unit_tests/executor/serializeUtilsTest.cpp @@ -18,7 +18,6 @@ #include "tensorrt_llm/executor/serializeUtils.h" #include "tensorrt_llm/batch_manager/kvCacheManager.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/executor.h" @@ -752,8 +751,7 @@ TEST(SerializeUtilsTest, ContextPhaseParams) { auto state = std::make_unique<texec::DataTransceiverState>(); state->setCommState(texec::kv_cache::CommState{12, "127.0.0.1"}); - state->setCacheState( - texec::kv_cache::CacheState{10, 12, 128, 128, 8, 8, 8, {4}, tensorrt_llm::DataType::kFLOAT}); + state->setCacheState(texec::kv_cache::CacheState{10, 12, 128, 128, 8, 8, 8, {4}, nvinfer1::DataType::kFLOAT}); auto stats = texec::ContextPhaseParams({10, 20, 30, 40, 50, 60}, 0, state.release(), VecTokens{10, 20}); auto stats2 = serializeDeserialize(stats); EXPECT_EQ(stats, stats2); @@ -1555,7 +1553,7 @@ TEST(SerializeUtilsTest, CacheStateIndexerKCache) texec::SizeType32 pp = 1; texec::SizeType32 cp = 1; std::vector<texec::SizeType32> attentionLayerNumPerPP{static_cast<texec::SizeType32>(nbKvHeadsPerLayer.size())}; - auto dataType = tensorrt_llm::DataType::kFLOAT; + auto dataType = nvinfer1::DataType::kFLOAT; auto attentionType = CacheState::AttentionType::kDEFAULT; int kvFactor = 2; bool enableAttentionDP = false; diff --git a/cpp/tests/unit_tests/executor/transferAgentTest.cpp b/cpp/tests/unit_tests/executor/transferAgentTest.cpp index dcf6dd5a6e1e..b915ec3bb9c4 100644 --- a/cpp/tests/unit_tests/executor/transferAgentTest.cpp +++ b/cpp/tests/unit_tests/executor/transferAgentTest.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -702,7 +702,8 @@ TEST(VmmDescSplitterTest, SplitTransferDescsDifferentChunkSizes) MemoryDescs srcInput{MemoryType::kVRAM, srcDescs}; MemoryDescs dstInput{MemoryType::kVRAM, dstDescs}; - auto [splitSrc, splitDst] = VmmDescSplitter::splitAndCoalesceTransferDescs(srcInput, dstInput, localMap, remoteMap); + auto [splitSrc, splitDst] + = VmmDescSplitter::splitTransferDescsWithRegionMaps(srcInput, dstInput, localMap, remoteMap); // dst has smaller chunks (512KB), so we get 4 pieces: 512K, 512K, 512K, 512K ASSERT_EQ(splitSrc.getDescs().size(), 4); @@ -734,7 +735,8 @@ TEST(VmmDescSplitterTest, SplitTransferDescsUnalignedBothSides) MemoryDescs srcInput{MemoryType::kVRAM, srcDescs}; MemoryDescs dstInput{MemoryType::kVRAM, dstDescs}; - auto [splitSrc, splitDst] = VmmDescSplitter::splitAndCoalesceTransferDescs(srcInput, dstInput, localMap, remoteMap); + auto [splitSrc, splitDst] + = VmmDescSplitter::splitTransferDescsWithRegionMaps(srcInput, dstInput, localMap, remoteMap); // src has 4MB chunks starting at srcBase → 1 piece from src side // dst has 2MB chunks starting at dstBase → 2 pieces from dst side @@ -758,7 +760,7 @@ TEST(VmmDescSplitterTest, SplitTransferDescsNoDstRegion) MemoryDescs dstInput{MemoryType::kVRAM, dstDescs}; auto [splitSrc, splitDst] - = VmmDescSplitter::splitAndCoalesceTransferDescs(srcInput, dstInput, localMap, emptyRemoteMap); + = VmmDescSplitter::splitTransferDescsWithRegionMaps(srcInput, dstInput, localMap, emptyRemoteMap); // Only src boundaries: 2 pieces of 1MB each ASSERT_EQ(splitSrc.getDescs().size(), 2); diff --git a/cpp/tests/unit_tests/executor/ucxCommTest.cpp b/cpp/tests/unit_tests/executor/ucxCommTest.cpp index 5d2bfc6e772f..51d1d84d2676 100644 --- a/cpp/tests/unit_tests/executor/ucxCommTest.cpp +++ b/cpp/tests/unit_tests/executor/ucxCommTest.cpp @@ -36,7 +36,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/mpi_utils/connection.h" #include "tensorrt_llm/executor/dataTransceiverState.h" #include "tensorrt_llm/executor/executor.h" @@ -133,11 +132,11 @@ TEST_F(UcxCommTest, Basic) tensorrt_llm::runtime::BufferManager bufferManager{std::make_shared<tensorrt_llm::runtime::CudaStream>()}; // Create and fill source CUDA buffer with random data - auto srcBuffer = bufferManager.gpu(buffer.size(), tensorrt_llm::DataType::kINT8); + auto srcBuffer = bufferManager.gpu(buffer.size(), nvinfer1::DataType::kINT8); bufferManager.copy(buffer.data(), *srcBuffer); bufferManager.getStream().synchronize(); - auto dstBuffer = bufferManager.gpu(buffer.size(), tensorrt_llm::DataType::kINT8); + auto dstBuffer = bufferManager.gpu(buffer.size(), nvinfer1::DataType::kINT8); // Send CUDA buffer using connection1 connection1->send(DataContext{0x75}, srcBuffer->data(), srcBuffer->getSizeInBytes()); @@ -205,14 +204,14 @@ TEST_F(UcxCommTest, multiSend) tensorrt_llm::runtime::BufferManager bufferManager{std::make_shared<tensorrt_llm::runtime::CudaStream>()}; - auto srcBuffer1 = bufferManager.gpu(buffer1.size(), tensorrt_llm::DataType::kINT8); - auto srcBuffer2 = bufferManager.gpu(buffer2.size(), tensorrt_llm::DataType::kINT8); + auto srcBuffer1 = bufferManager.gpu(buffer1.size(), nvinfer1::DataType::kINT8); + auto srcBuffer2 = bufferManager.gpu(buffer2.size(), nvinfer1::DataType::kINT8); bufferManager.copy(buffer1.data(), *srcBuffer1); bufferManager.copy(buffer2.data(), *srcBuffer2); bufferManager.getStream().synchronize(); - auto dstBuffer1 = bufferManager.gpu(buffer1.size(), tensorrt_llm::DataType::kINT8); - auto dstBuffer2 = bufferManager.gpu(buffer2.size(), tensorrt_llm::DataType::kINT8); + auto dstBuffer1 = bufferManager.gpu(buffer1.size(), nvinfer1::DataType::kINT8); + auto dstBuffer2 = bufferManager.gpu(buffer2.size(), nvinfer1::DataType::kINT8); connection1Peer->send(DataContext{0x75}, srcBuffer1->data(), srcBuffer1->getSizeInBytes()); connection2Peer->send(DataContext{0x75}, srcBuffer2->data(), srcBuffer2->getSizeInBytes()); diff --git a/cpp/tests/unit_tests/kernels/CMakeLists.txt b/cpp/tests/unit_tests/kernels/CMakeLists.txt index 5f121d2406a9..e593ce4b76b8 100644 --- a/cpp/tests/unit_tests/kernels/CMakeLists.txt +++ b/cpp/tests/unit_tests/kernels/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & # AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -35,14 +35,9 @@ if(USING_OSS_CUTLASS_MOE_GEMM) target_compile_definitions(mixtureOfExpertsTest PUBLIC USING_OSS_CUTLASS_MOE_GEMM) - # The internal-path variant includes headers (quantization.h) that only ship - # with the internal cutlass kernels sources; the prebuilt tarball provides the - # library and a reduced header set only. - if(INTERNAL_CUTLASS_KERNELS_PATH) - add_gtest(mixtureOfExpertsInternalTest mixtureOfExpertsTest.cu) - remove_compile_definition(mixtureOfExpertsInternalTest - USING_OSS_CUTLASS_MOE_GEMM) - endif() + add_gtest(mixtureOfExpertsInternalTest mixtureOfExpertsTest.cu) + remove_compile_definition(mixtureOfExpertsInternalTest + USING_OSS_CUTLASS_MOE_GEMM) endif() add_gtest(ropeTest ropeTest.cu) diff --git a/cpp/tests/unit_tests/kernels/banRepeatNGramsKernelsTest.cpp b/cpp/tests/unit_tests/kernels/banRepeatNGramsKernelsTest.cpp index 83e94ea2d6ad..567cb95e8e44 100644 --- a/cpp/tests/unit_tests/kernels/banRepeatNGramsKernelsTest.cpp +++ b/cpp/tests/unit_tests/kernels/banRepeatNGramsKernelsTest.cpp @@ -15,7 +15,6 @@ */ #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/banRepeatNgram.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -53,25 +52,24 @@ class BanRepeatNgramKernelsTest : public testing::Test SizeType32 const batchSize = outputIds.size(); auto const maxBatchSize = 2 * batchSize; - mLogits - = BufferManager::pinned(ITensor::makeShape({batchSize, mVocabSizePadded}), tensorrt_llm::DataType::kFLOAT); + mLogits = BufferManager::pinned(ITensor::makeShape({batchSize, mVocabSizePadded}), nvinfer1::DataType::kFLOAT); mSequenceLengths - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mBeamWidth}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mBeamWidth}), nvinfer1::DataType::kINT32); mFinished = BufferManager::pinned( ITensor::makeShape({maxBatchSize, mBeamWidth}), TRTDataType<tk::FinishedState::UnderlyingType>::value); mOutputIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize, mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); mOutputIdsPtr = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mBeamWidth}), ptrType); mParentIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize, mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); mParentIdsPtr = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mBeamWidth}), ptrType); - mNGramSizes = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mNGramSizes = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*mBatchSlots); for (SizeType32 bi = 0; bi < batchSize; ++bi) diff --git a/cpp/tests/unit_tests/kernels/cudaCoreGemm/cudaCoreGemmKernelTest.cpp b/cpp/tests/unit_tests/kernels/cudaCoreGemm/cudaCoreGemmKernelTest.cpp index f7dabf98a93e..05247e2d27a4 100644 --- a/cpp/tests/unit_tests/kernels/cudaCoreGemm/cudaCoreGemmKernelTest.cpp +++ b/cpp/tests/unit_tests/kernels/cudaCoreGemm/cudaCoreGemmKernelTest.cpp @@ -1,3 +1,4 @@ +#include <NvInferRuntime.h> #include <cublasLt.h> #include <cuda_fp8.h> #include <cuda_profiler_api.h> diff --git a/cpp/tests/unit_tests/kernels/decodingKernelTest.cpp b/cpp/tests/unit_tests/kernels/decodingKernelTest.cpp index a458f783ef65..4b94e67cb5a1 100644 --- a/cpp/tests/unit_tests/kernels/decodingKernelTest.cpp +++ b/cpp/tests/unit_tests/kernels/decodingKernelTest.cpp @@ -15,7 +15,6 @@ */ #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/kernels/decodingKernels.h" #include "tensorrt_llm/kernels/speculativeDecoding/externalDraftTokensKernels.h" @@ -217,15 +216,13 @@ class TestBeamHypothesesCopy : public ::testing::Test srcBeams.empty(*mBufferManager); srcBeams.reshape(batchSize, beamWidth, maxSeqLen); - mSrcCumLogProbs - = mBufferManager->gpu(ITensor::makeShape({batchSize, beamWidth}), tensorrt_llm::DataType::kFLOAT); + mSrcCumLogProbs = mBufferManager->gpu(ITensor::makeShape({batchSize, beamWidth}), nvinfer1::DataType::kFLOAT); setBuffers(srcBeams, mSrcCumLogProbs, 2); dstBeams.empty(*mBufferManager); dstBeams.reshape(batchSize, beamWidth, maxSeqLen); - mDstCumLogProbs - = mBufferManager->gpu(ITensor::makeShape({batchSize, beamWidth}), tensorrt_llm::DataType::kFLOAT); + mDstCumLogProbs = mBufferManager->gpu(ITensor::makeShape({batchSize, beamWidth}), nvinfer1::DataType::kFLOAT); setBuffers(dstBeams, mDstCumLogProbs, 1); } @@ -547,7 +544,7 @@ class TestGatherTree : public ::testing::Test SizeType32 constexpr nbRnnLayers{0}; SizeType32 constexpr nbHeads{16}; SizeType32 constexpr hiddenSize{1024}; - tensorrt_llm::DataType constexpr dtype{tensorrt_llm::DataType::kFLOAT}; + nvinfer1::DataType constexpr dtype{nvinfer1::DataType::kFLOAT}; ModelConfig modelConfig{ vocabSize, nbAttentionLayers + nbRnnLayers, nbAttentionLayers, nbRnnLayers, nbHeads, hiddenSize, dtype}; @@ -1142,35 +1139,32 @@ class DecodingKernelsTest : public testing::Test auto const ptrType = TRTDataType<T*>::value; mDraftTokens = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqlen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqlen}), nvinfer1::DataType::kINT32); mTargetTokens = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxTargetSeqlen}), tensorrt_llm::DataType::kINT32); - mOutputTokens = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxTargetSeqlen}), nvinfer1::DataType::kINT32); + mOutputTokens + = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); mNumsDraftTokens = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqPerStep}), tensorrt_llm::DataType::kINT32); - mSequenceLengths - = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mAcceptedLengths - = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mContextLengths - = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqPerStep}), nvinfer1::DataType::kINT32); + mSequenceLengths = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mAcceptedLengths = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mContextLengths = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mFinishedSteps = mBufferManager->pinnedPool(ITensor::makeShape({mMaxDraftTokens + 1, mMaxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); mFinishedFinal = mBufferManager->pinnedPool( ITensor::makeShape({mMaxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mFinishedSum = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mFinishedSum = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mPaths = mBufferManager->pinnedPool( - ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqPerStep, mMaxDraftTokens}), tensorrt_llm::DataType::kINT32); - mEndIds = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxDraftSeqPerStep, mMaxDraftTokens}), nvinfer1::DataType::kINT32); + mEndIds = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mBatchSlots = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); auto batchSlotsRange = BufferRange<SizeType32>(*mBatchSlots); std::iota(batchSlotsRange.begin(), batchSlotsRange.end(), 0); mCurandStates = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); + ITensor::makeShape({mMaxBatchSize, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); mAcceptedLen.resize(mMaxBatchSize); mOutputLen.resize(mMaxBatchSize); @@ -1200,9 +1194,8 @@ class DecodingKernelsTest : public testing::Test mMedusaInputLogitsPtrs = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize, mMaxNumHeads}), ptrType); mTokensPerStep - = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mBestPaths - = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mBestPaths = mBufferManager->pinnedPool(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); } } diff --git a/cpp/tests/unit_tests/kernels/eaglePackDataTest.cpp b/cpp/tests/unit_tests/kernels/eaglePackDataTest.cpp index 8ce24f813e65..bdf74efb59be 100644 --- a/cpp/tests/unit_tests/kernels/eaglePackDataTest.cpp +++ b/cpp/tests/unit_tests/kernels/eaglePackDataTest.cpp @@ -25,8 +25,9 @@ #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/runtimeKernels.h" +#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntimeBase.h> #include <algorithm> #include <cstdint> @@ -131,81 +132,81 @@ class EaglePackDataTest : public ::testing::Test { // inputs mBatchSlots = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mInputTemperatures = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kFLOAT); mInputRandomDataSample = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kFLOAT); mInputRandomDataValidation = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kFLOAT); + nvinfer1::DataType::kFLOAT); mInputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mInputNextDraftPaths = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mInputSpecDecodingGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mInputSpecDecodingPositionOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); auto const numPackedMasks = static_cast<SizeType32>(tensorrt_llm::common::divUp(mSamplingParams.getMaxDecodingTokens(), 32)); mInputSpecDecodingPackedMasks = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), numPackedMasks}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); // outputs mOutputTemperatures = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kFLOAT); mOutputRandomDataSample = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kFLOAT); mOutputRandomDataValidation = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kFLOAT); + nvinfer1::DataType::kFLOAT); mOutputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mOutputNextDraftLens = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mOutputNextDraftPaths = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mOutputSpecDecodingGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mOutputSpecDecodingPositionOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mOutputSpecDecodingPackedMasks = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), numPackedMasks}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); // workspace - mMaxGenerationLength = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + mMaxGenerationLength = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); mCumSumGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize() + 1}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getBatchSize() + 1}), nvinfer1::DataType::kINT32); mScanReduceTempStorageBytes = tksd::invokeScanReduceGenerationLengths( mSamplingParams.getBatchSize(), nullptr, nullptr, 0, nullptr, nullptr, mStream->get()); diff --git a/cpp/tests/unit_tests/kernels/mixtureOfExpertsTest.cu b/cpp/tests/unit_tests/kernels/mixtureOfExpertsTest.cu index 61c182dfd1ea..01cd1c4d792d 100644 --- a/cpp/tests/unit_tests/kernels/mixtureOfExpertsTest.cu +++ b/cpp/tests/unit_tests/kernels/mixtureOfExpertsTest.cu @@ -32,7 +32,6 @@ #endif #include "tensorrt_llm/kernels/cutlass_kernels/include/cutlass_kernel_selector.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include <tensorrt_llm/kernels/cutlass_kernels/cutlass_type_conversion.h> @@ -2509,31 +2508,31 @@ constexpr static auto typeToDtypeID() { if constexpr (std::is_same_v<T, SafeFP8>) { - return tensorrt_llm::DataType::kFP8; + return nvinfer1::DataType::kFP8; } else if constexpr (std::is_same_v<T, SafeFP4>) { - return tensorrt_llm::DataType::kFP4; + return nvinfer1::DataType::kFP4; } else if constexpr (std::is_same_v<T, uint8_t>) { - return tensorrt_llm::DataType::kINT8; + return nvinfer1::DataType::kINT8; } else if constexpr (std::is_same_v<T, cutlass::uint4b_t>) { - return tensorrt_llm::DataType::kINT4; + return nvinfer1::DataType::kINT4; } else if constexpr (std::is_same_v<T, nv_bfloat16>) { - return tensorrt_llm::DataType::kBF16; + return nvinfer1::DataType::kBF16; } else if constexpr (std::is_same_v<T, half>) { - return tensorrt_llm::DataType::kHALF; + return nvinfer1::DataType::kHALF; } else if constexpr (std::is_same_v<T, float>) { - return tensorrt_llm::DataType::kFLOAT; + return nvinfer1::DataType::kFLOAT; } else { @@ -2603,16 +2602,14 @@ TEST_F(MixtureOfExpertsProfilerTest, TestGeneratedProfilerDistribution) for (int ep : {1, 4, 8}) { #ifdef USING_OSS_CUTLASS_MOE_GEMM - backend.init(this->mMoERunner, GemmProfilerBackend::GemmToProfile::GEMM_1, tensorrt_llm::DataType::kHALF, - tensorrt_llm::DataType::kHALF, tensorrt_llm::DataType::kHALF, num_experts, k, 1024, 1024, 4096, - mGroupSize, {}, false, mUseLora, /*min_latency_mode=*/false, /*need_weights=*/true, - MOEParallelismConfig{1, 0, ep, 0}, + backend.init(this->mMoERunner, GemmProfilerBackend::GemmToProfile::GEMM_1, nvinfer1::DataType::kHALF, + nvinfer1::DataType::kHALF, nvinfer1::DataType::kHALF, num_experts, k, 1024, 1024, 4096, mGroupSize, {}, + false, mUseLora, /*min_latency_mode=*/false, /*need_weights=*/true, MOEParallelismConfig{1, 0, ep, 0}, /*enable_alltoall=*/false); #else - backend.init(this->mMoERunner, GemmProfilerBackend::GemmToProfile::GEMM_1, tensorrt_llm::DataType::kHALF, - tensorrt_llm::DataType::kHALF, tensorrt_llm::DataType::kHALF, num_experts, k, 1024, 4096, mGroupSize, - {}, false, mUseLora, /*min_latency_mode=*/false, /*need_weights=*/true, - MOEParallelismConfig{1, 0, ep, ep - 1}); + backend.init(this->mMoERunner, GemmProfilerBackend::GemmToProfile::GEMM_1, nvinfer1::DataType::kHALF, + nvinfer1::DataType::kHALF, nvinfer1::DataType::kHALF, num_experts, k, 1024, 4096, mGroupSize, {}, false, + mUseLora, /*min_latency_mode=*/false, /*need_weights=*/true, MOEParallelismConfig{1, 0, ep, ep - 1}); #endif auto ws_size = backend.getWorkspaceSize(num_tokens); diff --git a/cpp/tests/unit_tests/kernels/mlaChunkedPrefillTest.cu b/cpp/tests/unit_tests/kernels/mlaChunkedPrefillTest.cu index c1fa77239729..3e4e9a1da0a3 100644 --- a/cpp/tests/unit_tests/kernels/mlaChunkedPrefillTest.cu +++ b/cpp/tests/unit_tests/kernels/mlaChunkedPrefillTest.cu @@ -8,7 +8,6 @@ #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/mlaChunkedPrefill.cuh" #include "tensorrt_llm/runtime/cudaStream.h" #include <cstring> @@ -430,18 +429,18 @@ protected: using tensorrt_llm::runtime::ITensor; using tensorrt_llm::runtime::bufferCast; - auto dtype = tensorrt_llm::DataType::kHALF; + auto dtype = nvinfer1::DataType::kHALF; if constexpr (std::is_same_v<DataType, float>) { - dtype = tensorrt_llm::DataType::kFLOAT; + dtype = nvinfer1::DataType::kFLOAT; } else if constexpr (std::is_same_v<DataType, half>) { - dtype = tensorrt_llm::DataType::kHALF; + dtype = nvinfer1::DataType::kHALF; } else if constexpr (std::is_same_v<DataType, __nv_bfloat16>) { - dtype = tensorrt_llm::DataType::kBF16; + dtype = nvinfer1::DataType::kBF16; } else { @@ -450,11 +449,11 @@ protected: auto cacheType = dtype; if constexpr (std::is_same_v<TCache, __nv_fp8_e4m3>) { - cacheType = tensorrt_llm::DataType::kFP8; + cacheType = nvinfer1::DataType::kFP8; this->h_kv_scale_quant_orig - = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); - this->d_kv_scale_quant_orig = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); + = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); + this->d_kv_scale_quant_orig + = tensorrt_llm::runtime::BufferManager::gpuSync(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); auto* kv_scale_quant_orig_ptr = bufferCast<float>(*(this->h_kv_scale_quant_orig)); float kv_scale_orig_quant = 2.0F; kv_scale_quant_orig_ptr[0] = 1.0 / kv_scale_orig_quant; @@ -464,13 +463,13 @@ protected: // cu lens this->h_cu_kv_seq_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mBatchSize + 1}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({this->mBatchSize + 1}), nvinfer1::DataType::kINT64); this->h_cu_q_seq_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mBatchSize + 1}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({this->mBatchSize + 1}), nvinfer1::DataType::kINT64); this->d_cu_kv_seq_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_cu_kv_seq_lens->getShape(), tensorrt_llm::DataType::kINT64); + this->h_cu_kv_seq_lens->getShape(), nvinfer1::DataType::kINT64); this->d_cu_q_seq_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_cu_q_seq_lens->getShape(), tensorrt_llm::DataType::kINT64); + this->h_cu_q_seq_lens->getShape(), nvinfer1::DataType::kINT64); { this->mMaxSeqLen = 0; this->mMaxQSeqLen = 0; @@ -513,14 +512,14 @@ protected: int const total_cached_kv_len = this->mTotalKVLen - this->mTotalQLen; int const chunked_loop_num = (total_cached_kv_len + total_chunk_size - 1) / total_chunk_size; this->h_cu_chunk_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize + 1}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize + 1}), nvinfer1::DataType::kINT64); this->h_chunked_ld_global_offset = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize}), nvinfer1::DataType::kINT64); this->memsetZeroHost(this->h_chunked_ld_global_offset); this->d_cu_chunk_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_cu_chunk_lens->getShape(), tensorrt_llm::DataType::kINT64); + this->h_cu_chunk_lens->getShape(), nvinfer1::DataType::kINT64); this->d_chunked_ld_global_offset = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_chunked_ld_global_offset->getShape(), tensorrt_llm::DataType::kINT64); + this->h_chunked_ld_global_offset->getShape(), nvinfer1::DataType::kINT64); // kv cache this->mMaxBlockPerSeq = (this->mMaxSeqLen + this->mTokensPerBlock - 1) / this->mTokensPerBlock; @@ -540,13 +539,13 @@ protected: this->mLoraSize + this->mRopeSize}), cacheType); this->h_compressed_offset_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mBatchSize, 2, this->mMaxBlockPerSeq + 1}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({this->mBatchSize, 2, this->mMaxBlockPerSeq + 1}), nvinfer1::DataType::kINT32); this->d_kv_cache_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->h_kv_cache_tensor->getShape(), dtype); this->d_compressed_kv_cache_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->h_compressed_kv_cache_tensor->getShape(), cacheType); this->d_compressed_offset_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - this->h_compressed_offset_tensor->getShape(), tensorrt_llm::DataType::kINT32); + this->h_compressed_offset_tensor->getShape(), nvinfer1::DataType::kINT32); { auto* compressed_kv_cache_ptr = bufferCast<TCache>(*(this->h_compressed_kv_cache_tensor)); @@ -602,15 +601,15 @@ protected: this->m_h_output_tensor = tensorrt_llm::runtime::BufferManager::pinned( ITensor::makeShape({this->mTotalQLen, this->mNumHeads, this->mNopeSize}), dtype); this->m_h_softmax_sum_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({2, this->mTotalQLen, this->mNumHeads}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({2, this->mTotalQLen, this->mNumHeads}), nvinfer1::DataType::kFLOAT); this->m_h_softmax_sum_accum_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({2, this->mTotalQLen, this->mNumHeads}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({2, this->mTotalQLen, this->mNumHeads}), nvinfer1::DataType::kFLOAT); this->m_h_output_tensor_ref = tensorrt_llm::runtime::BufferManager::pinned( ITensor::makeShape({this->mTotalQLen, this->mNumHeads, this->mNopeSize}), dtype); this->m_h_output_tensor_accum = tensorrt_llm::runtime::BufferManager::pinned( ITensor::makeShape({this->mTotalQLen, this->mNumHeads, this->mNopeSize}), dtype); this->m_h_merge_op = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({chunked_loop_num + 1, this->mBatchSize}), nvinfer1::DataType::kINT64); this->m_d_q_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_q_tensor->getShape(), dtype); this->m_d_kv_full_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_kv_full_tensor->getShape(), dtype); @@ -619,13 +618,13 @@ protected: this->m_d_output_tensor = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_output_tensor->getShape(), dtype); this->m_d_softmax_sum_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - this->m_h_softmax_sum_tensor->getShape(), tensorrt_llm::DataType::kFLOAT); + this->m_h_softmax_sum_tensor->getShape(), nvinfer1::DataType::kFLOAT); this->m_d_softmax_sum_accum_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - this->m_h_softmax_sum_accum_tensor->getShape(), tensorrt_llm::DataType::kFLOAT); + this->m_h_softmax_sum_accum_tensor->getShape(), nvinfer1::DataType::kFLOAT); this->m_d_output_tensor_accum = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_output_tensor_accum->getShape(), dtype); - this->m_d_merge_op = tensorrt_llm::runtime::BufferManager::gpuSync( - this->m_h_merge_op->getShape(), tensorrt_llm::DataType::kINT64); + this->m_d_merge_op + = tensorrt_llm::runtime::BufferManager::gpuSync(this->m_h_merge_op->getShape(), nvinfer1::DataType::kINT64); { auto* q_ptr = bufferCast<DataType>(*(this->m_h_q_tensor)); diff --git a/cpp/tests/unit_tests/kernels/mlaPreprocessTest.cu b/cpp/tests/unit_tests/kernels/mlaPreprocessTest.cu index 3fc249a2f24a..f2c0863779bc 100644 --- a/cpp/tests/unit_tests/kernels/mlaPreprocessTest.cu +++ b/cpp/tests/unit_tests/kernels/mlaPreprocessTest.cu @@ -23,7 +23,6 @@ #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/runtime/bufferManager.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/mlaKernels.h" #include <random> @@ -233,18 +232,18 @@ protected: using tensorrt_llm::runtime::ITensor; using tensorrt_llm::runtime::bufferCast; - auto dtype = tensorrt_llm::DataType::kHALF; + auto dtype = nvinfer1::DataType::kHALF; if constexpr (std::is_same_v<DataType, float>) { - dtype = tensorrt_llm::DataType::kFLOAT; + dtype = nvinfer1::DataType::kFLOAT; } else if constexpr (std::is_same_v<DataType, half>) { - dtype = tensorrt_llm::DataType::kHALF; + dtype = nvinfer1::DataType::kHALF; } else if constexpr (std::is_same_v<DataType, __nv_bfloat16>) { - dtype = tensorrt_llm::DataType::kBF16; + dtype = nvinfer1::DataType::kBF16; } else { @@ -253,15 +252,15 @@ protected: auto cache_dtype = dtype; if constexpr (std::is_same_v<TCache, __nv_fp8_e4m3>) { - cache_dtype = tensorrt_llm::DataType::kFP8; + cache_dtype = nvinfer1::DataType::kFP8; this->h_kv_scale_orig_quant - = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); - this->d_kv_scale_orig_quant = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); + = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); + this->d_kv_scale_orig_quant + = tensorrt_llm::runtime::BufferManager::gpuSync(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); this->h_kv_scale_quant_orig - = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); - this->d_kv_scale_quant_orig = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); + = tensorrt_llm::runtime::BufferManager::pinned(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); + this->d_kv_scale_quant_orig + = tensorrt_llm::runtime::BufferManager::gpuSync(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); auto* kv_scale_orig_quant_ptr = bufferCast<float>(*(this->h_kv_scale_orig_quant)); auto* kv_scale_quant_orig_ptr = bufferCast<float>(*(this->h_kv_scale_quant_orig)); float kv_scale_orig_quant = 2.0f; @@ -277,13 +276,13 @@ protected: static_assert(std::is_same_v<DataType, TCache>, "TCache must be the same type as DataType"); } this->h_cu_seq_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mNumRequests + 1}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({this->mNumRequests + 1}), nvinfer1::DataType::kINT64); this->h_cu_ctx_cached_kv_lens = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mNumRequests + 1}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({this->mNumRequests + 1}), nvinfer1::DataType::kINT64); this->d_cu_seq_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({this->mNumRequests + 1}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({this->mNumRequests + 1}), nvinfer1::DataType::kINT64); this->d_cu_ctx_cached_kv_lens = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({this->mNumRequests + 1}), tensorrt_llm::DataType::kINT64); + ITensor::makeShape({this->mNumRequests + 1}), nvinfer1::DataType::kINT64); { // set random sequence length auto* cu_seq_lens_temp_ptr = bufferCast<int64_t>(*(this->h_cu_seq_lens)); @@ -334,9 +333,9 @@ protected: this->mTokensPerBlock, this->mLoraSize + this->mRopeSize}), cache_dtype); this->h_offset_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), nvinfer1::DataType::kINT32); this->h_compressed_offset_tensor = tensorrt_llm::runtime::BufferManager::pinned( - ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), nvinfer1::DataType::kINT32); this->d_kv_cache_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq, this->mNumHeadsUncompressed, this->mTokensPerBlock, this->mUncompressedHeadSize + this->mRopeSize}), @@ -350,9 +349,9 @@ protected: this->mTokensPerBlock, this->mLoraSize + this->mRopeSize}), cache_dtype); this->d_offset_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), nvinfer1::DataType::kINT32); this->d_compressed_offset_tensor = tensorrt_llm::runtime::BufferManager::gpuSync( - ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({this->mNumRequests, 2, this->mMaxBlockPerSeq}), nvinfer1::DataType::kINT32); { auto* kv_cache_ptr = bufferCast<DataType>(*(this->h_kv_cache_tensor)); auto* kv_cache_ref_ptr = bufferCast<DataType>(*(this->h_kv_cache_tensor_ref)); diff --git a/cpp/tests/unit_tests/kernels/prepareCustomMaskTest.cpp b/cpp/tests/unit_tests/kernels/prepareCustomMaskTest.cpp index 6616c9c47668..61617934f236 100644 --- a/cpp/tests/unit_tests/kernels/prepareCustomMaskTest.cpp +++ b/cpp/tests/unit_tests/kernels/prepareCustomMaskTest.cpp @@ -25,7 +25,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaKernels.h" #include "tensorrt_llm/kernels/trtllmGenKernels/fmha/fmhaRunnerParams.h" #include "tensorrt_llm/kernels/trtllmGenKernels/fmha/prepareCustomMask.h" @@ -294,8 +293,8 @@ class PrepareCustomMaskTest : public ::testing::Test int64_t totalMaskSize = static_cast<int64_t>(batchSize) * maxNumTilesQ * maxNumCustomMaskTilesKv * numInstsQ * numInstsKv * (tileSizeQ * tileSizeKvPadded) / 32; - auto customMaskOffsetsDevice = mBufferManager->gpu(batchSize, tensorrt_llm::DataType::kINT64); - auto customMaskDevice = mBufferManager->gpu(totalMaskSize, tensorrt_llm::DataType::kINT32); + auto customMaskOffsetsDevice = mBufferManager->gpu(batchSize, nvinfer1::DataType::kINT64); + auto customMaskDevice = mBufferManager->gpu(totalMaskSize, nvinfer1::DataType::kINT32); // Clear GPU buffers to ensure no stale data from previous tests cudaMemsetAsync(bufferCast<int64_t>(*customMaskOffsetsDevice), 0, batchSize * sizeof(int64_t), mStream->get()); diff --git a/cpp/tests/unit_tests/kernels/ropeTest.cu b/cpp/tests/unit_tests/kernels/ropeTest.cu index 36c91481722f..517b006e4fde 100644 --- a/cpp/tests/unit_tests/kernels/ropeTest.cu +++ b/cpp/tests/unit_tests/kernels/ropeTest.cu @@ -15,13 +15,11 @@ */ #include <gtest/gtest.h> -#include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/quantization.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" -#include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/kvCacheUtils.h" #include "tensorrt_llm/kernels/unfusedAttentionKernels.h" +#include "tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h" #include "tensorrt_llm/runtime/bufferManager.h" #include <random> @@ -31,7 +29,6 @@ #include <cuda_fp4.h> #endif -using namespace tensorrt_llm::common; using namespace tensorrt_llm::runtime; using namespace tensorrt_llm::kernels; @@ -505,27 +502,26 @@ protected: { auto const cu_seqlens_size = batch_size + 1; - cu_q_seqlens_tensor - = mBufferManager->pinned(ITensor::makeShape({cu_seqlens_size}), tensorrt_llm::DataType::kINT32); + cu_q_seqlens_tensor = mBufferManager->pinned(ITensor::makeShape({cu_seqlens_size}), nvinfer1::DataType::kINT32); cu_kv_seqlens_tensor - = mBufferManager->pinned(ITensor::makeShape({cu_seqlens_size}), tensorrt_llm::DataType::kINT32); - padding_offset_tensor = mBufferManager->pinned( - ITensor::makeShape({batch_size, input_seq_length}), tensorrt_llm::DataType::kINT32); - encoder_padding_offset_tensor = mBufferManager->pinned( - ITensor::makeShape({batch_size, cross_qkv_length}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({cu_seqlens_size}), nvinfer1::DataType::kINT32); + padding_offset_tensor + = mBufferManager->pinned(ITensor::makeShape({batch_size, input_seq_length}), nvinfer1::DataType::kINT32); + encoder_padding_offset_tensor + = mBufferManager->pinned(ITensor::makeShape({batch_size, cross_qkv_length}), nvinfer1::DataType::kINT32); fmha_tile_counter_ptr_tensor - = mBufferManager->pinned(ITensor::makeShape({mEnableContextFMHA ? 1 : 0}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({mEnableContextFMHA ? 1 : 0}), nvinfer1::DataType::kINT32); rotary_inv_freq_buf_tensor = mBufferManager->pinned( - ITensor::makeShape({batch_size, mRotaryEmbeddingDim / 2}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({batch_size, mRotaryEmbeddingDim / 2}), nvinfer1::DataType::kFLOAT); int const max_num_tokens = batch_size * input_seq_length; tokens_info_tensor - = mBufferManager->pinned(ITensor::makeShape({max_num_tokens, 2}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({max_num_tokens, 2}), nvinfer1::DataType::kINT32); #ifdef ENABLE_FP4 if constexpr (std::is_same_v<KVCacheType, __nv_fp4_e2m1>) { - global_scale_tensor = mBufferManager->pinned(ITensor::makeShape({2}), tensorrt_llm::DataType::kFLOAT); + global_scale_tensor = mBufferManager->pinned(ITensor::makeShape({2}), nvinfer1::DataType::kFLOAT); } #endif } @@ -588,7 +584,7 @@ protected: // // Rotary cos sin cache buffer to avoid re-computing. SizeType32 maxOutputSize{generateRandomSizeSmallerThan(1024)}; rotary_cos_sin_tensor = this->mBufferManager->pinned( - ITensor::makeShape({mRotaryEmbeddingMaxPositions, mRotaryEmbeddingDim}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({mRotaryEmbeddingMaxPositions, mRotaryEmbeddingDim}), nvinfer1::DataType::kFLOAT); rotary_fill_help = bufferCast<float>(*(rotary_cos_sin_tensor)); // createCosSinBuf(rotary_fill_help, mRotaryEmbeddingMaxPositions, mRotaryEmbeddingDim); //currently broken // fillWithOnesAndZerosInterleaved(rotary_fill_help, mRotaryEmbeddingMaxPositions* @@ -598,7 +594,7 @@ protected: batch_size = generateRandomSizeSmallerThan(12); - q_seq_lengths_tensor = mBufferManager->pinned(ITensor::makeShape({batch_size}), tensorrt_llm::DataType::kINT32); + q_seq_lengths_tensor = mBufferManager->pinned(ITensor::makeShape({batch_size}), nvinfer1::DataType::kINT32); q_seq_lengths = bufferCast<int32_t>(*(q_seq_lengths_tensor)); for (SizeType32 ii = 0; ii < batch_size; ++ii) diff --git a/cpp/tests/unit_tests/kernels/routing/routingDeepSeekTest.cpp b/cpp/tests/unit_tests/kernels/routing/routingDeepSeekTest.cpp index be5c8f3c48d1..78598fa4c417 100644 --- a/cpp/tests/unit_tests/kernels/routing/routingDeepSeekTest.cpp +++ b/cpp/tests/unit_tests/kernels/routing/routingDeepSeekTest.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ -#include "tensorrt_llm/common/tllmDataType.h" #include "tests/unit_tests/kernels/routing/routingTest.h" namespace tk = tensorrt_llm::kernels; @@ -157,8 +156,8 @@ class RoutingDeepSeekKernelTest : public RoutingKernelTest<T> { RoutingKernelTest<T>::allocateBuffers(param); int64_t scoresSize = param.numTokens * param.numExperts; - this->mPtrScoresHost = mBufferManager->pinned(ITensor::makeShape({scoresSize}), tensorrt_llm::DataType::kFLOAT); - this->mPtrScoresDevice = mBufferManager->gpu(ITensor::makeShape({scoresSize}), tensorrt_llm::DataType::kFLOAT); + this->mPtrScoresHost = mBufferManager->pinned(ITensor::makeShape({scoresSize}), nvinfer1::DataType::kFLOAT); + this->mPtrScoresDevice = mBufferManager->gpu(ITensor::makeShape({scoresSize}), nvinfer1::DataType::kFLOAT); this->mPtrRoutingBiasHost = mBufferManager->pinned(ITensor::makeShape({param.numExperts}), TRTDataType<T>::value); @@ -431,9 +430,9 @@ TYPED_TEST(RoutingDeepSeekKernelTest, ClusterLevelWithFloat32Bias) // the GPU kernel (using fp32 bias) and the host reference (using T-typed bias) // observe numerically equivalent inputs. auto float32BiasHost - = this->mBufferManager->pinned(ITensor::makeShape({param.numExperts}), tensorrt_llm::DataType::kFLOAT); + = this->mBufferManager->pinned(ITensor::makeShape({param.numExperts}), nvinfer1::DataType::kFLOAT); auto float32BiasDevice - = this->mBufferManager->gpu(ITensor::makeShape({param.numExperts}), tensorrt_llm::DataType::kFLOAT); + = this->mBufferManager->gpu(ITensor::makeShape({param.numExperts}), nvinfer1::DataType::kFLOAT); auto fp32BiasPtr = bufferCast<float>(*float32BiasHost); auto tBiasPtr = bufferCast<TypeParam>(*this->mPtrRoutingBiasHost); for (int i = 0; i < param.numExperts; i++) diff --git a/cpp/tests/unit_tests/kernels/routing/routingTest.cpp b/cpp/tests/unit_tests/kernels/routing/routingTest.cpp index c71510dc319c..ba5c020ade9e 100644 --- a/cpp/tests/unit_tests/kernels/routing/routingTest.cpp +++ b/cpp/tests/unit_tests/kernels/routing/routingTest.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ #include "tests/unit_tests/kernels/routing/routingTest.h" -#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::tests::kernels::routing { @@ -55,25 +54,25 @@ void RoutingKernelTest<T>::allocateBuffers(RoutingKernelTestParam const& param) { countsSize = 2 * 256; } - mPtrExpertCountsHost = mBufferManager->pinned(ITensor::makeShape({countsSize}), tensorrt_llm::DataType::kINT32); - mPtrExpertCountsDevice = mBufferManager->gpu(ITensor::makeShape({countsSize}), tensorrt_llm::DataType::kINT32); + mPtrExpertCountsHost = mBufferManager->pinned(ITensor::makeShape({countsSize}), nvinfer1::DataType::kINT32); + mPtrExpertCountsDevice = mBufferManager->gpu(ITensor::makeShape({countsSize}), nvinfer1::DataType::kINT32); int64_t permIdxSize = 1; - mPtrPermutedIdxSizeHost = mBufferManager->pinned(ITensor::makeShape({permIdxSize}), tensorrt_llm::DataType::kINT32); - mPtrPermutedIdxSizeDevice = mBufferManager->gpu(ITensor::makeShape({permIdxSize}), tensorrt_llm::DataType::kINT32); + mPtrPermutedIdxSizeHost = mBufferManager->pinned(ITensor::makeShape({permIdxSize}), nvinfer1::DataType::kINT32); + mPtrPermutedIdxSizeDevice = mBufferManager->gpu(ITensor::makeShape({permIdxSize}), nvinfer1::DataType::kINT32); int64_t expIdxToPermIdxSize = numTokens * topK; mPtrExpandedIdxToPermutedIdxHost - = mBufferManager->pinned(ITensor::makeShape({expIdxToPermIdxSize}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({expIdxToPermIdxSize}), nvinfer1::DataType::kINT32); mPtrExpandedIdxToPermutedIdxDevice - = mBufferManager->gpu(ITensor::makeShape({expIdxToPermIdxSize}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({expIdxToPermIdxSize}), nvinfer1::DataType::kINT32); // int64_t permIdxToTokenIdxSize = (numTokens * topK + (numExperts << paddingLog2) - numExperts); int64_t permIdxToTokenIdxSize = (numTokens * topK + (numExperts * tileTokensDim) - numExperts); mPtrPermutedIdxToTokenIdxHost - = mBufferManager->pinned(ITensor::makeShape({permIdxToTokenIdxSize}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({permIdxToTokenIdxSize}), nvinfer1::DataType::kINT32); mPtrPermutedIdxToTokenIdxDevice - = mBufferManager->gpu(ITensor::makeShape({permIdxToTokenIdxSize}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({permIdxToTokenIdxSize}), nvinfer1::DataType::kINT32); int64_t expWeightsSize = numTokens * topK; mPtrTopKWeightsHost = mBufferManager->pinned(ITensor::makeShape({expWeightsSize}), TRTDataType<T>::value); @@ -82,8 +81,8 @@ void RoutingKernelTest<T>::allocateBuffers(RoutingKernelTestParam const& param) if (useTopKAsInput) { int64_t topKIdsSize = numTokens * topK; - mPtrTopKIdsHost = mBufferManager->pinned(ITensor::makeShape({topKIdsSize}), tensorrt_llm::DataType::kINT32); - mPtrTopKIdsDevice = mBufferManager->gpu(ITensor::makeShape({topKIdsSize}), tensorrt_llm::DataType::kINT32); + mPtrTopKIdsHost = mBufferManager->pinned(ITensor::makeShape({topKIdsSize}), nvinfer1::DataType::kINT32); + mPtrTopKIdsDevice = mBufferManager->gpu(ITensor::makeShape({topKIdsSize}), nvinfer1::DataType::kINT32); } else { @@ -92,26 +91,23 @@ void RoutingKernelTest<T>::allocateBuffers(RoutingKernelTestParam const& param) } int64_t ctaIdxSize = numTokens * topK; - mPtrCtaIdxXyToBatchIdxHost - = mBufferManager->pinned(ITensor::makeShape({ctaIdxSize}), tensorrt_llm::DataType::kINT32); - mPtrCtaIdxXyToBatchIdxDevice - = mBufferManager->gpu(ITensor::makeShape({ctaIdxSize}), tensorrt_llm::DataType::kINT32); + mPtrCtaIdxXyToBatchIdxHost = mBufferManager->pinned(ITensor::makeShape({ctaIdxSize}), nvinfer1::DataType::kINT32); + mPtrCtaIdxXyToBatchIdxDevice = mBufferManager->gpu(ITensor::makeShape({ctaIdxSize}), nvinfer1::DataType::kINT32); - mPtrCtaIdxXyToMnLimitHost - = mBufferManager->pinned(ITensor::makeShape({ctaIdxSize}), tensorrt_llm::DataType::kINT32); - mPtrCtaIdxXyToMnLimitDevice = mBufferManager->gpu(ITensor::makeShape({ctaIdxSize}), tensorrt_llm::DataType::kINT32); + mPtrCtaIdxXyToMnLimitHost = mBufferManager->pinned(ITensor::makeShape({ctaIdxSize}), nvinfer1::DataType::kINT32); + mPtrCtaIdxXyToMnLimitDevice = mBufferManager->gpu(ITensor::makeShape({ctaIdxSize}), nvinfer1::DataType::kINT32); int64_t numNonExitingCtasSize = 1; mPtrNumNonExitingCtasHost - = mBufferManager->pinned(ITensor::makeShape({numNonExitingCtasSize}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({numNonExitingCtasSize}), nvinfer1::DataType::kINT32); mPtrNumNonExitingCtasDevice - = mBufferManager->gpu(ITensor::makeShape({numNonExitingCtasSize}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({numNonExitingCtasSize}), nvinfer1::DataType::kINT32); int64_t idxSize = numTokens * topK * sizeof(PackedType); - mPtrTopKPackedHost = mBufferManager->pinned(ITensor::makeShape({idxSize}), tensorrt_llm::DataType::kINT8); - mPtrTopKPackedDevice = mBufferManager->gpu(ITensor::makeShape({idxSize}), tensorrt_llm::DataType::kINT8); + mPtrTopKPackedHost = mBufferManager->pinned(ITensor::makeShape({idxSize}), nvinfer1::DataType::kINT8); + mPtrTopKPackedDevice = mBufferManager->gpu(ITensor::makeShape({idxSize}), nvinfer1::DataType::kINT8); mCurandStatesDevice - = mBufferManager->gpu(ITensor::makeShape({numTokens, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({numTokens, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); } template <typename T> @@ -131,19 +127,19 @@ void RoutingKernelTest<T>::computePermutation(RoutingKernelTestParam const& para PackedType* expIdxHostPtr = reinterpret_cast<PackedType*>(bufferCast<int8_t>(*this->mPtrTopKPackedHost)); auto tokenToExpertHost - = mBufferManager->pinned(ITensor::makeShape({param.numTokens * param.topK}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({param.numTokens * param.topK}), nvinfer1::DataType::kINT32); auto tokenToExpertHostPtr = bufferCast<int32_t>(*tokenToExpertHost); auto tokenToIdxInExpertHost - = mBufferManager->pinned(ITensor::makeShape({param.numTokens * param.topK}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({param.numTokens * param.topK}), nvinfer1::DataType::kINT32); auto tokenToIdxInExpertHostPtr = bufferCast<int32_t>(*tokenToIdxInExpertHost); auto expertScanCountsHost - = mBufferManager->pinned(ITensor::makeShape({param.numExperts + 1}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({param.numExperts + 1}), nvinfer1::DataType::kINT32); auto expertScanCountsHostPtr = bufferCast<int32_t>(*expertScanCountsHost); auto ctaScanCountsHost - = mBufferManager->pinned(ITensor::makeShape({param.numExperts + 1}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({param.numExperts + 1}), nvinfer1::DataType::kINT32); auto ctaScanCountsHostPtr = bufferCast<int32_t>(*ctaScanCountsHost); for (int ie = 0; ie < param.numExperts + 1; ++ie) @@ -411,7 +407,7 @@ void RoutingKernelTest<T>::runTest(RoutingKernelTestParam const& param) // Retrieve the workspace size of the routing kernel. auto const workspaceSize = getDeviceWorkspaceSize(param); TensorPtr workspaceDevice - = mBufferManager->gpu(ITensor::makeShape({static_cast<int64_t>(workspaceSize)}), tensorrt_llm::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({static_cast<int64_t>(workspaceSize)}), nvinfer1::DataType::kINT8); // Call tested function routing callTestedFunction(param, workspaceDevice); // Verify results diff --git a/cpp/tests/unit_tests/kernels/routing/routingTest.h b/cpp/tests/unit_tests/kernels/routing/routingTest.h index 8b24ee3aa24e..630cd72a5fcb 100644 --- a/cpp/tests/unit_tests/kernels/routing/routingTest.h +++ b/cpp/tests/unit_tests/kernels/routing/routingTest.h @@ -23,6 +23,7 @@ #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/runtimeKernels.h" +#include "tensorrt_llm/runtime/tllmLogger.h" #include <chrono> #include <cmath> #include <memory> //@todo check the usage of this diff --git a/cpp/tests/unit_tests/kernels/sampling/samplingPenaltyTest.cpp b/cpp/tests/unit_tests/kernels/sampling/samplingPenaltyTest.cpp index a188abf6bc31..8896dd005cf7 100644 --- a/cpp/tests/unit_tests/kernels/sampling/samplingPenaltyTest.cpp +++ b/cpp/tests/unit_tests/kernels/sampling/samplingPenaltyTest.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/penaltyTypes.h" #include "tests/unit_tests/kernels/sampling/samplingTest.h" @@ -162,14 +161,14 @@ class TemperaturePenaltyTest : public SamplingKernelTest<T> mLogitsPtrs = BufferManager::pinned(ITensor::makeShape({mBatchSize}), ptrType); mPenaltyWorkspaceDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep, mVocabSize * 2}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep, mVocabSize * 2}), nvinfer1::DataType::kINT32); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mBiasHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mVocabSizePadded}), dataType); mBiasDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mVocabSizePadded}), dataType); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); trk::invokeFill(*mLogitsRefHost, T{0.0f}, *mStream); trk::invokeFill(*mOutLogitsDevice, T{0.0f}, *mStream); @@ -205,7 +204,7 @@ class TemperaturePenaltyTest : public SamplingKernelTest<T> ASSERT_EQ(param.temperaturesSize, mMaxBatchSize) << "Invalid test configuration."; mTemperaturesDevice - = mBufferManager->gpu(ITensor::makeShape({param.temperaturesSize}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({param.temperaturesSize}), nvinfer1::DataType::kFLOAT); mBufferManager->copy(*param.temperatures, *mTemperaturesDevice); } @@ -282,8 +281,7 @@ TYPED_TEST(TemperaturePenaltyTest, NoPenalty) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 1.0f; @@ -299,8 +297,7 @@ TYPED_TEST(TemperaturePenaltyTest, LessThanOne) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 0.53f; @@ -316,8 +313,7 @@ TYPED_TEST(TemperaturePenaltyTest, GreaterThaneOne) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 2.01f; @@ -333,8 +329,7 @@ TYPED_TEST(TemperaturePenaltyTest, Mixed) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 0.53f + 0.2f * i; @@ -350,8 +345,7 @@ TYPED_TEST(TemperaturePenaltyTest, LargeVocab) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 0.53f + 0.2f * i; @@ -367,8 +361,7 @@ TYPED_TEST(TemperaturePenaltyTest, LargeVocabTokensPerStep) { int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; - TensorPtr temperaturesHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + TensorPtr temperaturesHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*temperaturesHost)[i] = 1.f; // 0.53f + 0.2f * i; @@ -548,25 +541,25 @@ class RepetitionPenaltyTest : public SamplingKernelTest<T> mLogitsPtrs = BufferManager::pinned(ITensor::makeShape({mBatchSize}), ptrType); mPenaltyWorkspaceDevice = mBufferManager->gpu( - ITensor::makeShape({mBatchSize, mMaxTokensPerStep, mVocabSize * 2}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mBatchSize, mMaxTokensPerStep, mVocabSize * 2}), nvinfer1::DataType::kINT32); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mOutputIdsHost = BufferManager::pinned( - ITensor::makeShape({mMaxBatchSize, mSequenceLength}), tensorrt_llm::DataType::kINT32); + mOutputIdsHost + = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mSequenceLength}), nvinfer1::DataType::kINT32); mOutputIdsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mSequenceLength}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mSequenceLength}), nvinfer1::DataType::kINT32); - mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mIdsPtrHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), ptrType); mIdsPtrDevice = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), ptrType); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*mBatchSlots); for (SizeType32 bi = 0; bi < mBatchSize; ++bi) @@ -613,13 +606,13 @@ class RepetitionPenaltyTest : public SamplingKernelTest<T> ASSERT_EQ(param.frequencyPenaltiesSize, mMaxBatchSize) << "Invalid test configuration."; ASSERT_EQ(param.promptIgnoreLengthsSize, mMaxBatchSize) << "Invalid test configuration."; mRepetitionPenaltiesDevice - = mBufferManager->gpu(ITensor::makeShape({param.repetitionPenaltiesSize}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({param.repetitionPenaltiesSize}), nvinfer1::DataType::kFLOAT); mPresencePenaltiesDevice - = mBufferManager->gpu(ITensor::makeShape({param.presencePenaltiesSize}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({param.presencePenaltiesSize}), nvinfer1::DataType::kFLOAT); mFrequencyPenaltiesDevice - = mBufferManager->gpu(ITensor::makeShape({param.frequencyPenaltiesSize}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({param.frequencyPenaltiesSize}), nvinfer1::DataType::kFLOAT); mPromptIgnoreLengthsDevice - = mBufferManager->gpu(ITensor::makeShape({param.promptIgnoreLengthsSize}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({param.promptIgnoreLengthsSize}), nvinfer1::DataType::kINT32); mBufferManager->copy(*param.repetitionPenalties, *mRepetitionPenaltiesDevice); mBufferManager->copy(*param.presencePenalties, *mPresencePenaltiesDevice); mBufferManager->copy(*param.frequencyPenalties, *mFrequencyPenaltiesDevice); @@ -747,13 +740,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchNoPenalty) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -780,13 +773,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchRepetitionLessThanOne) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53f; @@ -813,13 +806,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchRepetitionGreaterThaneOne) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 2.01f; @@ -846,13 +839,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchRepetitionMixed) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -879,13 +872,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchPresenceMixed) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -912,13 +905,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchPresenceHasDefaultValueZero2) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -945,13 +938,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchFrequencyMixed) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -978,13 +971,13 @@ TYPED_TEST(RepetitionPenaltyTest, BatchFrequencyHasDefaultValueZero2) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -1011,13 +1004,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeRepetitionPresence) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1044,13 +1037,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeRepetitionFrequency) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1077,13 +1070,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypePresenceFrequency) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 1.0f; @@ -1110,13 +1103,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeFull) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1143,13 +1136,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeFullTokensPerStep) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1177,13 +1170,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeFullWithPartialPromptIgnore) int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1210,13 +1203,13 @@ TYPED_TEST(RepetitionPenaltyTest, PenaltyTypeFullTokensPerStepWithFullPromptIgno int32_t batchSize = 6; int32_t maxBatchSize = 2 * batchSize; TensorPtr repetitionPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr presencePenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr frequencyPenaltyHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); TensorPtr promptIgnoreLengthsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); for (int32_t i = 0; i < maxBatchSize; ++i) { bufferCast<float>(*repetitionPenaltyHost)[i] = 0.53 + i * 0.2f; @@ -1340,23 +1333,23 @@ class MinLengthPenaltyTest : public SamplingKernelTest<T> mLogitsPtrs = BufferManager::pinned(ITensor::makeShape({mBatchSize}), ptrType); mPenaltyWorkspaceDevice = mBufferManager->gpu( - ITensor::makeShape({mBatchSize, mMaxTokensPerStep, mVocabSize}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mBatchSize, mMaxTokensPerStep, mVocabSize}), nvinfer1::DataType::kINT32); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mMinLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mMinLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mMinLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mMinLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mEndIdsHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIdsHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*mBatchSlots); for (SizeType32 bi = 0; bi < mBatchSize; ++bi) @@ -1558,30 +1551,30 @@ class MinLengthPenaltyOOBSafetyTest : public SamplingKernelTest<T> } // Defines currentStep. - mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); initConstant<int>(bufferCast<int32_t>(*mSeqLengthHost), mMaxBatchSize, 3); - mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mBufferManager->copy(*mSeqLengthHost, *mSeqLengthDevice); // Defines inputLength. - mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mContextLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); initConstant<int>(bufferCast<int32_t>(*mContextLengthHost), mMaxBatchSize, 2); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mBufferManager->copy(*mContextLengthHost, *mContextLengthDevice); // Defines minLength. - mMinLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mMinLengthHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); initConstant<int>(bufferCast<int32_t>(*mMinLengthHost), mMaxBatchSize, 10); - mMinLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mMinLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mBufferManager->copy(*mMinLengthHost, *mMinLengthDevice); // Defines endIds. - mEndIdsHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIdsHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); initConstant<int>(bufferCast<int32_t>(*mEndIdsHost), mMaxBatchSize, -1); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mBufferManager->copy(*mEndIdsHost, *mEndIdsDevice); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*mBatchSlots); for (SizeType32 bi = 0; bi < mBatchSize; ++bi) { diff --git a/cpp/tests/unit_tests/kernels/sampling/samplingTest.cpp b/cpp/tests/unit_tests/kernels/sampling/samplingTest.cpp index 1d4f19c45ad2..90da247f5203 100644 --- a/cpp/tests/unit_tests/kernels/sampling/samplingTest.cpp +++ b/cpp/tests/unit_tests/kernels/sampling/samplingTest.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ #include "tests/unit_tests/kernels/sampling/samplingTest.h" -#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::tests::kernels::sampling { @@ -52,78 +51,75 @@ void SamplingKernelTest<T>::allocateBuffers(SamplingKernelTestParam const& param auto const ptrType = TRTDataType<T*>::value; // Allocate GPU data - mSeqLengthsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); mFinishedHost = BufferManager::pinned( ITensor::makeShape({maxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); mFinishedDevice = mBufferManager->gpu( ITensor::makeShape({maxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mOutputIdsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); - mOutputIdsDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + mOutputIdsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); + mOutputIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); mProbsHost = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep, vocabSize}), dataType); mProbsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize, maxTokensPerStep, vocabSize}), dataType); mProbsPtrsDevice - = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT64); + = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep}), nvinfer1::DataType::kINT64); - mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); if (param.returnAllSelectedTokens) { SizeType32 maxTopK = param.topK == 0 ? vocabSize : param.topK; mOutputLogProbsDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxTopK}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxTopK}), nvinfer1::DataType::kFLOAT); } else { mOutputLogProbsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxSeqLen, maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({mMaxSeqLen, maxBatchSize}), nvinfer1::DataType::kFLOAT); } mZeroParentIdsDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxTokensPerStep}), nvinfer1::DataType::kINT32); mLogitsHost = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep, vocabSize}), dataType); mLogProbsHost = BufferManager::pinned(ITensor::makeShape({batchSize, maxTokensPerStep, vocabSize}), dataType); mIdsPtrHost = BufferManager::pinned(ITensor::makeShape({2 * maxBatchSize}), ptrType); - mEndIdsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIdsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mTopPsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); - mTopPsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + mTopPsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); + mTopPsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); - mTopKsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mTopKsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mTopKsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mTopKsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mSkipDecodeHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kBOOL); - mSkipDecodeDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kBOOL); + mSkipDecodeHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kBOOL); + mSkipDecodeDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kBOOL); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); - mExpectedCumLogProbsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + mExpectedCumLogProbsHost = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); if (param.returnAllSelectedTokens) { SizeType32 maxTopK = param.topK == 0 ? vocabSize : param.topK; mExpectedLogProbsHost - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, maxTopK}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, maxTopK}), nvinfer1::DataType::kFLOAT); } else { mExpectedLogProbsHost - = BufferManager::pinned(ITensor::makeShape({mMaxSeqLen, maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({mMaxSeqLen, maxBatchSize}), nvinfer1::DataType::kFLOAT); } mCurandStatesDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); } template <typename T> @@ -498,7 +494,7 @@ void SamplingKernelTest<T>::runTest(SamplingKernelTestParam const& param) // Retrieve the workspace size of the sampling kernel. auto const workspaceSize = getWorkspaceSize(param); TensorPtr workspaceDevice - = mBufferManager->gpu(ITensor::makeShape({static_cast<int32_t>(workspaceSize)}), tensorrt_llm::DataType::kINT8); + = mBufferManager->gpu(ITensor::makeShape({static_cast<int32_t>(workspaceSize)}), nvinfer1::DataType::kINT8); // Call tested function sampling callTestedFunction(param, workspaceDevice); diff --git a/cpp/tests/unit_tests/kernels/sampling/samplingTest.h b/cpp/tests/unit_tests/kernels/sampling/samplingTest.h index 74268bee092c..0c7f52ba369e 100644 --- a/cpp/tests/unit_tests/kernels/sampling/samplingTest.h +++ b/cpp/tests/unit_tests/kernels/sampling/samplingTest.h @@ -27,6 +27,7 @@ #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/runtimeKernels.h" +#include "tensorrt_llm/runtime/tllmLogger.h" namespace tensorrt_llm::tests::kernels::sampling { diff --git a/cpp/tests/unit_tests/kernels/sampling/samplingUtilsTest.cu b/cpp/tests/unit_tests/kernels/sampling/samplingUtilsTest.cu index f0f7690257e7..71a6d767171c 100644 --- a/cpp/tests/unit_tests/kernels/sampling/samplingUtilsTest.cu +++ b/cpp/tests/unit_tests/kernels/sampling/samplingUtilsTest.cu @@ -14,7 +14,6 @@ * limitations under the License. */ -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/gptDecoder.h" #include "tests/unit_tests/kernels/sampling/samplingTest.h" #include <random> @@ -58,8 +57,7 @@ TEST_F(SamplingUtilsKernelTest, CurandInitialize) sync_check_cuda_error(this->mStream->get()); // Generate random numbers using initialized curand states.MemoryType - auto randValsDevice - = this->mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + auto randValsDevice = this->mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); generateRandomNumber<<<1, batchSize, 0, this->mStream->get()>>>( bufferCast<int32_t>(*randValsDevice), batchSlotsPtr, curandStates, batchSize); auto randValsHost = this->mBufferManager->copyFrom(*randValsDevice, MemoryType::kCPU); @@ -99,7 +97,7 @@ TEST_F(SamplingUtilsKernelTest, CurandBatchInitialize) curandState_t* curandStates; cudaMalloc(&curandStates, sizeof(curandState_t) * 2 * batchSize); - auto randomSeedsHost = mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT64); + auto randomSeedsHost = mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT64); auto randomSeedsHostPtr = bufferCast<int64_t>(*randomSeedsHost); size_t const periodSize = 3; for (size_t i = 0; i < batchSize; ++i) @@ -108,7 +106,7 @@ TEST_F(SamplingUtilsKernelTest, CurandBatchInitialize) } auto randomSeedsDevice = mBufferManager->copyFrom(*randomSeedsHost, MemoryType::kGPU); - auto batchSlots = mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + auto batchSlots = mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); auto batchSlotsPtr = bufferCast<SizeType32>(*batchSlots); for (SizeType32 bi = 0; bi < batchSize; ++bi) @@ -122,7 +120,7 @@ TEST_F(SamplingUtilsKernelTest, CurandBatchInitialize) sync_check_cuda_error(mStream->get()); // Generate random numbers using initialized curand states. - auto randValsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + auto randValsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); generateRandomNumber<<<1, batchSize, 0, this->mStream->get()>>>( bufferCast<SizeType32>(*randValsDevice), batchSlotsPtr, curandStates, batchSize); auto const randValsHost = mBufferManager->copyFrom(*randValsDevice, MemoryType::kCPU); @@ -168,26 +166,25 @@ public: ITensor::makeShape({batchSize, maxBeamWidth, vocabSizePadded}), dataType); ITensor::SharedPtr logitsHostPtrs = this->mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), ptrType); auto refLogitsHost = this->mBufferManager->pinnedPool( - ITensor::makeShape({batchSize, maxBeamWidth, vocabSizePadded}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({batchSize, maxBeamWidth, vocabSizePadded}), nvinfer1::DataType::kFLOAT); auto refEntropyHost = this->mBufferManager->pinnedPool( - ITensor::makeShape({maxBatchSize, maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); - auto entropyDevice = this->mBufferManager->gpu( - ITensor::makeShape({maxBatchSize, maxBeamWidth}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({maxBatchSize, maxBeamWidth}), nvinfer1::DataType::kFLOAT); + auto entropyDevice + = this->mBufferManager->gpu(ITensor::makeShape({maxBatchSize, maxBeamWidth}), nvinfer1::DataType::kFLOAT); auto biasHost = this->mBufferManager->pinnedPool(ITensor::makeShape({vocabSize}), dataType); auto temperatureHost - = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kFLOAT); auto endIdsHost - = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); auto beamWidthsHost - = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = this->mBufferManager->pinnedPool(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); ITensor::SharedPtr finishedHost = this->mBufferManager->pinnedPool( ITensor::makeShape({maxBeamWidth, maxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - auto batchSlots - = this->mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + auto batchSlots = this->mBufferManager->pinnedPool(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); auto batchSlotsPtr = bufferCast<int32_t>(*batchSlots); auto beamWidthsHostPtr = bufferCast<SizeType32>(*beamWidthsHost); diff --git a/cpp/tests/unit_tests/kernels/shiftKCacheKernelTest.cu b/cpp/tests/unit_tests/kernels/shiftKCacheKernelTest.cu index 75fede666434..b1b3bd6234bf 100644 --- a/cpp/tests/unit_tests/kernels/shiftKCacheKernelTest.cu +++ b/cpp/tests/unit_tests/kernels/shiftKCacheKernelTest.cu @@ -2,7 +2,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decoderMaskedMultiheadAttentionUtils.h" #include "tensorrt_llm/kernels/gptKernels.h" #include "tensorrt_llm/kernels/kvCacheUtils.h" @@ -198,37 +197,37 @@ public: std::vector<int32_t> const& tokenSeqIdxs) { // allocate buffer - mSeqLengthsHost = mBufferManager->pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthsHost = mBufferManager->pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); - mInputLengthsHost = mBufferManager->pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); - mInputLengthsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + mInputLengthsHost = mBufferManager->pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); + mInputLengthsDevice = mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); - mKScaleQuantOrigDevice = mBufferManager->gpu(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); + mKScaleQuantOrigDevice = mBufferManager->gpu(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); mTokenReadIdxsHost = mBufferManager->pinned( - ITensor::makeShape({static_cast<int>(tokenReadIdxs.size())}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenReadIdxs.size())}), nvinfer1::DataType::kINT32); mTokenReadIdxsDevice = mBufferManager->gpu( - ITensor::makeShape({static_cast<int>(tokenReadIdxs.size())}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenReadIdxs.size())}), nvinfer1::DataType::kINT32); mTokenWriteIdxsHost = mBufferManager->pinned( - ITensor::makeShape({static_cast<int>(tokenWriteIdxs.size())}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenWriteIdxs.size())}), nvinfer1::DataType::kINT32); mTokenWriteIdxsDevice = mBufferManager->gpu( - ITensor::makeShape({static_cast<int>(tokenWriteIdxs.size())}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenWriteIdxs.size())}), nvinfer1::DataType::kINT32); mTokenPosIdxsHost = mBufferManager->pinned( - ITensor::makeShape({static_cast<int>(tokenPosIdxs.size())}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenPosIdxs.size())}), nvinfer1::DataType::kINT32); mTokenPosIdxsDevice = mBufferManager->gpu( - ITensor::makeShape({static_cast<int>(tokenPosIdxs.size())}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenPosIdxs.size())}), nvinfer1::DataType::kINT32); mTokenSeqIdxsHost = mBufferManager->pinned( - ITensor::makeShape({static_cast<int>(tokenSeqIdxs.size())}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenSeqIdxs.size())}), nvinfer1::DataType::kINT32); mTokenSeqIdxsDevice = mBufferManager->gpu( - ITensor::makeShape({static_cast<int>(tokenSeqIdxs.size())}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({static_cast<int>(tokenSeqIdxs.size())}), nvinfer1::DataType::kINT32); - // tensorrt_llm::DataType dataType = tensorrt_llm::DataType::kHALF - // tensorrt_llm::DataType::kHALF - // tensorrt_llm::DataType::kBF16 + // nvinfer1::DataType dataType = nvinfer1::DataType::kHALF + // nvinfer1::DataType::kHALF + // nvinfer1::DataType::kBF16 int32_t batchBeam = batchSize * beamWidth; if (pagedKvCache) { diff --git a/cpp/tests/unit_tests/kernels/sparseAttentionKernelsTest.cpp b/cpp/tests/unit_tests/kernels/sparseAttentionKernelsTest.cpp index 14aa58f04df5..508d157e1f61 100644 --- a/cpp/tests/unit_tests/kernels/sparseAttentionKernelsTest.cpp +++ b/cpp/tests/unit_tests/kernels/sparseAttentionKernelsTest.cpp @@ -1,6 +1,5 @@ #include <gtest/gtest.h> -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/sparseAttentionKernels.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" @@ -41,29 +40,28 @@ TEST_F(sparseAttentionKernelsTest, GatherKvPageOffsetsKernelTest) constexpr int total_sparse_tokens = 14; // Create input buffers - auto kv_page_offsets = mBufferManager->gpu( - ITensor::makeShape({batch_size, 2, max_num_pages_per_seq}), tensorrt_llm::DataType::kINT32); - auto seq_lengths = mBufferManager->gpu(ITensor::makeShape({batch_size}), tensorrt_llm::DataType::kINT32); + auto kv_page_offsets + = mBufferManager->gpu(ITensor::makeShape({batch_size, 2, max_num_pages_per_seq}), nvinfer1::DataType::kINT32); + auto seq_lengths = mBufferManager->gpu(ITensor::makeShape({batch_size}), nvinfer1::DataType::kINT32); // Shape: [num_head_kv, total_sparse_tokens] - flattened across all batches auto sparse_indices - = mBufferManager->gpu(ITensor::makeShape({num_head_kv, total_sparse_tokens}), tensorrt_llm::DataType::kINT32); - auto sparse_indices_offsets - = mBufferManager->gpu(ITensor::makeShape({batch_size + 1}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({num_head_kv, total_sparse_tokens}), nvinfer1::DataType::kINT32); + auto sparse_indices_offsets = mBufferManager->gpu(ITensor::makeShape({batch_size + 1}), nvinfer1::DataType::kINT32); // Create output buffers auto output_kv_page_offsets = mBufferManager->gpu( - ITensor::makeShape({num_head_kv, batch_size, 2, max_num_pages_per_seq}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({num_head_kv, batch_size, 2, max_num_pages_per_seq}), nvinfer1::DataType::kINT32); auto output_seq_lengths - = mBufferManager->gpu(ITensor::makeShape({num_head_kv, batch_size}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({num_head_kv, batch_size}), nvinfer1::DataType::kINT32); // Create pinned host buffers for data initialization auto kv_page_offsets_host = mBufferManager->pinned( - ITensor::makeShape({batch_size, 2, max_num_pages_per_seq}), tensorrt_llm::DataType::kINT32); - auto seq_lengths_host = mBufferManager->pinned(ITensor::makeShape({batch_size}), tensorrt_llm::DataType::kINT32); - auto sparse_indices_host = mBufferManager->pinned( - ITensor::makeShape({num_head_kv, total_sparse_tokens}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({batch_size, 2, max_num_pages_per_seq}), nvinfer1::DataType::kINT32); + auto seq_lengths_host = mBufferManager->pinned(ITensor::makeShape({batch_size}), nvinfer1::DataType::kINT32); + auto sparse_indices_host + = mBufferManager->pinned(ITensor::makeShape({num_head_kv, total_sparse_tokens}), nvinfer1::DataType::kINT32); auto sparse_indices_offsets_host - = mBufferManager->pinned(ITensor::makeShape({batch_size + 1}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({batch_size + 1}), nvinfer1::DataType::kINT32); // Initialize test data auto kv_page_offsets_ptr = bufferCast<int32_t>(*kv_page_offsets_host); @@ -145,9 +143,9 @@ TEST_F(sparseAttentionKernelsTest, GatherKvPageOffsetsKernelTest) // Copy results back to host for verification auto output_kv_page_offsets_host = mBufferManager->pinned( - ITensor::makeShape({num_head_kv, batch_size, 2, max_num_pages_per_seq}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({num_head_kv, batch_size, 2, max_num_pages_per_seq}), nvinfer1::DataType::kINT32); auto output_seq_lengths_host - = mBufferManager->pinned(ITensor::makeShape({num_head_kv, batch_size}), tensorrt_llm::DataType::kINT32); + = mBufferManager->pinned(ITensor::makeShape({num_head_kv, batch_size}), nvinfer1::DataType::kINT32); mBufferManager->copy(*output_kv_page_offsets, *output_kv_page_offsets_host); mBufferManager->copy(*output_seq_lengths, *output_seq_lengths_host); diff --git a/cpp/tests/unit_tests/kernels/stopCriteriaKernelsTest.cpp b/cpp/tests/unit_tests/kernels/stopCriteriaKernelsTest.cpp index 9fe4737e82c0..2fefae39552e 100644 --- a/cpp/tests/unit_tests/kernels/stopCriteriaKernelsTest.cpp +++ b/cpp/tests/unit_tests/kernels/stopCriteriaKernelsTest.cpp @@ -16,7 +16,6 @@ #include "tensorrt_llm/kernels/stopCriteriaKernels.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/runtime/bufferManager.h" @@ -61,35 +60,34 @@ class StopCriteriaKernelsTest : public testing::Test std::uniform_int_distribution<SizeType32> tokensPerStepDistr(1, mMaxTokensPerStep); mSequenceLengths - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), tensorrt_llm::DataType::kINT32); - mSequenceLengthLimits - = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), nvinfer1::DataType::kINT32); + mSequenceLengthLimits = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); mFinished = BufferManager::pinned( ITensor::makeShape({maxBatchSize, beamWidth}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mFinishedSum = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mFinishedSum = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); mOutputIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); mOutputIdsPtr - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), tensorrt_llm::DataType::kINT64); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), nvinfer1::DataType::kINT64); mParentIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); mParentIdsPtr - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), tensorrt_llm::DataType::kINT64); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, beamWidth}), nvinfer1::DataType::kINT64); mRefOutputIds = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize, beamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); - mStopWords = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, 2, maxStopWordsLen}), tensorrt_llm::DataType::kINT32); - mStopWordsPtr = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT64); - mStopWordsLen = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mStopWords + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, 2, maxStopWordsLen}), nvinfer1::DataType::kINT32); + mStopWordsPtr = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT64); + mStopWordsLen = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); - mEndIds = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); - mTokensPerStep = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIds = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); + mTokensPerStep = BufferManager::pinned(ITensor::makeShape({maxBatchSize}), nvinfer1::DataType::kINT32); auto batchSlotsPtr = bufferCast<SizeType32>(*mBatchSlots); for (SizeType32 bi = 0; bi < batchSize; ++bi) diff --git a/cpp/tests/unit_tests/layers/baseSamplingLayerTest.cpp b/cpp/tests/unit_tests/layers/baseSamplingLayerTest.cpp index 7886a6b54e2b..8bdacb2e9f6c 100644 --- a/cpp/tests/unit_tests/layers/baseSamplingLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/baseSamplingLayerTest.cpp @@ -15,7 +15,6 @@ */ #include "tests/unit_tests/layers/baseSamplingLayerTest.h" -#include "tensorrt_llm/common/tllmDataType.h" namespace tensorrt_llm::tests::layers::sampling { @@ -68,26 +67,26 @@ void BaseSamplingLayerTest<T>::setup(uint64_t seed, TestSamplingParams const& pa BaseSamplingLayerTest::mMaxOutputLen * params.beamWidth, mVocabSize); } - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), tensorrt_llm::DataType::kINT32); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), tensorrt_llm::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), nvinfer1::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), nvinfer1::DataType::kINT32); mFinishedDevice = params.isExternalDraftTokensLayerTest ? mBufferManager->gpu(ITensor::makeShape({mMaxTokensPerEngineStep, maxBatchSize()}), TRTDataType<tk::FinishedState::UnderlyingType>::value) : mBufferManager->gpu( ITensor::makeShape({maxBatchSize()}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mOutputIdsDevice = mBufferManager->gpu( - ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), tensorrt_llm::DataType::kINT32); + mOutputIdsDevice + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), nvinfer1::DataType::kINT32); mIdsPtrHost = mBufferManager->pinned(ITensor::makeShape({maxBatchSize()}), ptrType); - mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), tensorrt_llm::DataType::kFLOAT); + mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({maxBatchSize()}), nvinfer1::DataType::kFLOAT); mOutputLogProbsDevice - = mBufferManager->gpu(ITensor::makeShape({maxBatchSize(), mMaxSeqLen}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize(), mMaxSeqLen}), nvinfer1::DataType::kFLOAT); mBatchSlots - = mBufferManager->pinned(ITensor::makeShape({mBatchSize + mBatchSizeBadPad}), tensorrt_llm::DataType::kINT32); - mCurandStatesDevice = mBufferManager->gpu( - ITensor::makeShape({maxBatchSize(), sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); + = mBufferManager->pinned(ITensor::makeShape({mBatchSize + mBatchSizeBadPad}), nvinfer1::DataType::kINT32); + mCurandStatesDevice + = mBufferManager->gpu(ITensor::makeShape({maxBatchSize(), sizeof(curandState_t)}), nvinfer1::DataType::kINT8); auto const workspaceSize = mSamplingLayer->getWorkspaceSize(); @@ -153,11 +152,11 @@ void BaseSamplingLayerTest<T>::setup(uint64_t seed, TestSamplingParams const& pa setupParams = samplingSetupParams; mSrcCacheIndirection = mBufferManager->gpu( - ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); mTgtCacheIndirection = mBufferManager->gpu( - ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); mParentIds = mBufferManager->gpu( - ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize(), mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); auto constexpr nvTokenIdType = TRTDataType<TokenIdType>::value; auto constexpr nvSizeType = TRTDataType<SizeType32>::value; diff --git a/cpp/tests/unit_tests/layers/baseSamplingLayerTest.h b/cpp/tests/unit_tests/layers/baseSamplingLayerTest.h index 2bf782a5f5e8..5a375000a5e8 100644 --- a/cpp/tests/unit_tests/layers/baseSamplingLayerTest.h +++ b/cpp/tests/unit_tests/layers/baseSamplingLayerTest.h @@ -34,6 +34,7 @@ #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/runtimeKernels.h" +#include "tensorrt_llm/runtime/tllmLogger.h" #include "tensorrt_llm/common/tllmException.h" diff --git a/cpp/tests/unit_tests/layers/dynamicDecodeLayerTest.cpp b/cpp/tests/unit_tests/layers/dynamicDecodeLayerTest.cpp index cc3b9c411f5b..a3c2d56de16e 100644 --- a/cpp/tests/unit_tests/layers/dynamicDecodeLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/dynamicDecodeLayerTest.cpp @@ -15,7 +15,6 @@ */ #include "tests/unit_tests/layers/dynamicDecodeLayerTest.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/runtimeKernels.h" #include <algorithm> @@ -151,43 +150,43 @@ void DynamicDecodeLayerTest<T>::allocateData(TestSamplingParams const& params, T mRuntimeLogitsHost = BufferManager::pinned(ITensor::makeShape({mBatchSize, mBeamWidth, mVocabSizePadded}), dataType); - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mContextLengthDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mFinishedDevice = mBufferManager->gpu( ITensor::makeShape({mMaxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mFinishedSumDevice = BufferManager::pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kFLOAT); - mOutputIdsDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mBeamWidth, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + mFinishedSumDevice = BufferManager::pinned(ITensor::makeShape({1}), nvinfer1::DataType::kFLOAT); + mOutputIdsDevice + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mBeamWidth, mMaxSeqLen}), nvinfer1::DataType::kINT32); mNewTokens - = BufferManager::pinned(ITensor::makeShape({mMaxTokensPerStep, mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({mMaxTokensPerStep, mMaxBatchSize}), nvinfer1::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mEmbeddingBiasHost = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mVocabSizePadded}), dataType); mEmbeddingBiasDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mVocabSizePadded}), dataType); mRefLogProbsHost - = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kFLOAT); mOutputLogProbsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kFLOAT); mOutputLogProbsTiledDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxSeqLen, mMaxBatchSize}), tensorrt_llm::DataType::kFLOAT); + = mBufferManager->gpu(ITensor::makeShape({mMaxSeqLen, mMaxBatchSize}), nvinfer1::DataType::kFLOAT); - mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kFLOAT); + mCumLogProbsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kFLOAT); mMaxBadWordsLen = getMaxWordsLen(params.badWords); mMaxStopWordsLen = getMaxWordsLen(params.stopWords); - mBadWords = BufferManager::pinned( - ITensor::makeShape({mMaxBatchSize, 2, mMaxBadWordsLen}), tensorrt_llm::DataType::kINT32); - mBadWordsLens = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mBadWordsPtrs = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT64); + mBadWords + = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, 2, mMaxBadWordsLen}), nvinfer1::DataType::kINT32); + mBadWordsLens = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mBadWordsPtrs = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT64); - mStopWords = BufferManager::pinned( - ITensor::makeShape({mMaxBatchSize, 2, mMaxStopWordsLen}), tensorrt_llm::DataType::kINT32); - mStopWordsLens = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mStopWordsPtrs = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT64); + mStopWords + = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize, 2, mMaxStopWordsLen}), nvinfer1::DataType::kINT32); + mStopWordsLens = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mStopWordsPtrs = BufferManager::pinned(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT64); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); if (mDecodingMode.isMedusa()) { @@ -205,19 +204,19 @@ void DynamicDecodeLayerTest<T>::allocateMedusaData(TestSamplingParams const& par auto const dataType = TRTDataType<T>::value; mMaxMedusaHeads = params.maxNumMedusaHeads.value(); mPathsDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep, mMaxMedusaHeads + 1}), tensorrt_llm::DataType::kINT32); - mAcceptedLengths = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep, mMaxMedusaHeads + 1}), nvinfer1::DataType::kINT32); + mAcceptedLengths = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mMedusaLogitsDevice = BufferManager::pinned( ITensor::makeShape({mMaxMedusaHeads, mMaxBatchSize, mMaxTokensPerStep, mVocabSizePadded}), dataType); - mNextDraftTokensDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep - 1}), tensorrt_llm::DataType::kINT32); - mTokensPerStepDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); - mTreeIdsDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep - 1}), tensorrt_llm::DataType::kINT32); + mNextDraftTokensDevice + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep - 1}), nvinfer1::DataType::kINT32); + mTokensPerStepDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); + mTreeIdsDevice + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxTokensPerStep - 1}), nvinfer1::DataType::kINT32); mAcceptedLengthCumSumDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize + 1}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize + 1}), nvinfer1::DataType::kINT32); mPackedPathsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize * mMaxMedusaHeads}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize * mMaxMedusaHeads}), nvinfer1::DataType::kINT32); } template <typename T> @@ -605,7 +604,7 @@ template <typename T> void DynamicDecodeLayerTest<T>::batchCopy(SizeType32 step) { auto const logitsHost = ITensor::wrap(mTestLogitsInit.data() + step * mVocabSizePadded, - std::is_same_v<T, float> ? tensorrt_llm::DataType::kFLOAT : tensorrt_llm::DataType::kHALF, + std::is_same_v<T, float> ? nvinfer1::DataType::kFLOAT : nvinfer1::DataType::kHALF, ITensor::makeShape({mMaxTokensPerStep, mVocabSizePadded})); for (SizeType32 bi = 0; bi < mBatchSize; ++bi) { diff --git a/cpp/tests/unit_tests/layers/eagleLayerTest.cpp b/cpp/tests/unit_tests/layers/eagleLayerTest.cpp index 47bca93b47b4..bdb53f15618f 100644 --- a/cpp/tests/unit_tests/layers/eagleLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/eagleLayerTest.cpp @@ -22,8 +22,9 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/runtimeKernels.h" #include "tensorrt_llm/runtime/speculativeDecodingModule.h" +#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntimeBase.h> #include <algorithm> #include <cstdint> @@ -554,126 +555,126 @@ void EagleDecodingLayerTest<T>::allocateBuffers() // outputs mOutputIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxSeqLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mSeqLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mOutputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mOutputUnpackedNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mAcceptedLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mNextPosIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mPrevDraftLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mNextDraftLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mNextGenerationLengths - = mBufferManager->gpu(ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mNextGenerationLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mAcceptedLengthCumSum = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize() + 1}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize() + 1}), nvinfer1::DataType::kINT32); mPathsOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize() * mSamplingParams.getMaxDraftPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mPackedMasks = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), static_cast<SizeType32>(divUp(mSamplingParams.getMaxDecodingTokens(), 32))}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mRandomDataSample = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kFLOAT); mRandomDataValidation = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kFLOAT); + nvinfer1::DataType::kFLOAT); mOutputTemperatures = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kFLOAT); mOutputNextDraftPaths = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mEagleNetCtxRequestTypesHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mEagleNetCtxContextLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mEagleNetCtxPastKeyValueLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mEagleNetGenRequestTypesHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mEagleNetGenContextLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mEagleNetGenPastKeyValueLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); // inputs - mBatchSlots = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mBatchSlots + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); - mEndIds = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mEndIds + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mInputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); - mInputNextDraftLens = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mInputNextDraftLens + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mInputNextDraftPaths = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mInputLastDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); - mInputLastDraftLens = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mInputLastDraftLens + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mInputLastDraftPaths = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mInputAcceptedTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); - mInputAcceptedLens = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mInputAcceptedLens + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); - mInputAcceptedPathIds = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mInputAcceptedPathIds + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); - mChunkedContextNextTokens = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mChunkedContextNextTokens + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mDecodingWorkspace = std::make_shared<tensorrt_llm::runtime::DecodingLayerWorkspace>(mBufferManager, decodingDomain, TRTDataType<float>::value, mSamplingParams.getMaxBatchSize() * sizeof(curandState_t)); diff --git a/cpp/tests/unit_tests/layers/explicitDraftTokensLayerTest.cpp b/cpp/tests/unit_tests/layers/explicitDraftTokensLayerTest.cpp index 04d05e0d16a9..e7831b57f77f 100644 --- a/cpp/tests/unit_tests/layers/explicitDraftTokensLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/explicitDraftTokensLayerTest.cpp @@ -22,8 +22,9 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/runtimeKernels.h" #include "tensorrt_llm/runtime/speculativeDecodingModule.h" +#include "tensorrt_llm/runtime/tllmLogger.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntimeBase.h> #include <algorithm> #include <cstdint> @@ -654,29 +655,29 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() // outputs mOutputIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxSeqLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mSeqLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mAcceptedLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mNextDraftLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mPrevDraftLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mAcceptedLengthCumSum = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize() + 1}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize() + 1}), nvinfer1::DataType::kINT32); mOutputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingDraftTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mOutputPositionIdsBase = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mRandomDataSample = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), dataType); @@ -688,21 +689,21 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() mPackedMasks = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), static_cast<SizeType32>(divUp(mSamplingParams.getMaxDecodingTokens(), 32))}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mNextPosIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mOutputUnpackedNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mOutputUnpackedNextDraftIndices = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mOutputDraftProbs = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), @@ -712,68 +713,68 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() mOutputTemperatures = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), dataType); mOutputGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mOutputGenerationLengthsHost = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); - mMaxGenLengthHost = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + mMaxGenLengthHost = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); // inputs - mBatchSlots = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mBatchSlots + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mTokensPerStep = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mPathsOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize() * mSamplingParams.getMaxDraftPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mMasks = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kBOOL); + nvinfer1::DataType::kBOOL); mInputNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mLastDraftTokens = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mPackedPosIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mBestPathLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mBestPathIndices = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mSpecDecodingGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mNextFlatTokens = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize() * mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mInputPositionIdsBase = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); mNextDraftIndices = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mLastDraftIndices = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mNextDraftProbs = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getMaxBatchSize(), mSamplingParams.getMaxNumPaths(), @@ -781,20 +782,20 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() dataType); mEndIds = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mSamplingParams.getMaxBatchSize()}), nvinfer1::DataType::kINT32); - mMaxGenLengthDevice = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + mMaxGenLengthDevice = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); // Packed inputs - mMaxGenerationLength = BufferManager::pinnedPool(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); - mCumSumGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mMaxGenerationLength = BufferManager::pinnedPool(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); + mCumSumGenerationLengths + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); // Packed outputs - mPackedPositionIdsBase = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); - mPackedGenerationLengths = BufferManager::pinnedPool( - ITensor::makeShape({mSamplingParams.getBatchSize()}), tensorrt_llm::DataType::kINT32); + mPackedPositionIdsBase + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); + mPackedGenerationLengths + = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), nvinfer1::DataType::kINT32); mPackedRandomDataSample = BufferManager::pinnedPool(ITensor::makeShape({mSamplingParams.getBatchSize()}), dataType); mPackedRandomDataVerification = BufferManager::pinnedPool( ITensor::makeShape( @@ -803,21 +804,21 @@ void ExplicitDraftTokensLayerTest<T>::allocateBuffers() mPackedNextDraftTokens = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mPackedNextDraftIndices = BufferManager::pinnedPool( ITensor::makeShape( {mSamplingParams.getBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxPathLen()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mPackedPackedMasks = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens(), static_cast<SizeType32>(divUp(mSamplingParams.getMaxDecodingTokens(), 32))}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mPackedPositionOffsets = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mPackedPackedPosIds = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxDecodingTokens()}), - tensorrt_llm::DataType::kINT32); + nvinfer1::DataType::kINT32); mPackedDraftProbs = BufferManager::pinnedPool( ITensor::makeShape({mSamplingParams.getBatchSize(), mSamplingParams.getMaxNumPaths(), mSamplingParams.getMaxDraftPathLen(), mSamplingParams.getVocabSize()}), @@ -1563,6 +1564,7 @@ class FillRandDataTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro- void SetUp() override { + mLogger = std::make_shared<TllmLogger>(); mStream = std::make_shared<tensorrt_llm::runtime::CudaStream>(); mBufferManager = std::make_shared<tensorrt_llm::runtime::BufferManager>(mStream); } @@ -1574,12 +1576,12 @@ class FillRandDataTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro- { SizeType32* batchSlotsPtr{nullptr}; - auto curandState = mBufferManager->gpu(ITensor::makeShape({batchSize, 48}), tensorrt_llm::DataType::kUINT8); + auto curandState = mBufferManager->gpu(ITensor::makeShape({batchSize, 48}), nvinfer1::DataType::kUINT8); auto* curandStatePtr = reinterpret_cast<curandState_t*>(bufferCast<uint8_t>(*curandState)); if (batchInit) { - auto randomSeeds = mBufferManager->gpu(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT64); + auto randomSeeds = mBufferManager->gpu(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT64); trk::invokeFill(*randomSeeds, static_cast<int64_t>(randomSeed), *mStream); auto* randomSeedsPtr = bufferCast<uint64_t>(*randomSeeds); tk::invokeCurandBatchInitialize(curandStatePtr, batchSlotsPtr, batchSize, randomSeedsPtr, mStream->get()); @@ -1623,6 +1625,7 @@ class FillRandDataTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro- } private: + std::shared_ptr<nvinfer1::ILogger> mLogger; std::shared_ptr<tensorrt_llm::runtime::CudaStream> mStream; std::shared_ptr<tensorrt_llm::runtime::BufferManager> mBufferManager; }; diff --git a/cpp/tests/unit_tests/layers/externalDraftTokensLayerTest.cpp b/cpp/tests/unit_tests/layers/externalDraftTokensLayerTest.cpp index cdc913d3b3aa..c216a76bc2b8 100644 --- a/cpp/tests/unit_tests/layers/externalDraftTokensLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/externalDraftTokensLayerTest.cpp @@ -15,7 +15,6 @@ */ #include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tests/unit_tests/layers/baseSamplingLayerTest.h" @@ -89,7 +88,7 @@ class ExternalDraftTokensLayerTest : public BaseSamplingLayerTest<T> dataType); mDraftTokenIds = this->mBufferManager->gpu( - ITensor::makeShape({this->maxBatchSize(), mMaxDraftLen}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({this->maxBatchSize(), mMaxDraftLen}), nvinfer1::DataType::kINT32); mUseDraftLogits = this->mBufferManager->gpu(ITensor::makeShape({this->maxBatchSize()}), TRTDataType<bool>::value); mUseDraftLogitsHost diff --git a/cpp/tests/unit_tests/layers/lookaheadAlgorithmTest.cpp b/cpp/tests/unit_tests/layers/lookaheadAlgorithmTest.cpp index 7f4791a85ba4..cd1dc4799e6c 100644 --- a/cpp/tests/unit_tests/layers/lookaheadAlgorithmTest.cpp +++ b/cpp/tests/unit_tests/layers/lookaheadAlgorithmTest.cpp @@ -17,7 +17,6 @@ #include <tuple> #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/layers/lookaheadAlgorithm.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" @@ -88,25 +87,24 @@ TEST_P(LookaheadAlgorithmTest, predict) auto shape = ITensor::makeShape({maxTokensPerStep}); auto shape2d = ITensor::makeShape({maxTokensPerStep, maxTokensPerStep}); auto shapeSingle = ITensor::makeShape({1}); - TensorPtr posidMax = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); - TensorPtr attentionMaskMax = BufferManager::cpu(shape2d, tensorrt_llm::DataType::kBOOL); - TensorPtr inputLengthPtr = BufferManager::cpu(shapeSingle, tensorrt_llm::DataType::kINT32); + TensorPtr posidMax = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); + TensorPtr attentionMaskMax = BufferManager::cpu(shape2d, nvinfer1::DataType::kBOOL); + TensorPtr inputLengthPtr = BufferManager::cpu(shapeSingle, nvinfer1::DataType::kINT32); auto& inputLength(*BufferRange<SizeType32>(*inputLengthPtr).begin()); - TensorPtr outputMax = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); - TensorPtr endIdPtr = BufferManager::cpu(shapeSingle, tensorrt_llm::DataType::kINT32); + TensorPtr outputMax = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); + TensorPtr endIdPtr = BufferManager::cpu(shapeSingle, nvinfer1::DataType::kINT32); auto& endId(*BufferRange<TokenIdType>(*endIdPtr).begin()); endId = ascii->getEndToken(); - TensorPtr acceptedMax = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); - TensorPtr acceptedOffsetsMax = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); - TensorPtr acceptedLengthPtr = BufferManager::cpu(shapeSingle, tensorrt_llm::DataType::kINT32); + TensorPtr acceptedMax = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); + TensorPtr acceptedOffsetsMax = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); + TensorPtr acceptedLengthPtr = BufferManager::cpu(shapeSingle, nvinfer1::DataType::kINT32); auto& acceptedLength(*BufferRange<SizeType32>(*acceptedLengthPtr).begin()); - TensorPtr sequence - = BufferManager::cpu(ITensor::makeShape({maxSeqLen + maxDraftLen}), tensorrt_llm::DataType::kINT32); + TensorPtr sequence = BufferManager::cpu(ITensor::makeShape({maxSeqLen + maxDraftLen}), nvinfer1::DataType::kINT32); BufferRange<TokenIdType> sequenceRange(*sequence); - TensorPtr sequenceLengthPtr = BufferManager::cpu(shapeSingle, tensorrt_llm::DataType::kINT32); + TensorPtr sequenceLengthPtr = BufferManager::cpu(shapeSingle, nvinfer1::DataType::kINT32); auto& sequenceLength(*bufferCast<SizeType32>(*sequenceLengthPtr)); std::copy(promptRange.begin(), promptRange.end(), sequenceRange.begin()); @@ -226,13 +224,13 @@ TEST(LookaheadAlgorithmTest, treeEncodeTest) auto shape = inputTokens->getShape(); auto shape2d = ITensor::makeShape({shape.d[0], shape.d[0]}); - TensorPtr inputMasks = BufferManager::cpu(shape2d, tensorrt_llm::DataType::kBOOL); + TensorPtr inputMasks = BufferManager::cpu(shape2d, nvinfer1::DataType::kBOOL); LookaheadAlgorithm::posIdsToMask(inputMasks, inputPosIds); - TensorPtr outputTokens = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); - TensorPtr outputPosIds = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); - TensorPtr encodeMap = BufferManager::cpu(shape, tensorrt_llm::DataType::kINT32); - TensorPtr outputMasks = BufferManager::cpu(shape2d, tensorrt_llm::DataType::kBOOL); + TensorPtr outputTokens = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); + TensorPtr outputPosIds = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); + TensorPtr encodeMap = BufferManager::cpu(shape, nvinfer1::DataType::kINT32); + TensorPtr outputMasks = BufferManager::cpu(shape2d, nvinfer1::DataType::kBOOL); // auto len = LookaheadAlgorithm::treeEncode(outputTokens, outputPosIds, outputMasks, inputTokens, inputPosIds, // inputMasks, '$', 9); diff --git a/cpp/tests/unit_tests/layers/lookaheadDecodingLayerTest.cpp b/cpp/tests/unit_tests/layers/lookaheadDecodingLayerTest.cpp index 917f6dbdca55..414e6f101743 100644 --- a/cpp/tests/unit_tests/layers/lookaheadDecodingLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/lookaheadDecodingLayerTest.cpp @@ -23,7 +23,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/executor.h" #include "tensorrt_llm/layers/decodingParams.h" #include "tensorrt_llm/layers/lookaheadDecodingLayer.h" @@ -321,7 +320,7 @@ void LookaheadDecodingLayerTest::allocateBuffers() mLlm[gbi] = std::make_shared<LookaheadRandomLlm>(mAscii, mOracle[gbi], gbi); mScoreBoard[gbi] = std::ostringstream(); - mHistogram[gbi] = BufferManager::cpu(ITensor::makeShape({mTestParam.n + 1}), tensorrt_llm::DataType::kINT32); + mHistogram[gbi] = BufferManager::cpu(ITensor::makeShape({mTestParam.n + 1}), nvinfer1::DataType::kINT32); } switch (mTestParam.batchType) { @@ -349,50 +348,48 @@ void LookaheadDecodingLayerTest::allocateBuffers() auto maxBatchShape1D = ITensor::makeShape({maxBatchSize}); - mAlgoConfigBatch = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, 3}), tensorrt_llm::DataType::kINT32); + mAlgoConfigBatch = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, 3}), nvinfer1::DataType::kINT32); - mEndIds = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mTokensPerStep = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mEndIds = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); + mTokensPerStep = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); - mOutputIds - = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, maxBeamSize, mMaxSeqLen + mMaxTokensPerStep}), - tensorrt_llm::DataType::kINT32); - mSequenceLengths = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + mOutputIds = BufferManager::pinnedPool( + ITensor::makeShape({maxBatchSize, maxBeamSize, mMaxSeqLen + mMaxTokensPerStep}), nvinfer1::DataType::kINT32); + mSequenceLengths = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); mProbs = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, mMaxTokensPerStep, vocabSize}), tensorrt_llm::DataType::kFLOAT); + ITensor::makeShape({maxBatchSize, mMaxTokensPerStep, vocabSize}), nvinfer1::DataType::kFLOAT); mGoldenSampledTokens - = BufferManager::cpu(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); - mInputTokensBatch = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); - mPositionIdsBatch = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); + = BufferManager::cpu(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); + mInputTokensBatch + = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); + mPositionIdsBatch + = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); mNewTokens = BufferManager::pinnedPool( - ITensor::makeShape({mMaxTokensPerStep, maxBatchSize, 1}), tensorrt_llm::DataType::kINT32); - mNumNewTokens = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mDraftLengths = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mPrevDraftLengths = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mMaxTokensPerStep, maxBatchSize, 1}), nvinfer1::DataType::kINT32); + mNumNewTokens = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); + mDraftLengths = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); + mPrevDraftLengths = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); mDraftTokens - = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, maxDraftLen}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, maxDraftLen}), nvinfer1::DataType::kINT32); auto packedMaskShape = ITensor::makeShape( {maxBatchSize, mMaxTokensPerStep, static_cast<ITensor::DimType64>(common::divUp(mMaxTokensPerStep, 32))}); - mPackedMasks = BufferManager::pinnedPool(packedMaskShape, tensorrt_llm::DataType::kINT32); + mPackedMasks = BufferManager::pinnedPool(packedMaskShape, nvinfer1::DataType::kINT32); mPackedMasksBool = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, mMaxTokensPerStep, mMaxTokensPerStep}), tensorrt_llm::DataType::kBOOL); - mNumNewTokensCumSum - = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize + 1}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize, mMaxTokensPerStep, mMaxTokensPerStep}), nvinfer1::DataType::kBOOL); + mNumNewTokensCumSum = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize + 1}), nvinfer1::DataType::kINT32); mPathsOffsets = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), tensorrt_llm::DataType::kINT32); - mGenerationLengths = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - mPositionOffsets = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); - mPositionIds = BufferManager::pinnedPool( - ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), tensorrt_llm::DataType::kINT32); - mAttentionPackedMask = BufferManager::pinnedPool(packedMaskShape, tensorrt_llm::DataType::kINT32); - - mBatchSlotsMax = BufferManager::pinnedPool(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + ITensor::makeShape({maxBatchSize, maxAcceptedDraftLen}), nvinfer1::DataType::kINT32); + mGenerationLengths = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); + mPositionOffsets + = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); + mPositionIds + = BufferManager::pinnedPool(ITensor::makeShape({maxBatchSize, mMaxTokensPerStep}), nvinfer1::DataType::kINT32); + mAttentionPackedMask = BufferManager::pinnedPool(packedMaskShape, nvinfer1::DataType::kINT32); + + mBatchSlotsMax = BufferManager::pinnedPool(maxBatchShape1D, nvinfer1::DataType::kINT32); auto const batchSize = 0; auto batchShape1D = ITensor::makeShape({batchSize}); diff --git a/cpp/tests/unit_tests/layers/lookaheadRandomLlmTest.cpp b/cpp/tests/unit_tests/layers/lookaheadRandomLlmTest.cpp index 4ca7206c3bc8..2cc2523d08cb 100644 --- a/cpp/tests/unit_tests/layers/lookaheadRandomLlmTest.cpp +++ b/cpp/tests/unit_tests/layers/lookaheadRandomLlmTest.cpp @@ -15,7 +15,6 @@ */ #include <gtest/gtest.h> -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/samplingTopKKernels.h" #include "tensorrt_llm/layers/lookaheadAlgorithm.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" @@ -49,7 +48,7 @@ TEST(LookaheadRandomllm, forward) std::string str("hello world!"); TensorPtr logits = BufferManager::cpu(ITensor::makeShape({static_cast<SizeType32>(str.size()), ascii->getVocabSize()}), - tensorrt_llm::DataType::kFLOAT); + nvinfer1::DataType::kFLOAT); ascii->stringToLogits(logits, str); auto result = ascii->logitsToString(logits); EXPECT_EQ(result, str); @@ -68,7 +67,7 @@ TEST(LookaheadRandomllm, forward) std::vector<TokenIdType> positionIdVec({22, 23, 24, 23, 24, 25, 24, 25, 26, 25, 26, 27, 26, 27, 28}); TensorPtr positionIds = ITensor::wrap(positionIdVec, ITensor::makeShape({len})); TensorPtr outputLogits - = BufferManager::cpu(ITensor::makeShape({len, ascii->getVocabSize()}), tensorrt_llm::DataType::kFLOAT); + = BufferManager::cpu(ITensor::makeShape({len, ascii->getVocabSize()}), nvinfer1::DataType::kFLOAT); llm.forward(outputLogits, inputTokens, positionIds); @@ -124,29 +123,29 @@ TEST(LookaheadRandomllm, gpuSampling) SizeType32 workspaceSize = tensorrt_llm::kernels::getTopKWorkspaceSize<float>(maxBatchSize, maxTokensPerStep, mMaxTopK, vocabSizePadded); - TensorPtr workspaceDevice = mBufferManager->pinned( - ITensor::makeShape({static_cast<int32_t>(workspaceSize)}), tensorrt_llm::DataType::kINT8); + TensorPtr workspaceDevice + = mBufferManager->pinned(ITensor::makeShape({static_cast<int32_t>(workspaceSize)}), nvinfer1::DataType::kINT8); auto const dataType = TRTDataType<float>::value; auto const ptrType = TRTDataType<float*>::value; // Allocate GPU data - TensorPtr mSeqLengths = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kINT32); + TensorPtr mSeqLengths = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kINT32); TensorPtr mFinished = BufferManager::pinned(maxBatchShape1D, TRTDataType<tk::FinishedState::UnderlyingType>::value); - TensorPtr mEndIds = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - TensorPtr mTopPs = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kFLOAT); - TensorPtr mTopKs = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - TensorPtr mSkipDecode = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kBOOL); - TensorPtr mTokensPerStep = BufferManager::pinned(maxBatchShape1D, tensorrt_llm::DataType::kINT32); - - TensorPtr mCurandStates = BufferManager::pinned( - ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), tensorrt_llm::DataType::kINT8); + TensorPtr mEndIds = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kINT32); + TensorPtr mTopPs = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kFLOAT); + TensorPtr mTopKs = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kINT32); + TensorPtr mSkipDecode = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kBOOL); + TensorPtr mTokensPerStep = BufferManager::pinned(maxBatchShape1D, nvinfer1::DataType::kINT32); + + TensorPtr mCurandStates + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, sizeof(curandState_t)}), nvinfer1::DataType::kINT8); TensorPtr mOutputIds - = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + = BufferManager::pinned(ITensor::makeShape({maxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); TensorPtr mProbs = BufferManager::pinned(maxBatchShape3D, dataType); - TensorPtr mBatchSlots = BufferManager::pinned(batchShape1D, tensorrt_llm::DataType::kINT32); + TensorPtr mBatchSlots = BufferManager::pinned(batchShape1D, nvinfer1::DataType::kINT32); ///////////////////////////////////// std::copy(batchSlotsVec.begin(), batchSlotsVec.end(), BufferRange<SizeType32>(*mBatchSlots).begin()); diff --git a/cpp/tests/unit_tests/layers/medusaDecodeLayerTest.cpp b/cpp/tests/unit_tests/layers/medusaDecodeLayerTest.cpp index a93955d02c9a..37a479eb4214 100644 --- a/cpp/tests/unit_tests/layers/medusaDecodeLayerTest.cpp +++ b/cpp/tests/unit_tests/layers/medusaDecodeLayerTest.cpp @@ -15,7 +15,6 @@ */ #include "medusaDecodeLayerTest.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/decodingCommon.h" #include "tensorrt_llm/runtime/medusaModule.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -139,36 +138,35 @@ void MedusaDecodingLayerTest<T>::allocateBuffers() mFinishedDevice = mBufferManager->gpu( ITensor::makeShape({mMaxBatchSize}), TRTDataType<tk::FinishedState::UnderlyingType>::value); - mOutputIdsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), tensorrt_llm::DataType::kINT32); + mOutputIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxSeqLen}), nvinfer1::DataType::kINT32); - mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), tensorrt_llm::DataType::kINT32); + mBatchSlots = BufferManager::pinned(ITensor::makeShape({mBatchSize}), nvinfer1::DataType::kINT32); - mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mEndIdsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mPathsDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens, mMaxDraftPathLen + 1}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens, mMaxDraftPathLen + 1}), nvinfer1::DataType::kINT32); - mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mSeqLengthsDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mAcceptedLengths = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mAcceptedLengths = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); - mTreeIdsDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens - 1}), tensorrt_llm::DataType::kINT32); + mTreeIdsDevice + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens - 1}), nvinfer1::DataType::kINT32); mMedusaLogitsDevice = mBufferManager->gpu( ITensor::makeShape({mMaxDraftPathLen, mMaxBatchSize, mMaxDecodingTokens, mVocabSizePadded}), dataType); - mNextDraftTokensDevice = mBufferManager->gpu( - ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens - 1}), tensorrt_llm::DataType::kINT32); + mNextDraftTokensDevice + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize, mMaxDecodingTokens - 1}), nvinfer1::DataType::kINT32); - mTokensPerStepDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), tensorrt_llm::DataType::kINT32); + mTokensPerStepDevice = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize}), nvinfer1::DataType::kINT32); mAcceptedLengthCumSumDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize + 1}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize + 1}), nvinfer1::DataType::kINT32); mPackedPathsDevice - = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize * mMaxDraftPathLen}), tensorrt_llm::DataType::kINT32); + = mBufferManager->gpu(ITensor::makeShape({mMaxBatchSize * mMaxDraftPathLen}), nvinfer1::DataType::kINT32); for (int32_t bi = 0; bi < mBatchSize; ++bi) { @@ -217,7 +215,7 @@ void MedusaDecodingLayerTest<T>::setup(SamplingParams& params) for (SizeType32 bi = 0; bi < mBatchSize; ++bi) { auto const draftIdsHost = ITensor::wrap(reinterpret_cast<TokenIdType*>(params.draftIds[bi].data()), - tensorrt_llm::DataType::kINT32, ITensor::makeShape({1, mMaxDecodingTokens - 1})); + nvinfer1::DataType::kINT32, ITensor::makeShape({1, mMaxDecodingTokens - 1})); auto draftIdsDeviceSlice = ITensor::slice(mNextDraftTokensDevice, batchSlotsPtr[bi], 1); mBufferManager->copy(*draftIdsHost, *draftIdsDeviceSlice); } @@ -226,7 +224,7 @@ void MedusaDecodingLayerTest<T>::setup(SamplingParams& params) { auto& path = params.paths[bi]; auto const numPaths = static_cast<SizeType32>(params.paths[bi].size() / (mMaxDraftPathLen + 1)); - auto const pathsHost = ITensor::wrap(reinterpret_cast<SizeType32*>(path.data()), tensorrt_llm::DataType::kINT32, + auto const pathsHost = ITensor::wrap(reinterpret_cast<SizeType32*>(path.data()), nvinfer1::DataType::kINT32, ITensor::makeShape({1, numPaths, mMaxDraftPathLen + 1})); TensorPtr pathsDeviceSlice = ITensor::slice(mPathsDevice, batchSlotsPtr[bi], 1); pathsDeviceSlice->squeeze(0); diff --git a/cpp/tests/unit_tests/layers/randomLlm.cpp b/cpp/tests/unit_tests/layers/randomLlm.cpp index 63aa85eaad16..632cf8765c20 100644 --- a/cpp/tests/unit_tests/layers/randomLlm.cpp +++ b/cpp/tests/unit_tests/layers/randomLlm.cpp @@ -16,7 +16,6 @@ #include "tests/unit_tests/layers/randomLlm.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/common.h" @@ -35,7 +34,7 @@ TensorPtr initTensor(std::string str, std::optional<ITensor::Shape> shape) { TLLM_CHECK(ITensor::volume(shape1d) == ITensor::volume(shape.value())); } - TensorPtr tensor = BufferManager::cpu(shape.value_or(shape1d), tensorrt_llm::DataType::kINT32); + TensorPtr tensor = BufferManager::cpu(shape.value_or(shape1d), nvinfer1::DataType::kINT32); auto tensorRange = BufferRange<TokenIdType>(*tensor); std::copy(str.begin(), str.end(), tensorRange.begin()); return tensor; @@ -43,7 +42,7 @@ TensorPtr initTensor(std::string str, std::optional<ITensor::Shape> shape) TensorConstPtr RandomTokenLogits::tokenToLogits(TokenIdType token) const { - TensorPtr logits = BufferManager::cpu(mVocabulary->getShape(), tensorrt_llm::DataType::kFLOAT); + TensorPtr logits = BufferManager::cpu(mVocabulary->getShape(), nvinfer1::DataType::kFLOAT); tokenToLogits(logits, token); return logits; } @@ -153,7 +152,7 @@ void RandomTokenLogits::logitsToTensor(TensorPtr const& tokens, TensorConstPtr c TensorConstPtr RandomTokenLogits::logitsToTensor(TensorConstPtr const& logits) const { auto len = logits->getShape().d[0]; - TensorPtr result = BufferManager::cpu(ITensor::makeShape({len}), tensorrt_llm::DataType::kINT32); + TensorPtr result = BufferManager::cpu(ITensor::makeShape({len}), nvinfer1::DataType::kINT32); logitsToTensor(result, logits); return result; } @@ -210,7 +209,7 @@ bool RandomLlm::verify(SizeType32 const offset, TensorConstPtr const& script) co void RandomLlm::forward(TensorPtr const& output, runtime::SizeType32 startId, TensorConstPtr const& input, TensorConstPtr const& offsets, TensorConstPtr const mask) const { - TensorPtr posIds = BufferManager::cpu(input->getShape(), tensorrt_llm::DataType::kINT32); + TensorPtr posIds = BufferManager::cpu(input->getShape(), nvinfer1::DataType::kINT32); BufferRange<SizeType32> idRange(*posIds); BufferRange<SizeType32 const> offsetRange(*offsets); for (auto i = 0; i < idRange.size(); i++) @@ -227,7 +226,7 @@ void RandomLlm::forward(TensorPtr const& output, TensorConstPtr const& input, Te TLLM_CHECK(ITensor::volume(input->getShape()) == ITensor::volume(position->getShape())); TLLM_CHECK(ITensor::volume(output->getShape()) == ITensor::volume(input->getShape()) * mTable->getVocabSize()); - TensorPtr tokens = BufferManager::cpu(input->getShape(), tensorrt_llm::DataType::kINT32); + TensorPtr tokens = BufferManager::cpu(input->getShape(), nvinfer1::DataType::kINT32); foretell(tokens, input, position, mask); // foretellOld(tokens, input, position); mTable->tensorToLogits(output, tokens); @@ -248,7 +247,7 @@ void LookaheadRandomLlm::foretell(TensorPtr const& output, TensorConstPtr const& TLLM_CHECK(mask->getShape().d[1] >= len); } - TensorPtr maskRebuilt = BufferManager::cpu(ITensor::makeShape({len, len}), tensorrt_llm::DataType::kBOOL); + TensorPtr maskRebuilt = BufferManager::cpu(ITensor::makeShape({len, len}), nvinfer1::DataType::kBOOL); posIdsToMask(maskRebuilt, position); auto outputRange = BufferRange<TokenIdType>(*output); diff --git a/cpp/tests/unit_tests/layers/randomLlm.h b/cpp/tests/unit_tests/layers/randomLlm.h index b0f0564944dc..a6e898baedf5 100644 --- a/cpp/tests/unit_tests/layers/randomLlm.h +++ b/cpp/tests/unit_tests/layers/randomLlm.h @@ -18,7 +18,6 @@ #include <gtest/gtest.h> #include <list> -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/layers/lookaheadDecodingUtils.h" #include "tensorrt_llm/runtime/common.h" #include "tensorrt_llm/runtime/runtimeKernels.h" @@ -78,7 +77,7 @@ class AsciiRandomTokenLogits : public RandomTokenLogits : RandomTokenLogits( []() { - auto vocab = BufferManager::cpu(ITensor::makeShape({128}), tensorrt_llm::DataType::kINT32); + auto vocab = BufferManager::cpu(ITensor::makeShape({128}), nvinfer1::DataType::kINT32); auto vocabRange = BufferRange<TokenIdType>(*vocab); TokenIdType token{0}; std::for_each(vocabRange.begin(), vocabRange.end(), [&token](auto& v) { v = token++; }); diff --git a/cpp/tests/unit_tests/multi_gpu/cacheTransceiverTest.cpp b/cpp/tests/unit_tests/multi_gpu/cacheTransceiverTest.cpp index 1f216578f669..49afbf20dfae 100644 --- a/cpp/tests/unit_tests/multi_gpu/cacheTransceiverTest.cpp +++ b/cpp/tests/unit_tests/multi_gpu/cacheTransceiverTest.cpp @@ -36,7 +36,6 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/envUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/cache_transmission/agent_utils/connection.h" #include "tensorrt_llm/executor/cache_transmission/mpi_utils/connection.h" #include "tensorrt_llm/executor/dataTransceiverState.h" @@ -198,7 +197,7 @@ TEST_F(RequestInfoTest, Basic) } auto state = std::make_unique<texec::DataTransceiverState>(); state->setCommState(texec::kv_cache::CommState{12, "127.0.0.1"}); - state->setCacheState(texec::kv_cache::CacheState{10, 12, 128, 128, 8, 8, 8, {10}, tensorrt_llm::DataType::kFLOAT}); + state->setCacheState(texec::kv_cache::CacheState{10, 12, 128, 128, 8, 8, 8, {10}, nvinfer1::DataType::kFLOAT}); RequestInfo info{1, *state}; auto info2 = serializeDeserialize(info); EXPECT_EQ(info, info2); @@ -228,7 +227,7 @@ TEST_F(CacheConfigTest, EqualTo) constexpr SizeType32 nbRnnLayers{2}; constexpr SizeType32 nbHeads{12}; constexpr SizeType32 hiddenSize{768}; - constexpr tensorrt_llm::DataType dtype{tensorrt_llm::DataType::kFLOAT}; + constexpr nvinfer1::DataType dtype{nvinfer1::DataType::kFLOAT}; constexpr SizeType32 tokensPerBlock{64}; constexpr SizeType32 tensorParallelism{8}; constexpr SizeType32 pipelineParallelism{2}; @@ -286,7 +285,7 @@ class SymmetricalCacheTest : public ::testing::Test // NOLINT(cppcoreguidelines- return mWorldSize; } - void setUpCacheManager(bool enableBlockReuse = false) + void setUpCacheManager() { auto constexpr numLayers = 4; auto constexpr numHeads = 2; @@ -308,7 +307,8 @@ class SymmetricalCacheTest : public ::testing::Test // NOLINT(cppcoreguidelines- auto totalNumBlocks = mMaxNumSequences * numBlocksPerSeq; auto constexpr blocksInSecondaryPool = 0; - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr enableBlockReuse = false; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; using BlocksPerWindow = std::map<SizeType32, std::tuple<SizeType32, SizeType32>>; auto const blocksPerWindow = BlocksPerWindow{{maxAttentionWindow, {totalNumBlocks, blocksInSecondaryPool}}}; @@ -421,21 +421,6 @@ class SymmetricalCacheTest : public ::testing::Test // NOLINT(cppcoreguidelines- return std::make_unique<LlmRequest>(mRequestId++, std::move(request)); } - // Generation-only request whose DataTransceiverState carries the arbitrary-transfer - // provenance marker, as getSerializedDataTransceiverState would produce. - auto makeArbitraryLlmRequest(SizeType32 length, LlmRequest::RequestIdType arbitraryId) - { - constexpr SizeType32 maxNewTokens{1}; - texec::Request request{VecTokens(length, length), maxNewTokens}; - auto state = std::make_unique<texec::DataTransceiverState>(); - state->setCommState(*mContextCommState); - state->setCacheState(*mCacheState); - state->setIsArbitraryTransferState(true); - auto stats = texec::ContextPhaseParams({}, arbitraryId, state.release(), std::nullopt); - request.setContextPhaseParams(std::move(stats)); - return std::make_unique<LlmRequest>(arbitraryId, std::move(request)); - } - void addRequestAndTransportCache(std::shared_ptr<LlmRequest> const& llmRequest) { auto constexpr beamIdx{0}; @@ -533,83 +518,10 @@ TEST_F(SymmetricalCacheTest, SimpleTest) } } -TEST_F(SymmetricalCacheTest, ArbitraryTransferTest) -{ - auto worldSize = setUpCommunicator(); - if (worldSize != 2) - { - GTEST_SKIP() << "mpirun 2 processes is required to run this test."; - } - setUpCacheManager(/*enableBlockReuse=*/true); - setUpCacheTransceiver(); - - // 3 full blocks (tokensPerBlock = 8) so every requested chunk fully matches a stored block. - constexpr SizeType32 promptLen = 24; - constexpr SizeType32 missPromptLen = 40; - constexpr LlmRequest::RequestIdType arbitraryId = 4242; - auto constexpr beamIdx{0}; - auto constexpr beamWidth{1}; - - if (isSender) - { - // Store a patterned request in the reuse tree; no LlmRequest exists on the - // sender for the transfers below. - auto request = makeLlmRequest(promptLen); - mManager->addSequenceBatch( - {{{request->mRequestId, request->getNumTokens(beamIdx), beamWidth}}}, {std::ref(*request)}); - auto blockRange = BlockRange::fromAllBlockIds(*mManager, request->mRequestId); - for (auto const& windowSize : blockRange.getWindowSizes()) - { - auto blockRangeForWindow = blockRange.getBlockRangeForWindow(windowSize); - for (auto it = blockRangeForWindow.begin(); it != blockRangeForWindow.end(); ++it) - { - TLLM_CUDA_CHECK(cudaMemset(it->data(), request->getPromptLen(), it->getSizeInBytes())); - } - } - tensorrt_llm::testing::KvCacheManagerTestUtil::simulatePrefillCompletion(*request); - // A completed context request carries one generated token beyond the prompt; - // without it, storeBlocksForReuse drops the trailing prompt token and the - // final block never enters the reuse tree. - request->addNewToken(0, beamIdx); - mManager->removeSequence(request->mRequestId, request); - } - else - { - // Hit: the requested tokens are stored in the sender's reuse tree. - std::shared_ptr<LlmRequest> request = makeArbitraryLlmRequest(promptLen, arbitraryId); - mManager->addSequenceBatch( - {{{request->mRequestId, request->getNumTokens(beamIdx), beamWidth}}}, {std::ref(*request)}); - auto future = mRequester->receiveAsync(request); - future.get(); - TLLM_CUDA_CHECK(cudaDeviceSynchronize()); - auto blockRange = BlockRange::fromAllBlockIds(*mManager, request->mRequestId); - for (auto const& windowSize : blockRange.getWindowSizes()) - { - auto blockRangeForWindow = blockRange.getBlockRangeForWindow(windowSize); - for (auto it = blockRangeForWindow.begin(); it != blockRangeForWindow.end(); ++it) - { - std::vector<uint8_t> bytes(it->getSizeInBytes()); - TLLM_CUDA_CHECK(cudaMemcpy(bytes.data(), it->data(), it->getSizeInBytes(), cudaMemcpyDeviceToHost)); - EXPECT_TRUE(std::all_of(bytes.begin(), bytes.end(), [](uint8_t i) { return i == (promptLen & 0xff); })); - } - } - - // Miss: tokens never stored on the sender are rejected; the rejection - // surfaces as an exception on the receive future. - std::shared_ptr<LlmRequest> missRequest = makeArbitraryLlmRequest(missPromptLen, arbitraryId + 1); - mManager->addSequenceBatch( - {{{missRequest->mRequestId, missRequest->getNumTokens(beamIdx), beamWidth}}}, {std::ref(*missRequest)}); - auto missFuture = mRequester->receiveAsync(missRequest); - EXPECT_THROW(missFuture.get(), tensorrt_llm::common::TllmException); - } - // The sender must not tear down while the receiver's transfers are in flight. - tensorrt_llm::mpi::MpiComm::world().barrier(); -} - #if ENABLE_MULTI_DEVICE -using AsymmetricTestParam = std::tuple<int, int, int, int, int, int, int, int, int, int, tensorrt_llm::DataType, int, - bool, bool, bool, bool, bool, int, int>; +using AsymmetricTestParam = std::tuple<int, int, int, int, int, int, int, int, int, int, nvinfer1::DataType, int, bool, + bool, bool, bool, bool, int, int>; // CPMetaData struct to hold CP-specific information struct CPMetaData @@ -759,7 +671,7 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara } void setUpCacheManager(int numLayers, int numHeads, int sizePerHead, int tokensPerBlock, - tensorrt_llm::DataType dataType, int kvFactor = 2, bool isMLA = false, bool enableDPAttention = false, + nvinfer1::DataType dataType, int kvFactor = 2, bool isMLA = false, bool enableDPAttention = false, bool isWindow = false, bool isIndexerKCache = true, int indexerDimPerHead = 0, int indexerKCacheQuantBlockSize = 128) { @@ -970,18 +882,15 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara = [this, bufferManagers]() { return createCacheFormatter(mManager.get(), bufferManagers, mIsMLA); }; TLLM_LOG_DEBUG("setUpCacheTransceiver makeFormatter"); - // Generate a per-instance ID so each ctx/gen instance writes to - // its own CSV files (mirrors CacheTransceiver behaviour). - auto instanceId = "test_" + std::to_string(tensorrt_llm::mpi::MpiComm::world().getRank()); if (mIsContext) { - mSender = std::make_unique<CacheSender>(mConnectionManager.get(), mRankInInstance, - CacheTransferLayer(*mCacheState, makeFormatter()), instanceId); + mSender = std::make_unique<CacheSender>( + mConnectionManager.get(), mRankInInstance, CacheTransferLayer(*mCacheState, makeFormatter())); } else { - mRequester = std::make_unique<CacheReceiver>(mConnectionManager.get(), mRankInInstance, - CacheTransferLayer(*mCacheState, makeFormatter()), instanceId); + mRequester = std::make_unique<CacheReceiver>( + mConnectionManager.get(), mRankInInstance, CacheTransferLayer(*mCacheState, makeFormatter())); } TLLM_LOG_DEBUG("setUpCacheTransceiver mSender"); @@ -1062,18 +971,16 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara cpMetaData.emplace(length, tokensPerBlock, mCpRank, mCpSize); seqLen = cpMetaData.value().mSeqLenOnThisCPRank; } - + texec::Request request{VecTokens(seqLen, seqLen), maxNewTokens}; auto state = std::make_unique<texec::DataTransceiverState>(); + TLLM_CHECK(mContextCommState); state->setCommState(texec::kv_cache::CommState{*mContextCommState}); state->setCacheState(*mContextCacheState); auto stats = texec::ContextPhaseParams({}, mRequestId, state.release(), std::nullopt); + request.setContextPhaseParams(std::move(stats)); - tr::SamplingConfig samplingConfig{1}; - auto inputTokens = std::make_shared<VecTokens>(seqLen, seqLen); - auto llmRequestPtr = std::make_shared<LlmRequest>( - mRequestId++, maxNewTokens, inputTokens, samplingConfig, /*isStreaming=*/false); - llmRequestPtr->setContextPhaseParams(std::move(stats)); + auto llmRequestPtr = std::make_shared<LlmRequest>(mRequestId++, std::move(request)); return std::make_unique<WrappedLlmRequest>(std::move(llmRequestPtr), cpMetaData); } @@ -1089,6 +996,7 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara cpMetaData.emplace(length, tokensPerBlock, mCpRank, mCpSize); seqLen = cpMetaData.value().mSeqLenOnThisCPRank; } + texec::Request request{VecTokens(seqLen, seqLen), maxNewTokens}; auto state = std::make_unique<texec::DataTransceiverState>(); state->setCommState(texec::kv_cache::CommState{*mContextCommState}); @@ -1103,12 +1011,8 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara mContextCacheState->getParallelConfig().mTensorParallelism}; state->setCacheState(cacheState); auto stats = texec::ContextPhaseParams({}, requestId, state.release(), std::nullopt); - - tr::SamplingConfig samplingConfig{1}; - auto inputTokens = std::make_shared<VecTokens>(seqLen, seqLen); - auto llmRequestPtr - = std::make_shared<LlmRequest>(requestId, maxNewTokens, inputTokens, samplingConfig, /*isStreaming=*/false); - llmRequestPtr->setContextPhaseParams(std::move(stats)); + request.setContextPhaseParams(std::move(stats)); + auto llmRequestPtr = std::make_shared<LlmRequest>(requestId, std::move(request)); return std::make_unique<WrappedLlmRequest>(std::move(llmRequestPtr), cpMetaData); } @@ -1431,7 +1335,7 @@ class AsymmetricalCacheTest : public ::testing::TestWithParam<AsymmetricTestPara } std::variant<double, float, int16_t, int8_t, uint8_t> generateExpectedValue(size_t initial, int windowSize, - int tokenId, int layerId, int headId, int hiddenId, bool key, tensorrt_llm::DataType dataType) + int tokenId, int layerId, int headId, int hiddenId, bool key, nvinfer1::DataType dataType) { size_t seed = 0; std::size_t hashValue = std::hash<size_t>{}(initial); @@ -1507,7 +1411,7 @@ TEST_P(AsymmetricalCacheTest, TestCase) int numHeads = std::get<7>(param); int sizePerHead = std::get<8>(param); int tokensPerBlock = std::get<9>(param); - tensorrt_llm::DataType dataType = std::get<10>(param); + nvinfer1::DataType dataType = std::get<10>(param); int kvFactor = std::get<11>(param); bool isMLA = std::get<12>(param); @@ -1622,7 +1526,7 @@ TEST_P(AsymmetricalCacheTestWithDP, TestCase) int numHeads = std::get<7>(param); int sizePerHead = std::get<8>(param); int tokensPerBlock = std::get<9>(param); - tensorrt_llm::DataType dataType = std::get<10>(param); + nvinfer1::DataType dataType = std::get<10>(param); int kvFactor = std::get<11>(param); bool isMLA = std::get<12>(param); @@ -1759,7 +1663,7 @@ TEST_P(UnexpectedTerminationRaceTest, UnexpectedTerminationRaceTest) int numHeads = std::get<7>(param); int sizePerHead = std::get<8>(param); int tokensPerBlock = std::get<9>(param); - tensorrt_llm::DataType dataType = std::get<10>(param); + nvinfer1::DataType dataType = std::get<10>(param); int kvFactor = std::get<11>(param); bool isMLA = std::get<12>(param); @@ -1906,87 +1810,87 @@ TEST_P(UnexpectedTerminationRaceTest, UnexpectedTerminationRaceTest) INSTANTIATE_TEST_CASE_P(UnexpectedTerminationRaceTest, UnexpectedTerminationRaceTest, testing::Combine(testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), testing::Values(16), - testing::Values(tensorrt_llm::DataType::kFLOAT), testing::Values(2), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(0), testing::Values(128))); + testing::Values(nvinfer1::DataType::kFLOAT), testing::Values(2), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), + testing::Values(128))); // Waive off isWindow test for now INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0, AsymmetricalCacheTest, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(/*true,*/ false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(/*true,*/ false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithWindow, AsymmetricalCacheTest, testing::Combine(testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(5), testing::Values(4), testing::Values(4), testing::Values(8), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(2), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1, AsymmetricalCacheTest, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(8), testing::Values(4), testing::Values(4), testing::Values(8), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(2), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false /*, true*/), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1EvenLayer, AsymmetricalCacheTest, testing::Combine(testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(10), testing::Values(4), testing::Values(4), testing::Values(8), - testing::Values(tensorrt_llm::DataType::kFLOAT), testing::Values(2), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(0), testing::Values(128))); + testing::Values(nvinfer1::DataType::kFLOAT), testing::Values(2), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), + testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest2EvenLayer, AsymmetricalCacheTest, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(10), testing::Values(4), testing::Values(4), testing::Values(8), - testing::Values(tensorrt_llm::DataType::kFLOAT), testing::Values(2), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(0), testing::Values(128))); + testing::Values(nvinfer1::DataType::kFLOAT), testing::Values(2), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), + testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest2, AsymmetricalCacheTest, testing::Combine(testing::Values(1), testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(1, 4), testing::Values(1), testing::Values(16), testing::Values(16), testing::Values(4), - testing::Values(8), testing::Values(tensorrt_llm::DataType::kFLOAT), testing::Values(2), testing::Values(false), + testing::Values(8), testing::Values(nvinfer1::DataType::kFLOAT), testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0ForMLA, AsymmetricalCacheTest, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1ForMLA, AsymmetricalCacheTest, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(8), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1ForMLAEvenLayer, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(10), testing::Values(1), testing::Values(4), testing::Values(8), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false, true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest2ForMLAEvenLayer, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(10), testing::Values(1), testing::Values(4), testing::Values(8), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false, true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0ForMLAWithIndexerKCache, AsymmetricalCacheTest, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(true), testing::Values(256), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(true), testing::Values(256), testing::Values(128))); // Tests cases where there's non-trivial TP and PP on context side but only CP on gen side. INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0WithCPForMLA, AsymmetricalCacheTest, @@ -2000,7 +1904,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0WithCPForMLA, AsymmetricalCacheTest, /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -2022,7 +1926,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1WithCPForMLA, AsymmetricalCacheTest, /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -2044,7 +1948,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0WithCPForGQA, AsymmetricalCacheTest, /*numHeads*/ testing::Values(4), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(2), /*isMLA*/ testing::Values(false), /*contextDP*/ testing::Values(false), @@ -2063,7 +1967,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1WithCPForGQA, AsymmetricalCacheTest, /*numHeads*/ testing::Values(4), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(2), /*isMLA*/ testing::Values(false), /*contextDP*/ testing::Values(false), @@ -2082,7 +1986,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest0WithCPForMLAUnevenLayer, Asymmetrical /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -2104,7 +2008,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest1WithCPForMLAUnevenLayer, Asymmetrical /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -2126,7 +2030,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTest2WithCPForMLAUnevenLayer, Asymmetrical /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -2148,7 +2052,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForMLA0, AsymmetricalCacheT /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(false), @@ -2170,7 +2074,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForMLA1, AsymmetricalCacheT /*numHeads*/ testing::Values(1), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(1), /*isMLA*/ testing::Values(true), /*contextDP*/ testing::Values(true), @@ -2192,7 +2096,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForGQA0, AsymmetricalCacheT /*numHeads*/ testing::Values(4), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(2), /*isMLA*/ testing::Values(false), /*contextDP*/ testing::Values(false), @@ -2211,7 +2115,7 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForGQA1, AsymmetricalCacheT /*numHeads*/ testing::Values(4), /*sizePerHead*/ testing::Values(4), /*tokensPerBlock*/ testing::Values(8), - /*dataType*/ testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), + /*dataType*/ testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), /*kvFactor*/ testing::Values(2), /*isMLA*/ testing::Values(false), /*contextDP*/ testing::Values(true), @@ -2221,97 +2125,97 @@ INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithCPAndDPForGQA1, AsymmetricalCacheT INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA1, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(1), testing::Values(true), testing::Values(true), testing::Values(true), testing::Values(false), + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(true), testing::Values(true), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA2, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(1), testing::Values(true), testing::Values(true), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(true), testing::Values(true), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA3, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(true), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), + testing::Values(true), testing::Values(false), testing::Values(true), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA4, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(16), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForMLA5, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(16), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(1), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(1), testing::Values(true), testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLA, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(2), testing::Values(false), testing::Values(true), testing::Values(true), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(false), testing::Values(true), testing::Values(true), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLA1, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(2), testing::Values(false), testing::Values(true), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLA2, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(4), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(true), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(false), testing::Values(false), testing::Values(true), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate0, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(2), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(2), testing::Values(false), testing::Values(true, false), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(false), testing::Values(true, false), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate0EvenLayer, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(5), testing::Values(2), testing::Values(4), testing::Values(16), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(2), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), testing::Values(false), testing::Values(true, false), testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate1, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(1, 2), testing::Values(1, 2), testing::Values(1), testing::Values(2), testing::Values(2), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(2), testing::Values(false), testing::Values(true, false), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(false), testing::Values(true, false), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate2, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4, 2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(2), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate3, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(2), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(4), testing::Values(2), testing::Values(4), testing::Values(16), - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), testing::Values(2), + testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(true), testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); INSTANTIATE_TEST_CASE_P(AsymmetricCaseTestWithDPForNoMLADuplicate4, AsymmetricalCacheTestWithDP, testing::Combine(testing::Values(4), testing::Values(1), testing::Values(1), testing::Values(1, 2), testing::Values(2), testing::Values(1), testing::Values(4), testing::Values(1, 2), testing::Values(4), - testing::Values(16), testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kINT8), - testing::Values(2), testing::Values(false), testing::Values(false), testing::Values(false), - testing::Values(false), testing::Values(false), testing::Values(0), testing::Values(128))); + testing::Values(16), testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kINT8), testing::Values(2), + testing::Values(false), testing::Values(false), testing::Values(false), testing::Values(false), + testing::Values(false), testing::Values(0), testing::Values(128))); #endif @@ -2322,7 +2226,7 @@ TEST(targetTest, CacheStateNODP) int const numHeads = 2; int const sizePerHead = 64; int const tokensPerBlock = 64; - auto const dataType = tensorrt_llm::DataType::kFLOAT; + auto const dataType = nvinfer1::DataType::kFLOAT; bool const isMLA = true; int const kvFactor = 2; @@ -2612,7 +2516,7 @@ TEST(targetTest, CacheStateNODPForGQAWithCP) int const numHeads = 4; int const sizePerHead = 64; int const tokensPerBlock = 64; - auto const dataType = tensorrt_llm::DataType::kFLOAT; + auto const dataType = nvinfer1::DataType::kFLOAT; bool const isMLA = false; int const kvFactor = 2; @@ -2828,7 +2732,7 @@ TEST(targetTest, CacheStateContextDP) int const numHeads = 2; int const sizePerHead = 64; int const tokensPerBlock = 64; - auto const dataType = tensorrt_llm::DataType::kFLOAT; + auto const dataType = nvinfer1::DataType::kFLOAT; bool const isMLA = true; int const kvFactor = 2; diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceFusionTest.cu b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceFusionTest.cu index e95c84d7d05a..4b9c7af29a46 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceFusionTest.cu +++ b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceFusionTest.cu @@ -25,7 +25,6 @@ #include <random> #include <vector> -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceFusionKernels.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceWorkspace.h" #include "tensorrt_llm/kernels/quantization.h" @@ -236,28 +235,28 @@ template <> struct DTypeTraits<half> { static constexpr ncclDataType_t kNCCLDataType = ncclFloat16; - static constexpr tensorrt_llm::DataType kTRTDataType = tensorrt_llm::DataType::kHALF; + static constexpr nvinfer1::DataType kTRTDataType = nvinfer1::DataType::kHALF; }; template <> struct DTypeTraits<__nv_bfloat16> { static constexpr ncclDataType_t kNCCLDataType = ncclBfloat16; - static constexpr tensorrt_llm::DataType kTRTDataType = tensorrt_llm::DataType::kBF16; + static constexpr nvinfer1::DataType kTRTDataType = nvinfer1::DataType::kBF16; }; template <> struct DTypeTraits<float> { static constexpr ncclDataType_t kNCCLDataType = ncclFloat32; - static constexpr tensorrt_llm::DataType kTRTDataType = tensorrt_llm::DataType::kFLOAT; + static constexpr nvinfer1::DataType kTRTDataType = nvinfer1::DataType::kFLOAT; }; template <typename DType, ar_fusion::AllReduceFusionPattern Pattern> class TestRunner { static constexpr ncclDataType_t kNCCLDataType = DTypeTraits<DType>::kNCCLDataType; - static constexpr tensorrt_llm::DataType kTRTDataType = DTypeTraits<DType>::kTRTDataType; + static constexpr nvinfer1::DataType kTRTDataType = DTypeTraits<DType>::kTRTDataType; static constexpr bool kFP4QuantOutSupport = !std::is_same_v<DType, float>; static_assert(kFP4QuantOutSupport || Pattern != ar_fusion::AllReduceFusionPattern::kARResidualRMSNormFP4Quant, "kARResidualRMSNormFP4Quant is not supported for float dtype"); diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceKernelTest.cu b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceKernelTest.cu index 78747373accf..b3d120e7015c 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceKernelTest.cu +++ b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/allReduceKernelTest.cu @@ -32,7 +32,6 @@ #include <type_traits> #include <vector> -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/customAllReduceKernels.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" @@ -213,7 +212,7 @@ public: { } - void set_params(AllReduceParams& params, tensorrt_llm::DataType dataType, int token_num, int hidden_size, + void set_params(AllReduceParams& params, nvinfer1::DataType dataType, int token_num, int hidden_size, AllReduceFusionOp op) const { int world_size = world_config.getSize(); @@ -317,7 +316,7 @@ bool test(Workspace const& workspace, int token_num, int hidden_size, bool has_b in.copy_from(input_buffer.data()); AllReduceParams params; - workspace.set_params(params, tensorrt_llm::DataType::kHALF, token_num, hidden_size, fusion_op); + workspace.set_params(params, nvinfer1::DataType::kHALF, token_num, hidden_size, fusion_op); params.ranks_per_node = world_size; params.local_rank = rank; params.local_output_buffer_ptr = out.data(); @@ -335,21 +334,21 @@ bool test(Workspace const& workspace, int token_num, int hidden_size, bool has_b cudaEventCreate(&begin); cudaEventCreate(&end); lamportInitialize( - params.fusion_params.lamport_peer_comm_buffer_ptrs[rank], message_size, tensorrt_llm::DataType::kHALF, s); + params.fusion_params.lamport_peer_comm_buffer_ptrs[rank], message_size, nvinfer1::DataType::kHALF, s); lamportInitialize(params.fusion_params.lamport_peer_comm_buffer_ptrs[rank + MAX_RANKS_PER_NODE], message_size, - tensorrt_llm::DataType::kHALF, s); + nvinfer1::DataType::kHALF, s); lamportInitialize(params.fusion_params.lamport_peer_comm_buffer_ptrs[rank + MAX_RANKS_PER_NODE * 2], message_size, - tensorrt_llm::DataType::kHALF, s); + nvinfer1::DataType::kHALF, s); cudaDeviceSynchronize(); comm.barrier(); for (int i = 0; i < warmup; ++i) { - customAllReduce(params, tensorrt_llm::DataType::kHALF, runtime_strategy, config, fusion_op, s); + customAllReduce(params, nvinfer1::DataType::kHALF, runtime_strategy, config, fusion_op, s); } cudaEventRecord(begin, s); for (int i = 0; i < iter; ++i) { - customAllReduce(params, tensorrt_llm::DataType::kHALF, runtime_strategy, config, fusion_op, s); + customAllReduce(params, nvinfer1::DataType::kHALF, runtime_strategy, config, fusion_op, s); } cudaEventRecord(end, s); cudaEventSynchronize(end); @@ -463,7 +462,7 @@ bool test_prepostnorm(Workspace const& workspace, int token_num, int hidden_size in.copy_from(input_buffer.data()); AllReduceParams params; - workspace.set_params(params, tensorrt_llm::DataType::kHALF, token_num, hidden_size, fusion_op); + workspace.set_params(params, nvinfer1::DataType::kHALF, token_num, hidden_size, fusion_op); params.ranks_per_node = world_size; params.local_rank = rank; params.local_output_buffer_ptr = out.data(); @@ -485,12 +484,12 @@ bool test_prepostnorm(Workspace const& workspace, int token_num, int hidden_size comm.barrier(); for (int i = 0; i < warmup; ++i) { - customAllReduce(params, tensorrt_llm::DataType::kHALF, runtime_strategy, config, fusion_op, s); + customAllReduce(params, nvinfer1::DataType::kHALF, runtime_strategy, config, fusion_op, s); } cudaEventRecord(begin, s); for (int i = 0; i < iter; ++i) { - customAllReduce(params, tensorrt_llm::DataType::kHALF, runtime_strategy, config, fusion_op, s); + customAllReduce(params, nvinfer1::DataType::kHALF, runtime_strategy, config, fusion_op, s); } cudaEventRecord(end, s); cudaEventSynchronize(end); diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/gemmAllReduceTest.cu b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/gemmAllReduceTest.cu index 2ff2c8c37b9c..415a3c32da81 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/gemmAllReduceTest.cu +++ b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/gemmAllReduceTest.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2026, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,11 +25,12 @@ #else #include "allreduce_gemm_runner.h" #endif +#include "common.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/userbuffers/ub_interface.h" #include "tensorrt_llm/runtime/ipcNvlsMemory.h" #include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include <NvInferRuntime.h> #include "cute/tensor.hpp" #include "cutlass/cutlass.h" @@ -52,59 +53,8 @@ #include "cutlass/util/reference/host/gett.hpp" #include "cutlass/util/reference/host/tensor_fill.h" -namespace tensorrt_llm::testing -{ - -/** - * GPU timer for recording the elapsed time across kernel(s) launched in GPU stream - */ -struct GpuTimer -{ - cudaStream_t _stream_id; - cudaEvent_t _start; - cudaEvent_t _stop; - - /// Constructor - GpuTimer() - : _stream_id(0) - { - TLLM_CUDA_CHECK(cudaEventCreate(&_start)); - TLLM_CUDA_CHECK(cudaEventCreate(&_stop)); - } - - /// Destructor - ~GpuTimer() - { - TLLM_CUDA_CHECK(cudaEventDestroy(_start)); - TLLM_CUDA_CHECK(cudaEventDestroy(_stop)); - } - - /// Start the timer for a given stream (defaults to the default stream) - void start(cudaStream_t stream_id = 0) - { - _stream_id = stream_id; - TLLM_CUDA_CHECK(cudaEventRecord(_start, _stream_id)); - } - - /// Stop the timer - void stop() - { - TLLM_CUDA_CHECK(cudaEventRecord(_stop, _stream_id)); - } - - /// Return the elapsed time (in milliseconds) - float elapsed_millis() - { - float elapsed = 0.0; - TLLM_CUDA_CHECK(cudaEventSynchronize(_stop)); - TLLM_CUDA_CHECK(cudaEventElapsedTime(&elapsed, _start, _stop)); - return elapsed; - } -}; - -} // namespace tensorrt_llm::testing - using namespace cutlass; +using namespace nvinfer1; using namespace tensorrt_llm::mpi; using namespace tensorrt_llm::runtime; using namespace tensorrt_llm::common; @@ -288,7 +238,7 @@ struct ToType template <> struct ToType<cutlass::bfloat16_t> { - tensorrt_llm::DataType trt_value = tensorrt_llm::DataType::kBF16; + nvinfer1::DataType trt_value = nvinfer1::DataType::kBF16; ncclDataType_t nccl_value = ncclBfloat16; char const* str_value = "bf16"; }; @@ -296,7 +246,7 @@ struct ToType<cutlass::bfloat16_t> template <> struct ToType<cutlass::half_t> { - tensorrt_llm::DataType trt_value = tensorrt_llm::DataType::kHALF; + nvinfer1::DataType trt_value = nvinfer1::DataType::kHALF; ncclDataType_t nccl_value = ncclFloat16; char const* str_value = "fp16"; }; @@ -304,7 +254,7 @@ struct ToType<cutlass::half_t> template <> struct ToType<cutlass::float_e4m3_t> { - tensorrt_llm::DataType trt_value = tensorrt_llm::DataType::kFP8; + nvinfer1::DataType trt_value = nvinfer1::DataType::kFP8; ncclDataType_t nccl_value = ncclFloat8e4m3; char const* str_value = "fp8_e4m3"; }; diff --git a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/moeAllReduceFusionTest.cu b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/moeAllReduceFusionTest.cu index 4b81a4133fa9..8abccf214b7a 100644 --- a/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/moeAllReduceFusionTest.cu +++ b/cpp/tests/unit_tests/multi_gpu/kernels/allReduce/moeAllReduceFusionTest.cu @@ -24,7 +24,6 @@ #include <random> #include <vector> -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/kernels/communicationKernels/allReduceWorkspace.h" #include "tensorrt_llm/kernels/communicationKernels/moeAllReduceFusionKernels.h" #include "tensorrt_llm/kernels/quantization.h" @@ -393,8 +392,8 @@ class MoEARFuseTestRunner { static_assert(std::is_same_v<DType, half> || std::is_same_v<DType, __nv_bfloat16>); static constexpr ncclDataType_t kNCCLDataType = std::is_same_v<DType, half> ? ncclFloat16 : ncclBfloat16; - static constexpr tensorrt_llm::DataType kTRTDataType - = std::is_same_v<DType, half> ? tensorrt_llm::DataType::kHALF : tensorrt_llm::DataType::kBF16; + static constexpr nvinfer1::DataType kTRTDataType + = std::is_same_v<DType, half> ? nvinfer1::DataType::kHALF : nvinfer1::DataType::kBF16; public: MoEARFuseTestRunner(int max_token_num, int hidden_dim, int max_expert_num) diff --git a/cpp/tests/unit_tests/multi_gpu/mpiUtilsTest.cpp b/cpp/tests/unit_tests/multi_gpu/mpiUtilsTest.cpp index 941d7cb53ed6..b8596d511fab 100644 --- a/cpp/tests/unit_tests/multi_gpu/mpiUtilsTest.cpp +++ b/cpp/tests/unit_tests/multi_gpu/mpiUtilsTest.cpp @@ -20,6 +20,7 @@ #include "tensorrt_llm/runtime/utils/mpiUtils.h" #if ENABLE_MULTI_DEVICE +#include "tensorrt_llm/plugins/common/plugin.h" #include <nccl.h> #endif // ENABLE_MULTI_DEVICE @@ -116,6 +117,11 @@ TEST(MPIUtils, BroadcastNcclId) EXPECT_TRUE(std::any_of( id.internal, id.internal + sizeof(id.internal) / sizeof(id.internal[0]), [](auto x) { return x != 0; })); } + +TEST(MPIUtils, GlobalSessionHandle) +{ + EXPECT_EQ(tensorrt_llm::plugins::getCommSessionHandle(), &COMM_SESSION); +} #endif // ENABLE_MULTI_DEVICE template <typename T> diff --git a/cpp/tests/unit_tests/runtime/CMakeLists.txt b/cpp/tests/unit_tests/runtime/CMakeLists.txt index 3a171ee39877..c022ba31ebca 100644 --- a/cpp/tests/unit_tests/runtime/CMakeLists.txt +++ b/cpp/tests/unit_tests/runtime/CMakeLists.txt @@ -35,6 +35,7 @@ add_gtest(samplingConfigTest samplingConfigTest.cpp) add_gtest(samplingTest samplingTest.cpp) add_gtest(sanitizerTest sanitizerTest.cpp) add_gtest(tllmBuffersTest tllmBuffersTest.cpp) +add_gtest(tllmRuntimeTest tllmRuntimeTest.cpp) add_gtest(transposeKVKernelTest transposeKVKernelTest.cpp) add_gtest(utilsTest utilsTest.cpp) add_gtest(virtualMemoryTest virtualMemoryTest.cpp) diff --git a/cpp/tests/unit_tests/runtime/bufferManagerTest.cpp b/cpp/tests/unit_tests/runtime/bufferManagerTest.cpp index 8bdc6d0352a0..194cf88b6765 100644 --- a/cpp/tests/unit_tests/runtime/bufferManagerTest.cpp +++ b/cpp/tests/unit_tests/runtime/bufferManagerTest.cpp @@ -17,7 +17,6 @@ #include <gtest/gtest.h> #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaMemPool.h" @@ -122,7 +121,7 @@ TEST_F(BufferManagerTest, Pointers) static_assert(std::is_same_v<decltype(trtPointerType), BufferDataType const>); static_assert(trtPointerType.isPointer()); static_assert(trtPointerType.getDataType() == TRTDataType<cppBaseType>::value); - static_assert(static_cast<tensorrt_llm::DataType>(trtPointerType) == BufferDataType::kTrtPointerType); + static_assert(static_cast<nvinfer1::DataType>(trtPointerType) == BufferDataType::kTrtPointerType); static_assert(trtPointerType == BufferDataType::kTrtPointerType); // uses implicit type conversion // The C++ type corresponding to the TensorRT type for storing pointers (int64_t) using cppStorageType = DataTypeTraits<trtPointerType>::type; diff --git a/cpp/tests/unit_tests/runtime/decodingLayerWorkspaceTest.cpp b/cpp/tests/unit_tests/runtime/decodingLayerWorkspaceTest.cpp index 74f8faa37c87..bb6ce6410ad5 100644 --- a/cpp/tests/unit_tests/runtime/decodingLayerWorkspaceTest.cpp +++ b/cpp/tests/unit_tests/runtime/decodingLayerWorkspaceTest.cpp @@ -16,7 +16,6 @@ #include "tensorrt_llm/runtime/decodingLayerWorkspace.h" #include "tensorrt_llm/common/cudaUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/common/workspace.h" #include <gtest/gtest.h> #include <random> @@ -122,9 +121,8 @@ auto const tensorDataTypesTuples = testing::Combine(tensorDataTypes, tensorDataT auto const tensorShapeTuples = testing::Combine(tensorDimensions, tensorDimensions, tensorDimensions); auto const mirrorInWorkspaceParams = testing::Combine(tensorDataTypesTuples, tensorShapeTuples, randomSeeds); -using MirrorInWorkspaceParamType - = std::tuple<std::tuple<tensorrt_llm::DataType, tensorrt_llm::DataType, tensorrt_llm::DataType>, - std::tuple<std::int32_t, std::int32_t, std::int32_t>, std::uint64_t>; +using MirrorInWorkspaceParamType = std::tuple<std::tuple<nvinfer1::DataType, nvinfer1::DataType, nvinfer1::DataType>, + std::tuple<std::int32_t, std::int32_t, std::int32_t>, std::uint64_t>; class MirrorInWorkspaceTest : public testing::TestWithParam<MirrorInWorkspaceParamType> { diff --git a/cpp/tests/unit_tests/runtime/gptDecoderBatchedTest.cpp b/cpp/tests/unit_tests/runtime/gptDecoderBatchedTest.cpp index a979c9a4699f..15476899fce3 100644 --- a/cpp/tests/unit_tests/runtime/gptDecoderBatchedTest.cpp +++ b/cpp/tests/unit_tests/runtime/gptDecoderBatchedTest.cpp @@ -17,11 +17,10 @@ #include "tensorrt_llm/runtime/gptDecoderBatched.h" #include "tensorrt_llm/batch_manager/createNewDecoderRequests.h" #include "tensorrt_llm/batch_manager/decoderBuffers.h" -#include "tensorrt_llm/batch_manager/llmRequest.h" +#include "tensorrt_llm/batch_manager/makeDecodingBatchInputOutput.h" #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/common/logger.h" #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/common.h" @@ -48,62 +47,6 @@ using TensorPtr = ITensor::SharedPtr; namespace { -// Local copy of the former MakeDecodingBatchInputOutput::createDecoderBatchInputs -// helper, which was removed with the TensorRT-engine execution path. The decoder -// under test is backend-agnostic; this builds its step-batched inputs directly. -void createDecoderBatchInputs(tb::DecoderInputBuffers& inputBuffers, std::vector<SizeType32> const& activeSlots, - decoder::DecoderState const& decoderState) -{ - auto const& numDecodingEngineTokens = decoderState.getNumDecodingEngineTokens(); - auto const& maxDecodingEngineTokens = decoderState.getMaxDecodingEngineTokens(); - auto const& maxDecodingDecoderTokens = decoderState.getMaxDecodingDecoderTokens(); - auto const maxDecoderSteps = tc::ceilDiv(maxDecodingEngineTokens, maxDecodingDecoderTokens); - - auto& batchSlots = inputBuffers.forwardBatchSlots; - auto& decoderLogits = inputBuffers.decoderLogits; - - for (SizeType32 step = 0; step < maxDecoderSteps; ++step) - { - batchSlots.at(step)->resize(activeSlots.size()); - } - - auto constexpr singleRequest = 1; - - std::vector<SizeType32> batchSizes(maxDecoderSteps); - std::vector<std::vector<ITensor::SharedConstPtr>> batchLogits(maxDecoderSteps); - auto maxActiveDecoderSteps = 1; - for (size_t batchIdx = 0; batchIdx < activeSlots.size(); ++batchIdx) - { - auto const slot = activeSlots.at(batchIdx); - auto const& logits = decoderLogits.at(batchIdx); - - auto const numDecoderSteps = tc::ceilDiv(numDecodingEngineTokens.at(slot), maxDecodingDecoderTokens); - maxActiveDecoderSteps = std::max(maxActiveDecoderSteps, numDecoderSteps); - for (SizeType32 step = 0; step < numDecoderSteps; ++step) - { - auto batchSlotsRange = BufferRange<SizeType32>(*batchSlots.at(step)); - batchSlotsRange[batchSizes[step]] = slot; - batchSizes[step]++; - auto logitsSlice = ITensor::slice(logits, step, singleRequest); - batchLogits[step].emplace_back(std::move(logitsSlice)); - } - } - - for (SizeType32 step = 0; step < maxDecoderSteps; ++step) - { - batchSlots.at(step)->resize(batchSizes[step]); - } - batchLogits.resize(maxActiveDecoderSteps); - - inputBuffers.maxDecoderSteps = maxActiveDecoderSteps; - inputBuffers.batchLogits = batchLogits; -} - -} // namespace - -namespace -{ - std::shared_ptr<tb::LlmRequest> createLlmRequest(SizeType32 batchSlot, SizeType32 inputLengths, SizeType32 generatedTokensPerSteps, SizeType32 acceptedTokensPerStep, TokenIdType inputTokenId, TokenIdType expectedTokenId, SizeType32 maxNewTokens, SamplingConfig const& samplingConfig, TokenIdType endId) @@ -150,7 +93,7 @@ std::vector<std::shared_ptr<tb::LlmRequest>> createLlmRequests(std::vector<SizeT } void newRequests(std::vector<std::shared_ptr<tb::LlmRequest>> const& requests, TensorPtr const& batchSlots, - tensorrt_llm::DataType logitsType, ModelConfig const& modelConfig, WorldConfig const& worldConfig, + nvinfer1::DataType logitsType, ModelConfig const& modelConfig, WorldConfig const& worldConfig, tle::DecodingConfig const& decodingConfig, GptDecoderBatched& decoder, CudaStream const& runtimeStream, SizeType32 maxSequenceLength, tb::DecoderInputBuffers& inputBuffers, decoder::DecoderState& decoderState) { @@ -182,7 +125,7 @@ void newRequests(std::vector<std::shared_ptr<tb::LlmRequest>> const& requests, T } void createDecoderInputs(tb::DecoderInputBuffers& inputBuffers, SizeType32 batchSize, SizeType32 vocabSizePadded, - tensorrt_llm::DataType dataType, std::vector<SamplingConfig>& samplingConfigs, + nvinfer1::DataType dataType, std::vector<SamplingConfig>& samplingConfigs, std::vector<SizeType32> const& generatedTokensPerSteps, bool computeLogProbs, BufferManager& manager) { auto& logits = inputBuffers.decoderLogits; @@ -299,8 +242,8 @@ void verifyResults(BufferManager& manager, decoder::DecoderState const& decoderS } } -void testDecoder(tensorrt_llm::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, - SizeType32 maxBeamWidth, bool computeLogProbs) +void testDecoder(nvinfer1::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, SizeType32 maxBeamWidth, + bool computeLogProbs) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); SizeType32 constexpr tensorParallelism{1}; @@ -402,7 +345,7 @@ void testDecoder(tensorrt_llm::DataType const dtype, std::vector<SamplingConfig> auto activeSlots = std::vector<SizeType32>(batchSize); std::iota(activeSlots.begin(), activeSlots.end(), 0); - createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + tb::MakeDecodingBatchInputOutput::createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); decoder.forward(decoderState, inputBuffers); checkSequenceLengths(*decoderState.getSequenceLengths(), expectedLengths, manager); @@ -432,7 +375,7 @@ void testDecoder(tensorrt_llm::DataType const dtype, std::vector<SamplingConfig> EXPECT_FALSE(getFinished(*decoderState.getFinishedSum(), samplingConfigs, manager)[0]); } -void testDecoderWavefront(tensorrt_llm::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, +void testDecoderWavefront(nvinfer1::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, SizeType32 maxBeamWidth, bool computeLogProbs) { TLLM_LOG_TRACE("%s start", __PRETTY_FUNCTION__); @@ -531,7 +474,7 @@ void testDecoderWavefront(tensorrt_llm::DataType const dtype, std::vector<Sampli auto activeSlots = std::vector<SizeType32>(batchIdx + 1); std::iota(activeSlots.begin(), activeSlots.end(), 0); - createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + tb::MakeDecodingBatchInputOutput::createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); decoder.forward(decoderState, inputBuffers); advanceSequenceLengths( @@ -553,7 +496,7 @@ void testDecoderWavefront(tensorrt_llm::DataType const dtype, std::vector<Sampli auto finishedVec = getFinished(*decoderState.getFinishedSum(), samplingConfigs, manager); while (!std::all_of(expectedFinished.begin(), expectedFinished.end(), [](bool finish) { return finish; })) { - createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + tb::MakeDecodingBatchInputOutput::createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); decoder.forward(decoderState, inputBuffers); finishedVec = getFinished(*decoderState.getFinishedSum(), samplingConfigs, manager); @@ -583,7 +526,7 @@ void testDecoderWavefront(tensorrt_llm::DataType const dtype, std::vector<Sampli maxSeqLength, inputTokenId, expectedTokenId, endId); } -void testDecoderDraft(tensorrt_llm::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, +void testDecoderDraft(nvinfer1::DataType const dtype, std::vector<SamplingConfig>& samplingConfigs, SizeType32 maxBeamWidth, std::vector<SizeType32> const& generatedTokensPerSteps, std::vector<SizeType32> const& acceptedTokensPerStep, SizeType32 maxGeneratedTokensPerStep) { @@ -688,7 +631,7 @@ void testDecoderDraft(tensorrt_llm::DataType const dtype, std::vector<SamplingCo auto activeSlots = std::vector<SizeType32>(batchSize); std::iota(activeSlots.begin(), activeSlots.end(), 0); - createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); + tb::MakeDecodingBatchInputOutput::createDecoderBatchInputs(inputBuffers, activeSlots, decoderState); decoder.forward(decoderState, inputBuffers); checkSequenceLengths(*decoderState.getSequenceLengths(), expectedLengths, manager); EXPECT_THAT(getFinished(*decoderState.getFinishedSum(), samplingConfigs, manager), ::testing::Each(false)); @@ -705,11 +648,11 @@ struct BeamConfig std::vector<SizeType32> beamWidths; }; -using ParamType = std::tuple<tensorrt_llm::DataType, BeamConfig, bool>; +using ParamType = std::tuple<nvinfer1::DataType, BeamConfig, bool>; std::string generateTestName(testing::TestParamInfo<ParamType> const& info) { - std::string name{std::get<0>(info.param) == tensorrt_llm::DataType::kFLOAT ? "Float" : "Half"}; + std::string name{std::get<0>(info.param) == nvinfer1::DataType::kFLOAT ? "Float" : "Half"}; BeamConfig const beamConfig = std::get<1>(info.param); name.append("MaxBeamWidth" + std::to_string(beamConfig.maxBeamWidth)); for (auto const beamWdith : beamConfig.beamWidths) @@ -730,7 +673,7 @@ class ParamTest : public ::testing::TestWithParam<ParamType> TEST_P(ParamTest, Test) { - tensorrt_llm::DataType const dtype{std::get<0>(GetParam())}; + nvinfer1::DataType const dtype{std::get<0>(GetParam())}; BeamConfig const beamConfig{std::get<1>(GetParam())}; bool const computeLogProbs{std::get<2>(GetParam())}; std::vector<SamplingConfig> samplingConfigs; @@ -743,7 +686,7 @@ TEST_P(ParamTest, Test) } INSTANTIATE_TEST_SUITE_P(DecoderBwTest, ParamTest, - testing::Combine(testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF), + testing::Combine(testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF), testing::Values(BeamConfig{1, {1, 1, 1}}, BeamConfig{3, {3, 3, 3, 3}}, BeamConfig{4, {4, 4, 4}}, BeamConfig{10, {10, 10, 10}}), testing::Values(false, true)), @@ -755,7 +698,7 @@ class ParamWavefrontTest : public ::testing::TestWithParam<ParamType> TEST_P(ParamWavefrontTest, Test) { - tensorrt_llm::DataType const dtype{std::get<0>(GetParam())}; + nvinfer1::DataType const dtype{std::get<0>(GetParam())}; BeamConfig const beamConfig{std::get<1>(GetParam())}; bool const computeLogProbs{std::get<2>(GetParam())}; bool const normalizeLogProbs{true}; @@ -769,7 +712,7 @@ TEST_P(ParamWavefrontTest, Test) } INSTANTIATE_TEST_SUITE_P(DecoderBwTest, ParamWavefrontTest, - testing::Combine(testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF), + testing::Combine(testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF), testing::Values(BeamConfig{1, {1, 1, 1}}, BeamConfig{3, {3, 3, 3, 3}}, BeamConfig{4, {4, 4, 4}}, BeamConfig{10, {10, 10, 10}}), testing::Values(false, true)), @@ -782,7 +725,7 @@ struct DraftConfig std::vector<SizeType32> acceptedTokensPerStep; }; -using DraftTestParamType = std::tuple<tensorrt_llm::DataType, BeamConfig, DraftConfig>; +using DraftTestParamType = std::tuple<nvinfer1::DataType, BeamConfig, DraftConfig>; class ParamDraftTest : public ::testing::TestWithParam<DraftTestParamType> { @@ -790,7 +733,7 @@ class ParamDraftTest : public ::testing::TestWithParam<DraftTestParamType> TEST_P(ParamDraftTest, Test) { - tensorrt_llm::DataType const dtype{std::get<0>(GetParam())}; + nvinfer1::DataType const dtype{std::get<0>(GetParam())}; BeamConfig const beamConfig{std::get<1>(GetParam())}; DraftConfig const draftConfig{std::get<2>(GetParam())}; @@ -808,7 +751,7 @@ TEST_P(ParamDraftTest, Test) } INSTANTIATE_TEST_SUITE_P(DecoderTest, ParamDraftTest, - testing::Combine(testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF), + testing::Combine(testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF), testing::Values(BeamConfig{1, {1, 1, 1}}), testing::Values( // DraftConfig{2, {1, 1, 1}, {0, 0, 0}}, DraftConfig{2, {2, 2, 2}, {1, 1, 1}}, @@ -817,7 +760,7 @@ INSTANTIATE_TEST_SUITE_P(DecoderTest, ParamDraftTest, )), [](testing::TestParamInfo<DraftTestParamType> const& info) { - std::string name{std::get<0>(info.param) == tensorrt_llm::DataType::kFLOAT ? "Float" : "Half"}; + std::string name{std::get<0>(info.param) == nvinfer1::DataType::kFLOAT ? "Float" : "Half"}; BeamConfig const beamConfig = std::get<1>(info.param); DraftConfig const draftConfig = std::get<2>(info.param); name.append("MaxBeamWidth" + std::to_string(beamConfig.maxBeamWidth)); diff --git a/cpp/tests/unit_tests/runtime/gptDecoderTest.cpp b/cpp/tests/unit_tests/runtime/gptDecoderTest.cpp index 5f620aa4d9c4..e1fed49293bd 100644 --- a/cpp/tests/unit_tests/runtime/gptDecoderTest.cpp +++ b/cpp/tests/unit_tests/runtime/gptDecoderTest.cpp @@ -17,7 +17,6 @@ #include <gtest/gtest.h> #include "tensorrt_llm/common/memoryUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/gptDecoder.h" @@ -69,7 +68,7 @@ bool forwardAndSync(std::unique_ptr<IGptDecoder> const& decoder, DecodingOutput& } } -void testDecoder(tensorrt_llm::DataType const dtype, SamplingConfig const& samplingConfig) +void testDecoder(nvinfer1::DataType const dtype, SamplingConfig const& samplingConfig) { SizeType32 constexpr tensorParallelism{1}; SizeType32 constexpr pipelineParallelism{1}; @@ -141,21 +140,21 @@ void testDecoder(tensorrt_llm::DataType const dtype, SamplingConfig const& sampl if (beamWidth > 1) { auto srcCacheIndirection = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); manager.setZero(*srcCacheIndirection); inputs.cacheIndirection = srcCacheIndirection; } // set up outputs auto outputIds = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); manager.setZero(*outputIds); auto gatheredOutputIds = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); manager.setZero(*gatheredOutputIds); DecodingOutput outputs{outputIds, gatheredOutputIds}; auto newTokens - = std::shared_ptr(manager.gpu(ITensor::makeShape({batchSize, beamWidth}), tensorrt_llm::DataType::kINT32)); + = std::shared_ptr(manager.gpu(ITensor::makeShape({batchSize, beamWidth}), nvinfer1::DataType::kINT32)); manager.setZero(*newTokens); outputs.newTokens = newTokens; @@ -166,7 +165,7 @@ void testDecoder(tensorrt_llm::DataType const dtype, SamplingConfig const& sampl TRTDataType<tensorrt_llm::kernels::FinishedState::UnderlyingType>::value); inputs.finishReasons = ITensor::view(outputs.finishReasons); manager.setZero(*outputs.finishReasons); - outputs.finishedSum = BufferManager::pinnedPool(ITensor::makeShape({batchSize}), tensorrt_llm::DataType::kINT32); + outputs.finishedSum = BufferManager::pinnedPool(ITensor::makeShape({batchSize}), nvinfer1::DataType::kINT32); auto finishedSumHost = bufferCast<std::int32_t>(*outputs.finishedSum); for (SizeType32 bi = 0; bi < batchSize; ++bi) { @@ -176,17 +175,17 @@ void testDecoder(tensorrt_llm::DataType const dtype, SamplingConfig const& sampl if (beamWidth > 1) { auto tgtCacheIndirection = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); manager.setZero(*tgtCacheIndirection); outputs.cacheIndirection = tgtCacheIndirection; auto cumLogProbs - = std::shared_ptr(manager.gpu(ITensor::makeShape({batchSize, beamWidth}), tensorrt_llm::DataType::kFLOAT)); + = std::shared_ptr(manager.gpu(ITensor::makeShape({batchSize, beamWidth}), nvinfer1::DataType::kFLOAT)); manager.setZero(*cumLogProbs); outputs.cumLogProbs = cumLogProbs; auto parentIds = std::shared_ptr( - manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), tensorrt_llm::DataType::kINT32)); + manager.gpu(ITensor::makeShape({batchSize, beamWidth, maxSeqLength}), nvinfer1::DataType::kINT32)); manager.setZero(*parentIds); outputs.parentIds = parentIds; } @@ -246,13 +245,13 @@ void testDecoder(tensorrt_llm::DataType const dtype, SamplingConfig const& sampl } // namespace -class ParamTest : public ::testing::TestWithParam<std::tuple<tensorrt_llm::DataType, SizeType32>> +class ParamTest : public ::testing::TestWithParam<std::tuple<nvinfer1::DataType, SizeType32>> { }; TEST_P(ParamTest, Test) { - tensorrt_llm::DataType const dtype{std::get<0>(GetParam())}; + nvinfer1::DataType const dtype{std::get<0>(GetParam())}; SizeType32 const beamWidth{std::get<1>(GetParam())}; SamplingConfig const samplingConfig{beamWidth}; @@ -260,11 +259,10 @@ TEST_P(ParamTest, Test) } INSTANTIATE_TEST_SUITE_P(DecoderTest, ParamTest, - testing::Combine( - testing::Values(tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF), testing::Values(1, 3)), + testing::Combine(testing::Values(nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF), testing::Values(1, 3)), [](testing::TestParamInfo<ParamTest::ParamType> const& info) { - std::string name{std::get<0>(info.param) == tensorrt_llm::DataType::kFLOAT ? "Float" : "Half"}; + std::string name{std::get<0>(info.param) == nvinfer1::DataType::kFLOAT ? "Float" : "Half"}; auto const beamWidth = std::get<1>(info.param); name.append(beamWidth == 1 ? "Sampling" : "BeamWidth" + std::to_string(beamWidth)); return name; diff --git a/cpp/tests/unit_tests/runtime/iTensorTest.cpp b/cpp/tests/unit_tests/runtime/iTensorTest.cpp index 4637474f72c7..54ba8aa3beec 100644 --- a/cpp/tests/unit_tests/runtime/iTensorTest.cpp +++ b/cpp/tests/unit_tests/runtime/iTensorTest.cpp @@ -17,7 +17,6 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -26,7 +25,7 @@ using namespace tensorrt_llm::runtime; TEST(ITensorTest, SqueezeTensor) { auto dims = ITensor::makeShape({16, 1, 4}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor{BufferManager::cpu(dims, dataType)}; auto squeezeDim = 0; @@ -103,7 +102,7 @@ TEST(ITensorTest, UnsqueezeTensor) auto oldShape = ITensor::makeShape({2, 3, 4, 5}); { - auto tensor = BufferManager::cpu(oldShape, tensorrt_llm::DataType::kINT32); + auto tensor = BufferManager::cpu(oldShape, nvinfer1::DataType::kINT32); tensor->unsqueeze(0); auto shape = tensor->getShape(); @@ -115,7 +114,7 @@ TEST(ITensorTest, UnsqueezeTensor) EXPECT_EQ(shape.d[4], 5); } { - auto tensor = BufferManager::cpu(oldShape, tensorrt_llm::DataType::kINT32); + auto tensor = BufferManager::cpu(oldShape, nvinfer1::DataType::kINT32); tensor->unsqueeze(1); auto shape = tensor->getShape(); @@ -128,7 +127,7 @@ TEST(ITensorTest, UnsqueezeTensor) } { - auto tensor = BufferManager::cpu(oldShape, tensorrt_llm::DataType::kINT32); + auto tensor = BufferManager::cpu(oldShape, nvinfer1::DataType::kINT32); tensor->unsqueeze(4); auto shape = tensor->getShape(); @@ -145,7 +144,7 @@ TEST(ITensorTest, UnsqueezeTensor) { try { - auto tensor = BufferManager::cpu(oldShape, tensorrt_llm::DataType::kINT32); + auto tensor = BufferManager::cpu(oldShape, nvinfer1::DataType::kINT32); tensor->unsqueeze(invalidDim); FAIL() << "Expected failure"; } @@ -163,7 +162,7 @@ TEST(ITensorTest, UnsqueezeTensor) TEST(ITensorTest, TensorView) { auto const dims = ITensor::makeShape({16, 1, 4}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor = BufferManager::cpu(dims, dataType); auto const viewDims = ITensor::makeShape({16, 1, 2}); @@ -181,7 +180,7 @@ TEST(ITensorTest, TensorView) TEST(ITensorTest, TensorSlice) { auto dims = ITensor::makeShape({16, 8, 4}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor{BufferManager::cpu(dims, dataType)}; auto offset = dims.d[0] / 4; auto slice = ITensor::slice(tensor, offset); @@ -222,7 +221,7 @@ TEST(ITensorTest, TensorSlice) TEST(ITensorTest, TensorDimsSliceAtManual) { auto shape = ITensor::makeShape({5, 5, 5, 5, 5}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); auto offsetDims = ITensor::makeShape({4, 3, 3}); auto sizeDim = 2; @@ -283,7 +282,7 @@ TEST(ITensorTest, TensorDimsSliceAtManual) TEST(ITensorTest, TensorDimsSliceAtExtrame) { - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; { auto shape = ITensor::makeShape({5, 5, 5, 5, 5}); ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); @@ -541,7 +540,7 @@ TEST(ShapeRange, test) TEST(ITensorTest, TensorDimsSliceAt) { auto shape = ITensor::makeShape({5, 5, 5, 5}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); auto verify = [&shape, &tensor, &dataType](ITensor::Shape const& index) @@ -658,7 +657,7 @@ TEST(ITensorTest, TensorDimsSliceAt) TEST(BufferRangeTest, ConstType) { auto shape = ITensor::makeShape({5, 5, 5, 5, 5}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); ITensor::SharedConstPtr tensorConst = tensor; @@ -695,7 +694,7 @@ TEST(BufferRangeTest, ConstType) TEST(ITensorTest, GetDimension) { auto shape = ITensor::makeShape({10, 11, 12}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor(BufferManager::cpu(shape, dataType)); auto firstDimensionFromStart = tensor->getDimension<0>(); diff --git a/cpp/tests/unit_tests/runtime/loraCacheTest.cpp b/cpp/tests/unit_tests/runtime/loraCacheTest.cpp index 6a91d11df3db..4d4dc86dc824 100644 --- a/cpp/tests/unit_tests/runtime/loraCacheTest.cpp +++ b/cpp/tests/unit_tests/runtime/loraCacheTest.cpp @@ -28,7 +28,7 @@ #include "tensorrt_llm/runtime/utils/numpyUtils.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <gmock/gmock-matchers.h> #include <gmock/gmock.h> @@ -78,7 +78,7 @@ class LoraCacheTest : public ::testing::Test, void SetUp() override { - mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + mModelConfig = std::make_unique<ModelConfig>(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); mModelConfig->setMlpHiddenSize(32); mWorldConfig = std::make_unique<WorldConfig>(2, 1, 1, 0); std::vector<LoraModule> modules{ @@ -101,7 +101,7 @@ class LoraCacheTest : public ::testing::Test, mManager = std::make_unique<BufferManager>(mStream); auto pageConfig = LoraCachePageManagerConfig( - runtime::MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 2 * 8, 6, 64, 4 * 16, 1); + runtime::MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 2 * 8, 6, 64, 4 * 16, 1); pageConfig.setInitToZero(true); auto pageConfig2 = pageConfig; pageConfig2.setInitToZero(true); @@ -125,7 +125,7 @@ TEST_F(LoraCacheTest, LoraCachePageManagerTest) auto pageShape = ITensor::makeShape({maxAdapterSize, maxAdapterWeights}); LoraCachePageManagerConfig config( - runtime::MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 8, 6, maxAdapterSize, maxAdapterWeights, 1); + runtime::MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 8, 6, maxAdapterSize, maxAdapterWeights, 1); LoraCachePageManager manager(config, *mManager); auto block0 = manager.blockPtr(0); @@ -182,11 +182,11 @@ TEST_F(LoraCacheTest, LoraCachePageManagerTest) TEST_F(LoraCacheTest, determineNumPages) { - ModelConfig modelConfig(0, 2, 2, 0, 1, 4, tensorrt_llm::DataType::kFLOAT); + ModelConfig modelConfig(0, 2, 2, 0, 1, 4, nvinfer1::DataType::kFLOAT); modelConfig.setLoraModules(LoraModule::createLoraModules({"attn_dense", "attn_qkv"}, 4, 4, 1, 1, 2, 2, 0)); WorldConfig worldConfig(1, 1, 1, 0); - LoraCachePageManagerConfig pageConfig(MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 12393, 40, 80, 16, 1); + LoraCachePageManagerConfig pageConfig(MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 12393, 40, 80, 16, 1); LoraCache cache(pageConfig, modelConfig, worldConfig, *mManager); @@ -374,7 +374,7 @@ TEST_F(LoraCacheTest, basicPutGet) TEST_F(LoraCacheTest, splitTransposeCpu) { - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); auto worldConfig = WorldConfig(2, 1, 1, 0); SizeType32 const split{2}; @@ -391,8 +391,8 @@ TEST_F(LoraCacheTest, splitTransposeCpu) auto const outputShape = ITensor::makeShape({batchSize, inputLength / split}); auto inputTensor = mManager->copyFrom(input, inputShape, MemoryType::kCPU); - auto outputTensorRank0 = mManager->cpu(outputShape, tensorrt_llm::DataType::kINT32); - auto outputTensorRank1 = mManager->cpu(outputShape, tensorrt_llm::DataType::kINT32); + auto outputTensorRank0 = mManager->cpu(outputShape, nvinfer1::DataType::kINT32); + auto outputTensorRank1 = mManager->cpu(outputShape, nvinfer1::DataType::kINT32); mManager->setZero(*outputTensorRank0); mManager->setZero(*outputTensorRank1); @@ -416,8 +416,8 @@ TEST_F(LoraCacheTest, splitTransposeCpu) auto const outputShape = ITensor::makeShape({batchSize, inputLength / split}); auto inputTensor = mManager->copyFrom(input, inputShape, MemoryType::kCPU); - auto outputTensorRank0 = mManager->cpu(outputShape, tensorrt_llm::DataType::kINT32); - auto outputTensorRank1 = mManager->cpu(outputShape, tensorrt_llm::DataType::kINT32); + auto outputTensorRank0 = mManager->cpu(outputShape, nvinfer1::DataType::kINT32); + auto outputTensorRank1 = mManager->cpu(outputShape, nvinfer1::DataType::kINT32); mManager->setZero(*outputTensorRank0); mManager->setZero(*outputTensorRank1); @@ -438,7 +438,7 @@ TEST_F(LoraCacheTest, splitTransposeCpu) TEST_P(LoraCacheTest, copyToPages_tp1) { bool const isDora = GetParam(); - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(1, 1, 1, 0); std::vector<LoraModule> modules{ @@ -501,7 +501,7 @@ TEST_P(LoraCacheTest, copyToPages_tp1) TEST_P(LoraCacheTest, copyToPages_tp2_rank0) { bool const isDora = GetParam(); - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(2, 1, 1, 0); std::vector<LoraModule> modules{ @@ -562,7 +562,7 @@ TEST_P(LoraCacheTest, copyToPages_tp2_rank0) TEST_P(LoraCacheTest, copyToPages_tp2_rank1) { bool const isDora = GetParam(); - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(2, 1, 1, 1); std::vector<LoraModule> modules{ diff --git a/cpp/tests/unit_tests/runtime/loraManagerTest.cpp b/cpp/tests/unit_tests/runtime/loraManagerTest.cpp index 11c19d22efb0..6910719da76f 100644 --- a/cpp/tests/unit_tests/runtime/loraManagerTest.cpp +++ b/cpp/tests/unit_tests/runtime/loraManagerTest.cpp @@ -32,7 +32,6 @@ #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/utils/numpyUtils.h" #include <gtest/gtest.h> @@ -67,7 +66,7 @@ class LoraManagerTest { protected: LoraManagerTest() - : mModelConfig(1, 2, 2, 0, 1, 4, tensorrt_llm::DataType::kFLOAT) + : mModelConfig(1, 2, 2, 0, 1, 4, nvinfer1::DataType::kFLOAT) { } @@ -88,7 +87,7 @@ class LoraManagerTest PeftTable getPeftTable(SizeType32 tpRank = 0) { - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(2, 2, 1, 3); std::vector<LoraModule> modules{ @@ -103,7 +102,7 @@ class LoraManagerTest }; modelConfig.setLoraModules(modules); auto pageConfig = LoraCachePageManagerConfig( - runtime::MemoryType::kCPU, tensorrt_llm::DataType::kFLOAT, 2 * 8, 6, 64, 4 * 16, 1); + runtime::MemoryType::kCPU, nvinfer1::DataType::kFLOAT, 2 * 8, 6, 64, 4 * 16, 1); pageConfig.setInitToZero(true); LoraCache loraCache(pageConfig, modelConfig, worldConfig, *mManager); @@ -214,7 +213,7 @@ static void checkLoraTensors(LoraManager const& loraManager, std::vector<int64_t auto expectedTensor = expectedTensors.find(fieldName)->second; auto actualTensor = inputTensors.find(fieldName)->second; ITensor::shapeEquals(expectedTensor->getShape(), actualTensor->getShape()); - if (expectedTensor->getDataType() == tensorrt_llm::DataType::kINT64) + if (expectedTensor->getDataType() == nvinfer1::DataType::kINT64) { auto expT = bufferCast<int64_t>(*expectedTensor); auto actT = bufferCast<int64_t>(*actualTensor); @@ -309,7 +308,7 @@ TEST_P(LoraManagerTest, fillInputTensors) bool const isDora = GetParam(); LoraManager loraManager; - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, tensorrt_llm::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 16, nvinfer1::DataType::kFLOAT); modelConfig.setMlpHiddenSize(32); auto worldConfig = WorldConfig(1, 1, 1, 0); std::vector<LoraModule> modules{ @@ -333,9 +332,9 @@ TEST_P(LoraManagerTest, fillInputTensors) auto numLayers = static_cast<SizeType32>(modelConfig.getNbAttentionLayers()); SizeType32 numSeqs = 4; TensorPtr weightsPtrs - = mManager->cpu(ITensor::makeShape({numModules, numLayers, numSeqs, 3}), tensorrt_llm::DataType::kINT64); + = mManager->cpu(ITensor::makeShape({numModules, numLayers, numSeqs, 3}), nvinfer1::DataType::kINT64); TensorPtr adapterSizes - = mManager->cpu(ITensor::makeShape({numModules, numLayers, numSeqs}), tensorrt_llm::DataType::kINT32); + = mManager->cpu(ITensor::makeShape({numModules, numLayers, numSeqs}), nvinfer1::DataType::kINT32); mManager->setZero(*weightsPtrs); mManager->setZero(*adapterSizes); diff --git a/cpp/tests/unit_tests/runtime/loraUtilsTest.cpp b/cpp/tests/unit_tests/runtime/loraUtilsTest.cpp index 994a77acf818..a14fa7bb8c47 100644 --- a/cpp/tests/unit_tests/runtime/loraUtilsTest.cpp +++ b/cpp/tests/unit_tests/runtime/loraUtilsTest.cpp @@ -25,7 +25,7 @@ #include "tensorrt_llm/runtime/modelConfig.h" #include "tensorrt_llm/runtime/worldConfig.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <algorithm> #include <optional> @@ -53,7 +53,7 @@ class LoraUtilsTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-typ TEST_F(LoraUtilsTest, null_values) { std::optional<TensorPtr> optReqLoraWeights = std::nullopt; - std::optional<TensorPtr> optReqLoraConfig = mManager->emptyTensor(MemoryType::kCPU, tensorrt_llm::DataType::kHALF); + std::optional<TensorPtr> optReqLoraConfig = mManager->emptyTensor(MemoryType::kCPU, nvinfer1::DataType::kHALF); EXPECT_THAT([&]() { loraValidateRequestTensorDims(optReqLoraWeights, optReqLoraConfig); }, testing::Throws<std::runtime_error>()); @@ -66,35 +66,33 @@ TEST_F(LoraUtilsTest, null_values) TEST_F(LoraUtilsTest, dims_mem_type) { - std::optional<TensorPtr> optReqLoraWeights - = mManager->cpu(ITensor::makeShape({1, 2}), tensorrt_llm::DataType::kHALF); + std::optional<TensorPtr> optReqLoraWeights = mManager->cpu(ITensor::makeShape({1, 2}), nvinfer1::DataType::kHALF); std::optional<TensorPtr> optReqLoraConfig - = mManager->cpu(ITensor::makeShape({1, 2, 3}), tensorrt_llm::DataType::kINT32); + = mManager->cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); EXPECT_THAT([&]() { loraValidateRequestTensorDims(optReqLoraWeights, optReqLoraConfig); }, testing::Throws<std::runtime_error>()); - std::optional<TensorPtr> optGpuWeights - = mManager->gpu(ITensor::makeShape({1, 2, 50}), tensorrt_llm::DataType::kHALF); + std::optional<TensorPtr> optGpuWeights = mManager->gpu(ITensor::makeShape({1, 2, 50}), nvinfer1::DataType::kHALF); EXPECT_THAT([&]() { loraValidateRequestTensorDims(optGpuWeights, optReqLoraConfig); }, testing::Throws<std::runtime_error>()); - optReqLoraWeights = mManager->cpu(ITensor::makeShape({1, 2, 50}), tensorrt_llm::DataType::kHALF); - optReqLoraConfig = mManager->cpu(ITensor::makeShape({1, 2, 3}), tensorrt_llm::DataType::kINT32); + optReqLoraWeights = mManager->cpu(ITensor::makeShape({1, 2, 50}), nvinfer1::DataType::kHALF); + optReqLoraConfig = mManager->cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); loraValidateRequestTensorDims(optReqLoraWeights, optReqLoraConfig); } TEST_F(LoraUtilsTest, loraValidateRequestTensors) { - auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 4, tensorrt_llm::DataType::kFLOAT); + auto modelConfig = ModelConfig(0, 2, 2, 0, 1, 4, nvinfer1::DataType::kFLOAT); auto worldConfig = WorldConfig(); std::optional<TensorPtr> optReqLoraWeights - = mManager->cpu(ITensor::makeShape({1, 2, 32}), tensorrt_llm::DataType::kFLOAT); + = mManager->cpu(ITensor::makeShape({1, 2, 32}), nvinfer1::DataType::kFLOAT); std::optional<TensorPtr> optReqLoraConfig - = mManager->cpu(ITensor::makeShape({1, 2, 3}), tensorrt_llm::DataType::kINT32); + = mManager->cpu(ITensor::makeShape({1, 2, 3}), nvinfer1::DataType::kINT32); std::vector<int32_t> config{1, 0, 4, 1, 1, 4}; diff --git a/cpp/tests/unit_tests/runtime/medusaModuleTest.cpp b/cpp/tests/unit_tests/runtime/medusaModuleTest.cpp index 8e7ff6c75459..3cba1bf2994d 100644 --- a/cpp/tests/unit_tests/runtime/medusaModuleTest.cpp +++ b/cpp/tests/unit_tests/runtime/medusaModuleTest.cpp @@ -20,7 +20,7 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/utils/speculativeChoicesUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <gmock/gmock-matchers.h> #include <gmock/gmock.h> @@ -58,16 +58,14 @@ class MedusaModuleTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro- auto const tokensPerStep = medusaModule.getMaxDecodingTokens(); // batch size = 1 here. - TensorPtr medusaGenerationLengthsHost - = mManager->pinned(ITensor::makeShape({1}), tensorrt_llm::DataType::kINT32); + TensorPtr medusaGenerationLengthsHost = mManager->pinned(ITensor::makeShape({1}), nvinfer1::DataType::kINT32); TensorPtr medusaPositionOffsetsHost - = mManager->pinned(ITensor::makeShape({tokensPerStep}), tensorrt_llm::DataType::kINT32); - TensorPtr medusaTreeIdsHost - = mManager->pinned(ITensor::makeShape({tokensPerStep}), tensorrt_llm::DataType::kINT32); + = mManager->pinned(ITensor::makeShape({tokensPerStep}), nvinfer1::DataType::kINT32); + TensorPtr medusaTreeIdsHost = mManager->pinned(ITensor::makeShape({tokensPerStep}), nvinfer1::DataType::kINT32); TensorPtr medusaPathsHost - = mManager->pinned(ITensor::makeShape({tokensPerStep, medusaHeads + 1}), tensorrt_llm::DataType::kINT32); + = mManager->pinned(ITensor::makeShape({tokensPerStep, medusaHeads + 1}), nvinfer1::DataType::kINT32); TensorPtr attentionPackedMaskHost - = mManager->pinned(ITensor::makeShape({tokensPerStep, numPackedMasks}), tensorrt_llm::DataType::kINT32); + = mManager->pinned(ITensor::makeShape({tokensPerStep, numPackedMasks}), nvinfer1::DataType::kINT32); std::vector<SizeType32> topKs; utils::initTensorsFromChoices(medusaModule, choices, topKs, medusaGenerationLengthsHost, diff --git a/cpp/tests/unit_tests/runtime/runtimeKernelTest.cpp b/cpp/tests/unit_tests/runtime/runtimeKernelTest.cpp index 58372a6bd479..dd517c5de5a4 100644 --- a/cpp/tests/unit_tests/runtime/runtimeKernelTest.cpp +++ b/cpp/tests/unit_tests/runtime/runtimeKernelTest.cpp @@ -23,7 +23,7 @@ #include "tensorrt_llm/runtime/iTensor.h" #include "tensorrt_llm/runtime/runtimeKernels.h" -#include "tensorrt_llm/common/tllmDataType.h" +#include <NvInferRuntime.h> #include <gtest/gtest.h> #include <algorithm> @@ -85,7 +85,7 @@ TEST_F(RuntimeKernelTest, FillBufferInt8) { for (auto size : {123LLU, 1025LLU, 1LLU << 32}) { - auto buffer = mManager->gpu(size, tensorrt_llm::DataType::kINT8); + auto buffer = mManager->gpu(size, nvinfer1::DataType::kINT8); testFill<std::int8_t>(*buffer, *mManager, *mStream); } } @@ -94,7 +94,7 @@ TEST_F(RuntimeKernelTest, FillTensorInt8) { for (auto size : {123, 1025, std::numeric_limits<int32_t>::max()}) { - auto tensor = mManager->gpu(tr::ITensor::makeShape({size, 2}), tensorrt_llm::DataType::kINT8); + auto tensor = mManager->gpu(tr::ITensor::makeShape({size, 2}), nvinfer1::DataType::kINT8); testFill<std::int8_t>(*tensor, *mManager, *mStream); } } @@ -111,7 +111,7 @@ TEST_F(RuntimeKernelTest, ScatterHalf) auto const outputShape = tr::ITensor::makeShape({batchSize * beamWidth, inputLength}); auto inputTensor = mManager->copyFrom(input, inputShape, tr::MemoryType::kGPU); - auto outputTensor = mManager->gpu(outputShape, tensorrt_llm::DataType::kHALF); + auto outputTensor = mManager->gpu(outputShape, nvinfer1::DataType::kHALF); mManager->setZero(*outputTensor); tr::kernels::scatterTensor(*outputTensor, *inputTensor, beamWidth, *mStream); @@ -174,7 +174,7 @@ TEST_F(RuntimeKernelTest, TileInt32) auto const outputShape = tr::ITensor::makeShape({batchSize * beamWidth, inputLength}); auto inputTensor = mManager->copyFrom(input, inputShape, tr::MemoryType::kGPU); - auto outputTensor = mManager->gpu(outputShape, tensorrt_llm::DataType::kINT32); + auto outputTensor = mManager->gpu(outputShape, nvinfer1::DataType::kINT32); tr::kernels::tileTensor(*outputTensor, *inputTensor, beamWidth, *mStream); @@ -194,7 +194,7 @@ TEST_F(RuntimeKernelTest, TileHalf) auto const outputShape = tr::ITensor::makeShape({batchSize * beamWidth, inputLength}); auto inputTensor = mManager->copyFrom(input, inputShape, tr::MemoryType::kGPU); - auto outputTensor = mManager->gpu(outputShape, tensorrt_llm::DataType::kHALF); + auto outputTensor = mManager->gpu(outputShape, nvinfer1::DataType::kHALF); tr::kernels::tileTensor(*outputTensor, *inputTensor, beamWidth, *mStream); @@ -228,11 +228,11 @@ TEST_F(RuntimeKernelTest, TileInt8Large) // Scope the allocated tensors to ensure they are de-allocated before the test ends. { - auto inputTensor = mManager->gpu(inputShape, tensorrt_llm::DataType::kINT8); + auto inputTensor = mManager->gpu(inputShape, nvinfer1::DataType::kINT8); tr::kernels::invokeFill(*inputTensor, value, *mStream); mStream->synchronize(); - auto outputTensor = mManager->gpu(outputShape, tensorrt_llm::DataType::kINT8); + auto outputTensor = mManager->gpu(outputShape, nvinfer1::DataType::kINT8); tr::kernels::tileTensor(*outputTensor, *inputTensor, beamWidth, *mStream); mStream->synchronize(); @@ -257,11 +257,11 @@ void testCopyBatch(tr::SizeType64 stride, tr::BufferManager& manager, tr::CudaSt auto const bufferShape = tr::ITensor::makeShape({rows, stride}); auto const indicesShape = tr::ITensor::makeShape({numIndices}); - auto srcBufferHost = tr::BufferManager::cpu(bufferShape, tensorrt_llm::DataType::kINT32); - auto dstBufferDevice = manager.gpu(bufferShape, tensorrt_llm::DataType::kINT32); - auto srcOffsets = tr::BufferManager::pinned(indicesShape, tensorrt_llm::DataType::kINT64); - auto dstOffsets = tr::BufferManager::pinned(indicesShape, tensorrt_llm::DataType::kINT64); - auto sizes = tr::BufferManager::pinned(indicesShape, tensorrt_llm::DataType::kINT64); + auto srcBufferHost = tr::BufferManager::cpu(bufferShape, nvinfer1::DataType::kINT32); + auto dstBufferDevice = manager.gpu(bufferShape, nvinfer1::DataType::kINT32); + auto srcOffsets = tr::BufferManager::pinned(indicesShape, nvinfer1::DataType::kINT64); + auto dstOffsets = tr::BufferManager::pinned(indicesShape, nvinfer1::DataType::kINT64); + auto sizes = tr::BufferManager::pinned(indicesShape, nvinfer1::DataType::kINT64); tr::kernels::invokeFill(*dstBufferDevice, 0, stream); auto* srcBufferHostPtr = tr::bufferCast<std::int32_t>(*srcBufferHost); diff --git a/cpp/tests/unit_tests/runtime/samplingTest.cpp b/cpp/tests/unit_tests/runtime/samplingTest.cpp index bb93478cf8ff..dad99323164e 100644 --- a/cpp/tests/unit_tests/runtime/samplingTest.cpp +++ b/cpp/tests/unit_tests/runtime/samplingTest.cpp @@ -14,12 +14,12 @@ * limitations under the License. */ -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/executor/types.h" #include "tensorrt_llm/layers/dynamicDecodeLayer.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaStream.h" #include "tensorrt_llm/runtime/gptDecoder.h" +#include "tensorrt_llm/runtime/tllmLogger.h" #include <gtest/gtest.h> @@ -39,11 +39,14 @@ class SamplingTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type if (mDeviceCount == 0) GTEST_SKIP() << "No GPUs found"; + + mLogger = std::make_shared<TllmLogger>(); } void TearDown() override {} int mDeviceCount; + std::shared_ptr<nvinfer1::ILogger> mLogger; }; std::shared_ptr<tl::BaseDecodingOutputs> dynamicDecodeTest(std::shared_ptr<BufferManager> manager, size_t vocabSize, @@ -64,10 +67,10 @@ std::shared_ptr<tl::BaseDecodingOutputs> dynamicDecodeTest(std::shared_ptr<Buffe tk::FinishedState::UnderlyingType* gpuFinished = nullptr; - ITensor::SharedPtr gpuEndIds = manager->gpu(ITensor::makeShape({signedBatchSize}), tensorrt_llm::DataType::kINT32); + ITensor::SharedPtr gpuEndIds = manager->gpu(ITensor::makeShape({signedBatchSize}), nvinfer1::DataType::kINT32); manager->copy(cpuEndIds.data(), *gpuEndIds, MemoryType::kCPU); ITensor::SharedPtr gpuOutputIds = manager->gpu( - ITensor::makeShape({signedBatchSize, signedBeamWidth, signedMaxSeqLength}), tensorrt_llm::DataType::kINT32); + ITensor::makeShape({signedBatchSize, signedBeamWidth, signedMaxSeqLength}), nvinfer1::DataType::kINT32); manager->copy(cpuOutputIds.data(), *gpuOutputIds, MemoryType::kCPU); auto const decodingMode = beamWidth == 1 ? tle::DecodingMode::TopKTopP() : tle::DecodingMode::BeamSearch(); @@ -89,7 +92,7 @@ std::shared_ptr<tl::BaseDecodingOutputs> dynamicDecodeTest(std::shared_ptr<Buffe auto forwardParams = std::make_shared<tl::SamplingInputs>(gpuEndIds, batchSlots, step, ite, localBatchSize); auto logitsShape = ITensor::makeShape({signedBatchSize, static_cast<int64_t>(beamWidth), static_cast<int64_t>(vocabSizePadded)}); - ITensor::SharedPtr inputLogits = manager->gpu(logitsShape, tensorrt_llm::DataType::kFLOAT); + ITensor::SharedPtr inputLogits = manager->gpu(logitsShape, nvinfer1::DataType::kFLOAT); forwardParams->logits = inputLogits; manager->copy(cpuLogits.data(), *inputLogits, MemoryType::kCPU); @@ -98,10 +101,10 @@ std::shared_ptr<tl::BaseDecodingOutputs> dynamicDecodeTest(std::shared_ptr<Buffe forwardParams->stopCriteriaInputs = std::make_shared<tl::StopCriteriaDecodingInputs>(localBatchSize); auto outputParams = std::make_shared<tl::BaseDecodingOutputs>(gpuOutputIds); - outputParams->sequenceLength = manager->gpu(ITensor::makeShape({signedBatchSize}), tensorrt_llm::DataType::kINT32); + outputParams->sequenceLength = manager->gpu(ITensor::makeShape({signedBatchSize}), nvinfer1::DataType::kINT32); manager->copy(cpuSequenceLengths.data(), *outputParams->sequenceLength.value(), MemoryType::kCPU); outputParams->newTokens - = manager->gpu(ITensor::makeShape({signedBatchSize, signedBeamWidth}), tensorrt_llm::DataType::kINT32); + = manager->gpu(ITensor::makeShape({signedBatchSize, signedBeamWidth}), nvinfer1::DataType::kINT32); outputParams->finished = manager->gpu( ITensor::makeShape({signedBatchSize, signedBeamWidth}), TRTDataType<tk::FinishedState::UnderlyingType>::value); diff --git a/cpp/tests/unit_tests/runtime/tllmBuffersTest.cpp b/cpp/tests/unit_tests/runtime/tllmBuffersTest.cpp index 4080a0f29e9e..c901061695bb 100644 --- a/cpp/tests/unit_tests/runtime/tllmBuffersTest.cpp +++ b/cpp/tests/unit_tests/runtime/tllmBuffersTest.cpp @@ -19,7 +19,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/stringUtils.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/cudaMemPool.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -231,7 +230,7 @@ TEST_F(TllmBuffersTest, DeviceBuffer) { CudaAllocatorAsync allocator{mStream, mMemPool}; { - DeviceBuffer buffer{size, tensorrt_llm::DataType::kFLOAT, allocator}; + DeviceBuffer buffer{size, nvinfer1::DataType::kFLOAT, allocator}; testBuffer(buffer, sizeof(float)); } streamPtr->synchronize(); @@ -243,7 +242,7 @@ TEST_F(TllmBuffersTest, DeviceBuffer) { CudaAllocator allocator{}; { - StaticDeviceBuffer buffer{size, tensorrt_llm::DataType::kFLOAT, allocator}; + StaticDeviceBuffer buffer{size, nvinfer1::DataType::kFLOAT, allocator}; testBuffer(buffer, sizeof(float)); } streamPtr->synchronize(); @@ -264,10 +263,10 @@ TEST_F(TllmBuffersTest, DeviceTensor) GTEST_SKIP() << noPoolSkipReason; } auto streamPtr = std::make_shared<CudaStream>(); - tensorrt_llm::Dims constexpr dims{3, 16, 8, 4}; + nvinfer1::Dims constexpr dims{3, 16, 8, 4}; CudaAllocatorAsync allocator{streamPtr, mMemPool}; { - DeviceTensor tensor{dims, tensorrt_llm::DataType::kFLOAT, allocator}; + DeviceTensor tensor{dims, nvinfer1::DataType::kFLOAT, allocator}; EXPECT_EQ(tensor.getSize(), ITensor::volume(dims)); testBuffer(tensor, sizeof(float)); EXPECT_EQ(tensor.getSize(), ITensor::volume(tensor.getShape())); @@ -282,7 +281,7 @@ TEST_F(TllmBuffersTest, BufferSlice) { auto constexpr size = 1024; HostAllocator allocator{}; - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; auto buffer = std::make_shared<HostBuffer>(size, dataType, allocator); auto offset = size / 8; auto slice = IBuffer::slice(buffer, offset); @@ -320,7 +319,7 @@ TEST_F(TllmBuffersTest, BufferOutput) CudaAllocatorAsync allocator{streamPtr, mMemPool}; for (std::size_t size : {0, 16}) { - DeviceBuffer buffer{size, tensorrt_llm::DataType::kFLOAT, allocator}; + DeviceBuffer buffer{size, nvinfer1::DataType::kFLOAT, allocator}; TLLM_CUDA_CHECK(cudaMemsetAsync(buffer.data(), 0, buffer.getSizeInBytes(), streamPtr->get())); streamPtr->synchronize(); std::stringstream ss; @@ -344,11 +343,11 @@ TEST_F(TllmBuffersTest, TensorOutput) } auto streamPtr = std::make_shared<CudaStream>(); - tensorrt_llm::Dims constexpr dims{3, 16, 8, 4}; + nvinfer1::Dims constexpr dims{3, 16, 8, 4}; CudaAllocatorAsync allocator{streamPtr, mMemPool}; - for (auto dataType : {tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF, tensorrt_llm::DataType::kBOOL, - tensorrt_llm::DataType::kINT8, tensorrt_llm::DataType::kINT32, tensorrt_llm::DataType::kINT64, - tensorrt_llm::DataType::kUINT8}) + for (auto dataType : + {nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF, nvinfer1::DataType::kBOOL, nvinfer1::DataType::kINT8, + nvinfer1::DataType::kINT32, nvinfer1::DataType::kINT64, nvinfer1::DataType::kUINT8}) { DeviceTensor tensor{dims, dataType, allocator}; TLLM_CUDA_CHECK(cudaMemsetAsync(tensor.data(), 0, tensor.getSizeInBytes(), streamPtr->get())); @@ -484,8 +483,8 @@ TEST_F(TllmBuffersTest, PinnedPoolAllocator) EXPECT_EQ(segments.size(), 0); { - auto a = BufferManager::pinnedPool(ITensor::makeShape({512, 4, 4}), tensorrt_llm::DataType::kFLOAT); - auto b = BufferManager::pinnedPool(ITensor::makeShape({512, 10}), tensorrt_llm::DataType::kHALF); + auto a = BufferManager::pinnedPool(ITensor::makeShape({512, 4, 4}), nvinfer1::DataType::kFLOAT); + auto b = BufferManager::pinnedPool(ITensor::makeShape({512, 10}), nvinfer1::DataType::kHALF); pool.logSegments(); auto it = std::begin(segments); EXPECT_NE(it->tag, nullptr); @@ -513,7 +512,7 @@ TEST_F(TllmBuffersTest, PinnedPoolAllocator) std::size_t secondChunkSize; { // Test creating a new chunk - auto c = BufferManager::pinnedPool(ITensor::makeShape({initChunkSize + 1}), tensorrt_llm::DataType::kUINT8); + auto c = BufferManager::pinnedPool(ITensor::makeShape({initChunkSize + 1}), nvinfer1::DataType::kUINT8); pool.logSegments(); auto it = std::begin(segments); EXPECT_EQ(it->tag, nullptr); diff --git a/cpp/tests/unit_tests/runtime/tllmRuntimeTest.cpp b/cpp/tests/unit_tests/runtime/tllmRuntimeTest.cpp new file mode 100644 index 000000000000..b3a6f99146c2 --- /dev/null +++ b/cpp/tests/unit_tests/runtime/tllmRuntimeTest.cpp @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TOP_LEVEL_DIR +#error "Define TOP_LEVEL_DIR" +#endif + +#include <NvInfer.h> +#include <NvOnnxParser.h> +#include <gtest/gtest.h> + +#include "tensorrt_llm/common/cudaUtils.h" +#include "tensorrt_llm/runtime/rawEngine.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include "tensorrt_llm/runtime/tllmRuntime.h" + +#include <algorithm> +#include <array> +#include <filesystem> +#include <memory> +#include <vector> + +namespace fs = std::filesystem; +namespace trt = nvinfer1; + +namespace +{ +auto const TEST_RESOURCE_DIR = fs::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; +auto const MNIST_MODEL_PATH = TEST_RESOURCE_DIR / "models/mnist.onnx"; + +template <typename T> +std::unique_ptr<T> makeUnique(T* ptr) +{ + EXPECT_NE(ptr, nullptr); + return std::unique_ptr<T>(ptr); +} + +std::unique_ptr<trt::IHostMemory> buildMnistEngine(trt::ILogger& logger) +{ + EXPECT_TRUE(fs::exists(MNIST_MODEL_PATH)); + auto builder = makeUnique(trt::createInferBuilder(logger)); + auto const explicitBatch = 1U << static_cast<uint32_t>(trt::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH); + auto network = makeUnique(builder->createNetworkV2(explicitBatch)); + auto parser = makeUnique(nvonnxparser::createParser(*network, logger)); + auto const parsingSuccess = parser->parseFromFile( + MNIST_MODEL_PATH.string().c_str(), static_cast<int32_t>(trt::ILogger::Severity::kWARNING)); + EXPECT_TRUE(parsingSuccess); + auto config = makeUnique(builder->createBuilderConfig()); + return makeUnique(builder->buildSerializedNetwork(*network, *config)); +} +} // namespace + +using namespace tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; + +class TllmRuntimeTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-member-init) +{ +protected: + void SetUp() override + { + mDeviceCount = tc::getDeviceCount(); + + if (mDeviceCount == 0) + GTEST_SKIP(); + + mLogger.setLevel(trt::ILogger::Severity::kINFO); + mSerializedEngine = buildMnistEngine(mLogger); + ASSERT_NE(mSerializedEngine, nullptr); + } + + void TearDown() override {} + + int mDeviceCount; + TllmLogger mLogger{}; + std::unique_ptr<trt::IHostMemory> mSerializedEngine; +}; + +TEST_F(TllmRuntimeTest, SinglePass) +{ + EXPECT_TRUE(mSerializedEngine); + TllmRuntime rt{RawEngine(mSerializedEngine.get()), &mLogger, false, 1.0F}; + auto& engine = rt.getEngine(); + EXPECT_FALSE(engine.hasImplicitBatchDimension()); + EXPECT_EQ(rt.getNbProfiles(), engine.getNbOptimizationProfiles()); + EXPECT_EQ(rt.getNbContexts(), 0); + auto const nbIoTensors = engine.getNbIOTensors(); + EXPECT_EQ(nbIoTensors, 2); + rt.addContext(0); + EXPECT_EQ(rt.getNbContexts(), 1); + + auto constexpr dataType = trt::DataType::kFLOAT; + + auto const inputName = engine.getIOTensorName(0); + EXPECT_EQ(engine.getTensorIOMode(inputName), trt::TensorIOMode::kINPUT); + auto const inputDims = engine.getTensorShape(inputName); + std::array constexpr inputDimsExpected = {1, 1, 28, 28}; + EXPECT_EQ(inputDims.nbDims, inputDimsExpected.size()); + for (int i = 0; i < inputDims.nbDims; ++i) + { + EXPECT_EQ(inputDims.d[i], inputDimsExpected[i]); + } + EXPECT_EQ(engine.getTensorDataType(inputName), dataType); + + auto const outputName = engine.getIOTensorName(1); + EXPECT_EQ(engine.getTensorIOMode(outputName), trt::TensorIOMode::kOUTPUT); + auto const outputDims = engine.getTensorShape(outputName); + std::array constexpr outputDimsExpected = {1, 10}; + EXPECT_EQ(outputDims.nbDims, outputDimsExpected.size()); + for (int i = 0; i < outputDims.nbDims; ++i) + { + EXPECT_EQ(outputDims.d[i], outputDimsExpected[i]); + } + EXPECT_EQ(engine.getTensorDataType(outputName), dataType); + + auto& allocator = rt.getBufferManager(); + TllmRuntime::TensorMap tensorMap{}; + auto inputBuffer = std::shared_ptr<ITensor>{allocator.gpu(inputDims, dataType)}; + allocator.setZero(*inputBuffer); + tensorMap.insert(std::make_pair(inputName, inputBuffer)); + rt.setInputTensors(0, tensorMap); + rt.setOutputTensors(0, tensorMap); + ASSERT_NE(tensorMap.find(outputName), tensorMap.end()); + auto outputBuffer = tensorMap.at(outputName); + allocator.setZero(*outputBuffer); + rt.executeContext(0); + + std::vector<float> output(outputBuffer->getSize()); + allocator.copy(*outputBuffer, output.data()); + rt.getStream().synchronize(); + auto min = std::min_element(output.begin(), output.end()); + EXPECT_NEAR(*min, -0.126409f, 1e-5f); + auto max = std::max_element(output.begin(), output.end()); + EXPECT_NEAR(*max, 0.140218f, 1e-5f); +} diff --git a/cpp/tests/unit_tests/runtime/torchTest.cpp b/cpp/tests/unit_tests/runtime/torchTest.cpp index 4ca36d875d44..4aa498de8d25 100644 --- a/cpp/tests/unit_tests/runtime/torchTest.cpp +++ b/cpp/tests/unit_tests/runtime/torchTest.cpp @@ -17,7 +17,6 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/torch.h" #include "tensorrt_llm/runtime/torchView.h" @@ -53,7 +52,7 @@ class TorchTest : public ::testing::Test // NOLINT(cppcoreguidelines-pro-type-me namespace { -template <tensorrt_llm::DataType DType> +template <nvinfer1::DataType DType> void checkFilled(IBuffer& buffer, int fillValue) { if (DType == buffer.getDataType()) @@ -80,13 +79,13 @@ TEST_F(TorchTest, Aten) } auto constexpr fillValue = 1; - auto tensorHostBase = manager.allocate(MemoryType::kPINNED, shapeTllm, tensorrt_llm::DataType::kINT64); + auto tensorHostBase = manager.allocate(MemoryType::kPINNED, shapeTllm, nvinfer1::DataType::kINT64); for (auto memoryType : {MemoryType::kCPU, MemoryType::kGPU, MemoryType::kPINNED}) { - for (auto dtype : {tensorrt_llm::DataType::kFLOAT, tensorrt_llm::DataType::kHALF, tensorrt_llm::DataType::kINT8, - tensorrt_llm::DataType::kUINT8, tensorrt_llm::DataType::kINT32, tensorrt_llm::DataType::kINT64, - tensorrt_llm::DataType::kBF16, tensorrt_llm::DataType::kFP8, tensorrt_llm::DataType::kBOOL}) + for (auto dtype : {nvinfer1::DataType::kFLOAT, nvinfer1::DataType::kHALF, nvinfer1::DataType::kINT8, + nvinfer1::DataType::kUINT8, nvinfer1::DataType::kINT32, nvinfer1::DataType::kINT64, + nvinfer1::DataType::kBF16, nvinfer1::DataType::kFP8, nvinfer1::DataType::kBOOL}) { ITensor::SharedPtr tensorTllm{manager.allocate(memoryType, shapeTllm, dtype)}; @@ -99,20 +98,20 @@ TEST_F(TorchTest, Aten) EXPECT_THAT(tensorAten.sizes(), ::testing::ElementsAreArray(shapeAten)); EXPECT_EQ(tensorAten.data_ptr(), tensorTllm->data()); - if (dtype != tensorrt_llm::DataType::kFP8) + if (dtype != nvinfer1::DataType::kFP8) { tensorAten.fill_(c10::Scalar(fillValue)); auto tensorHost = ITensor::wrap(tensorHostBase->data(), dtype, shapeTllm); manager.copy(*tensorTllm, *tensorHost); mStream->synchronize(); - checkFilled<tensorrt_llm::DataType::kFLOAT>(*tensorHost, fillValue); - checkFilled<tensorrt_llm::DataType::kHALF>(*tensorHost, fillValue); - checkFilled<tensorrt_llm::DataType::kINT8>(*tensorHost, fillValue); - checkFilled<tensorrt_llm::DataType::kUINT8>(*tensorHost, fillValue); - checkFilled<tensorrt_llm::DataType::kINT32>(*tensorHost, fillValue); - checkFilled<tensorrt_llm::DataType::kINT64>(*tensorHost, fillValue); - checkFilled<tensorrt_llm::DataType::kBF16>(*tensorHost, fillValue); - checkFilled<tensorrt_llm::DataType::kBOOL>(*tensorHost, fillValue); + checkFilled<nvinfer1::DataType::kFLOAT>(*tensorHost, fillValue); + checkFilled<nvinfer1::DataType::kHALF>(*tensorHost, fillValue); + checkFilled<nvinfer1::DataType::kINT8>(*tensorHost, fillValue); + checkFilled<nvinfer1::DataType::kUINT8>(*tensorHost, fillValue); + checkFilled<nvinfer1::DataType::kINT32>(*tensorHost, fillValue); + checkFilled<nvinfer1::DataType::kINT64>(*tensorHost, fillValue); + checkFilled<nvinfer1::DataType::kBF16>(*tensorHost, fillValue); + checkFilled<nvinfer1::DataType::kBOOL>(*tensorHost, fillValue); } // Conversion back to TRT-LLM tensor diff --git a/cpp/tests/unit_tests/runtime/utilsTest.cpp b/cpp/tests/unit_tests/runtime/utilsTest.cpp index 58882824a7dc..8b69070c03d8 100644 --- a/cpp/tests/unit_tests/runtime/utilsTest.cpp +++ b/cpp/tests/unit_tests/runtime/utilsTest.cpp @@ -18,7 +18,6 @@ #error "Define TOP_LEVEL_DIR" #endif -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/iBuffer.h" #include "tensorrt_llm/runtime/iTensor.h" @@ -72,7 +71,7 @@ TEST_F(UtilsTest, LoadNpy) TEST_F(UtilsTest, LoadStoreNpy) { auto dims = ITensor::makeShape({2, 3, 4}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor{BufferManager::cpu(dims, dataType)}; auto tensorRange = BufferRange<float>(*tensor); std::iota(tensorRange.begin(), tensorRange.end(), 0); @@ -97,7 +96,7 @@ TEST_F(UtilsTest, LoadStoreNpy) TEST_F(UtilsTest, LoadStoreNpyGPU) { auto dims = ITensor::makeShape({2, 3, 4}); - auto constexpr dataType = tensorrt_llm::DataType::kFLOAT; + auto constexpr dataType = nvinfer1::DataType::kFLOAT; ITensor::SharedPtr tensor{BufferManager::cpu(dims, dataType)}; auto tensorRange = BufferRange<float>(*tensor); std::iota(tensorRange.begin(), tensorRange.end(), 0); diff --git a/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp b/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp index 159a07770694..f2045e7659d1 100644 --- a/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp +++ b/cpp/tests/unit_tests/runtime/virtualMemoryTest.cpp @@ -19,7 +19,6 @@ #include "tensorrt_llm/common/cudaUtils.h" #include "tensorrt_llm/common/nvmlWrapper.h" -#include "tensorrt_llm/common/tllmDataType.h" #include "tensorrt_llm/runtime/bufferManager.h" #include "tensorrt_llm/runtime/tllmBuffers.h" #include "tensorrt_llm/runtime/virtualMemory.h" @@ -1503,7 +1502,7 @@ TEST_F(VirtualMemoryManagerTest, TestCudaVirtualMemoryAllocator) // Create a buffer using the virtual address allocator auto buffer = std::make_unique<VirtualAddressDeviceBuffer>( - size, tensorrt_llm::DataType::kINT8, CudaVirtualMemoryAllocator{config}); + size, nvinfer1::DataType::kINT8, CudaVirtualMemoryAllocator{config}); auto memoryAfterAllocation = getCurrentProcessMemoryInfo(); if (memoryInfoAvailable()) @@ -1514,7 +1513,7 @@ TEST_F(VirtualMemoryManagerTest, TestCudaVirtualMemoryAllocator) // Test that we can access the buffer data ASSERT_NE(buffer->data(), nullptr) << "Buffer data should not be null"; ASSERT_EQ(buffer->getSize(), size) << "Buffer size should match requested size"; - ASSERT_EQ(buffer->getDataType(), tensorrt_llm::DataType::kINT8) << "Buffer data type should be INT8"; + ASSERT_EQ(buffer->getDataType(), nvinfer1::DataType::kINT8) << "Buffer data type should be INT8"; ASSERT_EQ(buffer->getMemoryType(), MemoryType::kGPU) << "Buffer memory type should be GPU"; // Test memory access by setting memory to a known pattern @@ -1575,7 +1574,7 @@ TEST_F(VirtualMemoryManagerTest, TestCudaVirtualMemoryAllocatorUnalignedSize) // Create a buffer using the virtual address allocator auto buffer = std::make_unique<VirtualAddressDeviceBuffer>( - size, tensorrt_llm::DataType::kINT8, CudaVirtualMemoryAllocator{config}); + size, nvinfer1::DataType::kINT8, CudaVirtualMemoryAllocator{config}); auto memoryAfterAllocation = getCurrentProcessMemoryInfo(); if (memoryInfoAvailable()) diff --git a/cpp/tests/unit_tests/utils/CMakeLists.txt b/cpp/tests/unit_tests/utils/CMakeLists.txt new file mode 100644 index 000000000000..2d7e9145c817 --- /dev/null +++ b/cpp/tests/unit_tests/utils/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +add_gtest(testUtilsTest utilsTest.cpp) diff --git a/cpp/tests/unit_tests/utils/utilsTest.cpp b/cpp/tests/unit_tests/utils/utilsTest.cpp new file mode 100644 index 000000000000..38587120ef84 --- /dev/null +++ b/cpp/tests/unit_tests/utils/utilsTest.cpp @@ -0,0 +1,55 @@ + +#include "common.h" +#include "tensorrt_llm/runtime/common.h" + +#include <gtest/gtest.h> + +#include <cstdint> +#include <numeric> + +struct RandomLogitsTestParameters +{ + using TupleT = std::tuple<int32_t, tensorrt_llm::runtime::SizeType32>; + + int32_t randomSeed; + tensorrt_llm::runtime::SizeType32 vocabSize; + + // Constructor that takes a tuple + RandomLogitsTestParameters( // NOLINT: implicit to allow gtest to convert from tuple generated by + // 'combine' + TupleT t) + : randomSeed(std::get<0>(t)) + , vocabSize(std::get<1>(t)) + { + } +}; + +class RandomLogits : public ::testing::Test, public ::testing::WithParamInterface<RandomLogitsTestParameters> +{ +protected: + static constexpr int randomSeed = 2345; +}; + +namespace +{ +constexpr int32_t kRandomSeed1 = 45; +constexpr int32_t kRandomSeed2 = 567; +auto const randomSeeds = ::testing::Values(kRandomSeed1, kRandomSeed2); + +constexpr tensorrt_llm::runtime::SizeType32 kMinVocabSize = 16; +constexpr tensorrt_llm::runtime::SizeType32 kMaxVocabSize = 100000; +auto const vocabSizes = ::testing::Values(kMinVocabSize, kMaxVocabSize); + +auto const paramGenerator + = ::testing::ConvertGenerator<RandomLogitsTestParameters::TupleT>(::testing::Combine(randomSeeds, vocabSizes)); +} // namespace + +TEST_P(RandomLogits, FloatSumToOne) +{ + auto rng = std::mt19937(randomSeed); + auto const randomLogits = tensorrt_llm::testing::randomLogits<std::mt19937, float>(456, &rng); + auto const sum = std::reduce(randomLogits.begin(), randomLogits.end()); + ASSERT_DOUBLE_EQ(sum, 1.0); +} + +INSTANTIATE_TEST_SUITE_P(Float, RandomLogits, paramGenerator); diff --git a/cpp/tests/utils/CMakeLists.txt b/cpp/tests/utils/CMakeLists.txt new file mode 100644 index 000000000000..0123ac1c4600 --- /dev/null +++ b/cpp/tests/utils/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +add_library(testingUtils common.cpp engines.cpp executorUtils.cpp) +target_link_libraries(testingUtils PUBLIC gtest_main ${SHARED_TARGET}) +target_include_directories(testingUtils PRIVATE ${MPI_C_INCLUDE_DIRS}) +target_compile_definitions(testingUtils PUBLIC TOP_LEVEL_DIR="${TOP_LEVEL_DIR}") diff --git a/cpp/tests/utils/common.cpp b/cpp/tests/utils/common.cpp new file mode 100644 index 000000000000..5640cef7b495 --- /dev/null +++ b/cpp/tests/utils/common.cpp @@ -0,0 +1,684 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common.h" + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/memoryUtils.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/utils/numpyUtils.h" +#include "tensorrt_llm/testing/modelSpec.h" +#include "tests/utils/common.h" + +#include <gtest/gtest.h> + +#include <algorithm> +#include <vector> + +namespace tensorrt_llm::testing +{ +namespace fs = std::filesystem; +namespace tr = tensorrt_llm::runtime; +namespace tc = tensorrt_llm::common; + +std::string PathUtil::FP16_GPT_ATTENTION_PACKED_DIR() +{ + return ModelSpec::getDefaultModelSpec().setKVCacheType(KVCacheType::kCONTINUOUS).getModelPath(); +} + +std::string PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DIR() +{ + return ModelSpec::getDefaultModelSpec().getModelPath(); +} + +std::string PathUtil::FP16_GPT_LORA_DIR() +{ + return ModelSpec::getDefaultModelSpec().useLoraPlugin().getModelPath(); +} + +std::string PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_DRAFT_TOKENS_DIR() +{ + return ModelSpec::getDefaultModelSpec().useDraftTokensExternalDecoding().getModelPath(); +} + +std::string PathUtil::FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().getModelPath(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_FILE() +{ + return ModelSpec::getDefaultModelSpec().getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_LONG_RESULT_FILE() +{ + return ModelSpec::getDefaultModelSpec().setInputFile("input_tokens_long.npy").getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().getGenerationLogitsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().getContextLogitsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_FILE() +{ + return ModelSpec::getDefaultModelSpec().getCumLogProbsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CUM_LOG_PROBS_FILE() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().getCumLogProbsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_FILE() +{ + return ModelSpec::getDefaultModelSpec().getLogProbsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_LOG_PROBS_FILE() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().getLogProbsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP1_FILE() +{ + return ModelSpec::getDefaultModelSpec().getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE() +{ + return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE() +{ + return ModelSpec::getDefaultModelSpec().useTensorParallelism(2).usePipelineParallelism(2).getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE() +{ + return ModelSpec::getDefaultModelSpec().usePipelineParallelism(4).getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE() +{ + return ModelSpec::getDefaultModelSpec().usePipelineParallelism(2).getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP1_FILE() +{ + return ModelSpec::getDefaultModelSpec().useTensorParallelism(2).getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_TP4_PP1_FILE() +{ + return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getContextLogitsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_TP4_PP1_FILE() +{ + return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getGenerationLogitsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_TP4_PP1_FILE() +{ + return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getCumLogProbsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_TP4_PP1_FILE() +{ + return ModelSpec::getDefaultModelSpec().useTensorParallelism(4).getLogProbsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_GATHER_CONTEXTFMHAFP32ACC_RESULT_FILE() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().enableContextFMHAFp32Acc().getResultsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXTFMHAFP32ACC_GENERATION_LOGITS_FILE() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().enableContextFMHAFp32Acc().getGenerationLogitsFile(); +} + +std::string PathUtil::FP16_PLUGIN_PACKED_PAGED_CONTEXTFMHAFP32ACC_CONTEXT_LOGITS_FILE() +{ + return ModelSpec::getDefaultModelSpec().gatherLogits().enableContextFMHAFp32Acc().getContextLogitsFile(); +} + +void TestData::loadLogProbs( + fs::path const& cumLogProbsFile, fs::path const& logProbsFile, tr::BufferManager const& manager) +{ + TLLM_CHECK_WITH_INFO( + cumLogProbsFile != "", "Testing return log probs, but missing the expected cum log probs results file."); + auto expectedCumLogProbsPtr + = std::shared_ptr(tr::utils::loadNpy(manager, cumLogProbsFile.string(), MemoryType::kCPU)); + + TLLM_CHECK_WITH_INFO( + logProbsFile != "", "Testing return log probs, but missing the expected log probs results file."); + auto expectedLogProbsPtr = std::shared_ptr(tr::utils::loadNpy(manager, logProbsFile.string(), MemoryType::kCPU)); + + for (SizeType32 inputIdx = 0; inputIdx < nbGivenInputs; ++inputIdx) + { + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto expectedCumLogProbsBatchSlice = std::shared_ptr(ITensor::slice(expectedCumLogProbsPtr, inputIdx, 1)); + expectedCumLogProbsBatchSlice->squeeze(0); // bs + expectedCumLogProbs[inputIdx] = expectedCumLogProbsBatchSlice; // shape: [beamWidth] + + auto expectedLogProbsBatchSlice = std::shared_ptr(ITensor::slice(expectedLogProbsPtr, inputIdx, 1)); + expectedLogProbsBatchSlice->squeeze(0); // bs + expectedLogProbs[inputIdx] = expectedLogProbsBatchSlice; // shape: [beamWidth, numOutputTokens] + } + } +} + +void TestData::loadContextLogits(fs::path const& contextLogitsFile, std::vector<SizeType32> const& givenInputLengths, + tr::BufferManager const& manager) +{ + TLLM_CHECK_WITH_INFO(contextLogitsFile != "", + "Testing with gather or replace logits, but missing the expected context logits results file."); + auto expectedContextLogitsPtr + = std::shared_ptr(tr::utils::loadNpy(manager, contextLogitsFile.string(), MemoryType::kCPU)); + + int promptOffset = 0; + for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) + { + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto expectedContextLogitBatchSlice + = std::shared_ptr(ITensor::slice(expectedContextLogitsPtr, promptOffset, givenInputLengths.at(bi))); + expectedContextLogits.at(bi) = expectedContextLogitBatchSlice; // shape: [prompt_length, vocab_size] + } + promptOffset += givenInputLengths.at(bi); + } +} + +void TestData::loadGenerationLogits(fs::path const& genLogitsFile, tr::BufferManager const& manager) +{ + TLLM_CHECK_WITH_INFO(genLogitsFile != "", + "Testing with gather or replace logits, but missing the expected generation logits results file."); + auto expectedGenerationLogitsPtr + = std::shared_ptr(tr::utils::loadNpy(manager, genLogitsFile.string(), MemoryType::kCPU)); + + for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) + { + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto expectedGenerationLogitBatchSlice + = std::shared_ptr(ITensor::slice(expectedGenerationLogitsPtr, bi, 1)); + expectedGenerationLogitBatchSlice->squeeze(0); // bs + expectedGenerationLogitBatchSlice->squeeze(0); // beam + expectedGenerationLogits.at(bi) = expectedGenerationLogitBatchSlice; // shape: [max_output_len, vocab_size] + } + } +} + +void TestData::makeDraft(SizeType32 maxDraftTokens, bool acceptDraftByLogits, fs::path const& genLogitsFile, + std::vector<SizeType32> const& givenInputLengths, tr::BufferManager const& manager) +{ + TLLM_CHECK(beamWidth == 1); + + ITensor::SharedPtr expectedGenerationLogitsPtr; + if (acceptDraftByLogits) + { + TLLM_CHECK_WITH_INFO( + genLogitsFile != "", "Testing Draft token, but missing the expected generation logits results file."); + expectedGenerationLogitsPtr + = std::shared_ptr(tr::utils::loadNpy(manager, genLogitsFile.string(), MemoryType::kCPU)); + } + + std::vector<SizeType32> draftLengths(givenInputLengths.size()); + // first draft length stays 0 + std::transform(givenInputLengths.begin() + 1, givenInputLengths.end(), draftLengths.begin() + 1, + [this, &maxDraftTokens](auto inputLength) + { return std::rand() % std::min((maxSeqLen - (inputLength + 1)), maxDraftTokens) + 1; }); + + auto* const expectedOutputData = tr::bufferCast<TokenIdType>(*expectedOutputIds); + for (SizeType32 bi = 0; bi < nbGivenInputs; ++bi) + { + SizeType32 constexpr beamIdx{0}; + auto const endId = endIds.at(bi); + auto const draftLen = draftLengths.at(bi); + auto acceptedLen = draftLen > 0 ? std::rand() % draftLen : 0; + + if (acceptDraftByLogits && draftLen > 0) + { + auto expectedLogitBatchSlice = std::shared_ptr(ITensor::slice(expectedGenerationLogitsPtr, bi, 1)); + expectedLogitBatchSlice->squeeze(0); // bs + expectedLogitBatchSlice->squeeze(0); // beam + auto expectedLogitBatchStepSlice = std::shared_ptr(ITensor::slice(expectedLogitBatchSlice, 1, draftLen)); + auto expectedLogitBatchStepView = ITensor::view(expectedLogitBatchStepSlice, + ITensor::makeShape({draftLen, 1, 1, expectedLogitBatchStepSlice->getShape().d[1]})); + draftLogits.at(bi) = manager.copyFrom(*expectedLogitBatchStepView, MemoryType::kCPU); + } + + for (SizeType32 si = 0; si < draftLen; ++si) + { + auto const draftIndex + = tc::flat_index3(bi, beamIdx, givenInputLengths.at(bi) + si + 1, beamWidth, maxSeqLen); + auto draftToken = expectedOutputData[draftIndex]; + if (draftToken == endId) + { + acceptedLen = std::min(acceptedLen, si); + } + if (si >= acceptedLen) + { + draftToken = -1; + if (acceptDraftByLogits) + { + auto vocabSizePadded = expectedGenerationLogitsPtr->getShape().d[3]; + auto* draftLogitsPtr = tr::bufferCast<float>(*draftLogits.at(bi)); + for (SizeType32 vi = 0; vi < vocabSizePadded; ++vi) + { + draftLogitsPtr[si * vocabSizePadded + vi] = 0.f; + } + } + } + draftTokens.at(bi).push_back(draftToken); + } + acceptedDraftTokensLengths.at(bi) = acceptedLen; + + auto const expectedLen = expectedOutputLengths.at(bi * beamWidth + beamIdx); + TLLM_CHECK(expectedLen > 0); + expectedOutputLengths[bi * beamWidth + beamIdx] + = draftLen > 0 ? std::min(expectedLen, (givenInputLengths.at(bi) + 1) + acceptedLen + 1) : expectedLen; + } +} + +template <typename T> +bool invokeCompareLogits(ITensor const& groundTruthLogits, ITensor const& outputLogits, float atol, float rtol) +{ + bool allMatch = true; + T const* const gtLogitsPtr = tr::bufferCast<T>(groundTruthLogits); + T const* const outputLogitsPtr = tr::bufferCast<T>(outputLogits); + + size_t outputSize = outputLogits.getSize(); + int errorNumber = 0; + + for (size_t i = 0; i < outputSize; i++) + { + if (!almostEqual(outputLogitsPtr[i], gtLogitsPtr[i], atol, rtol)) + { + TLLM_LOG_DEBUG("Mismatch value. Position of logits: %d, expected value: %f, output value: %f", i, + gtLogitsPtr[i], outputLogitsPtr[i]); + allMatch = false; + errorNumber++; + if (errorNumber == 10) + { + break; + } + } + } + return allMatch; +} + +bool compareLogits(ITensor const& groundTruthLogits, ITensor const& outputLogits, float atol, float rtol) +{ + EXPECT_EQ(groundTruthLogits.getDataType(), outputLogits.getDataType()); + switch (groundTruthLogits.getDataType()) + { + case nvinfer1::DataType::kFLOAT: return invokeCompareLogits<float>(groundTruthLogits, outputLogits, atol, rtol); + case nvinfer1::DataType::kHALF: return invokeCompareLogits<half>(groundTruthLogits, outputLogits, atol, rtol); + default: TLLM_THROW("Unsupported data type"); + } +} + +std::tuple<SizeType32, SizeType32> getRequestGivenInputIdxLength( + std::uint64_t requestId, SizeType32 nbGivenInputs, std::vector<SizeType32> const& givenInputLengths) +{ + auto const givenInputIdx = requestId % nbGivenInputs; + auto const inputLength = givenInputLengths.at(givenInputIdx); + return {givenInputIdx, inputLength}; +} + +std::tuple<std::vector<SizeType32>, SizeType32, SizeType32> getGivenInputLengths( + ITensor const& givenInput, SizeType32 padId) +{ + auto const& inputShape = givenInput.getShape(); + auto const nbGivenInputs = static_cast<SizeType32>(inputShape.d[0]); + auto const maxInputLength = static_cast<SizeType32>(inputShape.d[1]); + auto const* const givenInputData = tr::bufferCast<TokenIdType const>(givenInput); + + std::vector<SizeType32> givenInputLengths(nbGivenInputs); + for (SizeType32 i = 0; i < nbGivenInputs; ++i) + { + auto const* const seqBegin = givenInputData + i * maxInputLength; + auto const* const it = std::find(seqBegin, seqBegin + maxInputLength, padId); + givenInputLengths[i] = std::distance(seqBegin, it); + } + + return {givenInputLengths, nbGivenInputs, maxInputLength}; +} + +std::vector<executor::TokenIdType> createConsecutiveTokenSequence( + tr::SizeType32 length, tr::SizeType32 vocabSize, tr::TokenIdType firstTokenId) +{ + auto result = std::vector<executor::TokenIdType>(static_cast<size_t>(length), 0); + std::iota(result.begin(), result.end(), firstTokenId); + std::transform(result.begin(), result.end(), result.begin(), [&](auto const i) { return i % vocabSize; }); + return result; +} + +TestData TestData::loadTestData(BeamResult const& beamResults, ITensor const& givenInput, SizeType32 const maxBeamWidth, + tr::BufferManager& manager, executor::OutputConfig const& outConfig, ModelIds const& modelIds) +{ + auto const [givenInputLengths, nbGivenInputs, maxInputLength] = getGivenInputLengths(givenInput, modelIds.padId); + auto const& [beamWidth, resultsFile, contextLogitsFile, genLogitsFile, cumLogProbsFile, logProbsFile] = beamResults; + + TestData testData{nbGivenInputs, beamWidth}; + testData.expectedOutputIds = tr::utils::loadNpy(manager, resultsFile.string(), tr::MemoryType::kCPU); + + auto const& outputShape = testData.expectedOutputIds->getShape(); + EXPECT_EQ(outputShape.nbDims, 2); + EXPECT_EQ(nbGivenInputs * beamWidth, outputShape.d[0]); + testData.maxSeqLen = static_cast<SizeType32>(outputShape.d[1]); + EXPECT_LE(maxInputLength, testData.maxSeqLen); + EXPECT_LE(beamWidth, maxBeamWidth); + + auto const maxNewTokens = testData.maxSeqLen - maxInputLength; + + testData.endIds.insert(testData.endIds.end(), nbGivenInputs, modelIds.endId); + + if (outConfig.returnContextLogits && beamWidth == 1) + { + testData.loadContextLogits(contextLogitsFile, givenInputLengths, manager); + } + if (outConfig.returnGenerationLogits && beamWidth == 1) + { + testData.loadGenerationLogits(genLogitsFile, manager); + } + if (outConfig.returnLogProbs && beamWidth == 1) + { + testData.loadLogProbs(cumLogProbsFile, logProbsFile, manager); + } + + for (SizeType32 inputIdx = 0; inputIdx < nbGivenInputs; ++inputIdx) + { + for (SizeType32 beam = 0; beam < beamWidth; ++beam) + { + SizeType32 expectedLen = givenInputLengths[inputIdx] + maxNewTokens; + testData.expectedOutputLengths[inputIdx * beamWidth + beam] = expectedLen; + } + } + + return testData; +} + +void TestData::verifyOutput(std::unordered_map<SizeType32, std::vector<executor::BeamTokens>> const& resultTokens, + std::vector<SizeType32> const& givenInputLengths, bool streaming, bool excludeInputFromOutput, + FlakyTestInfo flakyTestInfo, bool isSpeculativeDecoding, SizeType32 reqBeamWidth, SizeType32 numReturnSequences, + bool isNonGreedySampling) +{ + for (auto const& [batchId, beamTokens] : resultTokens) + { + for (auto seqIdx = 0; seqIdx < numReturnSequences; seqIdx++) + { + auto const& tokens = beamTokens.at(seqIdx); + auto const inputLength = givenInputLengths.at(batchId); + SizeType32 const numReturnBeams = tokens.size(); + auto const* const expectedOutputData = tr::bufferCast<TokenIdType const>(*this->expectedOutputIds); + auto const expectedOutputLengths = this->expectedOutputLengths; + auto const endId = this->endIds[batchId]; + auto const maxSeqLen = this->maxSeqLen; + + for (SizeType32 beam = 0; beam < numReturnBeams; ++beam) + { + bool isFlaky = flakyTestInfo.batchIdBeams.count(std::make_pair(batchId, beam)); + if (isFlaky) + { + TLLM_LOG_WARNING("Disabling token comparison for batchId %d beam %d, test if flaky", batchId, beam); + } + + auto const expectInputOutputLength + = expectedOutputLengths[batchId * reqBeamWidth + beam]; // Ground truth output length + auto expectedOutputLength + = expectInputOutputLength - inputLength; // Number of new generated output tokens + + bool inputNotIncluded = (streaming || excludeInputFromOutput); + bool anyMismatch = false; + auto predictedTokens = tokens.at(beam); + // Remove the prompt + if (!inputNotIncluded) + { + predictedTokens.erase(predictedTokens.begin(), predictedTokens.begin() + inputLength); + } + + if (!isNonGreedySampling) + { + EXPECT_EQ(predictedTokens.size(), expectedOutputLength) + << "b: " << batchId << " seq: " << seqIdx << " beam: " << beam; + } + + auto numPredTokens = static_cast<SizeType32>(predictedTokens.size()); + + if (isSpeculativeDecoding) + { + // WAR to ensure bulk execution of spec decoding. + // We hope that no request in batch can finish 2x faster than any other request. + // For the cases when BS < 8, some predicted tokens are mismatched to reference data. + numPredTokens /= 2; + } + + for (auto i = 0; i < numPredTokens; ++i) + { + // Use the expected data for that beamWidth + auto const expectIndex = tc::flat_index3(batchId, beam, inputLength + i, reqBeamWidth, maxSeqLen); + auto const expectedToken = expectedOutputData[expectIndex]; + if (expectedToken == endId) + { + // TODO: can not find the error when (expectedToken == endId) && (predictedToken != endId) + break; + } + auto const predictedToken = predictedTokens.at(i); + if (!isFlaky && !isNonGreedySampling) + { + EXPECT_EQ(predictedToken, expectedToken) + << "b: " << batchId << " seq: " << seqIdx << " beam: " << beam << " i: " << i; + } + anyMismatch |= (predictedToken != expectedToken); + } + if (!isFlaky && !isNonGreedySampling) + { + EXPECT_FALSE(anyMismatch) << "b: " << batchId << " seq: " << seqIdx << " beam: " << beam; + } + else if (isNonGreedySampling) + { + EXPECT_TRUE(anyMismatch) << "b: " << batchId << " seq: " << seqIdx << " beam: " << beam; + } + } + } + } +} + +void TestData::verifyLogProbs(bool computeLogProbs, bool streaming, bool excludeInputFromOutput, SizeType32 inputLength, + SizeType32 beamWidth, executor::BeamTokens const& beamTokens, + std::optional<executor::VecLogProbs> const& cumLogProbs, + std::optional<std::vector<executor::VecLogProbs>> const& logProbs, SizeType32 batchId, FlakyTestInfo flakyTestInfo) +{ + auto expectedCumLogProbs = this->expectedCumLogProbs[batchId]; + auto expectedLogProbs = this->expectedLogProbs[batchId]; + auto const expectedOutputLengths = this->expectedOutputLengths; + auto const numReturnBeams = beamTokens.size(); + + if (computeLogProbs) + { + EXPECT_TRUE(cumLogProbs.has_value()) << "bid: " << batchId; + EXPECT_TRUE(logProbs.has_value()) << "bid: " << batchId; + EXPECT_EQ(cumLogProbs.value().size(), numReturnBeams) << "bid: " << batchId; + EXPECT_EQ(logProbs.value().size(), numReturnBeams) << "bid: " << batchId; + + bool removeInput = !excludeInputFromOutput && !streaming; + + for (SizeType32 beam = 0; beam < numReturnBeams; ++beam) + { + bool isFlaky = flakyTestInfo.batchIdBeams.count(std::make_pair(batchId, beam)); + if (isFlaky) + { + TLLM_LOG_WARNING("Disabling token comparison for batchId %d beam %d, test if flaky", batchId, beam); + } + + auto expectedOutputLength = expectedOutputLengths[batchId * beamWidth + beam]; + expectedOutputLength -= inputLength; + + auto numPredTokens = logProbs.value().at(beam).size(); + // Check shape + EXPECT_EQ(numPredTokens, beamTokens.at(beam).size() - (removeInput ? inputLength : 0)) + << "bid: " << batchId << " beam: " << beam; + + // If beamWidth == 1, compare log probs against python runtime + if (beamWidth == 1) + { + auto* const reqExpectedCumLogProbs = tr::bufferCast<float>(*expectedCumLogProbs); + // Only check cumLogProbs for the last generated token + if (numPredTokens == expectedOutputLength && !isFlaky) + { + EXPECT_TRUE(almostEqual(reqExpectedCumLogProbs[beam], cumLogProbs.value().at(beam), 2e-1, 5e-2)) + << "expectedCumLogProbs : " << reqExpectedCumLogProbs[beam] + << " cumlogProbs : " << cumLogProbs.value().at(beam); + } + + auto expectedLogProbsBeam = std::shared_ptr(tr::ITensor::slice(expectedLogProbs, beam, 1)); + expectedLogProbsBeam->squeeze(0); + auto* const reqExpectedLogProbs = tr::bufferCast<float>(*expectedLogProbsBeam); + for (auto i = 0; i < numPredTokens; ++i) + { + if (!isFlaky) + { + EXPECT_TRUE( + almostEqual(reqExpectedLogProbs[inputLength + i], logProbs.value()[beam][i], 5e-2, 5e-2)) + << "expectedLogProbs : " << reqExpectedLogProbs[inputLength + i] + << " logProbs : " << logProbs.value()[beam][i]; + } + } + } + } + } + else + { + EXPECT_FALSE(cumLogProbs.has_value()) << "bid: " << batchId; + EXPECT_FALSE(logProbs.has_value()) << "bid: " << batchId; + } +} + +void TestData::validateContextLogits(bool getContextLogits, SizeType32 inputLength, SizeType32 beamWidth, + std::optional<executor::Tensor> const& contextLogits, SizeType32 vocabSizePadded, SizeType32 batchId, float atol, + float rtol) +{ + if (getContextLogits) + { + EXPECT_TRUE(contextLogits.has_value()) << "bid: " << batchId; + EXPECT_EQ(contextLogits.value().getShape().size(), 2); + EXPECT_EQ(contextLogits.value().getShape()[0], inputLength); + EXPECT_EQ(contextLogits.value().getShape()[1], vocabSizePadded); + auto const expectedContextLogits = this->expectedContextLogits[batchId]; + + if (beamWidth == 1) + { + cudaDeviceSynchronize(); // Make sure the logits copy is complete. + EXPECT_TRUE(compareLogits( + *expectedContextLogits, *(executor::detail::toITensor(contextLogits.value())), atol, rtol)); + } + } + else + { + EXPECT_FALSE(contextLogits.has_value()) << "bid: " << batchId; + } +} + +void TestData::validateGenerationLogits(bool getGenLogits, bool isFinal, bool streaming, bool excludeInputFromOutput, + SizeType32 inputLength, SizeType32 maxOutputLen, SizeType32 beamWidth, executor::BeamTokens const& beamTokens, + std::optional<executor::Tensor> const& genLogits, SizeType32 vocabSizePadded, SizeType32 batchId, + bool const returnAllGeneratedTokens, float atol, float rtol) +{ + auto const numReturnBeams = beamTokens.size(); + + if (getGenLogits) + { + EXPECT_TRUE(genLogits.has_value()) << "bid: " << batchId; + EXPECT_EQ(genLogits.value().getShape().size(), 3); + + // Expected generation logits + auto const& expectedGenerationLogits + = this->expectedGenerationLogits[batchId]; // [maxOutputLen, vocabSizePadded] + // Output generation logits + // 1. non-streaming: [beamWidth, maxOutputLen, vocabSizePadded] + // 2. streaming: [maxOutputLen (or 1), beamWidth, vocabSizePadded] + auto const& outputGenerationLogits = executor::detail::toITensor(genLogits.value()); + + if (streaming) + { + EXPECT_EQ(genLogits.value().getShape()[1], numReturnBeams); + EXPECT_EQ(beamWidth, 1); // Only support streaming && beamWidth == 1 + + SizeType32 const beamIdx = 0; + bool removeInput = !excludeInputFromOutput && !streaming; + // If returnAllGeneratedTokens, will contain duplicate tokens + auto const& numPredTokens = beamTokens.at(beamIdx).size() - (removeInput ? inputLength : 0); + + SizeType32 numGeneratedToken = genLogits.value().getShape()[0]; + if (returnAllGeneratedTokens) + { + EXPECT_EQ(numGeneratedToken, numPredTokens); + } + else + { + EXPECT_EQ(numGeneratedToken, 1); + } + SizeType32 sliceOffset = returnAllGeneratedTokens ? 0 : numPredTokens - 1; + + auto const& expectedGenerationLogitsSlice + = std::shared_ptr(ITensor::slice(expectedGenerationLogits, sliceOffset, + numGeneratedToken)); // [numGeneratedToken, vocabSizePadded] + + cudaDeviceSynchronize(); // Make sure the logits copy is complete. + EXPECT_TRUE(compareLogits(*expectedGenerationLogitsSlice, *outputGenerationLogits, atol, rtol)); + } + else + { + // Non-streaming + EXPECT_EQ(genLogits.value().getShape()[0], numReturnBeams); + EXPECT_EQ(genLogits.value().getShape()[1], maxOutputLen); + + if (isFinal && beamWidth == 1) + { + cudaDeviceSynchronize(); // Make sure the logits copy is complete. + EXPECT_TRUE(compareLogits(*expectedGenerationLogits, *outputGenerationLogits, atol, rtol)); + } + } + EXPECT_EQ(genLogits.value().getShape()[2], vocabSizePadded); + } + else + { + EXPECT_FALSE(genLogits.has_value()) << "bid: " << batchId; + } +} + +} // namespace tensorrt_llm::testing diff --git a/cpp/tests/utils/common.h b/cpp/tests/utils/common.h new file mode 100644 index 000000000000..f7b73a9acea4 --- /dev/null +++ b/cpp/tests/utils/common.h @@ -0,0 +1,352 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifndef TOP_LEVEL_DIR +#error "Define TOP_LEVEL_DIR" +#endif + +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/runtime/bufferManager.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" + +#include <cmath> +#include <filesystem> +#include <random> +#include <set> +#include <string> +#include <utility> +#include <vector> + +namespace tensorrt_llm::testing +{ +namespace fs = std::filesystem; +namespace tr = tensorrt_llm::runtime; + +using tr::SizeType32; +using tr::TokenIdType; +using tr::ITensor; +using tr::MemoryType; + +auto const TEST_RESOURCE_PATH = fs::path{TOP_LEVEL_DIR} / "cpp/tests/resources"; + +auto const ENGINE_PATH = TEST_RESOURCE_PATH / "models/rt_engine"; +auto const GPT_MODEL_PATH = ENGINE_PATH / "gpt2"; +auto const LLAMA_MODEL_PATH = ENGINE_PATH / "Llama-3.2-1B"; +auto const MEDUSA_MODEL_PATH = ENGINE_PATH / "vicuna-7b-medusa"; +auto const CHATGLM_MODEL_PATH = ENGINE_PATH / "chatglm-6b"; +auto const CHATGLM2_MODEL_PATH = ENGINE_PATH / "chatglm2-6b"; +auto const CHATGLM3_MODEL_PATH = ENGINE_PATH / "chatglm3-6b"; +auto const GLM_MODEL_PATH = ENGINE_PATH / "glm-10b"; +auto const ENC_DEC_ENGINE_BASE = TEST_RESOURCE_PATH / "models/enc_dec/trt_engines"; + +auto const DATA_PATH = TEST_RESOURCE_PATH / "data"; +auto const GPT_DATA_PATH = DATA_PATH / "gpt2"; +auto const GPT_XGRAMMAR_TOKENIZER_INFO_PATH = GPT_DATA_PATH / "xgrammar_tokenizer_info.json"; +auto const LLAMA_DATA_PATH = DATA_PATH / "Llama-3.2-1B"; +auto const LLAMA_XGRAMMAR_TOKENIZER_INFO_PATH = LLAMA_DATA_PATH / "xgrammar_tokenizer_info.json"; +auto const MEDUSA_DATA_PATH = DATA_PATH / "vicuna-7b-medusa"; +auto const CHATGLM_DATA_PATH = DATA_PATH / "chatglm-6b"; +auto const CHATGLM2_DATA_PATH = DATA_PATH / "chatglm2-6b"; +auto const CHATGLM3_DATA_PATH = DATA_PATH / "chatglm3-6b"; +auto const GLM_DATA_PATH = DATA_PATH / "glm-10b"; +auto const ENC_DEC_DATA_BASE = DATA_PATH / "enc_dec"; + +auto constexpr T5_NAME = "t5-small"; +auto constexpr BART_NAME = "bart-large-cnn"; +auto constexpr LANGUAGE_ADAPTER_NAME = "language_adapter-enc_dec_language_adapter"; + +class PathUtil +{ +public: + static std::string EXECUTOR_WORKER_PATH() + { + return (std::filesystem::path{TOP_LEVEL_DIR} / "cpp/build/tensorrt_llm/executor_worker/executorWorker") + .string(); + } + + // model paths + static std::string FP16_GPT_ATTENTION_PACKED_DIR(); + static std::string FP16_GPT_ATTENTION_PACKED_PAGED_DIR(); + static std::string FP16_GPT_LORA_DIR(); + static std::string FP16_GPT_ATTENTION_PACKED_PAGED_DRAFT_TOKENS_DIR(); + static std::string FP16_GPT_ATTENTION_PACKED_PAGED_GATHER_DIR(); + static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_LONG_RESULT_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_GATHER_RESULT_FILE(); + // logits + static std::string FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_GATHER_CUM_LOG_PROBS_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_GATHER_LOG_PROBS_FILE(); + // results + static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP1_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP4_PP1_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP2_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP4_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP1_PP2_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_RESULT_TP2_PP1_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_CONTEXT_LOGITS_TP4_PP1_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_GENERATION_LOGITS_TP4_PP1_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_CUM_LOG_PROBS_TP4_PP1_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_LOG_PROBS_TP4_PP1_FILE(); + // GptExecutorTest.GenerationLogitsEarlyStop requires to use context_fmha_fp32_acc flag in runtime for better + // accuracy + static std::string FP16_PLUGIN_PACKED_PAGED_GATHER_CONTEXTFMHAFP32ACC_RESULT_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_CONTEXTFMHAFP32ACC_GENERATION_LOGITS_FILE(); + static std::string FP16_PLUGIN_PACKED_PAGED_CONTEXTFMHAFP32ACC_CONTEXT_LOGITS_FILE(); +}; + +class ModelIds +{ +public: + ModelIds() = default; + + constexpr ModelIds(TokenIdType endId, TokenIdType padId) + : endId{endId} + , padId{padId} + { + } + + TokenIdType endId{}; + TokenIdType padId{}; +}; + +class BeamResult +{ +public: + explicit BeamResult(SizeType32 beamWidth) + : beamWidth{beamWidth} {}; + + BeamResult(SizeType32 beamWidth, fs::path resultsFile, fs::path contextLogitsFile, fs::path genLogitsFile, + fs::path cumLogProbsFile, fs::path logProbsFile) + : beamWidth{beamWidth} + , resultsFile{std::move(resultsFile)} + , contextLogitsFile{std::move(contextLogitsFile)} + , genLogitsFile{std::move(genLogitsFile)} + , cumLogProbsFile{std::move(cumLogProbsFile)} + , logProbsFile{std::move(logProbsFile)} {}; + + SizeType32 beamWidth; + fs::path resultsFile; + + fs::path contextLogitsFile; + fs::path genLogitsFile; + + fs::path cumLogProbsFile; + fs::path logProbsFile; +}; + +using BeamResults = std::vector<BeamResult>; + +struct FlakyTestInfo +{ + // Pair of batch ID + beam which are flaky + std::set<std::pair<SizeType32, SizeType32>> batchIdBeams; +}; + +class TestData +{ +public: + explicit TestData(SizeType32 nbGivenInputs, SizeType32 beamWidth) + : nbGivenInputs{nbGivenInputs} + , beamWidth{beamWidth} + { + expectedOutputLengths.resize(nbGivenInputs * beamWidth); + + draftTokens.resize(nbGivenInputs); + draftLogits.resize(nbGivenInputs); + acceptedDraftTokensLengths.resize(nbGivenInputs); + expectedGenerationLogits.resize(nbGivenInputs); + expectedContextLogits.resize(nbGivenInputs); + expectedCumLogProbs.resize(nbGivenInputs); + expectedLogProbs.resize(nbGivenInputs); + } + + void loadLogProbs(fs::path const& cumLogProbsFile, fs::path const& logProbsFile, tr::BufferManager const& manager); + + void loadContextLogits(fs::path const& contextLogitsFile, std::vector<SizeType32> const& givenInputLengths, + tr::BufferManager const& manager); + void loadGenerationLogits(fs::path const& genLogitsFile, tr::BufferManager const& manager); + + void makeDraft(SizeType32 maxDraftTokens, bool acceptDraftByLogits, fs::path const& genLogitsFile, + std::vector<SizeType32> const& givenInputLengths, tr::BufferManager const& manager); + + static TestData loadTestData(BeamResult const& beamResults, ITensor const& givenInput, SizeType32 maxBeamWidth, + tr::BufferManager& manager, executor::OutputConfig const& outConfig, ModelIds const& modelIds); + + void verifyOutput(std::unordered_map<SizeType32, std::vector<executor::BeamTokens>> const& resultTokens, + std::vector<SizeType32> const& givenInputLengths, bool streaming, bool excludeInputFromOutput, + FlakyTestInfo flakyTestInfo, bool isSpeculativeDecoding, SizeType32 reqBeamWidth, SizeType32 numReturnSequences, + bool isNonGreedySampling); + + void verifyLogProbs(bool computeLogProbs, bool streaming, bool excludeInputFromOutput, SizeType32 inputLength, + SizeType32 beamWidth, executor::BeamTokens const& beamTokens, + std::optional<executor::VecLogProbs> const& cumLogProbs, + std::optional<std::vector<executor::VecLogProbs>> const& logProbs, SizeType32 batchId, + FlakyTestInfo flakyTestInfo); + + void validateContextLogits(bool getContextLogits, SizeType32 inputLength, SizeType32 beamWidth, + std::optional<executor::Tensor> const& contextLogits, SizeType32 vocabSizePadded, SizeType32 batchId, + float atol = 1e-2, float rtol = 1e-3); + + void validateGenerationLogits(bool getGenLogits, bool isFinal, bool streaming, bool excludeInputFromOutput, + SizeType32 inputLength, SizeType32 maxOutputLen, SizeType32 beamWidth, executor::BeamTokens const& beamTokens, + std::optional<executor::Tensor> const& genLogits, SizeType32 vocabSizePadded, SizeType32 batchId, + bool returnAllGeneratedTokens, float atol = 1e-2, float rtol = 1e-3); + + SizeType32 nbGivenInputs{}; + SizeType32 beamWidth{}; + SizeType32 maxSeqLen{}; + ITensor::SharedPtr expectedOutputIds; + std::vector<SizeType32> expectedOutputLengths; + std::vector<TokenIdType> endIds; + std::vector<tensorrt_llm::executor::VecTokens> draftTokens; + std::vector<ITensor::SharedPtr> draftLogits; + std::vector<SizeType32> acceptedDraftTokensLengths; + std::vector<ITensor::SharedPtr> expectedGenerationLogits; + std::vector<ITensor::SharedPtr> expectedContextLogits; + std::vector<ITensor::SharedPtr> expectedCumLogProbs; + std::vector<ITensor::SharedPtr> expectedLogProbs; +}; + +inline bool almostEqual(float a, float b, float atol = 1e-2, float rtol = 1e-3) +{ + // Params: a = value to compare and b = reference + // This function follows implementation of numpy.isclose(), which checks + // abs(a - b) <= (atol + rtol * abs(b)). + // Note that the inequality above is asymmetric where b is considered as + // a reference value. To account into both absolute/relative errors, it + // uses absolute tolerance and relative tolerance at the same time. The + // default values of atol and rtol borrowed from numpy.isclose(). For the + // case of nan value, the result will be true. + if (std::isnan(a) && std::isnan(b)) + { + return true; + } + return fabs(a - b) <= (atol + rtol * fabs(b)); +} + +bool compareLogits(ITensor const& groundTruthLogits, ITensor const& outputLogits, float atol = 1e-2, float rtol = 1e-3); + +std::tuple<SizeType32, SizeType32> getRequestGivenInputIdxLength( + std::uint64_t requestId, SizeType32 nbGivenInputs, std::vector<SizeType32> const& givenInputLengths); + +std::tuple<std::vector<SizeType32>, SizeType32, SizeType32> getGivenInputLengths( + ITensor const& givenInput, SizeType32 padId); + +/// @brief Generates a vector of floating point values summing to 1, that can be used as logits. +/// +/// @tparam TEngine The type of the random engine. +/// @tparam TLogits The type of floating point values. +/// @param vocabSize The vocabulary size, i.e. the size of the vector. +/// @param engine A random engine. +/// @return std::vector<TLogits> A vector of floating point values, summing to 1. +template <typename TEngine, typename TLogits> +std::vector<TLogits> randomLogits(runtime::SizeType32 vocabSize, TEngine* engine) +{ + if constexpr (std::disjunction_v<std::is_floating_point<TLogits>, std::is_same<TLogits, half>>) + { + // This algorithm ensures the resulting values sum to 1 by: + // 1. Sampling in the interval 0..1 + // 2. Sorting the sampled values and adding a last value equal to 1 + // 3. Calculating the adjacent differences of the sorted values + // Since the values are sorted and the last value is 1, we get that all the differences are positive and must + // sum to 1. It can be proven recursively by seeing that the first value sums to itself, and the n-1 first + // values must sum to the value at n, minus the difference between the n-th and n-1-th values. + // It is also helpful to convince yourself of it with a quick drawing. + auto distribution = std::uniform_real_distribution<float>(0, 1); + std::vector<float> samples(vocabSize); + samples.back() = 1.0; + std::transform(samples.begin(), samples.end() - 1, samples.begin(), + [&](auto const /*i*/) { return distribution(*engine); }); + std::sort(samples.begin(), samples.end() - 1); + std::vector<float> result(vocabSize); + std::adjacent_difference(samples.begin(), samples.end(), result.begin()); + if constexpr (std::is_same_v<TLogits, float>) + { + return result; + } + + if constexpr (std::is_same_v<TLogits, half>) + { + std::vector<half> halfResults(vocabSize); + std::transform( + result.begin(), result.end(), halfResults.begin(), [&](auto const f) { return __float2half(f); }); + return halfResults; + } + } + TLLM_THROW("Unsupported logits type."); +} + +std::vector<tensorrt_llm::executor::TokenIdType> createConsecutiveTokenSequence( + tr::SizeType32 length, tr::SizeType32 vocabSize, tr::TokenIdType firstTokenId); + +/** + * GPU timer for recording the elapsed time across kernel(s) launched in GPU stream + */ +struct GpuTimer +{ + cudaStream_t _stream_id; + cudaEvent_t _start; + cudaEvent_t _stop; + + /// Construct`or + GpuTimer() + : _stream_id(0) + { + TLLM_CUDA_CHECK(cudaEventCreate(&_start)); + TLLM_CUDA_CHECK(cudaEventCreate(&_stop)); + } + + /// Destructor + ~GpuTimer() + { + TLLM_CUDA_CHECK(cudaEventDestroy(_start)); + TLLM_CUDA_CHECK(cudaEventDestroy(_stop)); + } + + /// Start the timer for a given stream (defaults to the default stream) + void start(cudaStream_t stream_id = 0) + { + _stream_id = stream_id; + TLLM_CUDA_CHECK(cudaEventRecord(_start, _stream_id)); + } + + /// Stop the timer + void stop() + { + TLLM_CUDA_CHECK(cudaEventRecord(_stop, _stream_id)); + } + + /// Return the elapsed time (in milliseconds) + float elapsed_millis() + { + float elapsed = 0.0; + TLLM_CUDA_CHECK(cudaEventSynchronize(_stop)); + TLLM_CUDA_CHECK(cudaEventElapsedTime(&elapsed, _start, _stop)); + return elapsed; + } +}; + +} // namespace tensorrt_llm::testing diff --git a/cpp/tests/utils/engines.cpp b/cpp/tests/utils/engines.cpp new file mode 100644 index 000000000000..28f0d9b15935 --- /dev/null +++ b/cpp/tests/utils/engines.cpp @@ -0,0 +1,97 @@ +#include "engines.h" + +#include "tensorrt_llm/batch_manager/transformerBuffers.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/tllmLogger.h" + +#include <NvInfer.h> +#include <algorithm> +#include <memory> + +nvinfer1::ITensor& tensorrt_llm::testing::utils::engines::details::addInputIds( + EngineBuildState& buildState, runtime::SizeType32 maxNumTokens) +{ + auto* input_ids = buildState.networkDefinition->addInput(batch_manager::RuntimeBuffers::kInputIdsTensorName, + nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({-1})); + buildState.tensors.push_back(input_ids); + buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kInputIdsTensorName, + nvinfer1::OptProfileSelector::kMAX, runtime::ITensor::makeShape({maxNumTokens})); + buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kInputIdsTensorName, + nvinfer1::OptProfileSelector::kOPT, runtime::ITensor::makeShape({maxNumTokens / 2})); + buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kInputIdsTensorName, + nvinfer1::OptProfileSelector::kMIN, runtime::ITensor::makeShape({1})); + return *input_ids; +} + +nvinfer1::ITensor* tensorrt_llm::testing::utils::engines::details::addLastTokenIds( + EngineBuildState& buildState, runtime::SizeType32 maxBatchSize, runtime::SizeType32 maxBeamWidth) +{ + auto* last_token_ids + = buildState.networkDefinition->addInput(batch_manager::RuntimeBuffers::kLastTokenIdsTensorName, + nvinfer1::DataType::kINT32, runtime::ITensor::makeShape({-1})); + buildState.tensors.push_back(last_token_ids); + buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kLastTokenIdsTensorName, + nvinfer1::OptProfileSelector::kMAX, runtime::ITensor::makeShape({maxBatchSize * maxBeamWidth})); + buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kLastTokenIdsTensorName, + nvinfer1::OptProfileSelector::kOPT, runtime::ITensor::makeShape({maxBatchSize * maxBeamWidth / 2})); + buildState.profile->setDimensions(batch_manager::RuntimeBuffers::kLastTokenIdsTensorName, + nvinfer1::OptProfileSelector::kMIN, runtime::ITensor::makeShape({1})); + return last_token_ids; +} + +nvinfer1::ITensor& tensorrt_llm::testing::utils::engines::details::addKvCacheOffsets(EngineBuildState& buildState, + runtime::SizeType32 numPools, runtime::SizeType32 tokensPerBlock, runtime::SizeType32 maxBatchSize, + runtime::SizeType32 maxNumTokens, runtime::SizeType32 maxBeamWidth) +{ + auto* kvCacheOffsets = buildState.networkDefinition->addInput( + batch_manager::TransformerBuffers::kKvCacheBlockOffsetsTensorName, nvinfer1::DataType::kINT32, + runtime::ITensor::makeShape({numPools, -1, 2, -1})); // [numPools, maxBatch * maxBeamWidth, 2, maxBlocksPerSeq] + buildState.tensors.push_back(kvCacheOffsets); + auto const maxBlocksPerSeq = maxNumTokens / tokensPerBlock; + buildState.profile->setDimensions(batch_manager::TransformerBuffers::kKvCacheBlockOffsetsTensorName, + nvinfer1::OptProfileSelector::kMAX, + runtime::ITensor::makeShape({numPools, maxBatchSize * maxBeamWidth, 2, maxBlocksPerSeq})); + buildState.profile->setDimensions(batch_manager::TransformerBuffers::kKvCacheBlockOffsetsTensorName, + nvinfer1::OptProfileSelector::kOPT, + runtime::ITensor::makeShape({numPools, maxBatchSize * maxBeamWidth / 2, 2, maxBlocksPerSeq / 2})); + buildState.profile->setDimensions(batch_manager::TransformerBuffers::kKvCacheBlockOffsetsTensorName, + nvinfer1::OptProfileSelector::kMIN, runtime::ITensor::makeShape({numPools, 1, 2, 1})); + return *kvCacheOffsets; +} + +tensorrt_llm::testing::utils::engines::details::EngineBuildState +tensorrt_llm::testing::utils::engines::initializeEngineBuild(std::shared_ptr<runtime::TllmLogger> const& logger) +{ + auto* builder = nvinfer1::createInferBuilder(*logger); + auto* profile = builder->createOptimizationProfile(); + auto* network = builder->createNetworkV2( + 1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kSTRONGLY_TYPED)); + nvinfer1::IBuilderConfig* config = builder->createBuilderConfig(); + return {builder, network, profile, config}; +} + +nvinfer1::ITensor& tensorrt_llm::testing::utils::engines::details::addSingleOutputLayer( + tensorrt_llm::testing::utils::engines::details::EngineBuildState& buildState, nvinfer1::ILayer* layer) +{ + buildState.layers.push_back(layer); + auto* output = layer->getOutput(0); + buildState.tensors.push_back(output); + TLLM_LOG_INFO("Adding layer %s with output shape %s.", layer->getName(), + tensorrt_llm::runtime::ITensor::toString(output->getDimensions()).c_str()); + + return *output; +} + +tensorrt_llm::common::OptionalRef<nvinfer1::ITensor> tensorrt_llm::testing::utils::engines::details::getTensorByName( + tensorrt_llm::testing::utils::engines::details::EngineBuildState& buildState, std::string_view name) +{ + auto result = std::find_if(buildState.tensors.begin(), buildState.tensors.end(), + [name](auto const tensor) { return tensor->getName() == name; }); + if (result == buildState.tensors.end()) + { + return tensorrt_llm::common::OptionalRef<nvinfer1::ITensor>{}; + } + return **result; +} diff --git a/cpp/tests/utils/engines.h b/cpp/tests/utils/engines.h new file mode 100644 index 000000000000..54b53e5c6097 --- /dev/null +++ b/cpp/tests/utils/engines.h @@ -0,0 +1,356 @@ +#ifndef CA1B91B5_DF64_4CF8_948F_5AFF243A2555 +#define CA1B91B5_DF64_4CF8_948F_5AFF243A2555 + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/optionalRef.h" +#include "tensorrt_llm/runtime/common.h" +#include "tensorrt_llm/runtime/iBuffer.h" +#include "tensorrt_llm/runtime/iTensor.h" +#include "tensorrt_llm/runtime/tllmLogger.h" +#include <NvInfer.h> +#include <NvInferRuntime.h> +#include <algorithm> +#include <cstddef> +#include <memory> +#include <tensorrt_llm/batch_manager/runtimeBuffers.h> +#include <utility> +#include <vector> + +namespace tensorrt_llm::testing::utils::engines +{ + +namespace details +{ + +struct EngineBuildResource +{ + EngineBuildResource() = default; + virtual ~EngineBuildResource() = default; + EngineBuildResource(EngineBuildResource const& vector) = default; + EngineBuildResource& operator=(EngineBuildResource const& vector) = default; + EngineBuildResource(EngineBuildResource&& vector) noexcept = default; + EngineBuildResource& operator=(EngineBuildResource&& vector) noexcept = default; +}; + +template <typename TValue> +struct Vector : public EngineBuildResource +{ + explicit Vector(std::vector<TValue> values) + : values(std::move(values)){}; + Vector(Vector const& vector) = default; + Vector& operator=(Vector const& vector) = default; + Vector(Vector&& vector) noexcept = default; + Vector& operator=(Vector&& vector) noexcept = default; + ~Vector() override = default; + std::vector<TValue> values; +}; + +template <typename TValue, size_t Size> +struct Array : public EngineBuildResource +{ + explicit Array(std::array<TValue, Size> values) + : values(std::move(values)){}; + Array(Array const& vector) = default; + Array& operator=(Array const& vector) = default; + Array(Array&& vector) noexcept = default; + Array& operator=(Array&& vector) noexcept = default; + ~Array() override = default; + std::array<TValue, Size> values; +}; + +struct EngineBuildState +{ + EngineBuildState(nvinfer1::IBuilder* builder, nvinfer1::INetworkDefinition* networkDefinition, + nvinfer1::IOptimizationProfile* profile, nvinfer1::IBuilderConfig* builderConfig) + : builder(builder) + , networkDefinition(networkDefinition) + , profile(profile) + , builderConfig(builderConfig){}; + EngineBuildState(EngineBuildState const& vector) = delete; + EngineBuildState& operator=(EngineBuildState const& vector) = delete; + EngineBuildState(EngineBuildState&& vector) noexcept = default; + EngineBuildState& operator=(EngineBuildState&& vector) noexcept = default; + std::unique_ptr<nvinfer1::IBuilder> builder; + std::unique_ptr<nvinfer1::INetworkDefinition> networkDefinition; + nvinfer1::IOptimizationProfile* profile; + + // While building the engine, one might need some data for weights and such. Turns out, TensorRT does not keep a + // copy of those, so if you create them as temporaries and pass them to the TRT APIs, you will get UB. So we need + // some place where we can keep those things. + std::unique_ptr<nvinfer1::IBuilderConfig> builderConfig; + std::vector<std::unique_ptr<EngineBuildResource>> resources; + std::vector<nvinfer1::ITensor*> tensors; + std::vector<nvinfer1::ILayer*> layers; + + ~EngineBuildState() + { + // Builder needs to be deleteds last. + networkDefinition.reset(); + builderConfig.reset(); + builder.reset(); + } +}; + +common::OptionalRef<nvinfer1::ITensor> getTensorByName(EngineBuildState& buildState, std::string_view name); + +nvinfer1::ITensor& addSingleOutputLayer(EngineBuildState& buildState, nvinfer1::ILayer* layer); + +template <typename TResource> +TResource& addResource(EngineBuildState& buildState, TResource resource) +{ + return *dynamic_cast<TResource*>( + buildState.resources.emplace_back(std::make_unique<TResource>(std::move(resource))).get()); +} + +template <typename TValue> +Vector<TValue>& addSingleConstantVectorResource(EngineBuildState& buildState, TValue value, std::size_t length) +{ + std::vector<TValue> weights(length); + std::fill(weights.begin(), weights.end(), value); + return addResource(buildState, Vector<TValue>{weights}); +} + +template <typename TValue> +Vector<TValue>& addConstantVectorResource(EngineBuildState& buildState, std::vector<TValue> values) +{ + return addResource(buildState, Vector<TValue>{values}); +} + +template <typename TValue> +Array<TValue, 1>& addConstantScalarResource(EngineBuildState& buildState, TValue value) +{ + return addResource(buildState, Array<TValue, 1>{{value}}); +} + +nvinfer1::ITensor& addInputIds(EngineBuildState& buildState, runtime::SizeType32 maxNumTokens); +nvinfer1::ITensor* addLastTokenIds( + EngineBuildState& buildState, runtime::SizeType32 maxBatchSize, runtime::SizeType32 maxBeamWidth); +nvinfer1::ITensor& addKvCacheOffsets(EngineBuildState& buildState, runtime::SizeType32 numPools, + runtime::SizeType32 tokensPerBlock, runtime::SizeType32 maxBatchSize, runtime::SizeType32 maxNumTokens, + runtime::SizeType32 maxBeamWidth); + +template <typename TValue> +nvinfer1::ITensor& addSingleConstantVector(EngineBuildState& buildState, TValue value, runtime::SizeType32 length) +{ + auto& resourceWeights = addSingleConstantVectorResource(buildState, value, length); + auto const trtDatatype = runtime::TRTDataType<TValue>::value; + auto* layer = buildState.networkDefinition->addConstant(runtime::ITensor::makeShape({length}), + {trtDatatype, resourceWeights.values.data(), static_cast<runtime::ITensor::DimType64>(length)}); + return addSingleOutputLayer(buildState, layer); +} + +template <typename TValue> +nvinfer1::ITensor& addSingleConstantTensor(EngineBuildState& buildState, TValue value, runtime::SizeType32 length) +{ + auto& resourceWeights = addSingleConstantVectorResource(buildState, value, length); + auto const trtDatatype = runtime::TRTDataType<TValue>::value; + auto* layer = buildState.networkDefinition->addConstant(runtime::ITensor::makeShape({1, length}), + {trtDatatype, resourceWeights.values.data(), static_cast<runtime::ITensor::DimType64>(length)}); + return addSingleOutputLayer(buildState, layer); +} + +template <typename TValue> +nvinfer1::ITensor& addConstantVector(EngineBuildState& buildState, std::vector<TValue> values) +{ + auto& resourceWeights = addConstantVectorResource(buildState, values); + auto const trtDatatype = runtime::TRTDataType<TValue>::value; + auto const length = static_cast<runtime::ITensor::DimType64>(values.size()); + auto* layer = buildState.networkDefinition->addConstant( + runtime::ITensor::makeShape({length}), {trtDatatype, resourceWeights.values.data(), length}); + return addSingleOutputLayer(buildState, layer); +} + +template <typename TValue> +nvinfer1::ITensor& addConstantTensor( + EngineBuildState& buildState, std::vector<TValue> values, runtime::ITensor::Shape shape) +{ + auto& resourceWeights = addConstantVectorResource(buildState, values); + auto const trtDatatype = runtime::TRTDataType<TValue>::value; + auto const count = runtime::ITensor::volume(shape); + auto* layer = buildState.networkDefinition->addConstant(shape, {trtDatatype, resourceWeights.values.data(), count}); + return addSingleOutputLayer(buildState, layer); +} + +template <typename TValue> +nvinfer1::ITensor& addSingleConstantTensor(EngineBuildState& buildState, TValue value, runtime::ITensor::Shape shape) +{ + auto const count = runtime::ITensor::volume(shape); + auto& resourceWeights = addSingleConstantVectorResource(buildState, value, count); + auto const trtDatatype = runtime::TRTDataType<TValue>::value; + auto* layer = buildState.networkDefinition->addConstant(shape, {trtDatatype, resourceWeights.values.data(), count}); + return addSingleOutputLayer(buildState, layer); +} + +template <typename TValue> +nvinfer1::ITensor& addConstantScalar(EngineBuildState& buildState, TValue value) +{ + auto& resourceWeights = addConstantScalarResource<TValue>(buildState, value); + auto const trtDatatype = runtime::TRTDataType<TValue>::value; + auto* layer = buildState.networkDefinition->addConstant( + runtime::ITensor::makeShape({}), {trtDatatype, resourceWeights.values.data(), 1}); + return addSingleOutputLayer(buildState, layer); +} + +template <typename TValue> +nvinfer1::ITensor& oneHotEncode( + EngineBuildState& buildState, nvinfer1::ITensor& inputIds, runtime::SizeType32 vocabSize) +{ + auto const trtValueType = runtime::TRTDataType<TValue>::value; + auto& oneHotValues = addConstantVector<TValue>(buildState, {0, 1}); + auto& oneHotDepth = addConstantScalar(buildState, vocabSize); + auto* oneHotLayer = buildState.networkDefinition->addOneHot(inputIds, oneHotValues, oneHotDepth, 0); + return addSingleOutputLayer(buildState, oneHotLayer); +} +} // namespace details + +struct TrivialDecoderParameters +{ + TrivialDecoderParameters(runtime::SizeType32 vocabSize, runtime::SizeType32 maxBatchSize, + runtime::SizeType32 maxNumTokens, runtime::SizeType32 tokensPerBlock, runtime::SizeType32 maxBeamWidth, + bool gatherContextLogits) + : vocabSize(vocabSize) + , maxBatchSize(maxBatchSize) + , maxNumTokens(maxNumTokens) + , tokensPerBlock(tokensPerBlock) + , maxBeamWidth(maxBeamWidth) + , gatherContextLogits(gatherContextLogits){}; + runtime::SizeType32 vocabSize; + runtime::SizeType32 maxBatchSize; + runtime::SizeType32 maxNumTokens; + runtime::SizeType32 tokensPerBlock; + runtime::SizeType32 maxBeamWidth; + bool gatherContextLogits; +}; + +details::EngineBuildState initializeEngineBuild(std::shared_ptr<runtime::TllmLogger> const& logger); + +template <typename TLogits> +std::unique_ptr<nvinfer1::IHostMemory> createTrivialDecoder( + TrivialDecoderParameters parameters, std::shared_ptr<runtime::TllmLogger> const& logger) +{ + auto const trtLogitsType = runtime::TRTDataType<TLogits>::value; + auto buildState = initializeEngineBuild(logger); + auto* builder = buildState.builder.get(); + auto* profile = buildState.profile; + auto* network = buildState.networkDefinition.get(); + auto& inputIds = details::addInputIds(buildState, parameters.maxNumTokens); + auto& kvCacheOffsets = details::addKvCacheOffsets(buildState, 1, parameters.tokensPerBlock, parameters.maxBatchSize, + parameters.maxNumTokens, parameters.maxBeamWidth); + + auto& oneHotLayerOutput = details::oneHotEncode<TLogits>(buildState, inputIds, parameters.vocabSize); + oneHotLayerOutput.setName(batch_manager::RuntimeBuffers::kLogitsTensorName); + network->markOutput(oneHotLayerOutput); + + buildState.builderConfig->addOptimizationProfile(profile); + buildState.builderConfig->setProfilingVerbosity(nvinfer1::ProfilingVerbosity::kDETAILED); + auto* engine = builder->buildSerializedNetwork(*network, *buildState.builderConfig); + return std::unique_ptr<nvinfer1::IHostMemory>(engine); +} + +template <typename TLogits> +struct ConstantTrivialDecoderParameters +{ + ConstantTrivialDecoderParameters(TrivialDecoderParameters trivialDecoderParameters, std::vector<TLogits> logits) + : trivialDecoderParameters(trivialDecoderParameters) + , logits(logits) + { + auto const sizeTypeVocabSize = static_cast<std::size_t>(trivialDecoderParameters.vocabSize); + auto const logitsSize = logits.size(); + TLLM_CHECK_WITH_INFO(static_cast<std::size_t>(trivialDecoderParameters.vocabSize) == logits.size(), + "The size of the constant logits (%lu) has to be equal to the vocabulary size (%lu).", logitsSize, + sizeTypeVocabSize); + }; + + TrivialDecoderParameters trivialDecoderParameters; + std::vector<TLogits> logits; +}; + +template <typename TLogits> +details::EngineBuildState createConstantTrivialDecoderBase( + ConstantTrivialDecoderParameters<TLogits> parameters, std::shared_ptr<runtime::TllmLogger> const& logger) +{ + auto const trtLogitsType = runtime::TRTDataType<TLogits>::value; + auto buildState = initializeEngineBuild(logger); + auto* builder = buildState.builder.get(); + auto* profile = buildState.profile; + auto* network = buildState.networkDefinition.get(); + auto& inputIds = details::addInputIds(buildState, parameters.trivialDecoderParameters.maxNumTokens); + nvinfer1::ITensor* lastTokenIds = nullptr; + if (!parameters.trivialDecoderParameters.gatherContextLogits) + { + lastTokenIds = details::addLastTokenIds(buildState, parameters.trivialDecoderParameters.maxBatchSize, + parameters.trivialDecoderParameters.maxBeamWidth); + } + auto& kvCacheOffsets = details::addKvCacheOffsets(buildState, 1, parameters.trivialDecoderParameters.tokensPerBlock, + parameters.trivialDecoderParameters.maxBatchSize, parameters.trivialDecoderParameters.maxNumTokens, + parameters.trivialDecoderParameters.maxBeamWidth); + + auto const vocabSize = static_cast<runtime::ITensor::DimType64>(parameters.logits.size()); + + auto& constantLogitsPerToken = details::addConstantTensor<TLogits>( + buildState, parameters.logits, runtime::ITensor::makeShape({vocabSize, 1})); + auto& oneHotLayerOutput + = details::oneHotEncode<TLogits>(buildState, inputIds, parameters.trivialDecoderParameters.vocabSize); + auto& ones = details::addSingleConstantTensor<TLogits>(buildState, 1, runtime::ITensor::makeShape({1, vocabSize})); + auto* intermediateLayer1 = network->addMatrixMultiply( + ones, nvinfer1::MatrixOperation::kNONE, oneHotLayerOutput, nvinfer1::MatrixOperation::kNONE); + auto* intermediateLayer1Output = intermediateLayer1->getOutput(0); + + nvinfer1::ITensor* gatherLayerOutput = nullptr; + if (!parameters.trivialDecoderParameters.gatherContextLogits) + { + auto& one = details::addSingleConstantTensor<int32_t>(buildState, 1, runtime::ITensor::makeShape({1})); + auto* lastTokenIdsMinus1Layer + = network->addElementWise(*lastTokenIds, one, nvinfer1::ElementWiseOperation::kSUB); + auto* gatherLayer = network->addGather(*intermediateLayer1Output, *lastTokenIdsMinus1Layer->getOutput(0), 1); + gatherLayerOutput = gatherLayer->getOutput(0); + } + else + { + gatherLayerOutput = intermediateLayer1Output; + } + + auto* constLogitsLayer = network->addMatrixMultiply(*gatherLayerOutput, nvinfer1::MatrixOperation::kTRANSPOSE, + constantLogitsPerToken, nvinfer1::MatrixOperation::kTRANSPOSE); + auto* outputLogits = constLogitsLayer->getOutput(0); + network->markOutput(*outputLogits); + outputLogits->setName(batch_manager::RuntimeBuffers::kLogitsTensorName); + buildState.tensors.push_back(outputLogits); + return buildState; +} + +template <typename TLogits> +std::unique_ptr<nvinfer1::IHostMemory> createConstantTrivialDecoder( + ConstantTrivialDecoderParameters<TLogits> parameters, std::shared_ptr<runtime::TllmLogger> const& logger) +{ + auto buildState = createConstantTrivialDecoderBase<TLogits>(parameters, logger); + buildState.builderConfig->addOptimizationProfile(buildState.profile); + buildState.builderConfig->setProfilingVerbosity(nvinfer1::ProfilingVerbosity::kDETAILED); + auto* engine = buildState.builder->buildSerializedNetwork(*buildState.networkDefinition, *buildState.builderConfig); + return std::unique_ptr<nvinfer1::IHostMemory>(engine); +} + +template <typename TLogits> +std::unique_ptr<nvinfer1::IHostMemory> createConstantTrivialDecoderWithTopKLogits( + ConstantTrivialDecoderParameters<TLogits> parameters, runtime::SizeType32 numTopLogits, std::string_view outputName, + std::shared_ptr<runtime::TllmLogger> const& logger) +{ + auto buildState = createConstantTrivialDecoderBase<TLogits>(parameters, logger); + auto logits = details::getTensorByName(buildState, batch_manager::RuntimeBuffers::kLogitsTensorName); + TLLM_CHECK_WITH_INFO(static_cast<bool>(logits), + "You can only add topk logits on top of a network which contains a tensor named %s", + batch_manager::RuntimeBuffers::kLogitsTensorName); + auto* topKLayer = buildState.networkDefinition->addTopK( + logits.value(), nvinfer1::TopKOperation::kMAX, numTopLogits, 1UL << 1UL); + auto* topKLayerOutput = topKLayer->getOutput(0); + topKLayerOutput->setName(outputName.data()); + buildState.networkDefinition->markOutput(*topKLayerOutput); + auto* profile = buildState.profile; + buildState.builderConfig->addOptimizationProfile(profile); + buildState.builderConfig->setProfilingVerbosity(nvinfer1::ProfilingVerbosity::kDETAILED); + auto* engine = buildState.builder->buildSerializedNetwork(*buildState.networkDefinition, *buildState.builderConfig); + return std::unique_ptr<nvinfer1::IHostMemory>(engine); +} +} // namespace tensorrt_llm::testing::utils::engines + +#endif /* CA1B91B5_DF64_4CF8_948F_5AFF243A2555 */ diff --git a/cpp/tests/utils/executorUtils.cpp b/cpp/tests/utils/executorUtils.cpp new file mode 100644 index 000000000000..6dbd6223ef19 --- /dev/null +++ b/cpp/tests/utils/executorUtils.cpp @@ -0,0 +1,55 @@ +#include "executorUtils.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include <exception> +#include <future> + +std::unordered_map<tensorrt_llm::batch_manager::RequestIdType, std::vector<tensorrt_llm::executor::Response>> +tensorrt_llm::testing::runThroughRequests(executor::Executor& executor, std::vector<executor::Request> const& requests, + std::chrono::duration<float, std::milli> timeout) +{ + std::unordered_map<batch_manager::RequestIdType, std::vector<executor::Response>> accumulatedResponses; + auto responseReadFuture = std::async(std::launch::async, + [&]() -> std::optional<std::exception> + { + auto remainingRequests = requests.size(); + try + { + while (remainingRequests > 0) + { + auto const responses = executor.awaitResponses(); + for (auto const& response : responses) + { + auto const requestId = response.getRequestId(); + if (response.hasError()) + { + TLLM_LOG_ERROR("Error response received for request: %lu", requestId); + TLLM_THROW(response.getErrorMsg()); + } + auto const isFinal = response.hasError() || response.getResult().isFinal; + accumulatedResponses[requestId].emplace_back(response); + if (isFinal) + { + TLLM_LOG_DEBUG("Final response received for request: %lu", requestId); + --remainingRequests; + } + } + } + + return std::nullopt; + } + catch (std::exception const& e) + { + TLLM_LOG_EXCEPTION(e); + return e; + } + }); + auto const requestIds = executor.enqueueRequests(requests); + responseReadFuture.wait_for(timeout); + auto const readResult = responseReadFuture.get(); + if (readResult.has_value()) + { + throw std::exception(readResult.value()); + } + return accumulatedResponses; +} diff --git a/cpp/tests/utils/executorUtils.h b/cpp/tests/utils/executorUtils.h new file mode 100644 index 000000000000..97f154ca275d --- /dev/null +++ b/cpp/tests/utils/executorUtils.h @@ -0,0 +1,18 @@ +#ifndef A073F2DA_315E_434B_B811_D420F0A59DF3 +#define A073F2DA_315E_434B_B811_D420F0A59DF3 + +#include "tensorrt_llm/batch_manager/common.h" +#include "tensorrt_llm/executor/executor.h" +#include <ratio> +#include <unordered_map> + +namespace tensorrt_llm::testing +{ + +std::unordered_map<batch_manager::RequestIdType, std::vector<executor::Response>> runThroughRequests( + executor::Executor& executor, std::vector<executor::Request> const& requests, + std::chrono::duration<float, std::milli> timeout); + +} // namespace tensorrt_llm::testing + +#endif /* A073F2DA_315E_434B_B811_D420F0A59DF3 */ diff --git a/docker/Dockerfile.multi b/docker/Dockerfile.multi index 6ac69ef9c4fb..fa01610fafd1 100644 --- a/docker/Dockerfile.multi +++ b/docker/Dockerfile.multi @@ -31,6 +31,7 @@ FROM base AS devel # NB: PyTorch requires this to be < 1.0 ENV PYTORCH_ALLOC_CONF="garbage_collection_threshold:0.99999" +ARG TRT_VER ARG CUDA_VER ARG CUDNN_VER ARG NCCL_VER @@ -44,11 +45,11 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \ echo "Using Python version: ${PYTHON_VERSION}" && \ GITHUB_MIRROR=${GITHUB_MIRROR} \ PYTHON_VERSION=${PYTHON_VERSION} \ - CUDA_VER=${CUDA_VER} CUDNN_VER=${CUDNN_VER} \ + TRT_VER=${TRT_VER} CUDA_VER=${CUDA_VER} CUDNN_VER=${CUDNN_VER} \ NCCL_VER=${NCCL_VER} CUBLAS_VER=${CUBLAS_VER} \ TORCH_INSTALL_TYPE=${TORCH_INSTALL_TYPE} \ bash /opt/docker/common/install.sh --base --cmake --ccache --cuda_toolkit \ - --cuda_libs --polygraphy --mpi4py --pytorch + --tensorrt --polygraphy --mpi4py --pytorch # Install constraints after install.sh so cleanup() doesn't delete the file mid-RUN COPY constraints.txt /tmp/constraints.txt @@ -118,7 +119,7 @@ COPY .gitmodules setup.py requirements.txt requirements-dev.txt constraints.txt ENV CCACHE_DIR=/root/.cache/ccache # Build the TRT-LLM wheel ARG GITHUB_MIRROR="" -ARG BUILD_WHEEL_ARGS="--clean" +ARG BUILD_WHEEL_ARGS="--clean --benchmarks" ARG BUILD_WHEEL_SCRIPT="scripts/build_wheel.py" RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=${CCACHE_DIR} \ GITHUB_MIRROR=$GITHUB_MIRROR python3 ${BUILD_WHEEL_SCRIPT} ${BUILD_WHEEL_ARGS} @@ -127,9 +128,7 @@ FROM ${DEVEL_IMAGE} AS release WORKDIR /app/tensorrt_llm RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,from=wheel,source=/src/tensorrt_llm/build,target=/tmp/wheel \ - TRTLLM_WHEEL=$(find /tmp/wheel -maxdepth 1 -name 'tensorrt_llm*.whl' -print -quit) && \ - test -n "${TRTLLM_WHEEL}" && \ - pip install "${TRTLLM_WHEEL}[mx]" + pip install /tmp/wheel/tensorrt_llm*.whl RUN --mount=type=bind,source=README.md,target=/mnt/ctx/README.md \ --mount=type=bind,source=docs,target=/mnt/ctx/docs \ @@ -137,6 +136,7 @@ RUN --mount=type=bind,source=README.md,target=/mnt/ctx/README.md \ --mount=type=bind,source=examples,target=/mnt/ctx/examples \ --mount=type=bind,from=wheel,source=/src/tensorrt_llm/build,target=/mnt/wheel \ --mount=type=bind,from=wheel,source=/src/tensorrt_llm/benchmarks,target=/mnt/benchmarks \ + --mount=type=bind,from=wheel,source=/src/tensorrt_llm/cpp/build/benchmarks,target=/mnt/cpp_benchmarks \ # Copy build context files cp /mnt/ctx/README.md ./ && \ cp -r /mnt/ctx/docs ./docs && \ @@ -146,12 +146,24 @@ RUN --mount=type=bind,source=README.md,target=/mnt/ctx/README.md \ # Copy wheel stage outputs cp /mnt/wheel/tensorrt_llm*.whl ./ && \ cp -r /mnt/benchmarks ./benchmarks && \ - # Create a symlink to installed package libraries + mkdir -p benchmarks/cpp && \ + cp /mnt/cpp_benchmarks/bertBenchmark \ + /mnt/cpp_benchmarks/gptManagerBenchmark \ + /mnt/cpp_benchmarks/disaggServerBenchmark \ + benchmarks/cpp/ && \ + rm -v \ + benchmarks/cpp/bertBenchmark.cpp \ + benchmarks/cpp/gptManagerBenchmark.cpp \ + benchmarks/cpp/disaggServerBenchmark.cpp \ + benchmarks/cpp/CMakeLists.txt && \ + # Create symlinks to installed package binaries and libraries + ln -sv $(python3 -c 'import site; print(f"{site.getsitepackages()[0]}/tensorrt_llm/bin")') bin && \ + test -f bin/executorWorker && \ ln -sv $(python3 -c 'import site; print(f"{site.getsitepackages()[0]}/tensorrt_llm/libs")') lib && \ - test -f lib/libtensorrt_llm.so && \ + test -f lib/libnvinfer_plugin_tensorrt_llm.so && \ echo "/app/tensorrt_llm/lib" > /etc/ld.so.conf.d/tensorrt_llm.conf && \ ldconfig && \ - ! ( ldd -v lib/libth_common.so | grep tensorrt_llm | grep -q "not found" ) && \ + ! ( ldd -v bin/executorWorker | grep tensorrt_llm | grep -q "not found" ) && \ # Clean up caches and CVE workarounds rm -rf /root/.cache/uv/archive-v0 && \ # WAR against https://github.com/advisories/GHSA-58pv-8j8x-9vj2 diff --git a/docker/Makefile b/docker/Makefile index 7c8568ffe54c..0ca78eef80a0 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -47,6 +47,7 @@ CUDA_VERSION ?= CUDNN_VERSION ?= NCCL_VERSION ?= CUBLAS_VERSION ?= +TRT_VERSION ?= GIT_COMMIT ?= $(shell git rev-parse HEAD) TRT_LLM_VERSION ?= $(shell grep '^__version__' ../tensorrt_llm/version.py | grep -o '=.*' | tr -d '= "') GITHUB_MIRROR ?= @@ -98,6 +99,7 @@ base_pull: $(if $(CUDNN_VERSION), --build-arg CUDNN_VER="$(CUDNN_VERSION)") \ $(if $(NCCL_VERSION), --build-arg NCCL_VER="$(NCCL_VERSION)") \ $(if $(CUBLAS_VERSION), --build-arg CUBLAS_VER="$(CUBLAS_VERSION)") \ + $(if $(TRT_VERSION), --build-arg TRT_VER="$(TRT_VERSION)") \ $(if $(TRT_LLM_VERSION), --build-arg TRT_LLM_VER="$(TRT_LLM_VERSION)") \ $(if $(DEVEL_IMAGE), --build-arg DEVEL_IMAGE="$(DEVEL_IMAGE)") \ $(if $(GIT_COMMIT), --build-arg GIT_COMMIT="$(GIT_COMMIT)") \ diff --git a/docker/common/install.sh b/docker/common/install.sh index 5f0dc91c8ed8..0d12d812a354 100755 --- a/docker/common/install.sh +++ b/docker/common/install.sh @@ -11,7 +11,7 @@ base=0 cmake=0 ccache=0 cuda_toolkit=0 -cuda_libs=0 +tensorrt=0 polygraphy=0 mpi4py=0 pytorch=0 @@ -34,8 +34,8 @@ while [[ $# -gt 0 ]]; do cuda_toolkit=1 shift 1 ;; - --cuda_libs) - cuda_libs=1 + --tensorrt) + tensorrt=1 shift 1 ;; --polygraphy) @@ -55,7 +55,7 @@ while [[ $# -gt 0 ]]; do cmake=1 ccache=1 cuda_toolkit=1 - cuda_libs=1 + tensorrt=1 polygraphy=1 mpi4py=1 pytorch=1 @@ -92,9 +92,10 @@ if [ $cuda_toolkit -eq 1 ]; then bash $SCRIPT_DIR/install_cuda_toolkit.sh fi -if [ $cuda_libs -eq 1 ]; then - echo "Installing CUDA libraries (cuDNN/NCCL/cuBLAS)..." - bash $SCRIPT_DIR/install_cuda_libs.sh \ +if [ $tensorrt -eq 1 ]; then + echo "Installing TensorRT..." + bash $SCRIPT_DIR/install_tensorrt.sh \ + --TRT_VER=${TRT_VER} \ --CUDA_VER=${CUDA_VER} \ --CUDNN_VER=${CUDNN_VER} \ --NCCL_VER=${NCCL_VER} \ diff --git a/docker/common/install_cuda_libs.sh b/docker/common/install_tensorrt.sh similarity index 81% rename from docker/common/install_cuda_libs.sh rename to docker/common/install_tensorrt.sh index e5a8566c62d3..0c762e0a3ad3 100644 --- a/docker/common/install_cuda_libs.sh +++ b/docker/common/install_tensorrt.sh @@ -2,6 +2,7 @@ set -ex +TRT_VER="10.16.1.11" # Align with the pre-installed cuDNN / cuBLAS / NCCL versions from # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 CUDA_VER="13.2" # 13.2.1 @@ -18,6 +19,7 @@ CUDA_DRIVER_VERSION="595.58.03-1.el8" for i in "$@"; do case $i in + --TRT_VER=?*) TRT_VER="${i#*=}";; --CUDA_VER=?*) CUDA_VER="${i#*=}";; --CUDNN_VER=?*) CUDNN_VER="${i#*=}";; --NCCL_VER=?*) NCCL_VER="${i#*=}";; @@ -164,14 +166,47 @@ install_rockylinux_requirements() { ldconfig } +install_tensorrt() { + PY_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))') + PARSED_PY_VERSION=$(echo "${PY_VERSION//./}") + + TRT_CUDA_VERSION=${CUDA_VER} + TRT_VER_SHORT=$(echo $TRT_VER | cut -d. -f1-3) + + if [ -z "$RELEASE_URL_TRT" ];then + ARCH=${TRT_TARGETARCH} + if [ -z "$ARCH" ];then ARCH=$(uname -m);fi + if [ "$ARCH" = "arm64" ];then ARCH="aarch64";fi + if [ "$ARCH" = "amd64" ];then ARCH="x86_64";fi + RELEASE_URL_TRT="https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/${TRT_VER_SHORT}/tars/TensorRT-${TRT_VER}.Linux.${ARCH}-gnu.cuda-${TRT_CUDA_VERSION}.tar.gz" + fi + + wget --retry-connrefused --timeout=180 --tries=10 --continue ${RELEASE_URL_TRT} -O /tmp/TensorRT.tar + tar -xf /tmp/TensorRT.tar -C /usr/local/ + mv /usr/local/TensorRT-${TRT_VER} /usr/local/tensorrt + pip3 install --no-cache-dir /usr/local/tensorrt/python/tensorrt-*-cp${PARSED_PY_VERSION}-*.whl + rm -rf /tmp/TensorRT.tar + echo 'export LD_LIBRARY_PATH=/usr/local/tensorrt/lib:$LD_LIBRARY_PATH' >> "${ENV}" + + rm -f /usr/local/tensorrt/lib/libnvinfer_vc_plugin_static.a \ + /usr/local/tensorrt/lib/libnvinfer_plugin_static.a \ + /usr/local/tensorrt/lib/libnvinfer_static.a \ + /usr/local/tensorrt/lib/libnvinfer_dispatch_static.a \ + /usr/local/tensorrt/lib/libnvinfer_lean_static.a \ + /usr/local/tensorrt/lib/libnvonnxparser_static.a \ + /usr/local/tensorrt/lib/libnvinfer_builder_resource_win.so.* +} + # Install base packages depending on the base OS ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') case "$ID" in ubuntu) install_ubuntu_requirements + install_tensorrt ;; rocky) install_rockylinux_requirements + install_tensorrt ;; *) echo "Unable to determine OS..." diff --git a/docs/source/_ext/llmapi_config_telemetry.py b/docs/source/_ext/llmapi_config_telemetry.py index 0b0bf9e63427..01509c74a628 100644 --- a/docs/source/_ext/llmapi_config_telemetry.py +++ b/docs/source/_ext/llmapi_config_telemetry.py @@ -76,16 +76,19 @@ def _table(rows: list[dict]) -> str: def generate_telemetry_reference(repo_root: Path | str, output_path: Path | str) -> None: repo_root = Path(repo_root) golden = json.loads((repo_root / _GOLDEN_REL).read_text()) - rows = golden.get("TorchLlmArgs", []) - content = [ - _REFERENCE_PREAMBLE, - "### `TorchLlmArgs`", - "", - f"{len(rows)} captured fields.", - "", - _table(rows), - "", - ] + content = [_REFERENCE_PREAMBLE] + for args_class in ("TorchLlmArgs", "TrtLlmArgs"): + rows = golden.get(args_class, []) + content.extend( + [ + f"### `{args_class}`", + "", + f"{len(rows)} captured fields.", + "", + _table(rows), + "", + ] + ) output = Path(output_path) output.parent.mkdir(parents=True, exist_ok=True) output.write_text("\n".join(content)) diff --git a/docs/source/_static/config_db.json b/docs/source/_static/config_db.json index 8af081380d9c..7671934a6911 100644 --- a/docs/source/_static/config_db.json +++ b/docs/source/_static/config_db.json @@ -179,30 +179,6 @@ "model_display_name": "MiniMax-M3 (MXFP8)", "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8", "scenario": "Max Throughput" - }, - { - "command": "trtllm-serve deepseek-ai/DeepSeek-V4-Pro --config ${TRTLLM_DIR}/examples/configs/curated/deepseek-v4-pro-latency.yaml", - "config_filename": "deepseek-v4-pro-latency.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-v4-pro-latency.yaml", - "config_path": "examples/configs/curated/deepseek-v4-pro-latency.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/curated/deepseek-v4-pro-latency.yaml", - "gpu_compatibility": "B200", - "model": "deepseek-ai/DeepSeek-V4-Pro", - "model_display_name": "deepseek-ai/DeepSeek-V4-Pro", - "model_url": "", - "scenario": "Min Latency" - }, - { - "command": "trtllm-serve deepseek-ai/DeepSeek-V4-Pro --config ${TRTLLM_DIR}/examples/configs/curated/deepseek-v4-pro-throughput.yaml", - "config_filename": "deepseek-v4-pro-throughput.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-v4-pro-throughput.yaml", - "config_path": "examples/configs/curated/deepseek-v4-pro-throughput.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/curated/deepseek-v4-pro-throughput.yaml", - "gpu_compatibility": "B200", - "model": "deepseek-ai/DeepSeek-V4-Pro", - "model_display_name": "deepseek-ai/DeepSeek-V4-Pro", - "model_url": "", - "scenario": "Max Throughput" } ], "entries": [ @@ -1345,82 +1321,6 @@ "osl": 1024, "performance_profile": "Max Throughput" }, - { - "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", - "concurrency": 4, - "config_filename": "1k1k_tp4_conc4.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", - "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml", - "gpu": "B200_NVL", - "gpu_display": "4xB200_NVL", - "isl": 1024, - "model": "nvidia/DeepSeek-R1-0528-FP4-v2", - "model_display_name": "DeepSeek-R1 (NVFP4)", - "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", - "num_gpus": 4, - "osl": 1024, - "performance_profile": "Min Latency", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" - }, - { - "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", - "concurrency": 8, - "config_filename": "1k1k_tp4_conc8.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", - "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml", - "gpu": "B200_NVL", - "gpu_display": "4xB200_NVL", - "isl": 1024, - "model": "nvidia/DeepSeek-R1-0528-FP4-v2", - "model_display_name": "DeepSeek-R1 (NVFP4)", - "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", - "num_gpus": 4, - "osl": 1024, - "performance_profile": "Balanced", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" - }, - { - "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", - "concurrency": 16, - "config_filename": "1k1k_tp4_conc16.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", - "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml", - "gpu": "B200_NVL", - "gpu_display": "4xB200_NVL", - "isl": 1024, - "model": "nvidia/DeepSeek-R1-0528-FP4-v2", - "model_display_name": "DeepSeek-R1 (NVFP4)", - "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", - "num_gpus": 4, - "osl": 1024, - "performance_profile": "Balanced", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" - }, - { - "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", - "concurrency": 256, - "config_filename": "1k1k_tp4_conc256.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", - "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml", - "gpu": "B200_NVL", - "gpu_display": "4xB200_NVL", - "isl": 1024, - "model": "nvidia/DeepSeek-R1-0528-FP4-v2", - "model_display_name": "DeepSeek-R1 (NVFP4)", - "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", - "num_gpus": 4, - "osl": 1024, - "performance_profile": "Max Throughput", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" - }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp4_conc2048.yaml", "concurrency": 2048, @@ -1438,82 +1338,6 @@ "osl": 8192, "performance_profile": "High Throughput" }, - { - "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", - "concurrency": 4, - "config_filename": "8k1k_tp4_conc4.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", - "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml", - "gpu": "B200_NVL", - "gpu_display": "4xB200_NVL", - "isl": 8192, - "model": "nvidia/DeepSeek-R1-0528-FP4-v2", - "model_display_name": "DeepSeek-R1 (NVFP4)", - "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", - "num_gpus": 4, - "osl": 1024, - "performance_profile": "Min Latency", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" - }, - { - "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", - "concurrency": 8, - "config_filename": "8k1k_tp4_conc8.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", - "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml", - "gpu": "B200_NVL", - "gpu_display": "4xB200_NVL", - "isl": 8192, - "model": "nvidia/DeepSeek-R1-0528-FP4-v2", - "model_display_name": "DeepSeek-R1 (NVFP4)", - "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", - "num_gpus": 4, - "osl": 1024, - "performance_profile": "Low Latency", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" - }, - { - "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", - "concurrency": 16, - "config_filename": "8k1k_tp4_conc16.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", - "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml", - "gpu": "B200_NVL", - "gpu_display": "4xB200_NVL", - "isl": 8192, - "model": "nvidia/DeepSeek-R1-0528-FP4-v2", - "model_display_name": "DeepSeek-R1 (NVFP4)", - "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", - "num_gpus": 4, - "osl": 1024, - "performance_profile": "Balanced", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" - }, - { - "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", - "concurrency": 256, - "config_filename": "8k1k_tp4_conc256.yaml", - "config_github_url": "https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", - "config_path": "examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", - "config_raw_url": "https://raw.githubusercontent.com/NVIDIA/TensorRT-LLM/main/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml", - "gpu": "B200_NVL", - "gpu_display": "4xB200_NVL", - "isl": 8192, - "model": "nvidia/DeepSeek-R1-0528-FP4-v2", - "model_display_name": "DeepSeek-R1 (NVFP4)", - "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", - "num_gpus": 4, - "osl": 1024, - "performance_profile": "Balanced", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" - }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc1024.yaml", "concurrency": 1024, @@ -1529,7 +1353,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 4, "osl": 1024, - "performance_profile": "High Throughput" + "performance_profile": "Min Latency" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc2048.yaml", @@ -1597,9 +1421,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Low Latency", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "Low Latency" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml", @@ -1650,9 +1472,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Balanced", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "Balanced" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml", @@ -1669,9 +1489,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Balanced", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "Balanced" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml", @@ -1688,9 +1506,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "High Throughput" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml", @@ -1707,9 +1523,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "High Throughput" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc512.yaml", @@ -1998,9 +1812,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "Low Latency", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "Low Latency" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml", @@ -2068,9 +1880,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "High Throughput" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml", @@ -2087,9 +1897,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "High Throughput" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml", @@ -2106,9 +1914,7 @@ "model_url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", "num_gpus": 8, "osl": 1024, - "performance_profile": "High Throughput", - "validated_trtllm_commit": "93cb6518b6d6dbd6095748189e626db731f44545", - "validated_trtllm_version": "1.3.0rc14" + "performance_profile": "High Throughput" }, { "command": "trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --config ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc512.yaml", @@ -3726,10 +3532,6 @@ "display_name": "DeepSeek-R1", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" }, - "deepseek-ai/DeepSeek-V4-Pro": { - "display_name": "deepseek-ai/DeepSeek-V4-Pro", - "url": "" - }, "nvidia/DeepSeek-R1-0528-FP4-v2": { "display_name": "DeepSeek-R1 (NVFP4)", "url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2" diff --git a/docs/source/_static/config_selector.css b/docs/source/_static/config_selector.css index 43c5a4aa7bc6..ca84bb991f2a 100644 --- a/docs/source/_static/config_selector.css +++ b/docs/source/_static/config_selector.css @@ -1,6 +1,3 @@ -/* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ -/* SPDX-License-Identifier: Apache-2.0 */ - .trtllm-config-selector { border: 1px solid var(--pst-color-shadow); border-radius: 14px; @@ -32,10 +29,6 @@ align-items: start; } -.trtllm-config-selector__field[hidden] { - display: none; -} - .trtllm-config-selector__label { display: flex; align-items: center; diff --git a/docs/source/_static/config_selector.js b/docs/source/_static/config_selector.js index ca053655923a..e8ef62550a6b 100644 --- a/docs/source/_static/config_selector.js +++ b/docs/source/_static/config_selector.js @@ -1,18 +1,14 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - (function () { "use strict"; let dbPromise = null; let widgetId = 0; - const GROUP_ORDER = ["model", "topology", "islOsl", "concurrency", "profile"]; + const GROUP_ORDER = ["model", "topology", "islOsl", "concurrency"]; const GROUP_LABELS = { model: "Model", topology: "GPU(s)", islOsl: "ISL / OSL", concurrency: "Concurrency", - profile: "Profile", }; function $(root, sel) { @@ -62,7 +58,6 @@ state.concurrency != null && state.concurrency !== "" ? String(state.concurrency) : "", - profile: state.profile || "", }; } @@ -143,22 +138,9 @@ function profileLabel(profile) { const text = String(profile || "").trim(); - const labels = { - latency: "Latency", - balanced: "Balanced", - throughput: "Throughput", - }; - if (labels[text]) return labels[text]; return text || "Unknown Profile"; } - function profileOption(profile) { - return { - value: profile, - label: profileLabel(profile), - }; - } - function formatProfileSummary(profiles) { const labels = uniqBy(profiles.map((profile) => profileLabel(profile)), (label) => label); if (!labels.length) { @@ -212,12 +194,7 @@ .map((group) => concurrencyOption(group)) .sort((a, b) => sortNums(a.concurrency, b.concurrency)); - const profile = uniqBy( - entries.filter((entry) => entry.profile).map((entry) => profileOption(entry.profile)), - (option) => option.value - ); - - return { model, topology, islOsl, concurrency, profile }; + return { model, topology, islOsl, concurrency }; } function filterEntriesByState(entries, state) { @@ -237,9 +214,6 @@ if (normalizedState.concurrency) { if (String(entry.concurrency) !== normalizedState.concurrency) return false; } - if (normalizedState.profile && entry.profile !== normalizedState.profile) { - return false; - } return true; }); } @@ -388,8 +362,7 @@ normalizedState.model && normalizedState.topology && normalizedState.islOsl && - normalizedState.concurrency && - (!groups.profile.options.length || normalizedState.profile) + normalizedState.concurrency ) { return "Selection did not resolve to a single configuration."; } @@ -416,12 +389,6 @@ }; } - function validatedCommitUrl(commit) { - const normalized = String(commit || "").trim().toLowerCase(); - if (!/^[0-9a-f]{40}$/.test(normalized)) return ""; - return `https://github.com/NVIDIA/TensorRT-LLM/commit/${normalized}`; - } - function isFileProtocol() { return window.location.protocol === "file:"; } @@ -641,14 +608,14 @@ : allCurated ).map(normalizeEntry); - // curatedIndex lives outside normalizeState's scope and is preserved - // across Object.assign(state, view.state). + // curatedIndex lives outside normalizeState's scope — it is preserved + // across Object.assign(state, view.state) because normalizeState only + // touches the four filter keys. const state = { model: "", topology: "", islOsl: "", concurrency: "", - profile: "", curatedIndex: null, }; @@ -718,7 +685,6 @@ const selTopo = mkOptionGroup("GPU(s)", 2); const selSeq = mkOptionGroup("ISL / OSL", 3); const selConc = mkSelectField("Concurrency", `trtllm-conc-${id}`, 4); - const selProfile = mkSelectField("Profile", `trtllm-profile-${id}`, 5); form.appendChild(selModel.wrap); @@ -741,7 +707,6 @@ form.appendChild(selTopo.wrap); form.appendChild(selSeq.wrap); form.appendChild(selConc.wrap); - form.appendChild(selProfile.wrap); const output = el("div", { class: "trtllm-config-selector__output" }); const cmdPre = el("pre", { class: "trtllm-config-selector__cmd" }, [ @@ -870,10 +835,7 @@ return { type: "model-select" }; } if (activeEl === selConc.select) { - return { type: "select", key: "concurrency" }; - } - if (activeEl === selProfile.select) { - return { type: "select", key: "profile" }; + return { type: "select" }; } if ( activeEl.classList && @@ -895,8 +857,7 @@ return; } if (descriptor.type === "select") { - if (descriptor.key === "profile") selProfile.select.focus(); - else selConc.select.focus(); + selConc.select.focus(); return; } if (descriptor.type !== "button") return; @@ -1036,8 +997,8 @@ selectEl.dataset.status = (selectedOption && selectedOption.status) || "idle"; } - function setSelectOptions(selectEl, group, stateKey, placeholder) { - const previousValue = state[stateKey] || ""; + function setSelectOptions(selectEl, group) { + const previousValue = state.concurrency || ""; const visibleOptions = group.options.filter( (option) => option.status !== "incompatible" ); @@ -1045,7 +1006,7 @@ selectEl.appendChild( el("option", { value: "", - text: visibleOptions.length ? `Select ${placeholder}` : `No ${placeholder} available`, + text: visibleOptions.length ? "Select concurrency" : "No concurrency available", }) ); for (const option of visibleOptions) { @@ -1116,12 +1077,7 @@ setOptionButtons(selTopo.options, "topology", view.groups.topology); setOptionButtons(selSeq.options, "islOsl", view.groups.islOsl); - setSelectOptions(selConc.select, view.groups.concurrency, "concurrency", "concurrency"); - const hasProfileChoices = - Boolean(view.state.concurrency) && - view.groups.profile.options.some((option) => option.status !== "incompatible"); - selProfile.wrap.hidden = !hasProfileChoices; - setSelectOptions(selProfile.select, view.groups.profile, "profile", "profile"); + setSelectOptions(selConc.select, view.groups.concurrency); const code = cmdPre.querySelector("code"); if (curatedSelected) { @@ -1176,26 +1132,6 @@ } else { meta.appendChild(el("span", { text: e.config_path || "" })); } - if (e.validated_trtllm_version) { - meta.appendChild( - el("span", { - text: ` \u00b7 Validated with TensorRT-LLM ${e.validated_trtllm_version}`, - }) - ); - } - const commitUrl = validatedCommitUrl(e.validated_trtllm_commit); - if (commitUrl) { - meta.appendChild(el("span", { text: " \u00b7 Commit: " })); - meta.appendChild( - el("a", { - class: "trtllm-config-selector__configLink", - href: commitUrl, - target: "_blank", - rel: "noopener", - text: e.validated_trtllm_commit.slice(0, 12), - }) - ); - } currentEntry = e; resetYamlPanel(); @@ -1225,12 +1161,6 @@ selConc.select.addEventListener("change", () => { state.concurrency = selConc.select.value; - state.profile = ""; - render(); - }); - - selProfile.select.addEventListener("change", () => { - state.profile = selProfile.select.value; render(); }); @@ -1288,7 +1218,6 @@ formatCuratedCommand: formatCommand, nextStateAfterSelection, normalizeState, - validatedCommitUrl, }; } diff --git a/docs/source/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.md b/docs/source/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.md index 91d89d70cacf..22d4688c503d 100644 --- a/docs/source/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.md +++ b/docs/source/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.md @@ -92,7 +92,7 @@ Here we set `LOCAL_USER=1` argument to set up the local user instead of root acc Here we compile the source inside the container: ``` bash -python3 ./scripts/build_wheel.py --cuda_architectures "90-real;100-real" --clean +python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --cuda_architectures "90-real;100-real" --python_bindings --clean ``` You can set the cuda_architectures to "100-real" if targeting Blackwell only, and "90-real" to target Hopper only to save some build time. diff --git a/docs/source/blogs/media/tech_blog26_agentperf_closed_loop_workflow.svg b/docs/source/blogs/media/tech_blog26_agentperf_closed_loop_workflow.svg deleted file mode 100644 index f9d0003051bd..000000000000 --- a/docs/source/blogs/media/tech_blog26_agentperf_closed_loop_workflow.svg +++ /dev/null @@ -1,156 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- -Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -SPDX-License-Identifier: Apache-2.0 ---> -<svg xmlns="http://www.w3.org/2000/svg" width="1800" height="815" viewBox="0 0 1800 815" role="img" aria-labelledby="title desc"> - <title id="title">AA-AgentPerf closed-loop request workflow - A simulated agent sends the next recorded turn with its accumulated conversation. The inference deployment routes the request using conversation or cache affinity, reuses matching KV blocks, prefills only the uncached suffix, optionally retrieves the KV cache on the generation side in disaggregated serving, and streams the generated response. The client drains the complete response, simulates tool time when needed, advances the trajectory, and sends the next turn. - - - - - - - - - - - - - - - - - - - - - - - - - - CLIENT / AGENT HARNESS - - - - - INFERENCE DEPLOYMENT - - - - - 1 - Send recorded turn - Accumulated conversation - and the next request - - - - - 8 - Advance trajectory - Next recorded turn or - next assigned trajectory - - - - - 7 - Simulate tool time - Only for turns with a tool call - Client-side delay · no LLM compute - - - - - 6 - Receive complete response - Drain the stream and assemble chunks - Record TTFT and output speed - - - - - - - - - - 2 - Route + prefix lookup - Conversation-aware or cache-aware - worker and rank selection - Reuse matching resident KV blocks - - - - - 3 - Prefill uncached suffix - Reuse removes the cached prefix - from new prefill computation - Context side produces the first token - - - - - 4 - Retrieve KV cache - Returned context metadata locates - the KV cache for the generation side - Disaggregated serving only - - - - - 5 - Generate + stream - Continue decoding the response - on the same worker or a - separate generation worker - - - - - - - - - Disaggregated path - - - - Aggregated serving · same worker - - - - - - - Conversation affinity helps preserve reuse across turns - Later turns prefer the worker and rank that hold the accumulated conversation prefix. - - - - - Inference work - - - Client-simulated tool time - - - Conversation affinity and prefix reuse - - - diff --git a/docs/source/blogs/media/tech_blog26_deepseek_v4_hybrid_attention.png b/docs/source/blogs/media/tech_blog26_deepseek_v4_hybrid_attention.png deleted file mode 100644 index 8238521bcb40..000000000000 --- a/docs/source/blogs/media/tech_blog26_deepseek_v4_hybrid_attention.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0b9b5b49bba503ffcd6b0fb7993cf25e2c6537098ad4ffc232e9a89890fc218f -size 13509614 diff --git a/docs/source/blogs/media/tech_blog26_deepseek_v4_mhc_moe.png b/docs/source/blogs/media/tech_blog26_deepseek_v4_mhc_moe.png deleted file mode 100644 index 32a68bbe5038..000000000000 --- a/docs/source/blogs/media/tech_blog26_deepseek_v4_mhc_moe.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a034c944f943400df7b42b5a1f2321f7d3ee87eb80bd5c509dba5066cec5eee4 -size 1854626 diff --git a/docs/source/blogs/media/tech_blog26_dsv4_performance_evolution_8k1k.png b/docs/source/blogs/media/tech_blog26_dsv4_performance_evolution_8k1k.png deleted file mode 100644 index 3fbaede6ebe4..000000000000 Binary files a/docs/source/blogs/media/tech_blog26_dsv4_performance_evolution_8k1k.png and /dev/null differ diff --git a/docs/source/blogs/media/tech_blog26_dsv4_scratch_swa.png b/docs/source/blogs/media/tech_blog26_dsv4_scratch_swa.png deleted file mode 100644 index 37e806c18c77..000000000000 Binary files a/docs/source/blogs/media/tech_blog26_dsv4_scratch_swa.png and /dev/null differ diff --git a/docs/source/blogs/media/tech_blog26_two_level_routing.svg b/docs/source/blogs/media/tech_blog26_two_level_routing.svg deleted file mode 100644 index f032ffafa1ba..000000000000 --- a/docs/source/blogs/media/tech_blog26_two_level_routing.svg +++ /dev/null @@ -1,167 +0,0 @@ - - - - Two-level routing for context locality in disaggregated serving - An agentic request enters the TensorRT LLM disaggregated front-end. Instance-level routing first selects a context server, then the selected server's attention data-parallel router performs rank-level routing. Each level can use conversation-aware affinity or KV-cache-aware scoring. Full conversation-prefix reuse requires a later turn to return to both the context server and the rank that hold the corresponding KV blocks. The selected context rank transfers KV cache to a generation server, which decodes and streams the response. - - - - - - - - - - - - - - - - - - - - - - - - - - - - AGENT CLIENT - Next turn - - Conversation ID - - Accumulated prompt - - Cache salt (optional) - - - - request - - - - - TRTLLM-SERVE DISAGGREGATED - Front-end orchestrator - - LEVEL 1 · INSTANCE-LEVEL ROUTING - Use either routing policy to select a context server - - - - CONVERSATION-AWARE - Conversation ID → CTX server - Least-loaded first placement, then explicit affinity - - - - KV-CACHE-AWARE - Prompt blocks → cache view - Matched blocks plus active-request load - Content-derived affinity protects later turns - - - Selected destination: CTX server 1 - - - - - CONTEXT (CTX) TIER - - - - CTX server 0 - other rank-local caches - - CTX server 2 - other rank-local caches - - - - - SELECTED - CTX server 1 - - - - Attention data-parallel router - - LEVEL 2 · RANK-LEVEL ROUTING - Again, use either routing policy inside the selected server - - - CONVERSATION-AWARE - Conversation ID → rank - First turn: round-robin - - - KV-CACHE-AWARE - Rank-local prefix probe - Post-reuse work plus load - - - - - - rank-local reuse lost - - - Rank 0 - KV cache - - - Rank 1 - - prefix KV - - - Rank 2 - KV cache - - - Rank 3 - KV cache - - - - same server - - server-local - reuse lost - - - - - GEN SERVER - Generation - - Receive KV cache - - Continue decode - - Stream response - - - CTX → GEN KV transfer - - - - streamed response - diff --git a/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md b/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md deleted file mode 100644 index 260b94448f42..000000000000 --- a/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md +++ /dev/null @@ -1,490 +0,0 @@ - - -# DeepSeek-V4 on NVIDIA Blackwell: Model-Specific and Agentic-Workload Optimizations in TensorRT LLM - -By NVIDIA TensorRT LLM Team - -## Table of Contents - -- [Introduction](#introduction) -- [Part I. DeepSeek-V4 Model Support and Optimizations](#part-i-deepseek-v4-model-support-and-optimizations) - - [Building a Production-Ready DeepSeek-V4 Stack](#building-a-production-ready-deepseek-v4-stack) - - [Model support and parallelism](#model-support-and-parallelism) - - [Hybrid attention: SWA, CSA, and HCA](#hybrid-attention-swa-csa-and-hca) - - [Compressor implementation](#compressor-implementation) - - [Beyond attention: mHC, MoE, and speculative decoding](#beyond-attention-mhc-moe-and-speculative-decoding) - - [Parallelism and deployment notation](#parallelism-and-deployment-notation) - - [Cache management and runtime features](#cache-management-and-runtime-features) - - [Precision strategy and accuracy](#precision-strategy-and-accuracy) - - [DeepSeek-V4 Performance Optimizations](#deepseek-v4-performance-optimizations) - - [Optimize the CSA/HCA hot path](#optimize-the-csahca-hot-path) - - [Compressor and mHC](#compressor-and-mhc) - - [Compressor optimizations](#compressor-optimizations) - - [mHC optimizations](#mhc-optimizations) - - [MoE optimizations](#moe-optimizations) - - [Runtime optimization](#runtime-optimization) - - [DeepSeek-V4 Performance Evolution](#deepseek-v4-performance-evolution) - - [Experimental setup](#experimental-setup) - - [From Baseline to the Latest Measured Curve](#from-baseline-to-the-latest-measured-curve) -- [Part II. Agentic-Workload Optimizations](#part-ii-agentic-workload-optimizations) - - [AgentPerf Workflow](#agentperf-workflow) - - [Lessons from DeepSeek-V3.2 Agentic Workload Optimization](#lessons-from-deepseek-v32-agentic-workload-optimization) - - [End-to-End Optimizations for Agentic Serving](#end-to-end-optimizations-for-agentic-serving) - - [Preserve locality with two-level routing](#preserve-locality-with-two-level-routing) - - [Instance-level routing across context servers](#instance-level-routing-across-context-servers) - - [Rank-level routing within a context server](#rank-level-routing-within-a-context-server) - - [Routing policy selection](#routing-policy-selection) - - [KV cache reuse optimizations](#kv-cache-reuse-optimizations) - - [Optimize scheduling and remove end-to-end overhead](#optimize-scheduling-and-remove-end-to-end-overhead) - - [Optimize host overhead for CTX and GEN](#optimize-host-overhead-for-ctx-and-gen) - - [Optimize scheduling, orchestration, and protocol handling](#optimize-scheduling-orchestration-and-protocol-handling) - - [AgentPerf Results and External Validation](#agentperf-results-and-external-validation) -- [Reproduction and Future Work](#reproduction-and-future-work) - - [How to reproduce](#how-to-reproduce) - - [Future Work](#future-work) -- [Conclusion and Acknowledgments](#conclusion-and-acknowledgments) - -## Introduction - -DeepSeek-V4's hybrid attention, online compression, mHC, and MoE design make production inference a model-and-system co-design problem. This blog follows TensorRT LLM's optimization journey on NVIDIA Blackwell in two parts: first building and optimizing a production-ready DeepSeek-V4 model stack, then extending the optimization boundary to routing, KV reuse, host efficiency, and control-plane efficiency for agentic workloads. On GB300, the latest measured fixed-shape sweep increased peak output throughput from 984 to 1,618 tokens/s/GPU, a 64.5% improvement, while NVIDIA's GB300 AA-AgentPerf configurations, run and verified by Artificial Analysis, reached 57.5 concurrency per GPU (CPG) at SLO20 and 19.2 CPG at SLO60. - -## Part I. DeepSeek-V4 Model Support and Optimizations - -### Building a Production-Ready DeepSeek-V4 Stack - -Supporting DeepSeek-V4 required more than registering a new model class. Relative to DeepSeek-V3.2, the model replaces homogeneous sparse attention with a hybrid scheme that interleaves three attention modes across layers, adds an online sequence Compressor with its own persistent state, wraps every attention and MoE block in manifold-constrained hyper-connections (mHC), and updates the MoE routing and checkpoint layout. All of this must compose with the production features users expect from TensorRT LLM: Multi-Token Prediction (MTP), parallel execution, chunked prefill, KV-cache reuse, CUDA Graphs, the overlap scheduler, and disaggregated serving. - -TensorRT LLM implements DeepSeek-V4 as a dedicated PyTorch-backend model, `DeepseekV4ForCausalLM`, together with a DeepSeek-V4 sparse-attention backend and a specialized `DeepseekV4CacheManager`, targeting NVIDIA Blackwell GPUs (SM100+). This section describes that foundation. The performance optimizations built on top of it are covered in [**DeepSeek-V4 Performance Optimizations**](#deepseek-v4-performance-optimizations). - -#### Model support and parallelism - -DeepSeek-V4 is released in two model scales, each with Base and Instruct variants. TensorRT LLM loads both directly from their checkpoint metadata. - -| Checkpoint family | Total parameters | Activated parameters | Published precision | -| :--- | ---: | ---: | :--- | -| DeepSeek-V4-Flash-Base / -Flash | 284B | 13B | FP8 mixed / FP4 + FP8 mixed | -| DeepSeek-V4-Pro-Base / -Pro | 1.6T | 49B | FP8 mixed / FP4 + FP8 mixed | - -The implementation reads the attention layout from the checkpoint rather than hard-coding serving assumptions: the per-layer compression-ratio list (Flash begins `[0, 0, 4, 128, ...]`, while Pro begins `[128, 128, 4, 128, ...]`, and 0 marks a sliding-window-only layer), the 128-token sliding window, and the Indexer configuration including its Top-K (512 for Flash and 1024 for Pro). - -##### Hybrid attention: SWA, CSA, and HCA - -DeepSeek-V4's largest architectural change is its attention. DeepSeek-V3.2 introduced DeepSeek Sparse Attention (DSA), in which a learned Indexer scores the KV history and a Top-K selector supplies token indices to sparse MLA (see our [DeepSeek-V3.2 optimization blog](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog15_Optimizing_DeepSeek_V32_on_NVIDIA_Blackwell_GPUs.md)). TensorRT LLM implemented DSA on a general [sparse-attention framework](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog17_Sparse_Attention_in_TensorRT-LLM.md) that separates the algorithm-specific selection logic from the attention backend that consumes the selected indices. DeepSeek-V4 builds on the same separation but adds online compression and interleaves three layer types, determined by each layer's compression ratio: - -| Attention mode | Ratio | History visible to each query | Selection and computation | -| :--- | ---: | :--- | :--- | -| Sliding Window Attention (SWA) | 0 | The latest 128 raw tokens | Dense attention inside the window. No Compressor or Indexer. | -| Compressed Sparse Attention (CSA) | 4 | Latest 128 raw tokens + 4× compressed entries from completed compression groups | The Indexer scores the compressed entries. Sparse MLA attends to the window plus the Top-K selected entries. | -| Heavily Compressed Attention (HCA) | 128 | Latest 128 raw tokens + all 128× compressed entries from completed compression groups | Dense attention over all completed compressed entries. No Indexer or Top-K stage. | - -
-
- DeepSeek-V4 hybrid attention architecture. Panel a shows SWA attending over the latest 128 raw tokens. Panel b shows CSA combining the sliding window with 4× compressed KV entries selected by an Indexer and Top-K stage. Panel c shows HCA combining the sliding window with all 128× compressed KV entries without an Indexer. -
-
-

Figure 1. DeepSeek-V4 hybrid attention. (a) SWA attends densely within the 128-token sliding window. (b) CSA augments that window with Top-K entries selected from the 4× compressed history. (c) HCA attends to the window and the complete 128× compressed history without an Indexer.

- -At the kernel boundary, the history is represented as two cache pools, one for the sliding window and one for compressed entries, and a unified dual-pool MLA operator serves all three modes: it processes the sliding window first, then the selected CSA entries or the full HCA compressed stream, and combines both in one online-softmax reduction. An SWA-only layer simply passes an empty second pool. Selected compressed positions are converted from cache-page-local offsets to token-level global addresses before the attention kernel consumes them. Because KV compression makes the attention workload vary by query, TensorRT LLM passes a per-query `topk_lens` vector (`sparse_mla_topk_lens` in the implementation) to the FMHA kernel. It identifies the active SWA-plus-compressed index span within the fixed-width index buffer used for CUDA Graph compatibility. - -Checkpoint-provided attention-sink logits are also remapped and supplied to the backend as an additional softmax sink. This dual-pool design lets the heterogeneous layer types share one attention backend, and extends the sparse-attention framework from DSA's fine-grained token selection to DeepSeek-V4's combination of local attention, compressed memory, and optional sparse selection. - -##### Compressor implementation - -The Compressor produces the compressed KV entries that CSA and HCA attend to. It is a token-level, softmax-gated pooling ([DeepSeek-V4 paper](https://arxiv.org/abs/2606.19348)): each token's projected KV entry carries a learned compression-weight vector, a learnable per-position bias is added, and a softmax over the compression window reduces the group to one entry as a weighted sum. For CSA ($m{=}4$), each output pools projected entries from two adjacent $m$-token groups, giving a $2m$-token receptive field. Consecutive outputs overlap by $m$ raw-token positions through separate projection branches. HCA instead uses disjoint $m'$-token windows ($m'{=}128$). On CSA layers the same compression operation is applied a second time to produce the compressed Indexer keys that the Indexer scores for Top-K selection, so a CSA layer maintains two compressed streams: the attention KV entries and the Indexer keys. - -TensorRT LLM implements the Compressor as a fused `wkv_gate` projection plus dedicated CUDA kernels for prefill reduction, paged decode updates, and cache-write postprocessing. Tokens arrive incrementally: one at a time in decode, and chunk by chunk in chunked prefill, where chunk boundaries rarely align with compression windows. A compression window is often still incomplete when a forward pass ends. TensorRT LLM persists each token's KV and score projection outputs in FP32 Compressor-state buffers for the still-open windows. Once a window completes, the compression kernels finalize its entry from this state, preserving the semantics of compressing the whole sequence in one pass. Compressed entries are written in the dtype their consumer expects: BF16 or per-tensor FP8 for the main attention cache, and blockwise FP8 or packed MXFP4 for the Indexer-K stream. The kernel fusion work on this path is covered in [**Compressor and mHC**](#compressor-and-mhc). - -##### Beyond attention: mHC, MoE, and speculative decoding - -**mHC hyper-connections** ([mHC paper](https://arxiv.org/abs/2512.24880)). mHC widens the residual stream by a factor of 4 and mixes it around every sublayer through three dynamically generated mappings: a **pre-mapping** that reads the sublayer input from the expanded residual, a **residual-mixing** matrix projected onto the manifold of doubly stochastic matrices through 20 Sinkhorn-Knopp iterations, and a **post-mapping** that writes the sublayer output back into the residual streams. TensorRT LLM instantiates two mHC modules per layer (one around attention and one around the MoE block), plus an **HC head** that collapses the expanded residual to a single stream before the language-model head. It fuses adjacent post- and pre-mappings at the attention-to-MoE boundary and, where possible, across consecutive transformer layers. - -**MoE.** Both model scales activate six routed experts and one shared expert per token, selecting from 256 routed experts in Flash and 384 in Pro. The first three layers use hash routing, where a checkpoint-provided token-ID-to-expert table maps each input token directly to its experts. Later layers use a learned gate with Sqrt-Softplus affinity scores and a score-correction bias. TensorRT LLM implements both routing modes through a shared interface that supplies the selected experts and routing weights to its configurable MoE backends. TRTLLM-Gen, DeepGEMM, and the standard CuTe DSL fused MoE backend cover the corresponding DeepSeek-V4 precision paths and preserve the checkpoint-defined `swiglu_limit` activation clamp, including the uniform clamp used by the NVFP4 CuTe DSL path. DeepSeek-V4 is also registered with TensorRT LLM's expert parallel load balancer, allowing supported expert-parallel deployments to rebalance physical expert placement without changing the model's routing semantics. - -
-
- DeepSeek-V4 mHC architecture and MoE routing. Panel a shows the repeated transformer stack with mHC pre- and post-mappings around the attention and MoE blocks, followed by the HC head and language-model head. Panel b shows hash routing through a token-ID embedding table and learned routing through Sqrt-Softplus scores, correction bias, and Top-K expert selection. -
-
-

Figure 2. DeepSeek-V4 components beyond attention. (a) mHC wraps the attention and MoE sublayers with pre- and post-mappings, then uses an HC head to collapse the expanded residual stream before the language-model head. (b) The MoE router supports checkpoint-defined hash routing in the first three layers (left) and learned Sqrt-Softplus scoring with correction bias and Top-6 selection in later layers (right).

- -**Speculative decoding.** DeepSeek-V4 checkpoints with next-token prediction layers run MTP speculative decoding through the Eagle-style one-model path. Each MTP module consumes the expanded residual state from the target model or previous MTP module together with the next-token embedding, then runs the same mHC-wrapped attention-and-MoE structure as a target-model layer, using SWA for its attention. TensorRT LLM remaps the MTP weights, shares the embedding and language-model head with the target model, and extends the attention and cache metadata for the configured draft depth. The speculative-decoding performance results reported in this blog use this MTP path. DeepSeek has released dedicated [DSpark](https://arxiv.org/abs/2607.05147) checkpoints for DeepSeek-V4. DSpark adds a lightweight sequential head to a parallel draft backbone to model dependencies within a draft block, while performance tuning for this path remains in progress. - -##### Parallelism and deployment notation - -DeepSeek-V4 adopts the same parallel strategies as DeepSeek-R1 and DeepSeek-V3.2: attention data parallelism (ADP) combined with expert parallelism (EP) for throughput-oriented serving, and tensor parallelism (TP) for latency-oriented deployments, with pipeline parallelism available for fitting the largest checkpoints. See [Tech Blog 3](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog03_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.md) for the performance rationale and [Tech Blog 4](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog04_Scaling_Expert_Parallelism_in_TensorRT-LLM.md)/[8](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog08_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.md)/[14](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.md) for large-scale expert parallelism. Note that with ADP, `max_batch_size` is a per-rank limit, so benchmark concurrency must be scaled to populate all ranks. - -The rest of this blog uses a compact notation for these combinations. `TEP` shards both attention (TP) and experts (EP) across `N` ranks. `DEP` keeps attention data-parallel (ADP) while distributing experts across `N` ranks. Both compose with disaggregated serving, where context (CTX) and generation (GEN) servers are sized independently. For example, `DEP4 (CTX) + TEP8 (GEN)` runs a four-rank DEP context instance alongside an eight-rank TEP generation instance. - -#### Cache management and runtime features - -DeepSeek-V4's hybrid attention turns the "KV cache" into a collection of states with different shapes and lifetimes. Each attention mode owns a different set of logical cache layers: - -- an **SWA** layer owns only its sliding-window KV. -- a **CSA** layer adds the 4× compressed attention cache, Compressor KV/score state, compressed Indexer-K cache, and Indexer-Compressor KV/score state. -- an **HCA** layer adds the 128× compressed attention cache and Compressor KV/score state, with no Indexer-side cache. - -These tensors differ in dtype, bytes per entry, production rate, and lifetime. Treating them as a uniform KV tensor would either waste memory or give the scheduler an incorrect view of capacity. For the rest of this blog, we group them into two lifecycle categories. The **sliding-window cache** contains SWA's sliding-window KV and the short-lived Compressor KV/score state used by CSA, HCA, and the CSA Indexer. These buffers are needed only for bounded windows over recent raw-token positions, so older pages can be recycled. The **persistent compressed caches** contain the finalized 4× and 128× compressed attention entries and CSA's compressed Indexer-K entries. They represent retained history and remain available for attention and prefix reuse, so their storage grows with sequence length at the corresponding compression rate. These terms describe lifecycle categories rather than a one-to-one mapping to physical pools. `DeepseekV4CacheManager` describes every state as a buffer role with an explicit lifecycle and page-indexing mode, then groups compatible buffers into physical memory pools. Sliding-window cache buffers use per-layer page indices. Persistent compressed caches use shared indices across layers with the same compression layout. - -The Compressor breaks two assumptions made by a conventional paged KV cache: its intermediate state is short-lived, and its persistent output grows more slowly than the raw token sequence. TensorRT LLM maps both cases onto existing KV Cache Manager V2 abstractions. - -**Short-lived Compressor state.** A raw token's intermediate state is needed only while a compression window that contains it remains open. Once the last such window is finalized, that token's intermediate state can be recycled. This lifetime is naturally represented as a sliding window: 128 raw-token positions for HCA, and 8 for CSA because its overlapping Compressor keeps two 4-token groups live. The manager can therefore reuse the existing window-eviction and block-recycling machinery. - -**Slow-growing compressed caches.** A compressed stream produces one entry for every $r$ raw tokens. Rather than introduce a second coordinate system, the manager keeps page allocation in raw-token coordinates. For a page covering `tokens_per_block` raw positions, its compressed buffer stores `tokens_per_block / r` entries. The page still represents the same interval of the original sequence, but its storage cost reflects the 4× or 128× compression ratio. Allocation, block-table construction, prefix reuse, and capacity accounting can consequently share one page model across all streams. - -The physical pool split is workload-dependent. Fresh prefill creates the greatest pressure on the sliding-window cache, whereas persistent compressed caches grow with sequence length and decode concurrency. When `kv_cache_config.pool_ratio` is not specified, KV Cache Manager V2 derives an initial split from a representative mixed batch: one context request and up to `max_batch_size - 1` generation requests. `kv_cache_config.avg_seq_len` controls the representative total sequence length of a typical decode request, while additional constraints reserve enough capacity for a maximum-length decode request and a chunked-prefill step. Advanced deployments can override the initial split with `pool_ratio` (specified in pool-group order and summing to 1.0). An opt-in beta rebalancer, enabled with `enable_kv_pool_rebalance`, can later adjust the split from runtime statistics. It is disabled by default and currently targets a narrower set of aggregated-serving configurations. - -DeepSeek-V4 also enables SWA scratch reuse by default. During prefill, transient sliding-window pages can be recycled across compatible layers instead of being retained as independent long-lived allocations. This reduces the prefill peak that would otherwise dictate the entire pool split. Its performance impact is discussed in [**Runtime optimization**](#runtime-optimization). - -With this state model in place, the same attention implementation composes with the runtime features expected in production: - -- **Chunked prefill.** The Compressor and Indexer carry compressed lengths and open-window state across chunks, preserving the semantics of processing the prompt in one pass. -- **KV-cache reuse.** Prefix reuse restores the attention, Compressor, and Indexer state owned by each layer type, rather than treating the main attention KV as sufficient. -- **CUDA Graphs.** Top-K and Compressor workspaces use graph-safe buffers whose captured shapes remain stable for a given graph batch size. -- **Overlap scheduling.** Cache updates and sparse-attention preparation are designed to avoid host-device synchronizations on the critical path, allowing request preparation to overlap GPU execution. -- **Disaggregated serving.** The V2 transfer path describes cache contents by pool, layer, and buffer role, allowing context workers to transfer every DeepSeek-V4 cache type and generation workers to rebuild their local block tables consistently. - -Production support also extends to the API boundary. For Instruct checkpoints, selecting the `deepseek_v4` tokenizer wrapper applies the reference chat format, including thinking controls, tool definitions, and DSML, DeepSeek's special-token-delimited format for tool calls. Tool results are inserted with `` tags. Matching reasoning and tool parsers turn generated thinking and DSML tool-call blocks back into OpenAI-compatible response fields. Base checkpoints remain completion models and should receive raw prompts without this wrapper. - -#### Precision strategy and accuracy - -DeepSeek-V4 uses three checkpoint-level precision recipes, each mapping low precision to the components where it reduces weight or cache bandwidth while keeping higher precision for stateful reductions and numerically sensitive paths: - -| Component | FP8 (Base) | MXFP4 (Instruct) | NVFP4 (requantized) | -| :--- | :--- | :--- | :--- | -| MoE routed experts GEMM | Blockwise FP8 | MXFP8 activations × MXFP4 weights | NVFP4 | -| MoE shared expert GEMM | Blockwise FP8 | MXFP8 | MXFP8 | -| Attention QKV/O GEMM | Blockwise FP8 | MXFP8 | MXFP8 | -| Indexer Q projection | Blockwise FP8 | MXFP8 | MXFP8 | -| Indexer weights projection | BF16 | BF16 | BF16 | -| Compressor KV/score linears | BF16 | BF16 | BF16 | -| Main attention KV cache | BF16 or per-tensor FP8 | BF16 or per-tensor FP8 | BF16 or per-tensor FP8 | -| Indexer K cache | Blockwise FP8 or MXFP4 | Blockwise FP8 or MXFP4 | Blockwise FP8 or MXFP4 | -| Compressor state cache | FP32 | FP32 | FP32 | - -Apart from the main-transformer routed experts, the NVFP4 checkpoint keeps its tensors identical to the MXFP4 Instruct source. In particular, the MTP subtree retains the Instruct precision recipe. The two FP4 routed-expert formats are distinct recipes served by different kernels. The Instruct checkpoints publish packed MXFP4 routed-expert weights executed with MXFP8 activations and MXFP4 weights, while the main-layer experts in requantized checkpoints such as `nvidia/DeepSeek-V4-Pro-NVFP4` run through the NVFP4 MoE path. On the cache side, FP4 approximately halves the Indexer-K data payload and is TensorRT LLM's default Indexer-K format for DeepSeek-V4 on Blackwell. The attention projections follow the same philosophy of staying in low precision across kernel boundaries. For eligible context-only sparse MLA batches, query quantization is fused into the Q RMSNorm and RoPE kernels, which write the FP8 Q buffer consumed by attention. Generation and mixed batches use the unfused Q path. For eligible DeepSeek-V4 Pro configurations, inverse RoPE and FP8 quantization are further fused into the FMHA epilogue, while other configurations use the optimized standalone inverse-RoPE and quantization kernel before `o_a_proj`. These kernel fusions are discussed further in [**Optimize the CSA/HCA hot path**](#optimize-the-csahca-hot-path). - -Accuracy was validated across Flash and Pro checkpoints, the FP8 (Base), MXFP4 (Instruct), and NVFP4 recipes, MTP on and off, FP8 and BF16 KV cache, and aggregated and disaggregated serving, comparing GPQA-Diamond and LiveCodeBench scores against the paper baselines. In this blog, MTP-1 and MTP-3 denote speculative draft lengths of one and three tokens, respectively. Representative GPQA-Diamond results with MTP-3 and FP8 KV cache unless noted: - -| Configuration | GPQA-Diamond | -| :--- | ---: | -| DeepSeek-V4-Flash | 88.8 | -| DeepSeek-V4-Pro | 89.08 | -| DeepSeek-V4-Pro, disaggregated | 89.39 | -| DeepSeek-V4-Pro-NVFP4 | 89.39 | - -Production readiness also required testing well beyond single-feature correctness: the validation matrix spans CUDA Graph capture, chunked prefill with cached history, MTP, long contexts, high concurrency, disaggregated transfer and shutdown, and memory pressure during loading and autotuning, with model-specific CI covering aggregated and disaggregated serving on Blackwell. This coverage is what turned the initial functional implementation into a stack the subsequent performance work could safely optimize. - -### DeepSeek-V4 Performance Optimizations - -Building on the production-ready execution stack described above, TensorRT LLM optimizes DeepSeek-V4 across four areas: the CSA/HCA attention hot path, Compressor and mHC, MoE, and runtime execution and memory management. The following sections explain the key techniques in each area, while [**DeepSeek-V4 Performance Evolution**](#deepseek-v4-performance-evolution) presents their cumulative end-to-end impact under a consistent model workload. Performance numbers cited here are scoped operator measurements or A/B end-to-end comparisons, not cumulative speedups. - -#### Optimize the CSA/HCA hot path - -CSA and HCA share a dual-pool attention kernel over the 128-token sliding window and compressed history, while CSA additionally runs an Indexer and Top-K selection. TensorRT LLM optimizes this hot path through attention-kernel improvements, low-precision fusion, multi-stream scheduling, and Top-K optimization. - -**Sparse MLA kernel optimization.** The dual-pool kernel processes the 128-token sliding window first, then gathers selected CSA entries or streams all HCA entries from the compressed pool. The optimized TRTLLM-Gen kernel skips redundant dense and sliding-window softmax masking on full tiles and issues sparse V loads with less coordinate setup and register spill/reload pressure. On B200 with an FP8 KV cache, the measured FMHA kernel was up to 1.31× faster for CSA prefill and 1.21× faster for HCA generation in the tested shapes. These are operator-level results, and the model-level gain depends on the model's mix of SWA, CSA, and HCA layers. - -**Kernel fusion and low-precision dataflow.** The functional path initially relied on framework or generic operators around attention. TensorRT LLM adds a DeepSeek-V4-specific Q RMSNorm CUDA kernel and extends the MLA RoPE/assignment and FMHA epilogue kernels so that normalization, rotation, and quantization can be fused into their producers. The resulting optimizations are: - -- **Fused Q RMSNorm, RoPE, and FP8 quantization.** The initial context path used three kernels for Q RMSNorm, RoPE and assignment, and full-Q FP8 quantization. For eligible context-only sparse MLA batches with an FP8 KV cache, TensorRT LLM folds quantization of the 448-dimensional non-RoPE segment into the Q RMSNorm kernel, then folds quantization of the remaining 64-dimensional RoPE segment and BMM-scale generation into the RoPE and assignment kernel. These two producer kernels write directly into the same FP8 Q buffer, eliminating the standalone quantization kernel and the full higher-precision normalized Q tensor. Generation and mixed batches continue to use the unfused path. -- **Standalone inverse-RoPE and FP8 quantization.** DeepSeek-V4 first establishes an FP8-native input path for `o_a_proj` by combining inverse RoPE and 1×128 blockwise FP8 quantization in a standalone operator. This operator consumes the BF16 output from FMHA and writes E4M3 activations and FP32 scales directly in the layout required by the Blackwell `o_a_proj` BMM. Replacing the original Triton implementation with a shape-specialized CUDA kernel made the standalone operator 1.6–2.3× faster in kernel-only measurements across 1K–32K tokens on B200. This path remains the fallback when FMHA epilogue fusion is not applicable. -- **FMHA epilogue fusion.** Building on the same FP8 `o_a_proj` input contract, TensorRT LLM moves inverse RoPE and 1×128 E4M3 quantization into the FMHA correction epilogue for validated DeepSeek-V4 Pro configurations using the FP8 KV cache, sparse MLA, and attention data parallelism. FMHA then writes the E4M3 output and FP32 scales directly in the layout consumed by the `o_a_proj` BMM, eliminating both the standalone operator launch and the intermediate BF16 output write and read. The `o_a_proj` BMM remains a separate kernel. This fusion supports context-only and generation-only batches, while mixed batches and unsupported configurations use the standalone path. Relative to the combined time of standard FMHA and the standalone inverse-RoPE and quantization operator, the fused FMHA kernel was up to 1.62× faster for prefill and 1.34× faster for generation in the tested shapes. -- **Others.** For CSA layers with BF16, bias-free `q_b` projection weights, a shape-autotuned CuTe DSL GEMM replaces the generic linear path on supported SM100f systems and falls back when CuTe DSL is unavailable. The optimized attention path also removes a device-to-device copy after FMHA, and FP8 quantization kernels initialize their own scale-buffer padding instead of requiring a separate host-launched `zero_()` operation. - -**Top-K optimizations.** The CSA Indexer selects hundreds of entries per query, making Top-K particularly visible during long-context decode. TensorRT LLM accelerates it in two complementary ways: - -- **Top-K kernel optimization.** TensorRT LLM makes the standard Top-K path shape-aware and device-aware. Depending on the compressed sequence length and row count, the dispatcher uses insertion selection, a single-CTA radix kernel, or a multi-CTA split-and-merge implementation. The multi-CTA path exposes enough parallel work for small decode batches, while its launch policy uses the row count, compressed sequence length, configured split threshold, and the GPU's actual SM count. On B200 with FP32 logits, 196,608 columns, `next_n=1`, and no previous indices, the device-aware policy reduced the K=512 latency at batch size 148 from 384 µs to 112 µs. -- **GVR Top-K.** GVR, introduced in our [GVR technical blog](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog21_Temporal_Correlation_Meets_Sparse_Attention.md), exploits the overlap between the entries selected by adjacent decode steps. It uses the previous step's indices as candidates, verifies them against current scores, and refines or falls back when the fast-path conditions are not met. DeepSeek-V4 extends GVR to the Indexer's 4× compressed coordinate space and to checkpoint-defined Top-K values of 512 for Flash and 1024 for Pro. In kernel-only B300 tests, GVR was 1.40–2.17× faster than the radix-selection baseline for the tested Flash and Pro shapes. An end-to-end Flash test on eight B300 SXM6 GPUs with a 65K-token input, batch size 4, and MTP-3 reported 6.4% higher request throughput. - -**Multi-stream attention scheduling.** The CSA attention prologue contains the main-attention Compressor, the `q_b` projection, and an Indexer with only a few true dependencies between them. The main-attention Compressor and the query-independent part of the Indexer can start directly from the layer input, while the `q_b` projection and Q RMSNorm wait for the shared projection and `q_a` normalization. TensorRT LLM progressively exposes this parallelism at two levels: - -- **Initial outer overlap.** The first dependency-aware schedule runs the main-attention Compressor concurrently with the `q_b` projection and Q RMSNorm. It then adds a dedicated stream for the complete Indexer branch, allowing all three branches to make progress before sparse MLA consumes their outputs. -- **Nested Indexer overlap.** Inside the Indexer, query projection, RoPE, and Q quantization form one branch, while the query-independent weight projection, Indexer Compressor, and K-cache update form another. Events synchronize them only when Q scales are needed for weight scaling and when the logits and Top-K kernels require the updated cache. -- **Final schedule with earlier launches.** The refined schedule pre-launches the main-attention Compressor and the query-independent half of the Indexer before the shared `kv_a` projection and Q/KV normalizations. Once normalized Q is available, the query-dependent Indexer work runs on its dedicated stream, while the `q_b` projection and Q RMSNorm are queued after the main-attention Compressor on the Compressor stream. This is the final stream assignment, replacing the earlier three-way outer schedule. - -Correctness across streams requires explicit tensor-lifetime tracking. TensorRT LLM calls `record_stream()` on the precomputed Indexer tensors, normalized Q output, and Top-K indices when they move to the consuming stream, preventing the PyTorch caching allocator from reusing their storage before the downstream work completes. - -#### Compressor and mHC - -The Compressor and mHC introduce repeated chains of small, stateful, and reduction-heavy operations around attention and MoE. The Compressor turns projected KV and scores into persistent paged state and compressed cache entries, while mHC transforms the residual streams at each sublayer boundary. TensorRT LLM optimizes both in two layers: first fuse adjacent operations to remove launches and intermediate tensors, then specialize the fused kernels for DeepSeek-V4 shapes and execution regimes. - -##### Compressor optimizations - -The optimized Compressor pipeline uses fusion at three boundaries: - -- **KV and score projection.** A single `wkv_gate` projection produces both the KV vectors and their compression scores. -- **Compression and state update.** The prefill and decode kernels write new KV/score state into paged storage and perform the online-softmax reduction when a compression window completes, avoiding separate state-update and reduction passes. -- **Postprocessing and cache write.** One kernel applies RMSNorm, RoPE, the optional Hadamard transform, output quantization, and scatter into the compressed paged cache. - -**Kernel optimization.** The fused Compressor kernels support the different dtype and cache-layout requirements of both the main-attention Compressor and the Compressor inside the Indexer, including BF16, FP8, and MXFP4 paths. They accept BF16 projection output directly while retaining FP32 state updates and online-softmax accumulation where required, avoiding a full FP32 KV/score intermediate. The reduction kernels are specialized for the 4×-compression decode and MTP regime and the much longer 128×-compression prefill regime, with the latter also reducing newly arrived tokens while writing their paged state to avoid a later readback. - -##### mHC optimizations - -DeepSeek-V4 places mHC around both attention and MoE, making the boundary between consecutive sublayers the primary fusion opportunity. - -**Kernel fusion.** TensorRT LLM fuses the previous sublayer's post-mapping with the next sublayer's pre-mapping into shape-specialized CUDA kernels, with both two-kernel and all-in-one tactics for different token counts. The next sublayer's input RMSNorm is further folded into the fused mHC epilogue, eliminating a separate launch and an HBM round trip. Because mHC owns the residual connection at the attention-to-MoE boundary, the MoE path also skips separate residual handling. - -**Kernel optimization.** No single mHC kernel is optimal across the full range of token counts `M`. TensorRT LLM provides FMA implementations for small-`M` workloads and higher-throughput MMA implementations as `M` grows, with two-kernel and all-in-one variants, multiple tile configurations, and Split-K support. The autotuner evaluates the valid tactics for each workload and selects the best-performing implementation, while pruning choices that are not competitive in the corresponding `M` regime and caching the selected tactic for reuse. - -#### MoE optimizations - -Building on the routing design [described above](#beyond-attention-mhc-moe-and-speculative-decoding), TensorRT LLM optimizes DeepSeek-V4's MoE path through a custom router GEMM, the MXFP8 × MXFP4 DeepGEMM MegaMoE backend, streamlined input preparation, and ongoing NVFP4 MegaMoE integration. These efforts preserve the model's routing metadata, precision recipe, and checkpoint-defined `swiglu_limit` activation clamp. - -**MoE router GEMM optimization.** For the 256-expert Flash shape, the score-based router uses a custom GEMM when it receives one to 16 token rows. The kernel consumes BF16 activations and weights and produces FP32 logits directly, avoiding the full FP32 input and weight casts required by the original linear path. Unsupported shapes, including the 384-expert Pro router, continue to use cuBLAS. In the reported small-shape measurements, router latency decreased from approximately 8 µs to 3 µs. - -**MXFP8 × MXFP4 DeepGEMM MegaMoE.** For the W4A8 recipe with MXFP8 activations and MXFP4 weights, the DeepGEMM MegaMoE backend moves dispatch, the first expert GEMM, SwiGLU, the second expert GEMM, and combine into a fused-communication path that uses symmetric memory and in-kernel synchronization. In a 500-request DeepSeek-V4 Flash test on B200 with 1K-token inputs and outputs, TP4, EP4, and attention data parallelism, switching from the TensorRT LLM MoE backend to DeepGEMM MegaMoE increased throughput by 15.3% and reduced latency by 12.7%. TensorRT LLM also fuses the input preparation for this backend: instead of separately quantizing BF16 activations to MXFP8, converting selected expert indices, and copying activations, scales, indices, and routing weights into symmetric buffers, one CUDA kernel performs the quantization and writes all four outputs directly to their final buffers, reducing six GPU operations to one. No separate end-to-end gain is attributed to the preparation change. - -**NVFP4 MegaMoE (WIP).** For native NVFP4 checkpoints, TensorRT LLM is integrating the ported `Sm100MegaMoEKernel` as a CuTe DSL execution backend on the SM100f path. The integration covers BF16-to-NVFP4 input quantization, dispatch, the two expert GEMMs, SwiGLU, per-route combine writes with native NVFP4 weights, and a separate reduction of the Top-K route contributions. It also includes symmetric-memory exchange for multi-rank expert parallelism and workload-specific tactic selection through the TensorRT LLM autotuner. The NVFP4 and DeepGEMM MegaMoE paths have separate hardware, quantization, and topology requirements. This precision-specific backend remains under integration and validation. - -#### Runtime optimization - -After the major kernels are optimized, runtime overhead shifts to the boundaries between kernels, repeated metadata and data movement, and transient memory growth during long-context prefill. TensorRT LLM addresses these costs through earlier dependent launches, streamlined input preparation, and workload-aware memory control. - -**Programmatic Dependent Launch.** PDL allows a dependent consumer grid to launch before its producer grid fully retires, wait at an explicit dependency point, and continue as soon as the required producer state is ready. TensorRT LLM configures PDL globally for supported DeepGEMM operations and also applies it to the packed 1×128 FP8 quantization, MLA RoPE, sparse-index conversion, sparse FMHA, and fused mHC paths used by DeepSeek-V4. This shortens producer-to-consumer boundaries without changing the model's dependency graph. - -**Metadata preparation.** TensorRT LLM streamlines DeepSeek-V4 attention-metadata preparation by reusing tensors and consolidating the host- and device-side updates required for each step, reducing Python work, kernel launches, and data transfers. The cache manager also builds DeepSeek-V4 block tables on the GPU and uses a dedicated CUDA operator instead of compiling the earlier tensor implementation, reducing host-memory pressure while keeping block-table generation on the device. - -**SWA scratch reuse.** A long prefill step produces KV entries for every token in the input chunk, but sliding-window attention retains only the most recent window after the step. Among the new blocks allocated for the current chunk, blocks that will already be outside the final non-rewindable window are needed only by the currently executing layer. TensorRT LLM maps this scratch-eligible range through per-layer block tables to coalesced scratch subpages. Because these per-layer lifetimes do not overlap, the same physical scratch storage can be reused as execution advances across layers. Pre-existing history from earlier chunks remains in the normal KV cache and is never overwritten by scratch reuse. New blocks in the active window also remain in the normal cache, and the tail that MTP may rewind is excluded from scratch storage. As a result, the cache does not retain the full current prefill chunk independently for every compatible SWA layer, reducing long-context prefill memory consumption and leaving more capacity for larger chunks, longer prompts, or higher concurrency. - -
-
- SWA scratch reuse during long-context prefill. Scratch-eligible KV blocks produced by the current prefill chunk and falling outside the final retained window reuse the same physical storage across SWA layers, while blocks inside the sliding window remain in the normal KV cache. -
-
-

Figure 3. SWA scratch reuse during long-context prefill. Scratch-eligible blocks from the current prefill chunk reuse the same physical storage across SWA layers, while the final sliding-window state remains in the normal KV cache.

- -**Bounded Indexer workspace.** During long-context prefill, the Indexer's MQA-logits transient grows with both the query chunk and compressed history. TensorRT LLM controls this allocation at two levels. An outer heuristic reduces the configured prefill chunk when the compressed KV history crosses long-context thresholds. Within each chunk, query tiling imposes a hard per-call element budget so that only a bounded slice of the logits matrix is materialized at a time. Because logits and Top-K are computed independently for each query row, tiling preserves the exact selection result while enabling very long prompts without reducing the model-wide token budget. - -Together, these runtime optimizations help carry the preceding kernel gains through to the full model. Dependent work starts earlier, recurring host-side metadata work is reduced, and long-context memory follows a bounded working set instead of unchecked temporary growth. The next section, [**DeepSeek-V4 Performance Evolution**](#deepseek-v4-performance-evolution), shows how the kernel and runtime improvements accumulate at the full-model level. - -### DeepSeek-V4 Performance Evolution - -The preceding [**DeepSeek-V4 Performance Optimizations**](#deepseek-v4-performance-optimizations) section described the individual attention, Compressor, mHC, MoE, and runtime optimizations. Their real value, however, is whether they move the end-to-end serving frontier after the full system is re-tuned. We therefore tracked a sequence of complete TensorRT LLM Pareto sweeps rather than multiplying speedups from isolated operator benchmarks. Under the same DeepSeek-V4 Pro 8K/1K workload, the maximum observed output throughput increased from 984 to 1,618 tokens/s/GPU between the first validated functional baseline and the latest fully refreshed May 28 snapshot, a **64.5% improvement**. Additional optimizations have since been integrated into the newer DeepSeek-V4 stack, but have not yet been included in a new full Pareto sweep. - -#### Experimental setup - -The primary evaluation uses the InferenceX fixed-length 8K/1K workload, with exactly 8K input tokens and 1K output tokens per request. This shape stresses both sides of disaggregated serving: the context workers must process a substantial prompt, while the generation workers execute enough decode steps for sparse attention, MTP, and MoE efficiency to materially affect the result. - -| Item | Configuration | -| :--- | :--- | -| Model | DeepSeek-V4 Pro, 1.6T total parameters | -| Hardware | NVIDIA GB300 | -| Precision | Mixed-precision MXFP4 Instruct recipe: MXFP8 activations × MXFP4 weights for routed-expert GEMMs, with component-specific precision elsewhere | -| Workload | InferenceX, 8K input tokens and 1K output tokens per request | -| Serving mode | End-to-end disaggregated serving with independently sized context and generation workers | -| Sweep | TEP4/TEP8 for the latency-oriented end; DEP8/DEP16/DEP32 for the throughput-oriented end; MTP-3 for most latency and mid-frontier points and MTP-1 at the highest-concurrency points | -| Metrics | Output throughput per concurrent request stream (tokens/s/user) and output throughput per GPU (tokens/s/GPU) | - -Each curve in Figure 4 is a newly measured system Pareto sweep. A point can therefore change concurrency, MTP depth, parallel strategy, expert placement, and the context-to-generation resource ratio. The curve movement is the meaningful comparison; the peak values should not be interpreted as a component-by-component A/B test at an identical configuration. - -![Four DeepSeek-V4 Pro 8K/1K disaggregated-serving Pareto curves, showing peak output throughput increasing from 984 to 1,618 tokens/s/GPU](../media/tech_blog26_dsv4_performance_evolution_8k1k.png) - -

Figure 4. DeepSeek-V4 Pro 8K/1K disaggregated serving on GB300, using the mixed-precision MXFP4 Instruct recipe with MXFP8 × MXFP4 routed-expert GEMMs. Each line is a complete TensorRT LLM Pareto sweep from that milestone.

- -#### From Baseline to the Latest Measured Curve - -The four curves capture three distinct phases of optimization. The table reports the highest measured tokens/s/GPU on each sweep; because the maximizing configuration changes as the frontier moves, these values summarize system evolution rather than a strict single-configuration waterfall. - -| Milestone | Peak output throughput (tokens/s/GPU) | Gain from previous | Gain from baseline | Representative changes in this stage | -| :--- | ---: | ---: | ---: | :--- | -| April 28: validated baseline | 984 | — | — | Validated end-to-end disaggregated stack under the common model and workload definition | -| May 8: Top-K and Compressor | 1,167 | +18.6% | +18.6% | Small-batch exact Top-K, BF16 Compressor input and fusion, and disaggregated memory/correctness fixes | -| May 18: MegaMoE and EPLB | 1,505 | +29.0% | +53.0% | MegaMoE, EPLB-enabled DEP placements, fused mHC and RMSNorm, and expanded multi-stream attention | -| May 28: attention and runtime refinements | 1,618 | +7.5% | +64.5% | Deeper attention overlap, the optimized inverse-RoPE and FP8 `o_a_proj` path, PDL, and GPU-side scale-buffer and block-table cleanup | - -**The first measured curve establishes the validated functional baseline.** It represents the complete execution path after functional bring-up: hybrid SWA/CSA/HCA attention, persistent Compressor state, MTP, KV Cache Manager V2, and disaggregated cache transfer. We treat all work required to reach this point as part of the baseline rather than assigning it a performance gain. Every subsequent curve uses the same model, precision recipe, and fixed-shape workload definition. The reported evolution therefore measures optimization of a working system rather than the addition of missing model features. - -**Exact Indexer Top-K and Compressor improvements move the May 8 frontier.** The small-batch multi-CTA path increased GPU utilization for exact Top-K, while BF16 Compressor input and additional Compressor fusion removed conversion, launch, and intermediate-buffer overhead. Runtime fixes were equally important: correcting sliding-window over-allocation improved usable generation-side KV capacity, while the launcher fix ensured that the configured NUMA policy was applied to multi-process disaggregated runs. Together, the full re-tuned sweep raised the peak by 18.6%, from 984 to 1,167 tokens/s/GPU. - -**MegaMoE and EPLB drive the mid-May step.** The largest movement in the system curve came between May 8 and May 18. MegaMoE fused expert dispatch, the two expert GEMMs, activation, and combine into a communication-aware path using symmetric memory and in-kernel synchronization. At the throughput-oriented end of the curve, the May 18 sweep also moved to EPLB-enabled DEP placements to improve expert balance across ranks. In parallel, mHC absorbed the adjacent RMSNorm, and the Indexer and attention paths gained expanded dependency-aware overlap. Together, these changes produced a 29.0% stage-over-stage increase and a 53.0% increase over the initial baseline. Because the maximizing configuration changed across the complete sweeps, this movement is not a single-configuration decomposition of the individual optimizations. - -**Attention overlap and runtime cleanup extend the late-May frontier.** By May 28, deeper Indexer, Compressor, and Q-path overlap reduced exposed attention work. PDL, producer-side FP8 scale-buffer initialization, the optimized inverse-RoPE and FP8 `o_a_proj` input path, and cheaper GPU-side block-table preparation removed additional synchronization, launch, and data-movement overhead. The May 28 curve is roughly 7–8% above May 18 across most throughput-oriented DEP points, lifting the observed peak to 1,618 tokens/s/GPU. This broad movement is more informative than any single microbenchmark: it shows that the optimizations survived full-model scheduling, cache management, and context/generation rate matching. - -May 28 is the latest measured curve, not the current performance ceiling. The newer DeepSeek-V4 stack subsequently integrated sparse-MLA softmax and V-load improvements, fused MegaMoE input preparation, the optimized 128× Compressor prefill reduction, and later attention-epilogue fusion. These changes target attention, MoE preparation, and Compressor overhead, but the complete GB300 8K/1K Pareto sweep has not been rerun with the newer stack. None of their gains is included in the reported 64.5% improvement, and we do not extrapolate a new end-to-end number. We expect a refreshed curve to move further once it is measured under the same methodology. - -The latest measured fixed-shape InferenceX curve establishes the clean model/runtime baseline for [Part II](#part-ii-agentic-workload-optimizations). Agentic workloads change the optimization boundary: dynamic multi-turn prompts, very high KV reuse, conversation locality, routing, and serialization become part of the critical path. The next section, [**AgentPerf Workflow**](#agentperf-workflow), introduces that wider optimization boundary without re-counting the model-specific gains above. - - -## Part II. Agentic-Workload Optimizations - -### AgentPerf Workflow - -[AA-AgentPerf](https://artificialanalysis.ai/methodology/agentperf) measures how many active coding agents an inference deployment can support while meeting a model-specific service level objective (SLO). This [NVIDIA Developer Blog](https://developer.nvidia.com/blog/nvidia-achieves-leading-agentic-coding-performance-on-first-agentic-ai-benchmark/) introduces the benchmark and explains why prerecorded trajectories are needed to represent agentic inference. Unlike the fixed-shape InferenceX workload used in Part I, AA-AgentPerf replays multi-turn coding sessions with interleaved reasoning and tool calls. Each simulated agent advances sequentially and waits for the current response and any client-simulated tool time before sending its next request. - -| Property | Fixed-shape InferenceX | AA-AgentPerf | -| :--- | :--- | :--- | -| Unit of load | Independent request | Active agent following a trajectory | -| Request shape | Fixed input and output lengths | Growing input context and variable output lengths | -| Inter-request dependency | Requests are independent | Each turn waits for the response and any client-simulated tool time | -| Prefix reuse | No accumulated cross-turn history | Accumulated history enables reuse across turns | -| Objective | Throughput and latency at a fixed shape | Highest concurrency satisfying model-specific SLO thresholds | - -The following figure summarizes the closed-loop request path for aggregated and disaggregated serving. - -
-
- AA-AgentPerf closed-loop workflow. A simulated agent sends the next recorded turn with its accumulated conversation. The inference deployment routes the request, reuses matching KV blocks, prefills the uncached suffix, and streams the generated response. In disaggregated serving, the generation side retrieves the KV cache using context metadata. The client drains the complete response, simulates tool time when needed, advances the trajectory, and sends the next turn only after the current cycle completes. -
-
-

Figure 5. AA-AgentPerf closed-loop request workflow. Repeated conversation prefixes can be reused through the KV cache. Aggregated serving performs prefill and generation on the same worker. In disaggregated serving, context metadata allows the generation worker to retrieve the KV cache and continue decoding. Tool time is simulated by the client and consumes no LLM compute.

- -This closed loop makes the complete request cycle part of every agent's progress. Repeated prefixes make KV-cache reuse important because later turns can avoid recomputing accumulated history. Growing contexts and variable output lengths also exercise cache capacity and scheduling differently from fixed-shape requests. Simulated tool delays introduce idle periods between turns, while concurrent agents maintain sustained load on the deployment. - -AA-AgentPerf reports the highest concurrency that satisfies both the P25 of per-request output speed and the P95 of per-request time to first token (TTFT). Because the benchmark is continuously updated, this blog uses the [AA methodology](https://artificialanalysis.ai/methodology/agentperf) current at publication for the exact thresholds. We refer to the current DeepSeek-V4 Pro SLO #1 and SLO #2 tiers as **SLO20** and **SLO60**. SLO20 requires at least 20 tokens/s and P95 TTFT at most 10 seconds. SLO60 requires at least 60 tokens/s and P95 TTFT at most 5 seconds. SLO20 emphasizes serving capacity, while SLO60 preserves a faster interactive experience. - -For engineering comparisons, we use concurrency per GPU (CPG). It is the supported concurrency divided by the total number of GPUs in the deployment, including both context and generation GPUs. - -### Lessons from DeepSeek-V3.2 Agentic Workload Optimization - -Before DeepSeek-V4, we optimized DeepSeek-V3.2 for long, multi-turn agentic workloads in both aggregated and disaggregated serving. The durable value of that work was not a particular benchmark result or deployment configuration. It was a set of system-level rules that guided the DeepSeek-V4 effort: - -1. **High reuse creates a different bottleneck from raw GPU throughput.** High prefix reuse can leave only a small uncached suffix to compute while the accumulated conversation still occupies substantial KV-cache capacity. Scheduling and capacity planning should therefore consider the remaining compute tokens and resident KV footprint rather than raw prompt length alone. -2. **Amortize host overhead without pausing generation.** Two complementary approaches addressed this overhead. First, CUDA Graph capture and kernel fusion reduced recurring host and launch work. **Piecewise CUDA Graph capture** covered graphable regions in prefill and mixed prefill-decode iterations when the full iteration could not be captured as one graph. Generation-only iterations continued to use the regular monolithic CUDA Graph path. Second, context-only **Delay batching** briefly held small context requests so newly arriving context requests could join a larger batch. Generation requests continued to run during this wait. -3. **Treat routing as a trajectory-level balance between locality and load.** Later turns benefit from returning to the workers and ranks that hold their conversation prefix. That affinity must be balanced against load so that preserving reuse does not create persistent rank imbalance. -4. **Measure disaggregated serving end to end.** Front-end preprocessing and KV-cache movement can limit throughput even when context and generation GPUs appear underutilized. Asynchronous preprocessing, parallel data movement, and per-stage observability are therefore part of serving performance. - -DeepSeek-V4 changed the model path, cache layout, and serving control plane, so the earlier configuration could not simply be reused. We carried forward these rules rather than the exact parallel strategy, cache settings, batching parameters, or transfer settings. The following section applies them to DeepSeek-V4 routing, KV lifecycle, host overhead, and scheduling. - -### End-to-End Optimizations for Agentic Serving - -Starting from the final model/runtime stack in [Part I](#part-i-deepseek-v4-model-support-and-optimizations), this section applies the measurement methodology and system-level principles learned from [DeepSeek-V3.2](#lessons-from-deepseek-v32-agentic-workload-optimization) to DeepSeek-V4's AgentPerf-specific request path, without counting the model-specific gains again. In this workload, a routing miss can force a long conversation prefix to be recomputed, erasing gains from faster kernels. Hashing that same prefix token by token in Python can also serialize the orchestrator's request path. These bottlenecks expand the optimization boundary beyond GPU kernels. The remaining work spans three coupled areas: preserving locality through routing, keeping reusable KV blocks resident through lifecycle and capacity management, and removing overhead across the worker host paths, scheduler, orchestrator, and CTX-to-GEN protocol once reuse reduces the GPU work per turn. - -#### Preserve locality with two-level routing - -The DeepSeek-V3.2 study showed the central routing tradeoff: affinity without load control creates imbalance, while load balancing that overrides affinity destroys reuse. DeepSeek-V4 applies that lesson through two consecutive placement decisions. The front-end orchestrator first selects a context (CTX) server instance. The attention data-parallel (ADP) router then selects a rank within that instance. At either level, the router can use an explicit conversation identifier or infer locality from KV-cache contents. Reuse is preserved only when both decisions return a later turn to the server and rank that own its KV blocks. Instance-level affinity cannot compensate for a rank-level miss, and the correct rank is unreachable after the wrong server has been selected. - -
-
- Two-level routing for context locality in disaggregated serving. The front-end orchestrator first selects a context server, then the selected server's attention data-parallel router selects a rank. At each level, the router can use either conversation-aware affinity or KV-cache-aware scoring. Full conversation-prefix reuse requires both decisions to return to the context server and rank that own the corresponding KV blocks. The selected context rank transfers the KV cache to a generation server for decoding. -
-
-

Figure 6. Two-level routing for context locality in disaggregated serving. The front-end orchestrator first selects a CTX server, and that server's ADP router then selects a rank. Conversation-aware routing uses explicit conversation affinity, while KV-cache-aware routing infers placement from reusable prefix blocks and load. Full conversation-prefix reuse requires both decisions to return the request to the CTX server and ADP rank that hold the corresponding KV blocks.

- -##### Instance-level routing across context servers - -The first routing level distributes conversations across CTX server instances. It must preserve locality for later turns while balancing new conversations across servers. - -**Conversation-aware routing.** When the client provides a stable conversation identifier, the orchestrator assigns the first turn to the least-loaded CTX server, using round-robin among equally loaded candidates, and records the binding. For each later turn, the orchestrator uses that binding to send the request back to the same server, preserving reuse of the conversation prefix cached there as long as the server remains available. The affinity decision then becomes a session-table lookup, avoiding full-prompt tokenization and block-hash computation on the front-end event loop. - -**KV-cache-aware routing.** When a reliable conversation identifier is unavailable, the orchestrator infers locality by tokenizing the prompt, deriving its block keys, estimating the reusable prefix on each CTX server, and balancing reuse against current load. We made three improvements to keep this path efficient and stable. First, incremental tokenization reuses the token IDs for the stable prompt prefix and encodes only the appended suffix. The orchestrator passes those token IDs to the worker and applies the same tool definitions and chat-template arguments, avoiding repeated tokenization while ensuring that both sides derive the same block keys. Second, we refined candidate scoring, which selects a CTX server by weighing its reusable prefix against its current load. Instead of normalizing the cache match by the full prompt length, the new score uses the absolute number of matched blocks and a separate active-request penalty. This prevents the long system and tool prefix shared across requests from diluting the score advantage contributed by conversation-specific blocks. Third, we strengthened affinity when temporary cache eviction weakens the cache-match signal. A bounded mapping derived from stable early conversation content remembers the selected CTX server and continues to direct later turns to it even if eviction temporarily lowers the server's cache-match score. - -Cache-aware routing also requires the orchestrator to maintain a compatible and sufficiently current view of each worker's cache. Three design choices make that view both reliable and inexpensive to maintain. First, the orchestrator and workers use the same hash algorithm and cache namespace. An optional cache salt acts as a namespace identifier, allowing KV reuse only between requests that carry the same value. Second, the orchestrator adopts the worker's block size by default. When it tracks cache at a finer supported granularity, completed-request backfill adds the routed block keys to the selected server's cache view after successful execution, which lets the orchestrator track newly cached prefixes at its own granularity without translating between incompatible event boundaries. Third, cache metadata updates are batched, and worker events are refreshed in the background to keep this maintenance off the routing critical path. Together, these changes keep cache tracking useful without turning it into a new routing bottleneck. - -##### Rank-level routing within a context server - -After a CTX server has been selected, the second routing level places the request on one of its ADP ranks. It must preserve the exact rank that owns the reusable prefix while keeping new conversations distributed across the instance. - -**Conversation-aware routing.** The ADP router records a bounded `conversation ID → rank` mapping. It assigns the first turn of each new conversation by round-robin among ranks below a loose fair-share target, then pins later turns to the recorded rank. A returning conversation may exceed that soft target because moving it would discard rank-local reuse. Affinity is relaxed only when the assigned rank reaches the hard active-request limit. An overflow request does not overwrite the original binding, which allows a later turn to return when the rank has capacity again. - -**KV-cache-aware routing.** Without a reliable conversation ID, the ADP router must infer rank locality from the KV blocks that are actually present. For each new request, every rank probes its local radix tree to measure the reusable prefix without reserving any cache blocks. The probe uses the request's cache salt so that it searches the same cache namespace as the KV cache manager. The ranks then exchange their prefix-match lengths and current loads. For each candidate rank, the router estimates the remaining prefill work as the input length minus the matched prefix, then adds a normalized load penalty. A meaningful prefix match therefore favors the rank that can reuse it, while a weak match caused only by the shared system and tool scaffold is ignored in favor of load balancing. A loose fair-share cap provides a second guard against concentrating too many requests on a few warm ranks. When several related requests arrive in the same scheduling batch, the router groups them by their early prefix and places longer requests first. After each assignment, it updates the selected rank's load using only the tokens that still require computation. The next request is therefore routed against the work already placed in that batch rather than a stale load snapshot. - -**Protect rank balance.** A new conversation contains the shared system and tool scaffold but little conversation-specific history. Without explicit cold-start handling, the first few ranks that cache this common prefix can attract more new conversations, while the remaining ranks stay cold. Conversation-aware routing prevents this feedback loop by distributing first turns with round-robin before pinning later turns. KV-cache-aware routing can instead use an optional warmup phase that sends initial requests to ranks not yet selected, allowing every rank to cache the shared prefix before normal scoring begins. - -##### Routing policy selection - -Our evaluation results show that, with completed-request backfill, the cache-aware path achieved a cache hit rate comparable to conversation-aware routing, but required additional preprocessing and cache tracking. The two approaches therefore offer different ways to preserve locality: one uses explicit conversation metadata, while the other infers locality from cache contents. - -#### KV cache reuse optimizations - -Two-level routing keeps successive turns of the same conversation on the CTX server and ADP rank that hold the reusable prefix. Placement alone is not enough. The prefix must also remain resident until the next turn arrives. Analysis of early DeepSeek-V4 AgentPerf runs showed substantial optimization headroom in cache memory management. DeepSeek-V4 divides cache memory among the sliding-window cache and several persistent compressed caches with different storage costs and demand profiles, so reuse depends on both the cache lifecycle and how memory is budgeted across pools. We first added per-pool cache metrics to locate pressure, then used those signals to optimize the sliding-window cache lifecycle and tune cache pool ratios. - -**Add per-pool cache metrics.** An overall cache hit rate shows how much reuse was achieved, but it does not reveal which pool was constrained or whether the cause was sustained pressure or a short-lived capacity spike. We added per-pool telemetry for occupancy, allocation and commit volume, cached KV length, and data movement between HBM and host memory. Reuse counters distinguish fully reused, partially reused, and missed blocks. Peak watermarks and complete iteration histories capture transient pressure that periodic summaries can miss, while memory probes expose the HBM headroom before and after model loading. Host-tier gauges further distinguish evictable blocks from blocks dropped before reuse. Together, these signals identify the constrained pool, distinguish incorrect lifecycle retention from insufficient pool capacity, and provide the evidence for the next two optimizations. - -**Optimize the sliding-window cache lifecycle.** As described in [**Cache management and runtime features**](#cache-management-and-runtime-features), DeepSeek-V4's sliding-window cache includes SWA KV and short-lived Compressor KV/score state. These buffers should remain bounded by their active windows rather than grow with the full prompt. The previous reuse policy retained SWA KV across the full prompt, even though a later agent turn needs only the blocks within the active window. We introduced the `per_request` policy to select the sliding-window state needed by the next turn when the current prefill request completes, allowing older pages to be recycled. The same retention issue appeared at the CTX-to-GEN handoff in disaggregated serving. Before this optimization, GEN allocated sliding-window cache for the full transferred prompt, including blocks outside the active window. We changed GEN initialization to allocate cache only for blocks within the active window. Together, these changes keep the sliding-window cache bounded during prefill and disaggregated handoff. Persistent compressed caches still retain the long-term prefix and continue to grow with sequence length. - -**Tune cache pool ratios.** Correcting the cache lifecycle removes stale allocations, but reusable prefixes can still be evicted when the available HBM is divided poorly across pools. The two cache categories respond differently to workload shape. Sliding-window cache pressure is driven by the active-request count, while persistent compressed caches grow with retained sequence length and the number of concurrent conversations. Each allocation must be satisfied by its own pool group, so free pages in one group cannot cover a shortage in another. A split tuned for a different request-length distribution or serving role can therefore exhaust one pool and evict reusable blocks even while another pool has unused capacity. To address this imbalance, we introduced configurable pool-ratio controls for DeepSeek-V4. On CTX servers, prefix reuse makes pool demand highly dependent on the workload's prefix-reuse pattern, which the runtime cannot infer reliably. Users should therefore tune `pool_ratio` manually using the per-pool metrics described above. On GEN servers, users can set `avg_seq_len` to the workload's average total sequence length, allowing KV Cache Manager V2 to derive the initial pool ratio automatically. These controls direct capacity to the pools that need it and avoid evicting reusable history while memory remains unused elsewhere. An opt-in beta rebalancer can also adjust the split from runtime statistics. It is disabled by default and currently supports a narrower set of aggregated-serving configurations. Extending this mechanism to broader deployments remains ongoing work. - -**Host offloading and incremental transfer.** We also evaluated host KV offloading and incremental CTX-to-GEN transfer. Compared with models that retain full K/V history for every token, DeepSeek-V4 stores long-term history in compressed attention and Indexer caches while keeping raw-token KV and Compressor state within bounded sliding windows. Its KV-cache footprint per input token is therefore relatively small. This compact representation keeps more reusable KV blocks resident, leaving less additional reuse for host offloading to recover. It also reduces the CTX-to-GEN payload, limiting the data movement that incremental transfer can overlap. Neither mechanism therefore produced a meaningful end-to-end improvement for this AgentPerf configuration. The evaluation is still useful for workloads with larger KV-cache footprints or higher transfer volumes. - -#### Optimize scheduling and remove end-to-end overhead - -Routing and cache management make later turns with reusable prefixes much cheaper on the GPU. That saving reaches the user only if the rest of the serving path also avoids processing the full long prompt again. Profiling showed substantial host work during CTX prefill and a persistent gap between GEN inter-token latency and the speed-of-light (SOL) estimate. Long token arrays were still being hashed, copied, tokenized, serialized, and broadcast at several boundaries. We addressed this gap inside both the workers and the serving control plane. First, we removed host work from the CTX and GEN critical paths. We then optimized scheduling, orchestration, and protocol handling. - -##### Optimize host overhead for CTX and GEN - -Effective prefix reuse reduces the GPU work to the uncached suffix, but the surrounding host path can still scale with the full prompt. This is especially costly when the work runs in Python or is repeated on every distributed rank. We therefore moved per-token loops out of Python where possible and skipped host operations that could not produce useful GPU work. - -**CTX host path.** Profiling showed that CTX was paying host work at the wrong granularity. Block-key construction paid a conversion and SHA-256 update for every token, turning a long prompt into many small operations before prefill could begin. Distributed coordination was paid once per iteration, even when no new request was available. Those empty iterations could still enter prefix-gather and request-broadcast collectives. The first cost grew with the full prompt, while the second could create host bubbles without producing any GPU work. We changed each operation to follow the unit of useful work. Block-key construction now packs one block of tokens into a contiguous byte buffer and updates the hash once per block. All ranks perform a lightweight request-count probe before entering the larger collectives. If no new request exists, they take the same fast path and skip both prefix gathering and request broadcast. - -**GEN host path.** On GEN, profiling did not point to one dominant function. Instead, host time accumulated through three prompt-proportional costs as each long-prompt request crossed the CTX-to-GEN handoff, entered GEN, and was distributed across the ranks. First, GEN could repeat tokenization already completed by CTX. Second, request admission and routing repeatedly copied or materialized the full token list. Third, interprocess request transfer serialized token IDs as individual Python objects before rebuilding the full list on the receiving side. The latter two costs occurred on request-ingress or executor-loop paths while holding the Python GIL, so they could delay the next generation iteration. To reduce this accumulated host time, we introduced a set of targeted optimizations, each addressing one of these costs directly. (a) CTX now returns the prompt token IDs that it already computed, allowing GEN to bypass tokenization. (b) During admission, enqueue no longer deep-copies the token list, while routing obtains the token count without materializing the full list. (c) Serialization and C++ request construction use a compact token representation, avoiding per-token Python objects and full-list reconstruction. - -##### Optimize scheduling, orchestration, and protocol handling - -After reducing host work inside CTX and GEN, latency could still accumulate before a request entered a worker and while data moved between workers. We treated this serving control plane as three related problems. Scheduling determines which ready request runs next. The orchestrator performs routing and relays data between CTX and GEN. The protocol must minimize redundant payload while preserving request identity when retries occur. - -**Scheduling.** Under high concurrency, a GEN worker may have to choose between newly admitted requests waiting for their first token and requests that have already begun decoding. When the batch or token budget is tight, placing both groups in the same queue can leave a new request behind many ongoing conversations for several scheduling iterations. We added an opt-in policy for disaggregated GEN that gives first-token requests priority while preserving FIFO order within both groups. This can move a newly admitted request into an earlier feasible batch and reduce the queueing delay before its first decode step without adding GPU work. The policy is designed to improve first-token responsiveness and tail-latency SLO compliance rather than raw throughput. - -**Orchestrator.** Profiling exposed two sources of per-request overhead. First, **native block-key hashing** removed the routing hot spot. The instance-level KV router originally hashed every token ID in a Python loop on its single asyncio thread. This held the Global Interpreter Lock (GIL) and serialized concurrent requests before they could reach CTX. Replacing the loop with a native C++ block-key hasher preserved the cache-key format while moving prompt-length hashing out of the Python event loop. Second, **efficient relay serialization** reduced request-body processing when a request crossed the HTTP boundaries to CTX and GEN. The default path previously materialized a JSON-compatible Python object containing the full prompt-token list, after which the HTTP client traversed that object again to encode the request body. We changed this path to use `pydantic-core` to emit the JSON body directly. For deployments where request-body processing remains a bottleneck, an opt-in `msgspec` MessagePack transport provides an alternative internal orchestrator-to-worker path that removes JSON encoding and parsing from this hop. The external OpenAI API and the default direct-JSON path remain unchanged. - -**CTX-to-GEN protocol.** The CTX-to-GEN protocol had two issues: it could carry more request state than GEN needed, and a retried request could leave CTX and GEN with inconsistent identifiers. In supported text-only, non-Harmony deployments, an opt-in path removes earlier chat history from the GEN request while preserving the prompt token IDs, final message, tool definitions, and generation settings that GEN still needs. This reduces copying, serialization, and transfer without changing the request semantics supported by that path. We also hardened the retry path so that known transient connection failures do not leave CTX and GEN with inconsistent request identifiers. Together, these changes reduce protocol overhead while preserving consistency across the disaggregated handoff. - -Across the host and serving-control-plane paths, these changes reduce prompt-proportional processing, redundant serialization, and avoidable synchronization between GPU iterations. Together with routing and cache management, they allow prefix reuse to translate into end-to-end performance. The next section, [**AgentPerf Results and External Validation**](#agentperf-results-and-external-validation), presents the combined end-to-end results under frozen configurations and evaluates them against the SLO20 and SLO60 objectives. - -### AgentPerf Results and External Validation - -The previous section described how routing, KV-cache policy, host-path cleanup, scheduling, and orchestration removed distinct end-to-end bottlenecks. This section presents the DeepSeek-V4 Pro AgentPerf evaluation setup and the results reported by Artificial Analysis. We keep these agent-workload results separate from the model-specific results in [Part I](#part-i-deepseek-v4-model-support-and-optimizations) and report the best-performing configurations we developed for those AA-AgentPerf measurements. - -The GB300 NVL72 configurations served DeepSeek-V4 Pro at maximum thinking effort using TensorRT LLM with disaggregated serving. Several context (CTX) workers fed one generation (GEN) worker through NIXL KV-cache transfer. Each CTX worker used four GPUs with attention data parallelism and MoE expert parallelism. - -To select the end-to-end topology, we followed the speed-of-light (SOL) rate-matching workflow introduced in the [TensorRT LLM disaggregated serving tech blog](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog05_Disaggregated_Serving_in_TensorRT-LLM.md#measurement-methodology). We first measured CTX request throughput across candidate parallel mappings and batch settings that could satisfy the TTFT target, and independently measured GEN output throughput and per-agent output speed across mappings, batch sizes, and concurrency levels. We then matched the aggregate service rates of the two tiers to construct an idealized SOL Pareto curve. The final SLO20 configuration used DEP8 with a larger GEN batch to maximize capacity. The final SLO60 configuration used DEP16 with a smaller GEN batch to preserve higher per-agent decode speed. SOL assumes perfect rate matching and excludes practical end-to-end costs such as KV-cache transfer, routing, host work, and workload burstiness, so it served as a configuration-selection reference rather than a reported AgentPerf result. - -The same rate-matching calculation provided the starting `xPyD` ratio, where `xPyD` denotes `x` CTX/prefill instances feeding `y` GEN/decode instances. For each selected GEN operating point, we converted its output-token throughput into a request-equivalent service rate using the workload's output-length distribution, then compared it with the measured per-instance CTX request rate to estimate the ideal number of CTX instances. Because this ratio can be fractional while a deployment requires whole workers, we evaluated the neighboring integer `xPyD` topologies that fit the GPU budget. We selected the topology that provided the best end-to-end balance while satisfying both the output-speed and TTFT requirements, then swept concurrency to find its maximum passing load. - -The configurations used DEP4 CTX workers with `max_batch_size: 128` and the TensorRT LLM MoE backend. The capacity-oriented SLO20 configuration used a DEP8 GEN worker with `max_batch_size: 192`. The higher-speed SLO60 configuration used a DEP16 GEN worker with `max_batch_size: 32` and an offline EPLB map. Both GEN workers used the DeepGEMM MegaMoE backend. Both deployments used the MXFP4 recipe, MTP-3, FP8 KV cache, conversation-affinity routing at the orchestrator and ADP-rank levels, and heuristic GVR Top-K. - -As defined in [**AgentPerf Workflow**](#agentperf-workflow), we use concurrency per GPU (CPG) to compare deployment configurations. The GPU count here includes both the CTX and GEN tiers, and only points that pass the target SLO are reported. The Artificial Analysis article, [First results from AA-AgentPerf: the hardware benchmark for the agent era](https://artificialanalysis.ai/articles/aa-agentperf/), reports the GB300 configurations provided by NVIDIA and run and verified by Artificial Analysis. These results validate the complete workload path rather than an isolated TensorRT LLM microbenchmark. Artificial Analysis replayed the agent trajectories and applied its SLO search and steady-state methodology. The associated [serving-configuration browser](https://artificialanalysis.ai/benchmarks/hardware/configs) discloses the precision, topology, worker YAML, routing mode, MTP setup, cache policy, transfer backend, and launch commands. - -The GB300 results reached 57.5 CPG at SLO20 and 19.2 CPG at SLO60: - -| Target | CTX/GEN topology | Passing concurrency | GPUs used | CPG | Provenance | -| :--- | :--- | ---: | ---: | ---: | :--- | -| SLO20 | 6 × CTX DEP4 + 1 × GEN DEP8 | 1,840 | 32 | 57.5 | NVIDIA configuration, run and verified by AA | -| SLO60 | 6 × CTX DEP4 + 1 × GEN DEP16 | 768 | 40 | 19.2 | NVIDIA configuration, run and verified by AA | - -These optimizations are now also available in [NVIDIA Dynamo](https://github.com/ai-dynamo/dynamo) for readers looking to deploy in production. - -## Reproduction and Future Work - -### How to reproduce - -TensorRT LLM already provides the common operational steps for launching disaggregated serving. Follow the [disaggregated serving tech blog](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog05_Disaggregated_Serving_in_TensorRT-LLM.md) and the [disaggregated serving guide](https://nvidia.github.io/TensorRT-LLM/features/disagg-serving.html) for environment setup, CTX and GEN worker startup, NIXL KV-cache transfer, and orchestrator launch. Both result sets below build on this foundation but use different workload-specific references. The InferenceX path uses the linked GB300 recipes to reproduce the fixed-shape 8K/1K Pareto sweeps. The AA-AgentPerf path starts from the current TensorRT LLM `main` branch to evaluate the latest optimized stack and uses the linked AA configuration entries and methodology for the submitted topology, settings, and SLO evaluation. - -**InferenceX 8K/1K.** Reproduce the fixed-shape results from [**DeepSeek-V4 Performance Evolution**](#deepseek-v4-performance-evolution) with the [GB300 MXFP4 ISL8K/OSL1K MTP recipes](https://github.com/NVIDIA/srt-slurm/tree/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP). The directory contains the CTX/GEN topology sweep, MTP settings, benchmark concurrencies, and worker configurations used for the 8K/1K measurements. - -**AA-AgentPerf.** For the results reported above, use the corresponding entries in the [Artificial Analysis configuration browser](https://artificialanalysis.ai/benchmarks/hardware/configs). The 57.5-CPG SLO20 result uses the [GB300 DEP8 configuration](https://artificialanalysis.ai/benchmarks/hardware/configs#config-60987f21-a7db-4a8e-8d3a-810a7527de2f), while the 19.2-CPG SLO60 result uses the [GB300 DEP16 configuration](https://artificialanalysis.ai/benchmarks/hardware/configs#config-8e07c398-605a-457f-96c4-bb002691de79). Each entry provides the CTX and GEN worker YAML, disaggregated server configuration, topology, routing settings, and launch commands needed to reproduce the deployment setup. Use the [AA-AgentPerf methodology](https://artificialanalysis.ai/methodology/agentperf) for the benchmark procedure and SLO evaluation. - -### Future Work - -Our next phase continues to optimize DeepSeek-V4 as a complete serving system: improve the low-precision GPU execution path, reduce host and control-plane overhead, broaden hardware coverage, and expand speculative decoding support. These directions need to advance together, because faster model execution raises end-to-end capacity only when cache management, routing, and orchestration can keep the GPUs supplied efficiently. The following work is already underway. - -**Advance the NVFP4 execution path.** We are moving the performance path to the NVFP4 DeepSeek-V4 checkpoint, continuing to tune the NVFP4 MegaMoE backend, and evaluating a more aggressive quantization recipe that applies NVFP4 to additional linear layers. This work treats the checkpoint, kernel coverage, and quantization recipe as one end-to-end path. - -**Fuse more small kernels.** We will continue combining adjacent small operations to reduce intermediate memory traffic and the number of kernel launches. These fusions target both GPU execution time and the host launch overhead that remains visible in small effective batches. - -**Move the KV Cache Manager V2 hot path to C++.** KV Cache Manager V2 provides the flexible multi-pool model required by DeepSeek-V4, but its performance-critical control path is still primarily implemented in Python. Migrating this path to C++ will reduce GIL contention and recurring host overhead, which become especially visible when high cache reuse leaves only a small amount of GPU work per turn. - -**Optimize and scale the serving control plane.** Conversation-aware and KV-aware routing still have room to reduce TTFT and CPU overhead while preserving cache locality and balanced placement. At larger deployment scales, a multiprocess orchestrator with a coordinator-and-worker architecture is being developed to address the point at which a single orchestrator becomes the bottleneck. We will optimize these paths as one control plane, resolve the current multiprocess TTFT issues, and validate the complete design under high-concurrency, multi-turn workloads. - -**Add DeepSeek-V4 DSpark support.** We are integrating the DSpark speculative-decoding path for DeepSeek-V4 and validating it against the MTP baseline. The remaining work includes measuring acceptance behavior, tuning the runtime configuration, and quantifying the resulting end-to-end gain. - -**Extend DeepSeek-V4 support to Hopper.** The current performance work is centered on Blackwell. We are extending the model and runtime paths required for complete DeepSeek-V4 execution on Hopper, with platform-appropriate kernels and precision support. - -## Conclusion and Acknowledgments - -DeepSeek-V4 optimization in TensorRT LLM is an end-to-end systems effort. Building a production-ready model stack required correctness across hybrid attention, mHC, MTP, MoE, quantization, cache management, and disaggregated execution. Moving the performance frontier then required coordinated work from GPU kernels and execution overlap through KV reuse, host efficiency, routing, orchestration, configuration tuning, and benchmark methodology. The result is a stack that improves both fixed-shape inference and long-running agentic serving, demonstrating that delivered performance comes from optimizing the model and the serving system together. - -This work is the result of close collaboration across many teams and every layer of the inference stack. We sincerely thank everyone who contributed to model bring-up, accuracy validation, benchmark optimization, and infrastructure support. The results shared here came from repeated cycles of analysis, implementation, profiling, debugging, validation, and configuration tuning. We are grateful for the expertise, persistence, and teamwork that turned DeepSeek-V4 into a production-ready, high-performance TensorRT LLM stack. diff --git a/docs/source/commands/trtllm-bench.rst b/docs/source/commands/trtllm-bench.rst index 309422df3fba..fee60a9ab70c 100644 --- a/docs/source/commands/trtllm-bench.rst +++ b/docs/source/commands/trtllm-bench.rst @@ -20,10 +20,10 @@ Syntax Dataset preparation ------------------ -prepare-dataset -^^^^^^^^^^^^^^^ +prepare_dataset.py +^^^^^^^^^^^^^^^^^^ -trtllm-bench ships a ``prepare-dataset`` subcommand which generates benchmark datasets in the required format. It supports: +trtllm-bench is designed to work with the `prepare_dataset.py `_ script, which generates benchmark datasets in the required format. The prepare_dataset script supports: **Dataset Types:** @@ -38,17 +38,17 @@ trtllm-bench ships a ``prepare-dataset`` subcommand which generates benchmark da - Support for LoRA adapters and task IDs - Output in JSON format compatible with trtllm-bench -.. note:: - The tokenizer is taken from the model passed to ``trtllm-bench --model``. Use ``--output`` to write the dataset to a file, or ``--stdout`` to stream it with a JSON dataset entry on each line. +.. important:: + The ``--stdout`` flag is **required** when using prepare_dataset.py with trtllm-bench to ensure proper data streaming format. **Usage:** -prepare-dataset +prepare_dataset """"""""""""""" .. code-block:: bash - trtllm-bench --model prepare-dataset [OPTIONS] + python prepare_dataset.py [OPTIONS] **Options** @@ -60,10 +60,12 @@ prepare-dataset * - Option - Description + * - ``--tokenizer`` + - Tokenizer directory or HuggingFace model name (required) * - ``--output`` - Output JSON filename (default: preprocessed_dataset.json) * - ``--stdout`` - - Print output to stdout with a JSON dataset entry on each line instead of writing a file + - Print output to stdout with JSON dataset entry on each line (**required for trtllm-bench**) * - ``--random-seed`` - Random seed for token generation (default: 420) * - ``--task-id`` @@ -75,14 +77,14 @@ prepare-dataset * - ``--log-level`` - Logging level: info or debug (default: info) -real-dataset -"""""""""""" +dataset +""""""" Process real datasets from various sources. .. code-block:: bash - trtllm-bench --model prepare-dataset real-dataset [OPTIONS] + python prepare_dataset.py dataset [OPTIONS] **Options** @@ -106,14 +108,14 @@ Process real datasets from various sources. - Input format: json, jsonl, csv, or txt (default: auto-detect) -token-norm-dist +token_norm_dist """"""""""""""" Generate synthetic datasets with normal token distribution. .. code-block:: bash - trtllm-bench --model prepare-dataset token-norm-dist [OPTIONS] + python prepare_dataset.py token_norm_dist [OPTIONS] **Options** @@ -137,14 +139,14 @@ Generate synthetic datasets with normal token distribution. - Normal distribution standard deviation for output tokens (required) -token-unif-dist +token_unif_dist """"""""""""""" Generate synthetic datasets with uniform token distribution .. code-block:: bash - trtllm-bench --model prepare-dataset token-unif-dist [OPTIONS] + python prepare_dataset.py token_unif_dist [OPTIONS] **Options** diff --git a/docs/source/developer-guide/ci-overview.md b/docs/source/developer-guide/ci-overview.md index 907816f1e792..908315c3e3c1 100644 --- a/docs/source/developer-guide/ci-overview.md +++ b/docs/source/developer-guide/ci-overview.md @@ -98,8 +98,8 @@ Each line contains the fully qualified test name followed by an optional specific hardware family. Example: ```text -accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] SKIP (https://nvbugs/6120535) -full:A100/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) +examples/test_openai.py::test_llm_openai_triton_1gpu SKIP (https://nvbugspro.nvidia.com/bug/4963654) +full:GH200/examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (arm is not supported) ``` Changes to `waives.txt` should include a bug link or brief explanation so other @@ -109,17 +109,8 @@ developers understand why the test is disabled. ### Triggering Post-merge tests -Full `/bot run --post-merge` runs require the `ci: post-merge approved` PR -label because they can consume substantial shared GPU resources. The label is -intended to be applied by an active member of the -`NVIDIA/trt-llm-ci-approvers` GitHub team. A GitHub workflow validates the label -actor and normally removes invalid approvals. This is a best-effort resource -governance guard, not a strict authorization boundary. The label remains in -place when new commits are pushed and can be removed manually when the approval -no longer applies. - -When you only need to verify a handful of post-merge tests, specify exactly -which stages to run: +When you only need to verify a handful of post-merge tests, avoid the heavy +`/bot run --post-merge` command. Instead, specify exactly which stages to run: ```bash /bot run --stage-list "stage-A,stage-B" @@ -132,14 +123,8 @@ default pre-merge set: /bot run --extra-stage "stage-A,stage-B" ``` -Both options accept stage names and wildcard patterns defined in -`jenkins/L0_Test.groovy`. The `"*"`, `"*Post-Merge*"`, and `"*PerfSanity*"` -selectors require the same approval label, including when they appear in a -comma-separated list. Equivalent escaped or repeated-star forms are treated the -same. Other stage selectors, including explicit stage names and other limited -wildcard patterns, retain their existing behavior. - -Being selective keeps CI turnaround fast and conserves hardware resources. +Both options accept any stage name defined in `jenkins/L0_Test.groovy`. Being +selective keeps CI turnaround fast and conserves hardware resources. ### Avoiding unnecessary `--disable-fail-fast` usage diff --git a/docs/source/developer-guide/overview.md b/docs/source/developer-guide/overview.md index d8fe31631612..af7f44f139cf 100644 --- a/docs/source/developer-guide/overview.md +++ b/docs/source/developer-guide/overview.md @@ -101,6 +101,7 @@ Module names longer than 8 characters are abbreviated to fit the fixed-width tag | `deep_ep` | `deep_ep ` | | `deep_gemm` | `deepgemm` | | `executor` | `executor` | +| `executor_worker` | `exec_wkr` | | `flash_mla` | `flashmla` | | `kernels` | `kernels ` | | `layers` | `layers ` | diff --git a/docs/source/developer-guide/perf-benchmarking.md b/docs/source/developer-guide/perf-benchmarking.md index 5903c12e32ea..4eb04eef4d13 100644 --- a/docs/source/developer-guide/perf-benchmarking.md +++ b/docs/source/developer-guide/perf-benchmarking.md @@ -171,7 +171,7 @@ can simply read a line and assume a complete entry. When creating a dataset, be JSON entry is on every line. ``` -In order to prepare a synthetic dataset, you can use the provided script in the `benchmarks` +In order to prepare a synthetic dataset, you can use the provided script in the `benchmarks/cpp` directory. For example, to generate a synthetic dataset of 1000 requests with a uniform ISL/OSL of 128/128 for [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B), run: diff --git a/docs/source/developer-guide/perf-overview.md b/docs/source/developer-guide/perf-overview.md index 58de0107b3d8..223ac5e8e92b 100644 --- a/docs/source/developer-guide/perf-overview.md +++ b/docs/source/developer-guide/perf-overview.md @@ -268,7 +268,7 @@ Testing was performed using the PyTorch backend - this workflow does not require | Stage | Description | Command | | :- | - | - | -| [Dataset](#preparing-a-dataset) | Create a synthetic dataset | `trtllm-bench --model $model_name prepare-dataset --output $dataset_file token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0` | +| [Dataset](#preparing-a-dataset) | Create a synthetic dataset | `python benchmarks/cpp/prepare_dataset.py --tokenizer=$model_name --stdout token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0 > $dataset_file` | | [Run](#running-the-benchmark) | Run a benchmark with a dataset | `trtllm-bench --model $model_name throughput --dataset $dataset_file --backend pytorch --config $llm_options` | ### Variables @@ -281,18 +281,18 @@ Testing was performed using the PyTorch backend - this workflow does not require | `$pp_size` | Pipeline parallel mapping degree to run the benchmark with | | `$ep_size` | Expert parallel mapping degree to run the benchmark with | | `$model_name` | HuggingFace model name eg. meta-llama/Llama-2-7b-hf or use the path to a local weights directory | -| `$dataset_file` | Location of the dataset file generated by `trtllm-bench prepare-dataset` | +| `$dataset_file` | Location of the dataset file generated by `prepare_dataset.py` | | `$num_requests` | The number of requests to generate for dataset generation | | `$seq_len` | A sequence length of ISL + OSL | | `$llm_options` | (optional) A yaml file containing additional options for the LLM API | ### Preparing a Dataset -In order to prepare a dataset, use the `trtllm-bench prepare-dataset` subcommand. +In order to prepare a dataset, you can use the provided [script](source:benchmarks/cpp/prepare_dataset.py). To generate a synthetic dataset, run the following command: ```shell -trtllm-bench --model $model_name prepare-dataset --output $dataset_file token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0 +python benchmarks/cpp/prepare_dataset.py --tokenizer=$model_name --stdout token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0 > $dataset_file ``` The command will generate a text file located at the path specified `$dataset_file` where all requests are of the same diff --git a/docs/source/developer-guide/telemetry.md b/docs/source/developer-guide/telemetry.md index 9ca7b7aaa49f..207c4604cce1 100644 --- a/docs/source/developer-guide/telemetry.md +++ b/docs/source/developer-guide/telemetry.md @@ -28,7 +28,7 @@ unset or when the safety sanitizer rejects the runtime value. ### `TorchLlmArgs` -269 captured fields. +261 captured fields. | Captured key | Annotation | Kind | Converter | Allowed values | |--------------|------------|------|-----------|----------------| @@ -55,7 +55,7 @@ unset or when the safety sanitizer rejects the runtime value. | `cache_transceiver_config.kv_transfer_sender_future_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | | `cache_transceiver_config.kv_transfer_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | | `cache_transceiver_config.max_tokens_in_buffer` | `Optional[int]` | `value` | | | -| `cache_transceiver_config.transceiver_runtime` | `Optional[Literal['CPP', 'PYTHON', 'auto']]` | `categorical` | | `CPP`, `PYTHON`, `auto` | +| `cache_transceiver_config.transceiver_runtime` | `Optional[Literal['CPP', 'PYTHON']]` | `categorical` | | `CPP`, `PYTHON` | | `context_parallel_size` | `` | `value` | | | | `cp_config.block_size` | `Optional[int]` | `value` | | | | `cp_config.cp_anchor_size` | `Optional[int]` | `value` | | | @@ -71,7 +71,6 @@ unset or when the safety sanitizer rejects the runtime value. | `cuda_graph_config.mode` | `Literal['decode']` | `categorical` | | `decode`, `encode` | | `cuda_graph_config.num_tokens` | `Optional[List[Annotated[int, Gt(gt=0)]]]` | `value` | | | | `cuda_graph_config.seq_lens` | `Optional[List[Annotated[int, Gt(gt=0)]]]` | `value` | | | -| `disable_mm_encoder` | `` | `value` | | | | `disable_overlap_scheduler` | `` | `value` | | | | `dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32` | | `dwdp_config.contention_opt` | `` | `value` | | | @@ -105,7 +104,7 @@ unset or when the safety sanitizer rejects the runtime value. | `iter_stats_max_iterations` | `Optional[int]` | `value` | | | | `kv_cache_config.attention_dp_events_gather_period_ms` | `` | `value` | | | | `kv_cache_config.avg_seq_len` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `kv_cache_config.block_reuse_policy` | `Literal['all_reusable', 'per_request', 'per_conversation']` | `categorical` | | `all_reusable`, `per_request`, `per_conversation` | +| `kv_cache_config.block_reuse_policy` | `Literal['all_reusable', 'per_request']` | `categorical` | | `all_reusable`, `per_request` | | `kv_cache_config.copy_on_partial_reuse` | `` | `value` | | | | `kv_cache_config.cross_kv_cache_fraction` | `Optional[float]` | `value` | | | | `kv_cache_config.disk_cache_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | @@ -132,7 +131,7 @@ unset or when the safety sanitizer rejects the runtime value. | `kv_cache_config.secondary_offload_min_priority` | `Optional[int]` | `value` | | | | `kv_cache_config.sink_token_length` | `Optional[int]` | `value` | | | | `kv_cache_config.tokens_per_block` | `` | `value` | | | -| `kv_cache_config.use_kv_cache_manager_v2` | `Union[bool, Literal['auto']]` | `value` | | `auto` | +| `kv_cache_config.use_kv_cache_manager_v2` | `` | `value` | | | | `kv_cache_config.use_uvm` | `` | `value` | | | | `kv_connector_config.connector` | `Optional[str]` | `categorical` | allowlist | `lmcache`, `lmcache-mp`, `kvbm` | | `layer_wise_benchmarks_config.calibration_layer_indices` | `Optional[List[int]]` | `value` | | | @@ -157,13 +156,11 @@ unset or when the safety sanitizer rejects the runtime value. | `moe_config.use_low_precision_moe_combine` | `` | `value` | | | | `moe_expert_parallel_size` | `Optional[int]` | `value` | | | | `moe_tensor_parallel_size` | `Optional[int]` | `value` | | | -| `multimodal_config.encoder_cache_max_bytes` | `` | `value` | | | | `multimodal_config.encoder_side_stream_max_ahead` | `` | `value` | | | | `multimodal_config.video_pruning_rate` | `Optional[float]` | `value` | | | | `mx_config.preshard_strategy` | `` | `categorical` | allowlist | `per_module` | | `mx_config.server_query_timeout_s` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | | `num_postprocess_workers` | `` | `value` | | | -| `num_serve_frontends` | `` | `value` | | | | `nvfp4_gemm_config.allowed_backends` | `List[Literal['cutlass', 'cublaslt', 'cutedsl', 'cuda_core', 'marlin']]` | `value` | | `cutlass`, `cublaslt`, `cutedsl`, `cuda_core`, `marlin` | | `orchestrator_type` | `Optional[Literal['rpc', 'ray']]` | `categorical` | | `rpc`, `ray` | | `peft_cache_config.device_cache_percent` | `` | `value` | | | @@ -211,7 +208,6 @@ unset or when the safety sanitizer rejects the runtime value. | `sparse_attention_config.algorithm` | `Literal['dsa']` | `categorical` | | `dsa`, `deepseek_v4`, `minimax_m3`, `rocket`, `skip_softmax` | | `sparse_attention_config.compress_ratios` | `List[int]` | `value` | | | | `sparse_attention_config.enable_heuristic_topk` | `` | `value` | | | -| `sparse_attention_config.implementation` | `Literal['triton', 'msa']` | `categorical` | | `triton`, `msa` | | `sparse_attention_config.index_head_dim` | `Optional[int]` | `value` | | | | `sparse_attention_config.index_n_heads` | `Optional[int]` | `value` | | | | `sparse_attention_config.index_topk` | `Optional[int]` | `value` | | | @@ -220,8 +216,6 @@ unset or when the safety sanitizer rejects the runtime value. | `sparse_attention_config.indexer_rope_interleave` | `` | `value` | | | | `sparse_attention_config.kernel_size` | `Optional[int]` | `value` | | | | `sparse_attention_config.kt_cache_dtype` | `Optional[str]` | `categorical` | allowlist | `bfloat16`, `float8_e5m2` | -| `sparse_attention_config.num_attention_heads` | `Optional[int]` | `value` | | | -| `sparse_attention_config.num_key_value_heads` | `Optional[int]` | `value` | | | | `sparse_attention_config.page_size` | `Optional[int]` | `value` | | | | `sparse_attention_config.prompt_budget` | `Optional[int]` | `value` | | | | `sparse_attention_config.q_split_threshold` | `` | `value` | | | @@ -244,8 +238,7 @@ unset or when the safety sanitizer rejects the runtime value. | `speculative_config.acceptance_rate_window_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | | `speculative_config.allow_advanced_sampling` | `` | `value` | | | | `speculative_config.begin_thinking_phase_token` | `` | `value` | | | -| `speculative_config.block_size` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | -| `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `DSpark`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | +| `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | | `speculative_config.dynamic_tree_max_topK` | `Optional[int]` | `value` | | | | `speculative_config.eagle3_layers_to_capture` | `Optional[Set[int]]` | `value` | | | | `speculative_config.eagle3_model_arch` | `Literal['llama3', 'mistral_large3']` | `categorical` | | `llama3`, `mistral_large3` | @@ -258,8 +251,6 @@ unset or when the safety sanitizer rejects the runtime value. | `speculative_config.is_keep_all` | `` | `value` | | | | `speculative_config.is_public_pool` | `` | `value` | | | | `speculative_config.is_use_oldest` | `` | `value` | | | -| `speculative_config.markov_head_type` | `Optional[Literal['vanilla', 'gated', 'rnn']]` | `categorical` | | `vanilla`, `gated`, `rnn` | -| `speculative_config.markov_rank` | `Optional[int]` | `value` | | | | `speculative_config.mask_token_id` | `Optional[int]` | `value` | | | | `speculative_config.max_concurrency` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | | `speculative_config.max_draft_len` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | @@ -301,3 +292,290 @@ unset or when the safety sanitizer rejects the runtime value. | `use_cute_dsl_bf16_gemm` | `` | `value` | | | | `use_cute_dsl_blockscaling_bmm` | `` | `value` | | | | `use_cute_dsl_blockscaling_mm` | `` | `value` | | | + +### `TrtLlmArgs` + +280 captured fields. + +| Captured key | Annotation | Kind | Converter | Allowed values | +|--------------|------------|------|-----------|----------------| +| `backend` | `Optional[str]` | `categorical` | allowlist | `pytorch`, `tensorrt`, `_autodeploy` | +| `batching_type` | `Optional[tensorrt_llm.llmapi.llm_args.BatchingType]` | `categorical` | | `STATIC`, `INFLIGHT` | +| `build_config.dry_run` | `` | `value` | | | +| `build_config.enable_debug_output` | `` | `value` | | | +| `build_config.force_num_profiles` | `Optional[int]` | `value` | | | +| `build_config.gather_context_logits` | `` | `value` | | | +| `build_config.gather_generation_logits` | `` | `value` | | | +| `build_config.kv_cache_type` | `Optional[tensorrt_llm.llmapi.kv_cache_type.KVCacheType]` | `categorical` | | `continuous`, `paged`, `disabled` | +| `build_config.lora_config.lora_ckpt_source` | `Literal['hf', 'nemo']` | `categorical` | | `hf`, `nemo` | +| `build_config.lora_config.max_cpu_loras` | `Optional[int]` | `value` | | | +| `build_config.lora_config.max_lora_rank` | `` | `value` | | | +| `build_config.lora_config.max_loras` | `Optional[int]` | `value` | | | +| `build_config.lora_config.swap_gate_up_proj_lora_b_weight` | `` | `value` | | | +| `build_config.max_batch_size` | `` | `value` | | | +| `build_config.max_beam_width` | `` | `value` | | | +| `build_config.max_draft_len` | `` | `value` | | | +| `build_config.max_encoder_input_len` | `` | `value` | | | +| `build_config.max_input_len` | `` | `value` | | | +| `build_config.max_num_tokens` | `` | `value` | | | +| `build_config.max_prompt_embedding_table_size` | `` | `value` | | | +| `build_config.max_seq_len` | `Optional[int]` | `value` | | | +| `build_config.monitor_memory` | `` | `value` | | | +| `build_config.opt_batch_size` | `` | `value` | | | +| `build_config.opt_num_tokens` | `Optional[int]` | `value` | | | +| `build_config.plugin_config.bert_attention_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.bert_context_fmha_fp32_acc` | `` | `value` | | | +| `build_config.plugin_config.context_fmha` | `` | `value` | | | +| `build_config.plugin_config.dora_plugin` | `` | `value` | | | +| `build_config.plugin_config.fp8_rowwise_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.fuse_fp4_quant` | `` | `value` | | | +| `build_config.plugin_config.gemm_allreduce_plugin` | `Optional[Literal['float16', 'bfloat16', None]]` | `categorical` | | `float16`, `bfloat16`, `None` | +| `build_config.plugin_config.gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', 'fp8', 'nvfp4', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `fp8`, `nvfp4`, `None` | +| `build_config.plugin_config.gemm_swiglu_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | +| `build_config.plugin_config.gpt_attention_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.identity_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.layernorm_quantization_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.lora_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.low_latency_gemm_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | +| `build_config.plugin_config.low_latency_gemm_swiglu_plugin` | `Optional[Literal['fp8', None]]` | `categorical` | | `fp8`, `None` | +| `build_config.plugin_config.mamba_conv1d_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.manage_weights` | `` | `value` | | | +| `build_config.plugin_config.moe_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.multiple_profiles` | `` | `value` | | | +| `build_config.plugin_config.nccl_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.norm_quant_fusion` | `` | `value` | | | +| `build_config.plugin_config.paged_kv_cache` | `Optional[bool]` | `value` | | | +| `build_config.plugin_config.paged_state` | `` | `value` | | | +| `build_config.plugin_config.pp_reduce_scatter` | `` | `value` | | | +| `build_config.plugin_config.qserve_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.quantize_per_token_plugin` | `` | `value` | | | +| `build_config.plugin_config.quantize_tensor_plugin` | `` | `value` | | | +| `build_config.plugin_config.reduce_fusion` | `` | `value` | | | +| `build_config.plugin_config.remove_input_padding` | `` | `value` | | | +| `build_config.plugin_config.rmsnorm_quantization_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.smooth_quant_gemm_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.smooth_quant_plugins` | `` | `value` | | | +| `build_config.plugin_config.streamingllm` | `` | `value` | | | +| `build_config.plugin_config.tokens_per_block` | `` | `value` | | | +| `build_config.plugin_config.use_fp8_context_fmha` | `` | `value` | | | +| `build_config.plugin_config.use_fused_mlp` | `` | `value` | | | +| `build_config.plugin_config.use_paged_context_fmha` | `` | `value` | | | +| `build_config.plugin_config.user_buffer` | `` | `value` | | | +| `build_config.plugin_config.weight_only_groupwise_quant_matmul_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.plugin_config.weight_only_quant_matmul_plugin` | `Optional[Literal['auto', 'float16', 'float32', 'bfloat16', 'int32', None]]` | `categorical` | | `auto`, `float16`, `float32`, `bfloat16`, `int32`, `None` | +| `build_config.speculative_decoding_mode` | `` | `categorical` | | `NONE`, `DRAFT_TOKENS_EXTERNAL`, `MEDUSA`, `LOOKAHEAD_DECODING`, `EXPLICIT_DRAFT_TOKENS`, `EAGLE`, `NGRAM`, `USER_PROVIDED`, `SAVE_HIDDEN_STATES`, `AUTO` | +| `build_config.strongly_typed` | `` | `value` | | | +| `build_config.use_mrope` | `` | `value` | | | +| `build_config.use_refit` | `` | `value` | | | +| `build_config.use_strip_plan` | `` | `value` | | | +| `build_config.weight_sparsity` | `` | `value` | | | +| `build_config.weight_streaming` | `` | `value` | | | +| `cache_transceiver_config.backend` | `Optional[Literal['DEFAULT', 'UCX', 'NIXL', 'MOONCAKE', 'MPI']]` | `categorical` | | `DEFAULT`, `UCX`, `NIXL`, `MOONCAKE`, `MPI` | +| `cache_transceiver_config.kv_cache_bounce_size_mb` | `` | `value` | | | +| `cache_transceiver_config.kv_transfer_poll_interval_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `cache_transceiver_config.kv_transfer_sender_future_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `cache_transceiver_config.kv_transfer_timeout_ms` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `cache_transceiver_config.max_tokens_in_buffer` | `Optional[int]` | `value` | | | +| `cache_transceiver_config.transceiver_runtime` | `Optional[Literal['CPP', 'PYTHON']]` | `categorical` | | `CPP`, `PYTHON` | +| `calib_config.calib_batch_size` | `` | `value` | | | +| `calib_config.calib_batches` | `` | `value` | | | +| `calib_config.calib_max_seq_length` | `` | `value` | | | +| `calib_config.device` | `Literal['cuda', 'cpu']` | `categorical` | | `cuda`, `cpu` | +| `calib_config.random_seed` | `` | `value` | | | +| `calib_config.tokenizer_max_seq_length` | `` | `value` | | | +| `context_parallel_size` | `` | `value` | | | +| `cp_config.block_size` | `Optional[int]` | `value` | | | +| `cp_config.cp_anchor_size` | `Optional[int]` | `value` | | | +| `cp_config.cp_type` | `` | `categorical` | | `ULYSSES`, `STAR`, `RING`, `HELIX` | +| `cp_config.fifo_version` | `Optional[int]` | `value` | | | +| `cp_config.tokens_per_block` | `Optional[int]` | `value` | | | +| `cp_config.use_nccl_for_alltoall` | `Optional[bool]` | `value` | | | +| `dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32` | +| `embedding_parallel_mode` | `Literal['NONE', 'SHARDING_ALONG_VOCAB', 'SHARDING_ALONG_HIDDEN']` | `categorical` | | `NONE`, `SHARDING_ALONG_VOCAB`, `SHARDING_ALONG_HIDDEN` | +| `enable_attention_dp` | `` | `value` | | | +| `enable_build_cache.max_cache_storage_gb` | `` | `value` | | | +| `enable_build_cache.max_records` | `` | `value` | | | +| `enable_chunked_prefill` | `` | `value` | | | +| `enable_energy_metrics` | `` | `value` | | | +| `enable_lm_head_tp_in_adp` | `` | `value` | | | +| `enable_lora` | `` | `value` | | | +| `enable_prompt_adapter` | `` | `value` | | | +| `enable_tqdm` | `` | `value` | | | +| `extended_runtime_perf_knob_config.cuda_graph_cache_size` | `` | `value` | | | +| `extended_runtime_perf_knob_config.cuda_graph_mode` | `` | `value` | | | +| `extended_runtime_perf_knob_config.enable_context_fmha_fp32_acc` | `` | `value` | | | +| `extended_runtime_perf_knob_config.multi_block_mode` | `` | `value` | | | +| `fail_fast_on_attention_window_too_large` | `` | `value` | | | +| `fast_build` | `` | `value` | | | +| `gather_generation_logits` | `` | `value` | | | +| `gpus_per_node` | `Optional[int]` | `value` | | | +| `guided_decoding_backend` | `Optional[Literal['xgrammar', 'llguidance']]` | `categorical` | | `xgrammar`, `llguidance` | +| `iter_stats_max_iterations` | `Optional[int]` | `value` | | | +| `kv_cache_config.attention_dp_events_gather_period_ms` | `` | `value` | | | +| `kv_cache_config.avg_seq_len` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `kv_cache_config.block_reuse_policy` | `Literal['all_reusable', 'per_request']` | `categorical` | | `all_reusable`, `per_request` | +| `kv_cache_config.copy_on_partial_reuse` | `` | `value` | | | +| `kv_cache_config.cross_kv_cache_fraction` | `Optional[float]` | `value` | | | +| `kv_cache_config.disk_cache_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `kv_cache_config.disk_prefetch_num_reqs` | `` | `value` | | | +| `kv_cache_config.dtype` | `` | `categorical` | allowlist | `auto`, `float16`, `bfloat16`, `float32`, `fp8`, `nvfp4` | +| `kv_cache_config.enable_block_reuse` | `` | `value` | | | +| `kv_cache_config.enable_kv_pool_rebalance` | `` | `value` | | | +| `kv_cache_config.enable_partial_reuse` | `` | `value` | | | +| `kv_cache_config.enable_swa_scratch_reuse` | `` | `value` | | | +| `kv_cache_config.event_buffer_max_size` | `` | `value` | | | +| `kv_cache_config.free_gpu_memory_fraction` | `Optional[float]` | `value` | | | +| `kv_cache_config.host_cache_size` | `Optional[int]` | `value` | | | +| `kv_cache_config.iteration_stats_interval` | `` | `value` | | | +| `kv_cache_config.kv_cache_event_hash_algo` | `Literal['auto', 'v1_block_key', 'v2_sha256', 'v2_sha256_64']` | `categorical` | | `auto`, `v1_block_key`, `v2_sha256`, `v2_sha256_64` | +| `kv_cache_config.mamba_ssm_cache_dtype` | `Literal['auto', 'float16', 'bfloat16', 'float32']` | `categorical` | | `auto`, `float16`, `bfloat16`, `float32` | +| `kv_cache_config.mamba_ssm_philox_rounds` | `` | `value` | | | +| `kv_cache_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | +| `kv_cache_config.mamba_state_cache_interval` | `` | `value` | | | +| `kv_cache_config.max_attention_window` | `Optional[List[int]]` | `value` | | | +| `kv_cache_config.max_gpu_total_bytes` | `` | `value` | | | +| `kv_cache_config.max_tokens` | `Optional[int]` | `value` | | | +| `kv_cache_config.max_util_for_resume` | `` | `value` | | | +| `kv_cache_config.pool_ratio` | `Optional[List[float]]` | `value` | | | +| `kv_cache_config.secondary_offload_min_priority` | `Optional[int]` | `value` | | | +| `kv_cache_config.sink_token_length` | `Optional[int]` | `value` | | | +| `kv_cache_config.tokens_per_block` | `` | `value` | | | +| `kv_cache_config.use_kv_cache_manager_v2` | `` | `value` | | | +| `kv_cache_config.use_uvm` | `` | `value` | | | +| `load_format` | `Literal['auto', 'dummy']` | `categorical` | | `auto`, `dummy` | +| `lora_config.lora_ckpt_source` | `Literal['hf', 'nemo']` | `categorical` | | `hf`, `nemo` | +| `lora_config.max_cpu_loras` | `Optional[int]` | `value` | | | +| `lora_config.max_lora_rank` | `` | `value` | | | +| `lora_config.max_loras` | `Optional[int]` | `value` | | | +| `lora_config.swap_gate_up_proj_lora_b_weight` | `` | `value` | | | +| `max_batch_size` | `Optional[int]` | `value` | | | +| `max_beam_width` | `Optional[int]` | `value` | | | +| `max_input_len` | `Optional[int]` | `value` | | | +| `max_num_tokens` | `Optional[int]` | `value` | | | +| `max_prompt_adapter_token` | `` | `value` | | | +| `max_seq_len` | `Optional[int]` | `value` | | | +| `moe_cluster_parallel_size` | `Optional[int]` | `value` | | | +| `moe_expert_parallel_size` | `Optional[int]` | `value` | | | +| `moe_tensor_parallel_size` | `Optional[int]` | `value` | | | +| `normalize_log_probs` | `` | `value` | | | +| `num_postprocess_workers` | `` | `value` | | | +| `orchestrator_type` | `Optional[Literal['rpc', 'ray']]` | `categorical` | | `rpc`, `ray` | +| `peft_cache_config.device_cache_percent` | `` | `value` | | | +| `peft_cache_config.host_cache_size` | `` | `value` | | | +| `peft_cache_config.max_adapter_size` | `` | `value` | | | +| `peft_cache_config.max_pages_per_block_device` | `` | `value` | | | +| `peft_cache_config.max_pages_per_block_host` | `` | `value` | | | +| `peft_cache_config.num_copy_streams` | `` | `value` | | | +| `peft_cache_config.num_device_module_layer` | `` | `value` | | | +| `peft_cache_config.num_ensure_workers` | `` | `value` | | | +| `peft_cache_config.num_host_module_layer` | `` | `value` | | | +| `peft_cache_config.num_put_workers` | `` | `value` | | | +| `peft_cache_config.optimal_adapter_size` | `` | `value` | | | +| `perf_metrics_max_requests` | `` | `value` | | | +| `pipeline_parallel_size` | `` | `value` | | | +| `pp_partition` | `Optional[List[int]]` | `value` | | | +| `prometheus_metrics_config.e2e_request_latency_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_decode_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_inference_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_prefill_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.request_queue_time_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.time_per_output_token_buckets` | `Optional[List[float]]` | `value` | | | +| `prometheus_metrics_config.time_to_first_token_buckets` | `Optional[List[float]]` | `value` | | | +| `quant_config.clamp_val` | `Optional[List[float]]` | `value` | | | +| `quant_config.group_size` | `Optional[int]` | `value` | | | +| `quant_config.has_zero_point` | `` | `value` | | | +| `quant_config.kv_cache_quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `MXFP8`, `W4A16_NVFP4`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | +| `quant_config.mamba_ssm_philox_rounds` | `` | `value` | | | +| `quant_config.mamba_ssm_stochastic_rounding` | `` | `value` | | | +| `quant_config.pre_quant_scale` | `` | `value` | | | +| `quant_config.quant_algo` | `Optional[tensorrt_llm.quantization.mode.QuantAlgo]` | `categorical` | | `W8A16`, `W4A16`, `W4A16_AWQ`, `W4A8_AWQ`, `W8A16_GPTQ`, `W4A16_GPTQ`, `W8A8_SQ_PER_CHANNEL`, `W8A8_SQ_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN`, `W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN`, `W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN`, `W4A8_QSERVE_PER_GROUP`, `W4A8_QSERVE_PER_CHANNEL`, `FP8`, `FP8_PER_CHANNEL_PER_TOKEN`, `FP8_BLOCK_SCALES`, `INT8`, `MIXED_PRECISION`, `NVFP4`, `W4A8_NVFP4_FP8`, `W4A8_MXFP4_FP8`, `W4A8_MXFP4_MXFP8`, `W4A16_MXFP4`, `MXFP8`, `W4A16_NVFP4`, `NVFP4_AWQ`, `NVFP4_ARC`, `NO_QUANT` | +| `quant_config.smoothquant_val` | `` | `value` | | | +| `quant_config.use_meta_recipe` | `` | `value` | | | +| `reasoning_parser` | `Optional[str]` | `categorical` | allowlist | `auto`, `deepseek-r1`, `laguna`, `qwen3`, `qwen3_5`, `minimax_m2`, `minimax_m2_append_think`, `nano-v3`, `gemma4`, `kimi_k2`, `kimi_k25` | +| `request_stats_max_iterations` | `Optional[int]` | `value` | | | +| `return_perf_metrics` | `` | `value` | | | +| `scheduler_config.capacity_scheduler_policy` | `` | `categorical` | | `MAX_UTILIZATION`, `GUARANTEED_NO_EVICT`, `STATIC_BATCH` | +| `scheduler_config.context_chunking_policy` | `Optional[tensorrt_llm.llmapi.llm_args.ContextChunkingPolicy]` | `categorical` | | `FIRST_COME_FIRST_SERVED`, `EQUAL_PROGRESS`, `FORCE_CHUNK` | +| `scheduler_config.dynamic_batch_config.dynamic_batch_moving_average_window` | `` | `value` | | | +| `scheduler_config.dynamic_batch_config.enable_batch_size_tuning` | `` | `value` | | | +| `scheduler_config.dynamic_batch_config.enable_max_num_tokens_tuning` | `` | `value` | | | +| `scheduler_config.enable_prefix_aware_scheduling` | `` | `value` | | | +| `scheduler_config.use_python_scheduler` | `` | `value` | | | +| `scheduler_config.waiting_queue_policy` | `` | `categorical` | | `fcfs`, `priority` | +| `skip_tokenizer_init` | `` | `value` | | | +| `sparse_attention_config.algorithm` | `Literal['dsa']` | `categorical` | | `dsa`, `deepseek_v4`, `minimax_m3`, `rocket`, `skip_softmax` | +| `sparse_attention_config.compress_ratios` | `List[int]` | `value` | | | +| `sparse_attention_config.enable_heuristic_topk` | `` | `value` | | | +| `sparse_attention_config.index_head_dim` | `Optional[int]` | `value` | | | +| `sparse_attention_config.index_n_heads` | `Optional[int]` | `value` | | | +| `sparse_attention_config.index_topk` | `Optional[int]` | `value` | | | +| `sparse_attention_config.indexer_k_dtype` | `Literal['fp8', 'fp4']` | `categorical` | | `fp8`, `fp4` | +| `sparse_attention_config.indexer_max_chunk_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.indexer_rope_interleave` | `` | `value` | | | +| `sparse_attention_config.kernel_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.kt_cache_dtype` | `Optional[str]` | `categorical` | allowlist | `bfloat16`, `float8_e5m2` | +| `sparse_attention_config.page_size` | `Optional[int]` | `value` | | | +| `sparse_attention_config.prompt_budget` | `Optional[int]` | `value` | | | +| `sparse_attention_config.q_split_threshold` | `` | `value` | | | +| `sparse_attention_config.seq_len_threshold` | `Optional[int]` | `value` | | | +| `sparse_attention_config.skip_indexer_for_short_seqs` | `` | `value` | | | +| `sparse_attention_config.sparse_block_size` | `` | `value` | | | +| `sparse_attention_config.sparse_disable_index_value` | `` | `value` | | | +| `sparse_attention_config.sparse_index_dim` | `` | `value` | | | +| `sparse_attention_config.sparse_init_blocks` | `` | `value` | | | +| `sparse_attention_config.sparse_local_blocks` | `` | `value` | | | +| `sparse_attention_config.sparse_num_index_heads` | `` | `value` | | | +| `sparse_attention_config.sparse_score_type` | `Literal['max']` | `categorical` | | `max` | +| `sparse_attention_config.sparse_topk_blocks` | `` | `value` | | | +| `sparse_attention_config.topk` | `Optional[int]` | `value` | | | +| `sparse_attention_config.topr` | `Union[int, float, NoneType]` | `value` | | | +| `sparse_attention_config.use_cute_dsl_paged_mqa_logits` | `` | `value` | | | +| `sparse_attention_config.use_cute_dsl_topk` | `` | `value` | | | +| `sparse_attention_config.window_size` | `` | `value` | | | +| `speculative_config.acceptance_rate_threshold` | `Optional[float]` | `value` | | | +| `speculative_config.acceptance_rate_window_size` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `speculative_config.allow_advanced_sampling` | `` | `value` | | | +| `speculative_config.begin_thinking_phase_token` | `` | `value` | | | +| `speculative_config.decoding_type` | `Literal['AUTO']` | `categorical` | | `AUTO`, `DFlash`, `Draft_Target`, `Eagle3`, `Eagle`, `Lookahead`, `MTP`, `Medusa`, `NGram`, `PARD`, `SA`, `SaveState`, `User_Provided` | +| `speculative_config.dynamic_tree_max_topK` | `Optional[int]` | `value` | | | +| `speculative_config.eagle3_layers_to_capture` | `Optional[Set[int]]` | `value` | | | +| `speculative_config.eagle3_model_arch` | `Literal['llama3', 'mistral_large3']` | `categorical` | | `llama3`, `mistral_large3` | +| `speculative_config.eagle3_one_model` | `Optional[bool]` | `value` | | | +| `speculative_config.eagle_choices` | `Optional[List[List[int]]]` | `value` | | | +| `speculative_config.enable_global_pool` | `` | `value` | | | +| `speculative_config.end_thinking_phase_token` | `` | `value` | | | +| `speculative_config.global_pool_size` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `speculative_config.greedy_sampling` | `Optional[bool]` | `value` | | | +| `speculative_config.is_keep_all` | `` | `value` | | | +| `speculative_config.is_public_pool` | `` | `value` | | | +| `speculative_config.is_use_oldest` | `` | `value` | | | +| `speculative_config.mask_token_id` | `Optional[int]` | `value` | | | +| `speculative_config.max_concurrency` | `Optional[Annotated[int, Gt(gt=0)]]` | `value` | | | +| `speculative_config.max_draft_len` | `Optional[Annotated[int, Ge(ge=0)]]` | `value` | | | +| `speculative_config.max_matching_ngram_size` | `` | `value` | | | +| `speculative_config.max_ngram_size` | `` | `value` | | | +| `speculative_config.max_non_leaves_per_layer` | `Optional[int]` | `value` | | | +| `speculative_config.max_total_draft_tokens` | `Optional[int]` | `value` | | | +| `speculative_config.max_verification_set_size` | `` | `value` | | | +| `speculative_config.max_window_size` | `` | `value` | | | +| `speculative_config.medusa_choices` | `Optional[List[List[int]]]` | `value` | | | +| `speculative_config.mtp_eagle_one_model` | `` | `value` | | | +| `speculative_config.num_eagle_layers` | `Optional[int]` | `value` | | | +| `speculative_config.num_medusa_heads` | `Optional[int]` | `value` | | | +| `speculative_config.num_nextn_predict_layers` | `Optional[int]` | `value` | | | +| `speculative_config.posterior_threshold` | `Optional[float]` | `value` | | | +| `speculative_config.relaxed_delta` | `` | `value` | | | +| `speculative_config.relaxed_topk` | `` | `value` | | | +| `speculative_config.sa_config.enable_global_pool` | `` | `value` | | | +| `speculative_config.sa_config.threshold` | `` | `value` | | | +| `speculative_config.target_layer_ids` | `Optional[List[int]]` | `value` | | | +| `speculative_config.use_dynamic_tree` | `Optional[bool]` | `value` | | | +| `speculative_config.use_mtp_vanilla` | `` | `value` | | | +| `speculative_config.use_rejection_sampling` | `` | `value` | | | +| `speculative_config.use_relaxed_acceptance_for_thinking` | `` | `value` | | | +| `speculative_config.write_interval` | `` | `value` | | | +| `telemetry_config.disabled` | `` | `value` | | | +| `telemetry_config.usage_context` | `` | `categorical` | | `unknown`, `llm_class`, `cli_serve`, `cli_bench`, `cli_eval` | +| `tensor_parallel_size` | `` | `value` | | | +| `tokenizer_mode` | `Literal['auto', 'slow']` | `categorical` | | `auto`, `slow` | +| `trust_remote_code` | `` | `value` | | | diff --git a/docs/source/examples/customization.md b/docs/source/examples/customization.md index 3a7aa5b3f9e8..4c357554f504 100644 --- a/docs/source/examples/customization.md +++ b/docs/source/examples/customization.md @@ -2,51 +2,70 @@ ## Quantization -TensorRT LLM runs quantized models from pre-quantized checkpoints. Use a checkpoint quantized with [NVIDIA TensorRT Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer) (for example, the ready-made FP8/NVFP4 checkpoints published on the [NVIDIA Hugging Face hub](https://huggingface.co/nvidia)), and the quantization configuration is detected automatically when the model loads: +TensorRT LLM can quantize the Hugging Face model automatically. By setting the appropriate flags in the `LLM` instance. For example, to perform an Int4 AWQ quantization, the following code triggers the model quantization. Please refer to complete list of [supported flags](https://nvidia.github.io/TensorRT-LLM/_modules/tensorrt_llm/quantization/mode.html#QuantAlgo) and acceptable values. ``` python -from tensorrt_llm import LLM +from tensorrt_llm.llmapi import QuantConfig, QuantAlgo -llm = LLM("nvidia/Llama-3.1-8B-Instruct-FP8") -``` +quant_config = QuantConfig(quant_algo=QuantAlgo.W4A16_AWQ) -Refer to the [quantization feature documentation](../features/quantization.md) for the supported formats per GPU architecture and instructions on quantizing your own model. +llm = LLM(, quant_config=quant_config) +``` ## Sampling -SamplingParams can customize the sampling strategy to control LLM generated responses, such as beam search, temperature, and many others. +SamplingParams can customize the sampling strategy to control LLM generated responses, such as beam search, temperature, and [others](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/llmapi/utils.py#L55-L76). -As an example, to enable beam search with a beam width of 4, configure the engine limit with `max_beam_width` and request beam search through `SamplingParams`: +As an example, to enable beam search with a beam size of 4, set the `sampling_params` as follows: ```python -from tensorrt_llm import LLM, SamplingParams +from tensorrt_llm.llmapi import LLM, SamplingParams, BuildConfig -llm = LLM(, max_beam_width=4) +build_config = BuildConfig() +build_config.max_beam_width = 4 + +llm = LLM(, build_config=build_config) # Let the LLM object generate text with the default sampling strategy, or # you can create a SamplingParams object as well with several fields set manually -sampling_params = SamplingParams(n=4, use_beam_search=True) +sampling_params = SamplingParams(beam_width=4) # current limitation: beam_width should be equal to max_beam_width for output in llm.generate(, sampling_params=sampling_params): print(output) ``` -Refer to the [class documentation](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html#tensorrt_llm.llmapi.SamplingParams) for the complete list of fields. +`SamplingParams` manages and dispatches fields to C++ classes including: + +* [SamplingConfig](https://nvidia.github.io/TensorRT-LLM/_cpp_gen/runtime.html#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE) +* [OutputConfig](https://nvidia.github.io/TensorRT-LLM/_cpp_gen/executor.html#_CPPv4N12tensorrt_llm8executor12OutputConfigE) + +Refer to the [class documentation](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html#tensorrt_llm.llmapi.SamplingParams) for more details. + +## Build Configuration + +Apart from the arguments mentioned above, you can also customize the build configuration with the `build_config` class and other arguments borrowed from the trtllm-build CLI. These build configuration options provide flexibility in building engines for the target hardware and use cases. Refer to the following example: + +```python +llm = LLM(, + build_config=BuildConfig( + max_num_tokens=4096, + max_batch_size=128, + max_beam_width=4)) +``` +Refer to the [buildconfig documentation](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/builder.py#L470-L501) for more details. ## Runtime Customization -Runtime behavior such as KV cache management and GPU memory allocation can be customized with dedicated configuration classes like `kv_cache_config` and `peft_cache_config` passed to the `LLM` constructor. Refer to the following example: +Similar to `build_config`, you can also customize the runtime configuration with the `runtime_config`, `peft_cache_config` or other [arguments](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/llmapi/llm_utils.py#L186-L223) borrowed from the Executor APIs. These runtime configuration options provide additional flexibility with respect to KV cache management, GPU memory allocation and so on. Refer to the following example: + ```python -from tensorrt_llm import LLM -from tensorrt_llm.llmapi import KvCacheConfig +from tensorrt_llm.llmapi import LLM, KvCacheConfig llm = LLM(, kv_cache_config=KvCacheConfig( free_gpu_memory_fraction=0.8)) ``` -Refer to the [LLM API reference](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) for all available configuration classes. - ## Tokenizer Customization By default, the LLM API uses transformers’ `AutoTokenizer`. You can override it with your own tokenizer by passing it when creating the LLM object. Refer to the following example: @@ -71,8 +90,8 @@ for output in llm.generate([32, 12]): For performance considerations, you can disable the tokenizer by passing `skip_tokenizer_init=True` when creating `LLM`. In this case, `LLM.generate` and `LLM.generate_async` will expect prompt token ids as input. Refer to the following example: ```python -llm = LLM(, skip_tokenizer_init=True) -for output in llm.generate([[32, 12]]): +llm = LLM() +for output in llm.generate([[32, 12]], skip_tokenizer_init=True): print(output) ``` diff --git a/docs/source/features/auto_deploy/advanced/testing_strategy.md b/docs/source/features/auto_deploy/advanced/testing_strategy.md index b60eada7e722..d65841862ee6 100644 --- a/docs/source/features/auto_deploy/advanced/testing_strategy.md +++ b/docs/source/features/auto_deploy/advanced/testing_strategy.md @@ -117,8 +117,8 @@ Format: `path/to/test_file.py::test_function_name[param_id]` Example from `l0_a30.yml`: ```yaml -- accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype -- unittest/_torch/modeling -k "modeling_llama" +- accuracy/test_cli_flow.py::TestLlama3_1_8BInstruct::test_medusa_fp8_prequantized +- examples/test_multimodal.py::test_llm_multimodal_general[Qwen2-VL-7B-Instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:4] ``` ### Example: Adding an Accuracy Test diff --git a/docs/source/features/checkpoint-loading.md b/docs/source/features/checkpoint-loading.md index fa617b43dbdf..6caa3a0fcd95 100644 --- a/docs/source/features/checkpoint-loading.md +++ b/docs/source/features/checkpoint-loading.md @@ -83,15 +83,6 @@ Currently, HF checkpoint loader is the primary built-in format, supporting: - **Configuration parser** - Parsing HF stored configuration information to TRTLLM `ModelConfig` object - **Weights Mapping** - Converting HF weights into TRTLLM compatible representation -### ModelExpress (MX) Loading Path - -The PyTorch backend can use ModelExpress (MX) for peer-to-peer weight transfer -from a running TensorRT-LLM source instance before falling back to Hugging Face -checkpoint loading. Selecting MX does not require an MX-specific on-disk -checkpoint or conversion of the Hugging Face checkpoint. For installation, MX -service deployment, and configuration details, see -[ModelExpress (MX) Checkpoint Loading](./model-express.md). - ## Using Checkpoint Loaders ### Basic Usage diff --git a/docs/source/features/disagg-serving.md b/docs/source/features/disagg-serving.md index 9ed621158a26..f38e3c818f57 100644 --- a/docs/source/features/disagg-serving.md +++ b/docs/source/features/disagg-serving.md @@ -94,19 +94,11 @@ The optimizations required for KV cache transmission vary depending on whether i ### Unique Global Request ID -The context and generation phases of one request must share a single request ID: the ctx↔gen KV-cache transfer is keyed by it, so a collision (two in-flight requests with the same ID) corrupts the transfer. This shared ID is carried on `DisaggregatedParams.disagg_request_id`. - -The disaggregated server generates this ID itself as a **snowflake** — a self-contained 64-bit positive integer that is unique without any cross-process coordination. The bit layout is: - -``` -[ 0 (1 bit) | timestamp_ms (39 bits) | node_id (8 bits) | process_id (6 bits) | counter (10 bits) ] -``` - -- `node_id` (0–255) identifies the node (defaults to a hash of the MAC address; overridable via `node_id` in the disaggregated config). -- `process_id` (0–63) identifies the orchestrator process on that node. In a [coordinator + worker fleet](#coordinator-and-worker-fleet) each fleet worker receives a distinct value, so co-located workers never emit the same ID in the same millisecond. It is set from the `TRTLLM_DISAGG_WORKER_PROCESS_ID` environment variable (assigned automatically per worker by the launcher). -- The `(node_id, process_id)` pair therefore makes the ID unique across all orchestrator processes without a shared counter or an extra network round trip — each worker mints its own IDs locally. - -Global disaggregated IDs occupy the range `[1 << 40, 2**63)`; worker-local and warm-up request IDs occupy the disjoint range `[0, 1 << 40)`, so the two never collide. If a client supplies its own positive `disagg_request_id`, that value is used verbatim and must be globally unique; when unset, the server mints a snowflake ID as above. +A disaggregated-serving request can provide a unique global request ID via `DisaggregatedParams.disagg_request_id`. +When this field is a positive integer, the context and generation requests share that value as their internal request ID, which enables end-to-end tracking. +To avoid collisions with worker-local or warm-up requests, it is recommended to use a value larger than `1 << 42 = 4398046511104`. +If the field is unset or non-positive, the context and generation requests instead receive separate local sequence IDs, rotating within the range `(0, 1<<42)`, assigned by the respective workers. When `disagg_request_id` is specified, do not route the context and generation requests to the same worker. +This field is optional at present; however, some forthcoming features will depend on this unique identifier. ## Usage @@ -209,7 +201,7 @@ When routing requests to the context servers, the disaggregated server will mark when routing requests to the generation servers, the disaggregated server will mark the requests as "generation-only" to skip the context phase. Clients can then send requests to the disaggregated server at `localhost:8000`, which is an OpenAI-compatible endpoint. For example, you can send requests to the disaggregated server using curl: -``` +```bash curl http://localhost:8000/v1/completions \ -H "Content-Type: application/json" \ -d '{ @@ -241,95 +233,6 @@ Example (two-node deployment): - **Client entrypoint** - Send requests or use a load balancer forwarding to `node-a:8000` and `node-b:8000` -### Coordinator and Worker Fleet - -A single disaggregated server process is itself a single-threaded orchestrator and can become a throughput bottleneck (it terminates every client connection, runs routing, and proxies the ctx→gen hop). To scale the orchestrator on one node without standing up multiple independent instances, `trtllm-serve disaggregated` can run a **fleet** of stateless disaggregated-server worker processes behind a shared **coordinator**. - -The two roles split as follows: - -- **Coordinator** — a single process that owns all cluster state: the ctx/gen routers, worker readiness, and (for the KV-cache-aware router) the single ZMQ event-ingest endpoint. It exposes an internal coordination API (`/select`, `/finish`, `/cluster_info`, `/health`). -- **Fleet workers** — `num_workers` stateless disaggregated servers that share the public port via `SO_REUSEPORT` (each worker is its own process binding the same port, so the kernel load-balances incoming connections across them by 4-tuple hash). Each holds a lightweight delegating client: it computes the routing key locally (e.g. block hashes) and delegates the placement decision to the coordinator over HTTP. Workers own no routing state, so routing stays globally consistent no matter which worker terminates a connection. Each worker also gets a distinct `process_id` for the [global request ID](#unique-global-request-id). - -This is controlled by two fields in the disaggregated config: - -- `num_workers` (int, default `1`) — number of disaggregated-server worker processes to run on the public port. -- `disagg_coordinator_url` (str, optional) — URL of an already-running coordinator. When set, this process starts **no** coordinator and its fleet delegates to that external one. - -The three resulting topologies: - -| `num_workers` | `disagg_coordinator_url` | Behavior | -|---------------|--------------------------|----------| -| `1` | unset | Single self-contained server with an in-process coordinator (the default; unchanged from earlier examples). | -| `> 1` | unset | An **implicit** coordinator starts in this process (on `port - 1`) and a fleet of `num_workers` delegating servers runs on the public port. | -| any | set | **No** coordinator starts here; a fleet of `num_workers` delegating servers points at the external `disagg_coordinator_url`. | - -```{note} -The fleet is most useful with a *stateful* router (`kv_cache_aware`, `conversation`) where placement must be globally consistent — that decision is delegated to the coordinator. With a *stateless* router (`round_robin`, `load_balancing`) each worker simply places locally and no coordinator round-trip occurs. -``` - -#### Example: implicit coordinator + 4-worker fleet - -Extend the `disagg_config.yaml` from the [trtllm-serve](#trtllm-serve) example with `num_workers` and a router type: - -```yaml -hostname: localhost -port: 8000 -backend: pytorch -# Run 4 stateless disaggregated-server workers on port 8000, with an implicit -# coordinator started in-process on port 7999 (port - 1). -num_workers: 4 -context_servers: - num_instances: 2 - urls: - - "localhost:8001" - - "localhost:8002" - router: - type: kv_cache_aware -generation_servers: - num_instances: 1 - urls: - - "localhost:8003" - router: - type: kv_cache_aware -``` - -Launch it exactly as before — the coordinator and fleet are started for you: - -```bash -trtllm-serve disaggregated -c disagg_config.yaml -``` - -Clients still send requests to the public endpoint (`localhost:8000`); the fleet transparently delegates routing to the coordinator. - -#### Example: external coordinator - -To point a fleet at a coordinator already running elsewhere (for example, one shared across nodes), set `disagg_coordinator_url` and omit the coordinator from this process: - -```yaml -hostname: localhost -port: 8000 -backend: pytorch -num_workers: 4 -disagg_coordinator_url: "http://coordinator-host:7999" -context_servers: - num_instances: 2 - urls: - - "localhost:8001" - - "localhost:8002" - router: - type: kv_cache_aware -generation_servers: - num_instances: 1 - urls: - - "localhost:8003" - router: - type: kv_cache_aware -``` - -```{note} -A fleet worker fails fast if its coordinator is unreachable: on startup it probes the coordinator's `/cluster_info` with bounded retry (up to `--server_start_timeout` seconds) and exits with an error rather than coming up and returning `Cluster is not ready` for every request. -``` - ## Environment Variables TRT-LLM uses some environment variables to control the behavior of disaggregated service. diff --git a/docs/source/features/feature-combination-matrix.md b/docs/source/features/feature-combination-matrix.md index d91322fc8c11..b56c1b219af9 100644 --- a/docs/source/features/feature-combination-matrix.md +++ b/docs/source/features/feature-combination-matrix.md @@ -14,7 +14,7 @@ | Speculative Decoding — Linear | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | --- | | | | | | | | | | | Speculative Decoding — Dynamic Trees | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | No | --- | | | | | | | | | | Speculative Decoding — Legacy Path (NGram, user-provided) | Yes | Yes | Yes | No | Yes | No | Yes | Yes | Yes | No | No | --- | | | | | | | | -| Torch Sampler | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes (MTP dynamic tree: greedy only) | Yes | --- | | | | | | | +| Torch Sampler | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | --- | | | | | | | | TLLM C++ Sampler | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | No | No | No | --- | | | | | | | KV Cache Reuse | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | --- | | | | | | Sliding Window Attention | Yes | Yes | Yes | Yes | Yes | Untested | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | --- | | | | diff --git a/docs/source/features/model-express.md b/docs/source/features/model-express.md deleted file mode 100644 index 3007b1402c4a..000000000000 --- a/docs/source/features/model-express.md +++ /dev/null @@ -1,152 +0,0 @@ - - -# ModelExpress (MX) Checkpoint Loading - -The MX checkpoint-loading integration is intended to reduce repeated disk -reads when multiple TensorRT LLM workers load the same model. A worker that -loads from disk can publish its weights as an MX source, and later workers can -receive those weights directly through MX. - -TensorRT LLM can use ModelExpress (MX) as a checkpoint-loading path for -PyTorch backend deployments. `checkpoint_format="MX"` selects this loading -path; it does not identify an MX-specific on-disk checkpoint format, and no -checkpoint conversion is required. TensorRT LLM attempts to fetch compatible -weights from another running TensorRT LLM instance through the MX server. If -no compatible source is available, or if MX transfer fails, loading falls back -to the provided Hugging Face checkpoint. - -## Current Support Scope - -The post-transform MX receive path currently supports only -`LlamaForCausalLM` with transform protocol version 1. TensorRT LLM publishes -post-transform weights together with source-identity and layout metadata. A -receiver whose model family is not allow-listed does not consume those bytes; -it falls back to the standard Hugging Face checkpoint path. - -Loads that require a separately loaded draft model also fall back to the -standard checkpoint path. Target-plus-draft post-transform transfer remains -disabled until layout state is tracked and qualified independently for each -submodel. - -### Adding a Model Family - -Support for another model family requires a focused qualification change: - -1. Audit every post-load hook in the family and its nested modules. Move - structural wiring to `setup_aliases()`, one-time tensor-layout changes to - `transform_weights()`, and process-local derived state to - `cache_derived_state()`. -2. Verify that every one-time transform is guarded by `_weights_transformed` - and that the staged receiver can skip `transform_weights()` without - changing aliases, derived state, tensor layout, or outputs. -3. Add the model class and transform protocol version to the MX staged-receiver - allow-list only after full-load and staged-load equivalence tests pass. -4. Cover compatible transfer, source-identity mismatch, unsupported layout or - protocol, and non-allow-listed fallback. Keep target-plus-draft loading - disabled unless that combination has its own mixed-layout tests. -5. Run a real ModelExpress donor/receiver test with the model configurations - being claimed, including the supported quantization and TP/PP/EP layouts. - Compare deterministic output token IDs with the standard Hugging Face load - path before documenting the family as supported. - -## Installation - -The official TensorRT LLM release container includes the MX Python client. No -additional Python package installation is required in that container. MX -remains opt-in at runtime: TensorRT LLM uses the client only when the MX -checkpoint-loading path and a server URL are configured. Installing the client -does not expand the model support scope described above. - -For pip installations outside the official release container, install the MX -Python client through the optional `mx` extra: - -```bash -pip install "tensorrt-llm[mx]" -``` - -The extra pins the ModelExpress client to version `0.4.1`, matching the client -API qualified by this integration. Deploy a compatible MX server version. -The extra can be added to an existing TensorRT LLM installation. If the MX -loading path is configured but the client cannot be imported, TensorRT LLM -fails with an actionable installation message instead of silently loading from -the Hugging Face checkpoint. Source discovery and transfer failures continue to -use the Hugging Face fallback described above. - -## Deploy the MX Service - -Deploy the MX server and its Redis metadata backend independently of -TensorRT LLM. One MX service can be shared by multiple TensorRT LLM launches, -provided every instance can reach the MX endpoint. TensorRT LLM does not start, -stop, or otherwise manage either service. - -The following commands illustrate a standalone Docker deployment. Production -deployments should manage service lifecycle, persistence, networking, and -security according to their environment. - -```bash -docker network create modelexpress -docker run -d --name modelexpress-redis \ - --network modelexpress \ - redis:8-alpine -docker run -d --name modelexpress-server \ - --network modelexpress \ - -p 8001:8001 \ - -e MODEL_EXPRESS_SERVER_PORT=8001 \ - -e MODEL_EXPRESS_LOG_LEVEL=info \ - -e MX_METADATA_BACKEND=redis \ - -e REDIS_URL=redis://modelexpress-redis:6379 \ - nvcr.io/nvidia/ai-dynamo/modelexpress-server:0.4.1 -``` - -## Configure TensorRT LLM - -Select the MX checkpoint-loading path and provide the MX server URL in a -`trtllm-serve` config. The model argument remains a standard Hugging Face model -ID or checkpoint path: - -```yaml -checkpoint_format: MX -mx_config: - server_url: http://mx-server.example.com:8001 -``` - -```bash -trtllm-serve /path/to/model --config config.yaml -``` - -The `MODEL_EXPRESS_URL` environment variable can also provide the server URL -when `mx_config.server_url` is not set. - -Multiple TensorRT LLM launches can use the same configuration. A worker that -does not find a compatible source loads from Hugging Face storage and publishes -its weights through MX. Later compatible workers can receive those weights by -P2P transfer. - -If neither `mx_config.server_url` nor `MODEL_EXPRESS_URL` is set, MX transfer is -not attempted and checkpoint loading falls back to the standard Hugging Face -path. - -## Configuration - -| Field | Default | Description | -|-------|---------|-------------| -| `mx_config.server_url` | `null` | URL of the separately managed MX server. | -| `mx_config.server_query_timeout_s` | `null` | Timeout for MX source discovery. When unset, TensorRT LLM uses a short fallback cap when no source exists and otherwise lets MX wait for long donor loads. | - -## Notes and Limitations - -- Post-transform MX reception is currently limited to the Llama model family. - Other model families safely fall back to Hugging Face loading until they are - explicitly qualified and added to the staged-receiver allow-list. -- The MX server and Redis lifecycle is external to TensorRT LLM. Every - TensorRT LLM instance must be able to reach the configured MX server URL. -- The MX server coordinates source discovery but does not store model weights. - A source TensorRT LLM process must remain running and network-reachable until - receiver transfers finish. -- The first worker may still load weights from disk if no compatible MX source - is already registered. -- This page describes the MX checkpoint-loading path only. GPU Memory Service - (GMS) integration is configured separately. diff --git a/docs/source/features/sampling.md b/docs/source/features/sampling.md index 72ad4a62ad41..d5a9eaef0685 100644 --- a/docs/source/features/sampling.md +++ b/docs/source/features/sampling.md @@ -75,8 +75,7 @@ llm.generate(["Hello, my name is", * The sampling is controlled via `SamplingParams`. -* By default (`temperature = top_p = top_k = None`), greedy sampling is used - (unless top-p decay is active, see below). +* By default (`temperature = top_p = top_k = None`), greedy sampling is used. * If either `temperature = 0`, `top_p = 0`, and/or `top_k = 1`, is specified, sampling is greedy, irrespective of the values of the remaining parameters. @@ -102,19 +101,6 @@ llm.generate(["Hello, my name is", * The implementation does not guarantee any particular treatment of tied probabilities. -* Top-P decay is supported: if `top_p_decay < 1` is specified, the effective `top_p` is - multiplied by `top_p_decay` after every sampled token, bounded from below by `top_p_min` - (default `1e-6`), and reset to the initial `top_p` whenever the token `top_p_reset_ids` - is sampled (default `-1`, which never matches a token). Out-of-range values - (`top_p_decay` or `top_p_min` outside `(0, 1]`, negative `top_p_reset_ids`) are rejected. - - * An active top-p decay implies top-p sampling even if `top_p` is unspecified or `top_p = 1` - (the initial `top_p` then defaults to 1). However, explicitly requested greedy sampling - (`temperature = 0`, `top_p = 0`, and/or `top_k = 1`) takes precedence over top-p decay. - - * Top-P decay is not supported in combination with beam search or with speculative decoding - modes that route draft tokens through the Torch Sampler; such requests are rejected. - ### Performance The Torch Sampler leverages the optimized sampling kernels provided by diff --git a/docs/source/index.rst b/docs/source/index.rst index 15cfe320bd53..011d222644fd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -80,7 +80,6 @@ Welcome to TensorRT LLM's Documentation! features/guided-decoding.md features/speculative-decoding.md features/checkpoint-loading.md - features/model-express.md features/auto_deploy/auto-deploy.md features/auto_deploy/transforms.rst features/ray-orchestrator.md diff --git a/docs/source/legacy/performance/perf-benchmarking.md b/docs/source/legacy/performance/perf-benchmarking.md index 8d1567aee020..4fc460596f22 100644 --- a/docs/source/legacy/performance/perf-benchmarking.md +++ b/docs/source/legacy/performance/perf-benchmarking.md @@ -202,7 +202,7 @@ can simply read a line and assume a complete entry. When creating a dataset, be JSON entry is on every line. ``` -In order to prepare a synthetic dataset, you can use the provided script in the `benchmarks` +In order to prepare a synthetic dataset, you can use the provided script in the `benchmarks/cpp` directory. For example, to generate a synthetic dataset of 1000 requests with a uniform ISL/OSL of 128/128 for [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B), run: @@ -481,7 +481,7 @@ The PyTorch workflow supports benchmarking with LoRA (Low-Rank Adaptation) adapt Use `prepare_dataset.py` with LoRA-specific options to generate requests with LoRA metadata: ```shell -python3 benchmarks/prepare_dataset.py \ +python3 benchmarks/cpp/prepare_dataset.py \ --stdout \ --rand-task-id 0 1 \ --tokenizer /path/to/tokenizer \ @@ -555,7 +555,7 @@ To benchmark multi-modal models with PyTorch workflow, you can follow the simila First, prepare the dataset: ``` -python ./benchmarks/prepare_dataset.py \ +python ./benchmarks/cpp/prepare_dataset.py \ --tokenizer Qwen/Qwen2-VL-2B-Instruct \ --stdout \ dataset \ @@ -846,7 +846,7 @@ The following table summarizes the commands needed for running benchmarks: | Scenario | Phase | Command | | - | - | - | -| Dataset | Preparation | `python benchmarks/prepare_dataset.py --stdout --tokenizer $HF_MODEL token-norm-dist --input-mean $ISL --output-mean $OSL --input-stdev 0 --output-stdev 0 --num-requests $NUM_REQUESTS > $DATASET_PATH` | +| Dataset | Preparation | `python benchmarks/cpp/prepare_dataset.py --stdout --tokenizer $HF_MODEL token-norm-dist --input-mean $ISL --output-mean $OSL --input-stdev 0 --output-stdev 0 --num-requests $NUM_REQUESTS > $DATASET_PATH` | | Throughput | Build | `trtllm-bench --model $HF_MODEL build --dataset $DATASET_PATH` | | Throughput | Benchmark | `trtllm-bench --model $HF_MODEL throughput --dataset $DATASET_PATH --engine_dir $ENGINE_DIR` | | Latency | Build | See [section about building low latency engines](#low-latency-tensorrt-llm-engine-for-llama-3-70b) | diff --git a/docs/source/legacy/performance/performance-tuning-guide/benchmarking-default-performance.md b/docs/source/legacy/performance/performance-tuning-guide/benchmarking-default-performance.md index 7277b0afa5b0..17cb9aef45ba 100644 --- a/docs/source/legacy/performance/performance-tuning-guide/benchmarking-default-performance.md +++ b/docs/source/legacy/performance/performance-tuning-guide/benchmarking-default-performance.md @@ -86,7 +86,7 @@ The README in the examples folder for supported models walks through building en `trtllm-bench` expects to be passed in a dataset of requests to run through the model. This guide creates a dummy dataset of 1000 requests with every request having input and output sequence length of 2048. TensorRT-LLM provides the `prepare_dataset.py` script to produce the dataset. To use it clone the TensorRT-LLM Repo and run the following command: -`python benchmarks/prepare_dataset.py --stdout --tokenizer /path/to/hf/Llama-3.3-70B-Instruct/ token-norm-dist --input-mean 2048 --output-mean 2048 --input-stdev 0 --output-stdev 0 --num-requests 1000 > synthetic_2048_2048.txt` +`python benchmarks/cpp/prepare_dataset.py --stdout --tokenizer /path/to/hf/Llama-3.3-70B-Instruct/ token-norm-dist --input-mean 2048 --output-mean 2048 --input-stdev 0 --output-stdev 0 --num-requests 1000 > synthetic_2048_2048.txt` `trtllm-bench` can also take in real data, see [`trtllm-bench` documentation](../perf-benchmarking.md) for more details on the required format. diff --git a/docs/source/models/encoder-decoder.md b/docs/source/models/encoder-decoder.md deleted file mode 100644 index 30fa8d5695b9..000000000000 --- a/docs/source/models/encoder-decoder.md +++ /dev/null @@ -1,580 +0,0 @@ - - -# Use encoder-decoder models with the PyTorch backend - -TensorRT LLM can run supported Hugging Face encoder-decoder checkpoints directly -with the PyTorch backend. You do not need to convert the checkpoint or build a -TensorRT engine. The LLM API treats the supplied prompt as the encoder input and -automatically starts the decoder with the checkpoint's -`decoder_start_token_id` or `bos_token_id`. - -This guide covers text-to-text generation with the following Hugging Face -architectures: - -| Hugging Face architecture | Model families and examples | -| --- | --- | -| `T5ForConditionalGeneration` | T5, Flan-T5, and ByT5, for example `google/flan-t5-small` | -| `BartForConditionalGeneration` | BART checkpoints | -| `MBartForConditionalGeneration` | mBART checkpoints | - -Whisper is not currently supported by the PyTorch encoder-decoder path. Support -is in progress. - -mBART architecture loading is available. When a BART or mBART checkpoint -defines `forced_bos_token_id`, the PyTorch backend seeds that token in the -decoder prefix. Source- and target-language selection remains -checkpoint-specific, so validate the configured language tokens before -deployment. Refer to [Understand BART and mBART decoder tokens](#understand-bart-and-mbart-decoder-tokens) -for BOS, EOS, and output-limit behavior. - -## Feature support - -The following table describes the supported and recommended configurations. - -| Feature | Support | Notes | -| --- | --- | --- | -| KV cache manager V1 | Yes; recommended | This is the default. It supports greedy decoding, beam search, batching, the overlap scheduler, decoder CUDA graphs, and tensor parallelism. | -| KV cache manager V2 | Yes | Set `use_kv_cache_manager_v2=True`. It currently requires `max_beam_width=1`, so use greedy or sampling with a single sequence rather than beam search. | -| Greedy decoding | Yes | Set `temperature=0.0`. | -| Beam search | Yes with V1 | Configure `max_beam_width` when constructing `LLM`, then set `use_beam_search=True` in `SamplingParams`. | -| Attention backend | `TRTLLM` | Use this backend for encoder-decoder models. It is required when `tensor_parallel_size > 1`. | -| Decoder CUDA graphs | Yes | `CudaGraphConfig` captures decoder work. V1 supports greedy and beam search; V2 supports its single-beam path. | -| Encoder CUDA graphs | No | `EncodeCudaGraphConfig` is disabled for encoder-decoder models. The encoder runs eagerly. | -| Overlap scheduler | Yes | Enabled by default. V1 supports greedy decoding and beam search; V2 remains limited to `max_beam_width=1`. | -| Tensor parallelism | Yes | Use `tensor_parallel_size > 1` with `attn_backend="TRTLLM"`. Attention head counts must be divisible by the TP size. | -| Pipeline parallelism | No | Keep `pipeline_parallel_size=1`. | -| Context parallelism | No | Keep `context_parallel_size=1`. | -| Attention data parallelism | No | Keep `enable_attention_dp=False`. | -| Chunked prefill | Not supported for the encoder phase | Set `enable_chunked_prefill=False`. The complete encoder input must fit in the iteration token budget. | -| Piecewise CUDA graph | No | Do not set `torch_compile_config.enable_piecewise_cuda_graph=True`. | - -BF16 is the recommended model dtype. Validate accuracy with your checkpoint and -task before deploying a different precision or quantization configuration. - -## Choose the attention backend - -Use `attn_backend="TRTLLM"` for encoder-decoder models. T5 self-attention needs -this backend to apply relative attention bias, and tensor parallel -encoder-decoder execution explicitly requires it. - -The `TRTLLM` backend can internally select optimized kernels when the hardware -and request are eligible. For example, compatible operations on Blackwell can -use FlashInfer TRTLLM-Gen kernels. This internal selection is different from -setting `attn_backend="FLASHINFER"`. Cases such as T5 relative attention bias or -beam-expanded self-attention can fall back to another kernel within the -`TRTLLM` backend; no customer-side backend change is needed. - -## Run basic generation - -Install TensorRT LLM using the [installation guide](../installation/installation-guide.md) -and make sure the checkpoint is accessible either from the Hugging Face Hub or -from a local directory. - -The following example uses KV cache manager V1, greedy decoding, and the overlap -scheduler: - -```python -from tensorrt_llm.llmapi import LLM, KvCacheConfig, SamplingParams, SchedulerConfig - - -model = "google/flan-t5-small" - -with LLM( - model=model, - backend="pytorch", - max_batch_size=4, - max_input_len=512, - max_num_tokens=2048, - max_seq_len=512, - kv_cache_config=KvCacheConfig( - enable_block_reuse=False, - free_gpu_memory_fraction=0.8, - cross_kv_cache_fraction=0.5, - use_kv_cache_manager_v2=False, - ), - scheduler_config=SchedulerConfig(use_python_scheduler=True), -) as llm: - sampling_params = SamplingParams( - max_tokens=64, - temperature=0.0, - ) - result = llm.generate( - "translate English to German: The house is wonderful.", - sampling_params=sampling_params, - use_tqdm=False, - ) - print(result.outputs[0].text) -``` - -Use the task format expected by the checkpoint. For example, T5 translation -checkpoints commonly expect a task prefix such as `translate English to -German:`, while a summarization checkpoint expects the source document. - -The LLM API performs these encoder-decoder-specific steps automatically: - -1. Tokenizes the supplied string as the encoder input. -2. Runs the encoder once and retains its output for cross-attention. -3. Initializes the decoder from the checkpoint's decoder start token. -4. Generates decoder tokens and returns the detokenized decoder output. - -Do not prepend a decoder start token to the source prompt. If you pass token IDs -instead of text, pass only the encoder-side token IDs: - -```python -from transformers import AutoTokenizer - - -tokenizer = AutoTokenizer.from_pretrained(model) -source_text = "translate English to German: The house is wonderful." -source_token_ids = tokenizer.encode(source_text, add_special_tokens=True) -result = llm.generate(source_token_ids, sampling_params=sampling_params) -``` - -### Configure an mBART tokenizer - -mBART tokenization depends on the source language. Create the Hugging Face -tokenizer with `src_lang` and pass that tokenizer to `LLM` so string prompts -receive the correct source-language token: - -```python -from transformers import AutoTokenizer - -from tensorrt_llm.llmapi import LLM, KvCacheConfig, SamplingParams - - -model = "/path/to/mbart-large-50-many-to-one-mmt" -tokenizer = AutoTokenizer.from_pretrained(model, src_lang="ro_RO") - -with LLM( - model=model, - tokenizer=tokenizer, - backend="pytorch", - attn_backend="TRTLLM", - dtype="bfloat16", - enable_chunked_prefill=False, - kv_cache_config=KvCacheConfig(cross_kv_cache_fraction=0.5), -) as llm: - result = llm.generate( - "Şeful ONU spune că nu există o soluţie militară în Siria.", - sampling_params=SamplingParams(max_tokens=64, temperature=0.0), - use_tqdm=False, - ) - print(result.outputs[0].text) -``` - -For this many-to-one checkpoint, `generation_config.json` selects English with -the `en_XX` forced BOS token. For other mBART checkpoints, confirm that -`decoder_start_token_id`, `forced_bos_token_id`, `eos_token_id`, and the -tokenizer language settings select the source and target languages you intend -to serve. - -### Understand BART and mBART decoder tokens - -When a BART or mBART checkpoint defines `forced_bos_token_id`, the PyTorch -backend initializes the decoder with the following internal prefix: - -```text -[decoder_start_token_id, forced_bos_token_id] -``` - -For example, BART-large-CNN uses `[2, 0]`. Customers provide only the encoder -input; do not prepend either decoder token. By default, the returned token IDs -exclude `decoder_start_token_id` but include `forced_bos_token_id`, so the -BART-large-CNN output begins with token ID 0. - -The forced BOS token counts against `SamplingParams.max_tokens`. Consequently, -a request using this prefix requires `max_tokens` to be at least 2. The runtime -uses the remaining token budget for model-selected tokens. This behavior is the -same for greedy decoding and beam search and does not require a customer logits -processor. - -EOS is a stopping token rather than a forced final token. The runtime uses -`SamplingParams.end_id`, which defaults to the tokenizer's `eos_token_id`. If -the model generates EOS before the output limit, the returned token IDs include -EOS and `finish_reason` is `"stop"`. Set `ignore_eos=True` to continue decoding -past EOS. - -The runtime does not inject `forced_eos_token_id` when a sequence reaches -`max_tokens`. It preserves the model-selected final token and reports -`finish_reason="length"`. - -## Run a batch - -Pass a list of strings to batch inputs. The strings can have different tokenized -lengths: - -```python -sources = [ - "translate English to German: The house is wonderful.", - "translate English to German: The book is on the table.", -] - -results = llm.generate(sources, sampling_params=sampling_params, use_tqdm=False) -for source, result in zip(sources, results): - print(f"source={source!r} output={result.outputs[0].text!r}") -``` - -`max_num_tokens` must cover the encoder tokens admitted in an iteration as well -as decoder work. Increase it for larger batches or longer source sequences. - -## Choose KV cache manager V1 or V2 - -Encoder-decoder execution uses two KV cache pools: - -- The self-attention pool stores decoder-side KV states. -- The cross-attention pool stores encoder-derived K/V states used by every - decoder layer. - -`cross_kv_cache_fraction` is required for every encoder-decoder model. It divides -the configured KV cache memory budget between the two pools. A value of `0.5` -is a reasonable starting point: - -```python -kv_cache_config = KvCacheConfig( - free_gpu_memory_fraction=0.8, - cross_kv_cache_fraction=0.5, - use_kv_cache_manager_v2=False, -) -``` - -Increase `cross_kv_cache_fraction` when long encoder inputs exhaust the cross -pool. Decrease it when long decoder outputs or wide beams exhaust the -self-attention pool. The two fractions are related as follows: - -```text -cross-attention pool = total KV cache budget * cross_kv_cache_fraction -self-attention pool = total KV cache budget * (1 - cross_kv_cache_fraction) -``` - -V1 is the default and should be the first choice for production deployments. -To evaluate V2, change only the manager selection and keep beam width equal to -one: - -```python -kv_cache_config = KvCacheConfig( - free_gpu_memory_fraction=0.8, - cross_kv_cache_fraction=0.5, - use_kv_cache_manager_v2=True, -) - -llm = LLM( - model=model, - backend="pytorch", - attn_backend="TRTLLM", - max_beam_width=1, - kv_cache_config=kv_cache_config, -) -``` - -KV cache manager V2 is a prototype feature and rejects configurations with a -maximum beam width greater than one. - -## Use beam search - -Beam search requires KV cache manager V1. The maximum beam width is a runtime -capacity setting and must be specified when constructing `LLM`: - -```python -beam_width = 4 - -with LLM( - model="/path/to/bart-large-cnn", - backend="pytorch", - attn_backend="TRTLLM", - dtype="bfloat16", - max_beam_width=beam_width, - enable_chunked_prefill=False, - kv_cache_config=KvCacheConfig( - free_gpu_memory_fraction=0.8, - cross_kv_cache_fraction=0.5, - use_kv_cache_manager_v2=False, - ), -) as llm: - beam_params = SamplingParams( - best_of=beam_width, - max_tokens=64, - n=beam_width, - temperature=0.0, - use_beam_search=True, - ) - result = llm.generate( - "The engineering team released a faster inference service on Monday. " - "The update improves batching, lowers latency, and adds detailed " - "monitoring for operators.", - sampling_params=beam_params, - use_tqdm=False, - ) - - for hypothesis in result.outputs: - print(hypothesis.text) -``` - -`best_of` sets the beam width and must not exceed `LLM.max_beam_width`. `n` -sets the number of returned hypotheses and must not exceed `best_of`. Set `n=1` -to return only the best hypothesis. - -Beam search expands decoder-side cache and compute requirements. Include this -expansion when sizing the self-attention KV pool and CUDA graph batch sizes. - -## Enable decoder CUDA graphs - -Pass `CudaGraphConfig` to capture and replay decoder iterations: - -```python -from tensorrt_llm.llmapi import CudaGraphConfig - - -llm = LLM( - model=model, - backend="pytorch", - attn_backend="TRTLLM", - max_batch_size=8, - cuda_graph_config=CudaGraphConfig( - max_batch_size=8, - enable_padding=True, - ), - kv_cache_config=KvCacheConfig( - free_gpu_memory_fraction=0.8, - cross_kv_cache_fraction=0.5, - ), -) -``` - -This configuration captures decoder work only; the encoder continues to run -eagerly. With beam search, graph batch sizes must cover the active decoder -sequences after beam expansion. Padding lets nearby runtime batch sizes reuse a -captured graph. - -Do not use `EncodeCudaGraphConfig` for an encoder-decoder model. The runtime -warns and disables it. Piecewise CUDA graphs through `TorchCompileConfig` are -also unsupported for this model type. - -## Control the overlap scheduler - -The PyTorch backend enables the overlap scheduler by default. The examples set -`disable_overlap_scheduler=False` explicitly to make that choice visible: - -```python -llm = LLM( - model=model, - backend="pytorch", - disable_overlap_scheduler=False, - kv_cache_config=KvCacheConfig(cross_kv_cache_fraction=0.5), -) -``` - -Overlap is not restricted to KV cache manager V1. Both V1 and V2 enter the same -overlap executor loop, and that loop contains V2-specific resource handling. -V2 remains limited to `max_beam_width=1`. Set -`disable_overlap_scheduler=True` when debugging. - -## Use tensor parallelism - -Set `tensor_parallel_size` to the number of GPUs over which to shard the model: - -```python -with LLM( - model=model, - backend="pytorch", - attn_backend="TRTLLM", - tensor_parallel_size=2, - pipeline_parallel_size=1, - context_parallel_size=1, - enable_attention_dp=False, - enable_chunked_prefill=False, - kv_cache_config=KvCacheConfig( - free_gpu_memory_fraction=0.8, - cross_kv_cache_fraction=0.5, - use_kv_cache_manager_v2=False, - ), -) as llm: - result = llm.generate(source_text, sampling_params=sampling_params) -``` - -For single-node execution through the LLM API, do not add an `mpirun` prefix. -TensorRT LLM starts the worker processes. The selected TP size must divide the -encoder and decoder attention head counts. Cross-attention KV head duplication -is not supported, so its KV head count must also be divisible by the TP size. - -Tensor parallelism currently requires `attn_backend="TRTLLM"`. Pipeline -parallelism, context parallelism, and attention DP are rejected for -encoder-decoder models. - -## Serve an encoder-decoder model - -The following configuration starts a greedy Flan-T5 service with the PyTorch -backend. Save it as `enc-dec-config.yaml`: - -```yaml -attn_backend: TRTLLM -dtype: bfloat16 -disable_overlap_scheduler: false -enable_chunked_prefill: false -max_beam_width: 1 -max_input_len: 512 -max_num_tokens: 2048 -max_seq_len: 512 -kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - cross_kv_cache_fraction: 0.5 - use_kv_cache_manager_v2: false -scheduler_config: - use_python_scheduler: true -``` - -Start the server: - -```bash -trtllm-serve google/flan-t5-small \ - --backend pytorch \ - --max_batch_size 4 \ - --config enc-dec-config.yaml -``` - -Send the source text through the completions endpoint: - -```bash -curl http://localhost:8000/v1/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "google/flan-t5-small", - "prompt": "translate English to German: The house is wonderful.", - "max_tokens": 64, - "temperature": 0.0 - }' -``` - -To serve beam search, use KV cache manager V1, restart the server with -`max_beam_width` set to the desired maximum, and add the following request -fields: - -```json -{ - "use_beam_search": true, - "best_of": 4, - "n": 1 -} -``` - -For tensor parallel serving, add `--tp_size ` to `trtllm-serve` and keep the -attention backend set to `TRTLLM`. - -## Size the runtime - -Use these guidelines as a starting point: - -- Set `max_input_len` to at least the maximum tokenized encoder input length. -- Set `max_seq_len` to at least the larger of the maximum encoder input length - and maximum decoded sequence length. The current encoder-decoder runtime uses - this value while sizing both phases. -- Set `max_num_tokens` high enough for all encoder tokens admitted together and - for the active decoder tokens. This is especially important for mixed-length - batches. -- Increase `max_batch_size` for more concurrent requests. Beam width multiplies - the number of active decoder sequences but not the number of source requests. -- Tune `free_gpu_memory_fraction` first, then tune - `cross_kv_cache_fraction` based on whether the cross-attention or - self-attention pool is exhausted. - -## Performance - -The following benchmarks compare the PyTorch backend with the legacy TensorRT -encoder-decoder path for large-batch inference. The measurements use BF16 on -one H100 80 GB GPU with greedy decoding, an output limit of 128 tokens, and -mixed encoder input lengths from 260 to 440 tokens. The Flan-T5-XL results are -the average of ten timed runs after three warmup runs. The BART results are the -average of 20 timed runs after five warmup runs. Executed-token throughput -includes the terminal EOS token when a sequence emits it. - -The PyTorch configuration uses the `TRTLLM` attention backend, the overlap -scheduler, the Python scheduler, decoder CUDA graphs with padding, KV cache -manager V1, `max_input_len=512`, `max_seq_len=1024`, and -`max_num_tokens=65536`. Block reuse and chunked prefill are disabled. The KV -cache uses `free_gpu_memory_fraction=0.3` and -`cross_kv_cache_fraction=0.5`. - -The legacy TensorRT configuration uses separate BF16 encoder and decoder -engines built for batch size 128 and beam width 1. The encoder supports 512 -input tokens and 65,536 tokens per iteration; the decoder supports a sequence -length of 129. The benchmark runs these engines through `ModelRunnerCpp` with -greedy `top_k=1` decoding and the same KV cache fractions. For BART, the legacy -TensorRT benchmark starts the decoder with token IDs `[2, 0]` and generates at -most 127 more tokens. The PyTorch LLM API applies the same decoder prefix -internally and counts token ID 0 as the first output token; customers do not -need to provide the decoder prefix. Both paths use token ID 2 as EOS and stop -when the model generates it naturally. If a sequence reaches the output limit, -it retains the model-selected final token and reports a length stop instead of -forcing EOS. This setup also lets beam search begin after the shared decoder -prefix without a per-step Python logits processor. - -### Flan-T5-XL - -For Flan-T5-XL, the PyTorch backend performs on par with the legacy TensorRT -path, with slightly lower latency and higher executed-token throughput across -the tested batch sizes. - -| Batch size | Legacy TensorRT latency | PyTorch latency | PyTorch latency improvement over legacy TensorRT | Legacy TensorRT executed tokens/s | PyTorch executed tokens/s | -| ---: | ---: | ---: | ---: | ---: | ---: | -| 32 | 727.6 ms | 706.1 ms | 3.0% | 3,153 | 3,312 | -| 64 | 1,225.0 ms | 1,136.7 ms | 7.2% | 3,863 | 4,184 | -| 128 | 2,056.8 ms | 1,999.3 ms | 2.8% | 4,601 | 4,768 | - -### BART-large-CNN - -For BART-large-CNN, the PyTorch backend has 21.9% to 36.0% higher latency than -the legacy TensorRT path across the tested batch sizes. - -| Batch size | Legacy TensorRT latency | PyTorch latency | PyTorch latency difference | Legacy TensorRT executed tokens/s | PyTorch executed tokens/s | -| ---: | ---: | ---: | ---: | ---: | ---: | -| 32 | 229.9 ms | 280.2 ms | 21.9% slower | 12,611 | 10,662 | -| 64 | 252.2 ms | 343.0 ms | 36.0% slower | 22,007 | 16,209 | -| 128 | 352.3 ms | 472.7 ms | 34.2% slower | 31,544 | 23,518 | - -Performance depends on the model, request distribution, decoding settings, and -GPU configuration. Benchmark with a representative workload before deployment. - -## Troubleshooting - -### `cross_kv_cache_fraction` is required - -Every encoder-decoder runtime needs a cross-attention KV pool. Add -`KvCacheConfig(cross_kv_cache_fraction=...)`; `0.5` is a reasonable initial -value. Do not set this field for a decoder-only model. - -### `decoder_start_token_id` is required - -The checkpoint must define `decoder_start_token_id` or `bos_token_id` in its -Hugging Face model or generation configuration. Use a checkpoint with a complete -`config.json` and, when applicable, `generation_config.json`. - -### KV cache manager V2 fails with beam search - -V2 currently requires `max_beam_width=1`. Select V1 by setting -`use_kv_cache_manager_v2=False` before enabling beam search. - -### Tensor parallel initialization is rejected - -Check all of the following: - -- `attn_backend` is `TRTLLM`. -- Encoder, decoder, and cross-attention head counts are divisible by the TP - size. -- `pipeline_parallel_size=1` and `context_parallel_size=1`. -- `enable_attention_dp=False`. - -### CUDA graphs do not capture the encoder - -This is expected. `CudaGraphConfig` accelerates decoder iterations only. The -encoder path runs eagerly, and `EncodeCudaGraphConfig` is disabled for -encoder-decoder models. - -### Output quality differs from the Hugging Face example - -Confirm that the source uses the task prefix and language settings expected by -the checkpoint. Also compare the same model dtype, beam width, length penalty, -EOS stopping behavior, and forced BOS configuration. Small numerical -differences can change lower-ranked beam hypotheses when scores are close. diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index b03f9961f7cc..6fa3dc971fba 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -80,8 +80,8 @@ Note: Support for other models may vary. Features marked "N/A" are not applicabl | `Step3p7ForConditionalGeneration`| Yes | Yes | Yes | Untested | Untested | Yes | No | No | No | Yes | Untested | Untested | Yes | Untested | Untested | | `MiniMaxM3SparseForConditionalGeneration` [^12] | Yes | Yes | Yes | Untested | Untested | No | No | No | No | Yes | Untested | No | N/A | Untested | Untested | -[^1]: Chunked Prefill for MLA can only be enabled on SM90/SM100/SM103/SM120. -[^2]: KV cache reuse for MLA can only be enabled on SM90/SM100/SM103/SM120/SM121 and in BF16/FP8 KV cache dtype. +[^1]: Chunked Prefill for MLA can only be enabled on SM100/SM103. +[^2]: KV cache reuse for MLA can only be enabled on SM90/SM100/SM103 and in BF16/FP8 KV cache dtype. [^3]: Qwen3-Next-80B-A3B exhibits relatively low accuracy on the SciCode-AA-v2 benchmark. [^5]: Supported via the [AutoDeploy](../features/auto_deploy/auto-deploy.md) backend. See [AD Configs](../../../examples/auto_deploy/model_registry/configs). [^6]: Also supports text-only inference via the [AutoDeploy](../features/auto_deploy/auto-deploy.md) backend. @@ -124,26 +124,6 @@ Note: - V: Video - A: Audio -## Multimodal Encoder Optimizations - -The following optimizations are available to models that implement -`MultimodalModelMixin`. Currently, only `Mistral3ForConditionalGeneration` supports them. - -| Model Architecture | Multimodal Encoder Side Stream | Multimodal Embeddings Cache | -| ------------------ | ------------------------------ | --------------------------- | -| `Mistral3ForConditionalGeneration` | Yes | Yes | - -- **Multimodal encoder side stream** prefetches encoder work for pending requests on a separate - CUDA stream, allowing it to overlap with work on the main stream. Set - `multimodal_config.encoder_side_stream_max_ahead` to a positive value to enable it; the value - limits the number of prefetched requests that can be ahead of admission. This option is mutually - exclusive with `multimodal_config.encoder_cuda_graph` and can increase peak GPU memory use. -- **Multimodal embeddings cache** is a per-model, cross-request LRU cache of encoder embeddings. - Set `multimodal_config.encoder_cache_max_bytes` to its capacity (for example, `"512MiB"`), or - `0` to disable it. Entries are cached per multimodal item, but a request reuses cached embeddings - only when all of its items hit the cache. At present, only single-modality requests are cacheable; - mixed-modality requests bypass the cache. - # Visual Generation Models TensorRT-LLM provides beta support for diffusion-based image and video generation. diff --git a/docs/source/torch/adding_custom_kernels.md b/docs/source/torch/adding_custom_kernels.md index 7a3bd12104ec..bed8ee59da6c 100644 --- a/docs/source/torch/adding_custom_kernels.md +++ b/docs/source/torch/adding_custom_kernels.md @@ -258,30 +258,6 @@ The same pattern (`@torch.library.custom_op("trtllm::...")`, `register_fake`, co - **DLPack/CUDA Graphs.** When exporting tensors via DLPack, use the stream override that mimics the `CUDAGraphCompatibleWrapper` in `argmax.py` so the capture replays cleanly. - **JIT compile cache.** Always cache compiled kernels by the keys that affect codegen (dtype, last-dim size, hardware variant). `argmax.py` and the `cute_dsl_custom_ops.py` runners are good references. -### 4.5 Pruning autotuner tactics with nvMatmulHeuristics - -A CuTe DSL runner that exposes many tactics (tile / cluster / swap / prefetch combinations) can be expensive to autotune, because the AutoTuner JIT-compiles and benchmarks every candidate returned by `get_valid_tactics`. A runner can therefore rank and prune its own candidate list with [nvMatmulHeuristics](https://pypi.org/project/nvidia-matmul-heuristics/) *inside* `get_valid_tactics`, before the tactics ever reach the compile+benchmark loop — all shapes and inputs are already visible at that point, so no extra AutoTuner hook is needed. The pruning is a no-op by default and must degrade gracefully to the full list on any failure, so tuning never loses a valid candidate. - -Today this is wired up for the SM100/SM103 (Blackwell B200/B300) NVFP4 dense GEMM: `CuteDSLNVFP4BlackwellRunner.get_valid_tactics` builds the full sweep and then hands it to the private `_rank_prune_tactics` helper. The pruned set is a strict, re-validated subset of the full sweep — it never introduces a tactic the kernel validator rejected — so correctness is unchanged and only autotuner wall-time is affected. On a 4096³ NVFP4 GEMM this cuts a cold autotune from ~120 tactics to ~5 (roughly 7× less wall-time) while keeping the selected kernel within a few percent of the full-sweep best. - -The behavior is controlled entirely by environment variables (all read at tune time; the package must be installed, otherwise the path silently falls back to the full sweep): - -| Environment variable | Default | Meaning | -|---|---|---| -| `TRTLLM_CUTEDSL_NVMMH_ENABLE` | `0` | Master opt-in switch. Set to `1` to enable heuristic tactic pruning. When unset/`0`, or when `nvidia-matmul-heuristics` is not installed, the full tactic sweep runs unchanged. | -| `TRTLLM_CUTEDSL_NVMMH_FIELDS` | `tile,cluster` | Comma-separated list of the knobs nvMatmulHeuristics drives; any knob not listed is swept by the runner. Recognized tokens: `tile`, `cluster`, `swizzle`, `cta_order`. `tile` and `cluster` are a coupled pair (the 2-SM encoding maps the joint `(cta, cluster)`), so naming either drives both. Accepted aliases: `cta_tile` / `mma_tiler` → `tile`. Unknown tokens are warned once and ignored. **Top-K tile/cluster pruning happens only when `tile`/`cluster` is selected; with scheduler-only fields (`swizzle` and/or `cta_order`) every valid tile/cluster is swept and merely annotated with the model's scheduler knobs.** | -| `TRTLLM_CUTEDSL_NVMMH_MAX_TACTICS` | `5` | Top-K distinct `(tile, cluster, swap)` keys to keep from the model's ranking (applies only when `tile`/`cluster` is in `FIELDS`). Both `use_prefetch` variants of a kept key are profiled on top (prefetch is not modeled), so the final tactic count can be up to ~2× this. Larger K = closer to the full-sweep best but slower autotune; smaller K = faster autotune with a larger potential perf gap. | - -Example (enable pruning, keep the top 8 tile/cluster candidates): - -```bash -export TRTLLM_CUTEDSL_NVMMH_ENABLE=1 -export TRTLLM_CUTEDSL_NVMMH_FIELDS=tile,cluster -export TRTLLM_CUTEDSL_NVMMH_MAX_TACTICS=8 -``` - -The adapter and env helpers live in [`tensorrt_llm/_torch/custom_ops/cutedsl_matmul_heuristics.py`](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/_torch/custom_ops/cutedsl_matmul_heuristics.py); the `get_valid_tactics` / `_rank_prune_tactics` pruning is in [`cute_dsl_custom_ops.py`](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py). - --- ## 5. Example walkthrough: `indexer_k_cache_scatter_op` diff --git a/examples/auto_deploy/README.md b/examples/auto_deploy/README.md index 343ad8c87f35..d1485fdd494b 100644 --- a/examples/auto_deploy/README.md +++ b/examples/auto_deploy/README.md @@ -2,7 +2,7 @@ This folder contains runnable examples for **AutoDeploy** as it ships inside [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). For general AutoDeploy documentation, motivation, support matrix, and feature overview, please see the [official docs](https://nvidia.github.io/TensorRT-LLM/features/auto_deploy/auto-deploy.html). -> Looking for the lightweight standalone package (no TRT-LLM required)? See **Paragraf** at [github.com/NVIDIA/llm-compiler](https://github.com/NVIDIA/llm-compiler). That repo is generated from this source tree by [`paragraf/create_standalone_package.py`](./paragraf/create_standalone_package.py). +> Looking for the lightweight standalone package (no TRT-LLM required)? See **LLM Compiler** at [github.com/NVIDIA/llm-compiler](https://github.com/NVIDIA/llm-compiler). That repo is generated from this source tree by [`llmc/create_standalone_package.py`](./llmc/create_standalone_package.py). ______________________________________________________________________ diff --git a/examples/auto_deploy/paragraf/CONTRIBUTING.md b/examples/auto_deploy/llmc/CONTRIBUTING.md similarity index 66% rename from examples/auto_deploy/paragraf/CONTRIBUTING.md rename to examples/auto_deploy/llmc/CONTRIBUTING.md index 0df6dc35400c..d4b027ca2e54 100644 --- a/examples/auto_deploy/paragraf/CONTRIBUTING.md +++ b/examples/auto_deploy/llmc/CONTRIBUTING.md @@ -1,10 +1,10 @@ -# Contributing to paragraf +# Contributing to llm-compiler ## Where to send PRs -The `paragraf` standalone package is **generated from the AutoDeploy source tree inside [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)**. The mirror repo on GitHub is **read-only** — it is overwritten on every release by [`create_standalone_package.py`](./create_standalone_package.py). +The `llmc` standalone package is **generated from the AutoDeploy source tree inside [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM)**. The mirror repo on GitHub is **read-only** — it is overwritten on every release by [`create_standalone_package.py`](./create_standalone_package.py). -**Do not open pull requests against the standalone `paragraf` repo.** Any changes pushed there will be lost the next time the package is regenerated. +**Do not open pull requests against the standalone `llmc` repo.** Any changes pushed there will be lost the next time the package is regenerated. The supported workflow is: @@ -13,7 +13,7 @@ The supported workflow is: 1. **Optional:** validate your change against the standalone repo by regenerating it locally and running its test suite (see below). 1. Open a PR against `main` of `NVIDIA/TensorRT-LLM`. -You're welcome to fork the standalone `paragraf` repo to experiment, iterate, or build on top — just remember the upstream source of truth is TensorRT-LLM. +You're welcome to fork the standalone `llmc` repo to experiment, iterate, or build on top — just remember the upstream source of truth is TensorRT-LLM. ## Validating a change with the standalone build @@ -27,20 +27,20 @@ python scripts/check_auto_deploy_imports.py pytest tests/unittest/auto_deploy/standalone/ -q ``` -`tests/unittest/auto_deploy/standalone/test_standalone_package.py` regenerates the `paragraf` tree, installs it in an isolated venv, and runs the copied unit tests against the standalone install. This is the same job that gates merges. +`tests/unittest/auto_deploy/standalone/test_standalone_package.py` regenerates the `llmc` tree, installs it in an isolated venv, and runs the copied unit tests against the standalone install. This is the same job that gates merges. To regenerate the standalone tree without running tests: ```bash -python examples/auto_deploy/paragraf/create_standalone_package.py \ - --output-dir /path/to/paragraf_pkg +python examples/auto_deploy/llmc/create_standalone_package.py \ + --output-dir /path/to/llmc_pkg ``` ## PR conventions PRs against `NVIDIA/TensorRT-LLM` follow the project-wide rules: -- **Title format:** `[JIRA/NVBUG/None][type] description` (e.g. `[None][feat] paragraf: add foo transform`). +- **Title format:** `[JIRA/NVBUG/None][type] description` (e.g. `[None][feat] llmc: add foo transform`). - **DCO sign-off** is required: commit with `git commit -s`. Don't add AI tools or co-authors to the sign-off line. - The pre-commit hooks (formatting, lint, the `auto-deploy-import-discipline` hook) run on commit. If they modify files, re-stage and commit again. @@ -48,7 +48,7 @@ For the rest (full coding guidelines, branching policy, CI commands), see [the T ## Import discipline -`paragraf` source is the same files as `tensorrt_llm/_torch/auto_deploy/` — copied verbatim, never rewritten. For that to work cleanly: +`llmc` source is the same files as `tensorrt_llm/_torch/auto_deploy/` — copied verbatim, never rewritten. For that to work cleanly: - Imports **inside** the auto_deploy package must be **relative** (`from ..foo import bar`). - Imports **to the rest of TensorRT-LLM** must be **absolute** (`from tensorrt_llm.X import Y`) and gated at runtime by `_compat.TRTLLM_AVAILABLE` so they fail gracefully in standalone mode. diff --git a/examples/auto_deploy/paragraf/README.md b/examples/auto_deploy/llmc/README.md similarity index 69% rename from examples/auto_deploy/paragraf/README.md rename to examples/auto_deploy/llmc/README.md index 07d96ae6fc10..25d5b89b9e99 100644 --- a/examples/auto_deploy/paragraf/README.md +++ b/examples/auto_deploy/llmc/README.md @@ -1,13 +1,8 @@ -# 🔥🚀⚡ Paragraf +# 🔥🚀⚡ LLM Compiler -**Paragraf** (Python package: `paragraf`) is the standalone, lightweight distribution of the AutoDeploy graph-transformation pipeline. It exposes the same `ModelFactory` and `InferenceOptimizer` building blocks as TensorRT-LLM's AutoDeploy backend, but with a minimal dependency footprint (PyTorch + Triton + FlashInfer), so you can prototype optimization pipelines or run inference without a full TRT-LLM install. +**LLM Compiler** (Python package: `llmc`) is the standalone, lightweight distribution of the AutoDeploy graph-transformation pipeline. It exposes the same `ModelFactory` and `InferenceOptimizer` building blocks as TensorRT-LLM's AutoDeploy backend, but with a minimal dependency footprint (PyTorch + Triton + FlashInfer), so you can prototype optimization pipelines or run inference without a full TRT-LLM install. -The `paragraf` package is **generated** from the AutoDeploy source tree inside the [TensorRT-LLM repo](https://github.com/NVIDIA/TensorRT-LLM). The standalone repo is **read-only** — see [`CONTRIBUTING.md`](./CONTRIBUTING.md) for how to land changes. - -During the rename transition, `llmc` remains a deprecated import alias for `paragraf`, -the distribution metadata remains `nvidia-llmc`, and the legacy redirect environment -variable and runner entry point remain supported. New code should use the Paragraf -names shown below. +The `llmc` package is **generated** from the AutoDeploy source tree inside the [TensorRT-LLM repo](https://github.com/NVIDIA/TensorRT-LLM). The standalone repo is **read-only** — see [`CONTRIBUTING.md`](./CONTRIBUTING.md) for how to land changes. For general AutoDeploy documentation (motivation, support matrix, feature overview), see the [official docs](https://nvidia.github.io/TensorRT-LLM/features/auto_deploy/auto-deploy.html). @@ -42,49 +37,37 @@ uv pip install -e ".[dev]" ### Sanity check ```python -from paragraf._compat import TRTLLM_AVAILABLE +from llmc._compat import TRTLLM_AVAILABLE print(f"TRT-LLM available: {TRTLLM_AVAILABLE}") # False in standalone mode ``` ### Use the TensorRT-LLM runtime TensorRT-LLM releases that still import their bundled AutoDeploy package can be redirected to use -Paragraf as the compiler implementation: +LLMC as the compiler implementation: ```bash -TRTLLM_REDIRECT_AD_TO_PARAGRAF=true \ -python runners/trtllm/build_and_run_paragraf_trtllm.py ... +TRTLLM_REDIRECT_AD_TO_LLMC=true \ +python runners/trtllm/build_and_run_llmc_trtllm.py ... ``` -The redirect is disabled by default and must be enabled before importing either Paragraf or bundled +The redirect is disabled by default and must be enabled before importing either LLMC or bundled AutoDeploy. It maps `tensorrt_llm._torch.auto_deploy.*` imports to the corresponding canonical -`paragraf.*` modules in the parent process and TensorRT-LLM MPI workers. +`llmc.*` modules in the parent process and TensorRT-LLM MPI workers. In standalone mode the package uses the PyTorch, Triton, and FlashInfer kernel paths. TRT-LLM-only kernels (custom CUDA, optimized all-reduce, MoE fused kernels, the `pyexecutor` runtime) are skipped at registration time. ### Run the bundled tests -The generated repository contains every AutoDeploy test selected by the -TensorRT-LLM test-name classifier. Tests that need TensorRT-LLM skip cleanly -when its optional wheel is absent. - ```bash pytest tests/ ``` -To include tests that exercise TensorRT-LLM kernels, runtime, or test helpers, -install the optional wheel and explicitly enable the Paragraf redirect: - -```bash -uv pip install -e ".[dev,trtllm]" -TRTLLM_REDIRECT_AD_TO_PARAGRAF=true pytest tests/ -``` - ______________________________________________________________________ ## Building a custom inference pipeline -The two core abstractions in `paragraf` are: +The two core abstractions in `llmc` are: - **`ModelFactory`** — wraps a HuggingFace checkpoint (or any other source) and produces an initialized `nn.Module` plus dynamic-shape metadata for export. - **`InferenceOptimizer`** — runs a configured pipeline of graph transforms (export → fuse → quantize → shard → compile → cache) over a model produced by a factory, against a `CachedSequenceInterface` that defines the input contract. @@ -94,10 +77,10 @@ The code below builds a custom optimization pipeline end-to-end, without going t ```python import torch -from paragraf.models.factory import ModelFactoryRegistry -from paragraf.shim.interface import CachedSequenceInterface -from paragraf.transform.optimizer import InferenceOptimizer -from paragraf.utils.dist_config import DistConfig +from llmc.models.factory import ModelFactoryRegistry +from llmc.shim.interface import CachedSequenceInterface +from llmc.transform.optimizer import InferenceOptimizer +from llmc.utils.dist_config import DistConfig # 1. Build a ModelFactory. # @@ -125,7 +108,7 @@ cache_seq = CachedSequenceInterface( max_seq_len=2048, max_batch_size=4, device="cuda", - kv_cache_config=None, # or paragraf._compat.KvCacheConfig(...) + kv_cache_config=None, # or llmc._compat.KvCacheConfig(...) max_num_tokens=4 * 2048, vocab_size_padded=factory.vocab_size_padded, ) @@ -135,7 +118,7 @@ cache_seq = CachedSequenceInterface( # Each entry maps a registered transform name to its config. # `stage` controls ordering across the pipeline; transforms are # re-sorted by stage at construction time. The names below are -# illustrative — see `paragraf.transform.library` for the full set. +# illustrative — see `llmc.transform.library` for the full set. transforms_config = { "export_to_gm": {"stage": "export"}, "fuse_gemms": {"stage": "post_export"}, @@ -172,11 +155,11 @@ print(logits.shape) | Topic | Path | |-------|------| -| Available transforms | `paragraf/transform/library/` | -| Available factories | `paragraf/models/`, `paragraf/models/custom/` | -| Custom ops & backends | `paragraf/custom_ops/` | -| Compile backends | `paragraf/compile/backends/` | -| Type compatibility shims | `paragraf/_compat.py` | -| Configuration data classes | `paragraf/transform/interface.py`, `paragraf/llm_args.py` | - -For higher-level usage, the `paragraf.LLM` class (available when running inside TRT-LLM) wraps all of the above behind a familiar generate API. In pure-standalone mode use `InferenceOptimizer` directly as shown. +| Available transforms | `llmc/transform/library/` | +| Available factories | `llmc/models/`, `llmc/models/custom/` | +| Custom ops & backends | `llmc/custom_ops/` | +| Compile backends | `llmc/compile/backends/` | +| Type compatibility shims | `llmc/_compat.py` | +| Configuration data classes | `llmc/transform/interface.py`, `llmc/llm_args.py` | + +For higher-level usage, the `llmc.LLM` class (available when running inside TRT-LLM) wraps all of the above behind a familiar generate API. In pure-standalone mode use `InferenceOptimizer` directly as shown. diff --git a/examples/auto_deploy/paragraf/_license_data.py b/examples/auto_deploy/llmc/_license_data.py similarity index 99% rename from examples/auto_deploy/paragraf/_license_data.py rename to examples/auto_deploy/llmc/_license_data.py index f0322edf4abb..1b986d5a81aa 100644 --- a/examples/auto_deploy/paragraf/_license_data.py +++ b/examples/auto_deploy/llmc/_license_data.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""License and attribution data and generators for the standalone paragraf package. +"""License and attribution data and generators for the standalone llmc package. Separated from create_standalone_package.py to keep data and logic apart. """ diff --git a/examples/auto_deploy/llmc/create_standalone_package.py b/examples/auto_deploy/llmc/create_standalone_package.py index c0e53741a4c4..3368cd373bd2 100644 --- a/examples/auto_deploy/llmc/create_standalone_package.py +++ b/examples/auto_deploy/llmc/create_standalone_package.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,17 +14,888 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Forward the legacy standalone generator entry point to Paragraf.""" +"""Create a standalone llmc package from the TensorRT-LLM source tree. -import runpy +This script copies the ``tensorrt_llm/_torch/auto_deploy`` source tree and +tests into a standalone pip-installable package. The output directory can +be pushed directly to the read-only standalone repository. + +Usage: + python create_standalone_package.py [--output-dir /path/to/output] + +The generated package uses ``llmc`` as the top-level Python package name and +``nvidia-llmc`` as the distribution name: + + from llmc._compat import TRTLLM_AVAILABLE + from llmc.custom_ops.attention_interface import SequenceInfo + +The ``auto_deploy`` source tree itself is copied verbatim — internal imports +must already be relative (enforced by the +``auto-deploy-import-discipline`` pre-commit hook), so no source rewriting +is required. Test files use absolute ``tensorrt_llm._torch.auto_deploy`` +imports by design and ARE rewritten to ``llmc`` on copy. +""" + +import argparse +import os +import re +import shutil import sys -from pathlib import Path +import textwrap + +from _license_data import VENDORED_PROJECTS, generate_attributions, generate_license + +# --------------------------------------------------------------------------- +# Path constants +# --------------------------------------------------------------------------- +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) +AUTO_DEPLOY_SRC = os.path.join(REPO_ROOT, "tensorrt_llm", "_torch", "auto_deploy") +TRTLLM_REQUIREMENTS = os.path.join(REPO_ROOT, "requirements.txt") +TRTLLM_DEV_REQUIREMENTS = os.path.join(REPO_ROOT, "requirements-dev.txt") +TRTLLM_LICENSE = os.path.join(REPO_ROOT, "LICENSE") +TRTLLM_GITIGNORE = os.path.join(REPO_ROOT, ".gitignore") +TRTLLM_EDITORCONFIG = os.path.join(REPO_ROOT, ".editorconfig") +TRTLLM_CODE_OF_CONDUCT = os.path.join(REPO_ROOT, "CODE_OF_CONDUCT.md") +TRTLLM_SECURITY = os.path.join(REPO_ROOT, "SECURITY.md") +TRTLLM_ATTRIBUTIONS_PYTHON = os.path.join(REPO_ROOT, "ATTRIBUTIONS-Python.md") +LLMC_README = os.path.join(SCRIPT_DIR, "README.md") +LLMC_CONTRIBUTING = os.path.join(SCRIPT_DIR, "CONTRIBUTING.md") + +# Test source directories +AD_TESTS_DIR = os.path.join(REPO_ROOT, "tests", "unittest", "auto_deploy") +AD_UTILS_TEST_DIR = os.path.join(AD_TESTS_DIR, "_utils_test") +AD_TORCH_TESTS_DIR = os.path.join(REPO_ROOT, "tests", "unittest", "_torch", "auto_deploy") + +# Example/e2e harness sources (Tier-1 e2e: build_and_run_ad.py + model registry). +# These ship with the package so the standalone install can run e2e models via +# the same entrypoint as TRT-LLM. Python is rewritten auto_deploy -> llmc; the +# model_registry YAML is data and copied verbatim. +AD_EXAMPLES_SRC = os.path.join(REPO_ROOT, "examples", "auto_deploy") +# Source filename -> destination filename in the standalone package. The e2e +# entrypoint is renamed to make its scope explicit in the llmc distribution. +EXAMPLE_FILES = {"build_and_run_ad.py": "build_and_run_llmc_trtllm.py"} +EXAMPLE_DIRS = ["model_registry"] + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- +COPY_EXTENSIONS = {".py", ".yaml", ".yml", ".json", ".txt", ".md"} +EXCLUDE_PATTERNS = {"__pycache__", ".pyc", ".pyo"} + +# Standalone runtime dependencies (version pins pulled from TRT-LLM requirements.txt) +STANDALONE_DEPS = [ + "torch", + "transformers", + "pydantic", + "pydantic-settings", + "triton", + "flashinfer-python", + "safetensors", + "accelerate", + "huggingface-hub", + "omegaconf", + "pyyaml", + "numpy", + "pillow", + "einops", + # Transitive deps that may not be pulled in by all installers/platforms + "six", + "importlib-metadata", + "werkzeug", + "StrEnum", + "graphviz", +] + +# Dev/test dependencies (version pins pulled from TRT-LLM requirements-dev.txt) +DEV_DEPS = [ + "pytest", + "pytest-timeout", + "pytest-xdist", + "pytest-cov", + "pytest-mock", + "parameterized", +] + +# Test directories to exclude from the standalone package (require TRT-LLM runtime) +EXCLUDE_TEST_DIRS = {"smoke", "shim", "standalone"} + +# Individual test files to exclude (require TRT-LLM runtime/kernels or external scripts) +EXCLUDE_TEST_FILES = { + # TRT-LLM kernel tests + "test_trtllm_moe.py", + "test_trtllm_attention_op.py", + "test_fla_cached_gated_delta_rule.py", + "test_flashinfer_trtllm_mla_op.py", + "test_trtllm_mla_op.py", + "test_fuse_trtllm_attention_quant_fp8.py", + "test_fuse_relu2_quant_nvfp4.py", + "test_moe_fusion.py", + "test_trtllm_gen_diag.py", + # Standalone flashinfer ROPE path has a known BF16 strided-interleaved mismatch. + "test_rope_op_variants.py", + # QKV fusion → trtllm cache insertion (TRT-LLM attention backend only) + "test_gemm_fusion_trtllm.py", + # Require TRT-LLM LlmArgs / runtime + "test_eagle.py", + "test_modeling_nemotron_h.py", + "test_example_configs.py", + "test_hybrid_patches.py", + "test_captured_graph.py", + # Require external scripts (build_and_run_ad.py) + "test_llama4_vlm_patch.py", + "test_mistral3_patches.py", + # Require TRT-LLM test utils not in standalone + "test_ad_moe_op.py", + "test_triton_moe.py", + # Require onnx (optional dep) + "test_export_fp8_linear_to_onnx.py", + # Uses hardcoded TRT-LLM repo path + "test_mrope_delta_cache.py", + # Depend on TRT-LLM mamba/fla kernels (relative imports beyond auto_deploy) + "test_mamba_rms_norm.py", + "test_triton_rms_norm.py", + "test_fuse_rmsnorm.py", + "test_fused_add_rms_norm.py", + "test_fuse_l2norm.py", + # Require TRT-LLM KVCacheManager or runtime + "test_kv_cache.py", + "test_torch_gated_delta_rule_cache.py", + "test_gated_delta_rule_cache.py", + "test_kv_cache_transformers.py", + # trtllm attention backend (insert_cached_attention backend=trtllm) not available standalone + "test_kv_cache_trtllm_multipool.py", + # Require TRT-LLM CUDA causal conv / mamba kernels (ops not registered standalone) + "test_cuda_causal_conv_cached_op.py", + "test_triton_causal_conv_cached_op.py", + "test_triton_mamba_cached_op.py", + "test_flashinfer_mamba_cached_op.py", + # Require TRT-LLM custom ops (dsv3_router_gemm_op, noaux_tc_op, etc.) + "test_deepseek_custom.py", + "test_glm4_moe_modeling.py", + "test_glm4_moe_lite_modeling.py", + "test_glm_moe_dsa_modeling.py", + # Full-model tests hit standalone-incompatible HF cache behavior. + "test_granite_moe_hybrid_modeling.py", + # Imports triton_kernels, which is not a standalone dependency. + "test_mxfp4_moe_layout.py", + # Require TRT-LLM distributed ops (trtllm_dist_all_gather) + "test_gather_logits_before_lm_head.py", + # Multimodal processors depend on TensorRT-LLM multimodal request types. + "test_gemma4_modeling.py", + "test_qwen3_5_moe.py", + # Hardware-specific (requires H100+ shared memory) + "test_triton_mla_op.py", + # Require TRT-LLM ops (noaux_tc_op) — split from test_export.py + "test_export_glm4_moe_lite.py", + # fuse_fp8_linear / fuse_nvfp4_linear / fuse_finegrained_fp8_linear transforms + # live in fuse_quant.py which imports tensorrt_llm.quantization.utils.fp8_utils; + # the module is silently skipped in standalone so the transforms aren't registered. + "test_quant_fusion.py", + # Imports utils.util.skip_pre_blackwell (not shipped in standalone) and exercises + # fuse_finegrained_fp8_swiglu which depends on TRT-LLM runtime. + "test_finegrained_fp8_swiglu.py", + # Exercise trtllm-gen MXFP4 MoE kernels (Blackwell-only) and import the + # prepare_trtllm_gen_moe_mxfp4_weights / utils.util helpers not in standalone. + "test_fuse_mxfp4_moe.py", + "test_trtllm_quant_mxfp4_trtllm_gen_moe.py", +} + +# Single-GPU smoke tests. Some run with LLMC alone; others exercise the optional +# TensorRT-LLM wheel/CLI path and are gated by TRTLLM_REDIRECT_AD_TO_LLMC. +STANDALONE_SINGLEGPU_SMOKE_TEST_FILES = ( + "smoke/test_ad_build_small_single.py", + "smoke/test_ad_guided_decoding_regex.py", + "smoke/test_ad_speculative_decoding.py", + "smoke/test_ad_trtllm_bench.py", + "smoke/test_ad_trtllm_sampler.py", + "smoke/test_ad_trtllm_serve.py", + "smoke/test_disagg.py", +) + +# Multi-GPU tests that exercise only AutoDeploy and its standalone dependencies. +# Keep this list explicit: the remaining multi-GPU tests depend on TensorRT-LLM +# runtime components, kernels, or test infrastructure that LLMC does not ship. +STANDALONE_MULTIGPU_TEST_FILES = ( + "custom_ops/test_dist.py", + "custom_ops/test_sharded_rmsnorm.py", + "smoke/test_ad_build_small_multi.py", + "transformations/library/test_apply_sharding_hints.py", + "transformations/library/test_bmm_sharding.py", + "transformations/library/test_ep_sharding.py", + "transformations/library/test_rmsnorm_sharding.py", + "transformations/library/test_sharding_num_correctness.py", + "transformations/library/test_step3p7_sharding_ir.py", + "transformations/library/test_tp_sharding.py", +) + +# Pytest support files required by the allowlisted multi-GPU tests. +STANDALONE_MULTIGPU_SUPPORT_FILES = ("transformations/library/conftest.py",) + +# Newer AD unit tests live under tests/unittest/_torch/auto_deploy. Copy only +# tracked tests whose dependencies are available in LLMC plus the optional +# TensorRT-LLM wheel. +STANDALONE_TORCH_UNIT_TEST_FILES = ("unit/singlegpu/models/test_gpt_oss_modeling.py",) + +# Import path rewrite: old -> new (applied to test files only). +_IMPORT_REWRITE = "tensorrt_llm._torch.auto_deploy" +_IMPORT_TARGET = "llmc" +_BUILD_AND_RUN_AD_IMPORT = "from build_and_run_ad import ExperimentConfig, main" +_TRTLLM_IMPORT_RE = re.compile( + r"(?m)^(?:from|import) " + r"(?:tensorrt_llm(?:\.|\b)|llmc\.models\.custom\.modeling_gpt_oss(?:\.|\b))" +) +_LLMC_OPTIONAL_TRTLLM_GUARD = """ +_trtllm_redirect_value = os.environ.get("TRTLLM_REDIRECT_AD_TO_LLMC", "").lower() +if _trtllm_redirect_value not in {"1", "true", "yes", "on"}: + pytest.skip( + "LLMC optional TRT-LLM tests require TRTLLM_REDIRECT_AD_TO_LLMC=true", + allow_module_level=True, + ) +pytest.importorskip("tensorrt_llm")""" +_LLMC_TRTLLM_RUNNER_IMPORT = ( + "from runners.trtllm.build_and_run_llmc_trtllm import ExperimentConfig, main" +) + +# Paths that the script owns and regenerates on every run. +# Everything else in the output directory (e.g., .git/, .github/) is preserved +# and owned by the standalone repo itself. +_MANAGED_PATHS = [ + "llmc", + "tests", + "runners", + "pyproject.toml", + "README.md", + "LICENSE", + "ATTRIBUTIONS-Python.md", + "CONTRIBUTING.md", + ".gitignore", + ".editorconfig", + "CODE_OF_CONDUCT.md", + "SECURITY.md", + "ATTRIBUTIONS-Python.md", +] + + +# --------------------------------------------------------------------------- +# Helper functions +# --------------------------------------------------------------------------- +def _should_copy(filepath: str) -> bool: + for pattern in EXCLUDE_PATTERNS: + if pattern in filepath: + return False + _, ext = os.path.splitext(filepath) + return ext in COPY_EXTENSIONS + + +def _copy_tree(src_dir: str, dst_dir: str) -> int: + """Copy files from src_dir to dst_dir, preserving directory structure.""" + count = 0 + for root, dirs, files in os.walk(src_dir): + dirs[:] = [d for d in dirs if d != "__pycache__"] + for filename in files: + src_path = os.path.join(root, filename) + if not _should_copy(src_path): + continue + rel_path = os.path.relpath(src_path, src_dir) + dst_path = os.path.join(dst_dir, rel_path) + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copy2(src_path, dst_path) + count += 1 + return count + + +def _rewrite_imports_in_file(filepath: str, *, optional_trtllm_guards: bool = True) -> int: + """Rewrite imports in a copied test file for standalone mode. + + Source files inside ``tensorrt_llm/_torch/auto_deploy`` already use + relative imports (enforced by the ``auto-deploy-import-discipline`` + pre-commit hook), so no rewriting is needed for them. Tests, however, + are written against the canonical absolute path + ``tensorrt_llm._torch.auto_deploy`` and need to be rewritten to + ``llmc``. Cross-package types (e.g. ``KvCacheConfig``, + ``ActivationType``) are sourced via ``..._torch.auto_deploy._compat``, + so the primary rewrite handles them too. + + Returns the number of line-level changes made. + """ + with open(filepath) as f: + content = f.read() + + original = content + content = content.replace(_IMPORT_REWRITE, _IMPORT_TARGET) + + def ensure_imports(before_pos: int, *imports: str) -> None: + nonlocal content + prefix = content[:before_pos] + missing_imports = [ + import_name for import_name in imports if f"import {import_name}\n" not in prefix + ] + if not missing_imports: + return + first_import = re.search(r"(?m)^(?:import|from) ", content) + if first_import is None: + raise ValueError(f"No import block found in {filepath}") + content = ( + content[: first_import.start()] + + "\n".join(f"import {import_name}" for import_name in missing_imports) + + "\n" + + content[first_import.start() :] + ) + + def insert_optional_trtllm_guard() -> None: + nonlocal content + if _LLMC_OPTIONAL_TRTLLM_GUARD in content: + return + pytest_import = re.search(r"(?m)^import pytest\n", content) + if pytest_import is None: + raise ValueError(f"No pytest import found in {filepath}") + content = ( + content[: pytest_import.end()] + + _LLMC_OPTIONAL_TRTLLM_GUARD + + "\n" + + content[pytest_import.end() :] + ) + + if optional_trtllm_guards and _BUILD_AND_RUN_AD_IMPORT in content: + build_import_pos = content.index(_BUILD_AND_RUN_AD_IMPORT) + ensure_imports(build_import_pos, "os", "pytest") + insert_optional_trtllm_guard() + content = content.replace(_BUILD_AND_RUN_AD_IMPORT, _LLMC_TRTLLM_RUNNER_IMPORT) + elif optional_trtllm_guards: + trtllm_import = _TRTLLM_IMPORT_RE.search(content) + if trtllm_import is not None: + ensure_imports(trtllm_import.start(), "os", "pytest") + insert_optional_trtllm_guard() + + if optional_trtllm_guards: + # The standalone package can rely on the installed trtllm-bench entrypoint, + # but it does not ship TensorRT-LLM's source-tree benchmarks/cpp directory. + content = content.replace( + ' script_dir = Path(root_dir, "benchmarks", "cpp")\n', + " script_dir = Path(temp_dir)\n", + ) + + replacements = sum(1 for a, b in zip(original, content) if a != b) # rough count + if content != original: + with open(filepath, "w") as f: + f.write(content) + # Count actual line-level changes + replacements = sum(1 for a, b in zip(original.splitlines(), content.splitlines()) if a != b) + + return replacements + + +def _rewrite_imports_in_dir(directory: str, *, optional_trtllm_guards: bool = True) -> int: + """Rewrite imports in all .py files in a directory tree.""" + total = 0 + for root, _, files in os.walk(directory): + for filename in files: + if filename.endswith(".py"): + total += _rewrite_imports_in_file( + os.path.join(root, filename), + optional_trtllm_guards=optional_trtllm_guards, + ) + return total + + +def _read_pinned_versions(req_file: str) -> dict: + """Read requirements.txt and extract package->version-spec mapping.""" + versions = {} + if not os.path.exists(req_file): + return versions + with open(req_file) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or line.startswith("-"): + continue + line = line.split("#")[0].strip() + # Handle semicolons (environment markers like ; python_version >= "3.10") + line = line.split(";")[0].strip() + match = re.match(r"^([a-zA-Z0-9_-]+(?:\[[^\]]+\])?)(.*)", line) + if match: + pkg_name = match.group(1).split("[")[0].lower() + version_spec = match.group(2).strip() + if version_spec: + versions[pkg_name] = version_spec + return versions + + +def _resolve_dependencies(dep_names: list, pinned: dict) -> list: + """Resolve dependency list by adding version pins from requirements.""" + resolved = [] + for name in dep_names: + extras = "" + if "[" in name: + base, extras_part = name.split("[", 1) + extras = f"[{extras_part}" + else: + base = name + version = pinned.get(base.lower(), "") + resolved.append(f"{base}{extras}{version}") + return resolved + + +# --------------------------------------------------------------------------- +# Test copying +# --------------------------------------------------------------------------- +def _should_exclude_test(filepath: str) -> bool: + """Check if a test file should be excluded from the standalone package.""" + basename = os.path.basename(filepath) + if basename in EXCLUDE_TEST_FILES: + return True + parts = filepath.replace("\\", "/").split("/") + return any(d in EXCLUDE_TEST_DIRS for d in parts) + + +def _copy_tests(output_dir: str) -> int: + """Copy auto_deploy test files to the standalone package tests/ directory.""" + tests_dst = os.path.join(output_dir, "tests") + count = 0 + + # Copy singlegpu tests (excluding TRT-LLM-only dirs/files) + singlegpu_src = os.path.join(AD_TESTS_DIR, "singlegpu") + if os.path.isdir(singlegpu_src): + for root, dirs, files in os.walk(singlegpu_src): + # Skip excluded directories + dirs[:] = [d for d in dirs if d not in EXCLUDE_TEST_DIRS and d != "__pycache__"] + + for filename in files: + src_path = os.path.join(root, filename) + if not _should_copy(src_path) or _should_exclude_test(src_path): + continue + rel_path = os.path.relpath(src_path, AD_TESTS_DIR) + dst_path = os.path.join(tests_dst, rel_path) + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copy2(src_path, dst_path) + count += 1 + + # Copy all single-GPU smoke tests. Tests that require the optional + # TensorRT-LLM wheel are guarded during import rewriting. + singlegpu_smoke_files = STANDALONE_SINGLEGPU_SMOKE_TEST_FILES + for rel_path in singlegpu_smoke_files: + src_path = os.path.join(singlegpu_src, rel_path) + if not os.path.isfile(src_path): + raise FileNotFoundError( + f"Allowlisted single-GPU smoke test file does not exist: {src_path}" + ) + dst_path = os.path.join(tests_dst, "singlegpu", rel_path) + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copy2(src_path, dst_path) + count += 1 + + # Copy only the explicitly standalone-compatible multi-GPU tests. Unlike + # singlegpu/, this tree contains several tests that require TensorRT-LLM. + multigpu_src = os.path.join(AD_TESTS_DIR, "multigpu") + multigpu_files = STANDALONE_MULTIGPU_TEST_FILES + STANDALONE_MULTIGPU_SUPPORT_FILES + for rel_path in multigpu_files: + src_path = os.path.join(multigpu_src, rel_path) + if not os.path.isfile(src_path): + raise FileNotFoundError(f"Allowlisted multi-GPU test file does not exist: {src_path}") + dst_path = os.path.join(tests_dst, "multigpu", rel_path) + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copy2(src_path, dst_path) + count += 1 + + # Copy standalone-compatible tests from the newer _torch/auto_deploy unit tree. + for rel_path in STANDALONE_TORCH_UNIT_TEST_FILES: + src_path = os.path.join(AD_TORCH_TESTS_DIR, rel_path) + if not os.path.isfile(src_path): + raise FileNotFoundError( + f"Allowlisted _torch AutoDeploy unit test does not exist: {src_path}" + ) + dst_path = os.path.join(tests_dst, "_torch", "auto_deploy", rel_path) + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copy2(src_path, dst_path) + count += 1 + + # Copy test utilities + if os.path.isdir(AD_UTILS_TEST_DIR): + utils_dst = os.path.join(tests_dst, "_utils_test") + for filename in os.listdir(AD_UTILS_TEST_DIR): + src_path = os.path.join(AD_UTILS_TEST_DIR, filename) + if os.path.isfile(src_path) and _should_copy(src_path): + dst_path = os.path.join(utils_dst, filename) + os.makedirs(utils_dst, exist_ok=True) + shutil.copy2(src_path, dst_path) + count += 1 + + # Create conftest.py for test discovery and imports + _create_test_conftest(tests_dst) + + # Create a stub for test_common.llm_data (used by some model tests) + _create_test_common_stub(tests_dst) + _create_test_utils_stub(tests_dst) + + return count + + +def _create_test_conftest(tests_dir: str) -> None: + """Create a conftest.py that configures the test environment for standalone mode.""" + content = textwrap.dedent("""\ + # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + \"\"\"Conftest for standalone auto_deploy tests.\"\"\" + import importlib.util + import os + import sys + from pathlib import Path + + import pytest + + _allow_trtllm_redirect = ( + os.environ.get("TRTLLM_REDIRECT_AD_TO_LLMC", "").lower() + in {"1", "true", "yes", "on"} + ) + _trtllm_spec = importlib.util.find_spec("tensorrt_llm") + if _trtllm_spec is not None and not _allow_trtllm_redirect: + raise RuntimeError( + "Standalone llmc tests must not be able to import tensorrt_llm; " + "set TRTLLM_REDIRECT_AD_TO_LLMC=true only for optional TRT-LLM tests; " + f"found {getattr(_trtllm_spec, 'origin', None)!r}" + ) + + _tests_dir = os.path.dirname(__file__) + _package_root = os.path.dirname(_tests_dir) + + # Add generated package/test roots to the Python path so tests can import + # local llmc, runners, and _utils_test even under safe-path settings. + sys.path.insert(0, _package_root) + sys.path.insert(0, _tests_dir) + sys.path.insert(0, os.path.join(_tests_dir, "_utils_test")) + + + @pytest.fixture(scope="module") + def llm_root(): + env_root = os.environ.get("LLM_ROOT") + if env_root: + return Path(env_root) + return Path(_package_root) + """) + with open(os.path.join(tests_dir, "conftest.py"), "w") as f: + f.write(content) + + +def _create_test_common_stub(tests_dir: str) -> None: + """Create a stub for test_common.llm_data (provides HF model path resolution). + + In standalone mode, tests that need local model weights will be skipped + unless LLM_MODELS_ROOT is set. + """ + stub_dir = os.path.join(tests_dir, "test_common") + os.makedirs(stub_dir, exist_ok=True) + + with open(os.path.join(stub_dir, "__init__.py"), "w") as f: + f.write( + "# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION" + " & AFFILIATES. All rights reserved.\n" + "# SPDX-License-Identifier: Apache-2.0\n" + ) + + content = textwrap.dedent("""\ + # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + \"\"\"Stub for test_common.llm_data in standalone mode.\"\"\" + import os + from pathlib import Path + from unittest.mock import patch + + LLM_MODELS_ROOT = os.environ.get("LLM_MODELS_ROOT") + + + def llm_models_root(): + return Path(LLM_MODELS_ROOT) if LLM_MODELS_ROOT else None + + + def hf_id_to_local_model_dir(hf_id: str): + root = llm_models_root() + if root is None: + return hf_id # Fall back to HF hub download + # Try direct match + candidate = root / hf_id.split("/")[-1] + if candidate.exists(): + return str(candidate) + return hf_id + + + def with_mocked_hf_download_for_single_gpu(func): + return func # No-op in standalone mode + """) + with open(os.path.join(stub_dir, "llm_data.py"), "w") as f: + f.write(content) + + +def _create_test_utils_stub(tests_dir: str) -> None: + """Create minimal TensorRT-LLM unittest utility shims used by copied tests.""" + utils_dir = os.path.join(tests_dir, "utils") + os.makedirs(utils_dir, exist_ok=True) + + with open(os.path.join(utils_dir, "__init__.py"), "w") as f: + f.write( + "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION" + " & AFFILIATES. All rights reserved.\n" + "# SPDX-License-Identifier: Apache-2.0\n" + ) + + content = textwrap.dedent("""\ + # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + \"\"\"Minimal unittest utility shims for standalone LLMC tests.\"\"\" + + import pytest + import torch + + + def _sm_version() -> int: + if not torch.cuda.is_available(): + return 0 + major, minor = torch.cuda.get_device_capability(0) + return major * 10 + minor + + + skip_pre_hopper = pytest.mark.skipif( + _sm_version() < 90, + reason="This test is not supported in pre-Hopper architecture", + ) + """) + with open(os.path.join(utils_dir, "util.py"), "w") as f: + f.write(content) + + +# --------------------------------------------------------------------------- +# Example / e2e harness copying +# --------------------------------------------------------------------------- +def _copy_runners(output_dir: str) -> int: + """Copy the Tier-1 e2e harness into the standalone package under ``runners/trtllm/``. + + ``examples/auto_deploy/build_and_run_ad.py`` is copied to + ``runners/trtllm/build_and_run_llmc_trtllm.py`` (see ``EXAMPLE_FILES``) together + with its sibling ``model_registry/``. Because the script resolves the registry + relative to its own location (``Path(__file__).parent / "model_registry"``), + the rename + relocation are safe and ``--use-registry`` keeps working. + ``.py`` files get the usual ``auto_deploy -> llmc`` import rewrite (applied by + the caller); the ``model_registry`` YAML is data, copied verbatim. + """ + runners_dst = os.path.join(output_dir, "runners", "trtllm") + count = 0 + for src_name, dst_name in EXAMPLE_FILES.items(): + src = os.path.join(AD_EXAMPLES_SRC, src_name) + if os.path.isfile(src): + os.makedirs(runners_dst, exist_ok=True) + shutil.copy2(src, os.path.join(runners_dst, dst_name)) + count += 1 + for dname in EXAMPLE_DIRS: + src = os.path.join(AD_EXAMPLES_SRC, dname) + if os.path.isdir(src): + count += _copy_tree(src, os.path.join(runners_dst, dname)) + return count + + +# --------------------------------------------------------------------------- +# Package generation +# --------------------------------------------------------------------------- +def _create_pyproject_toml(output_dir: str, dependencies: list, dev_dependencies: list) -> None: + """Create a pyproject.toml for the standalone package.""" + deps_lines = "\n".join(f' "{dep}",' for dep in dependencies) + dev_deps_lines = "\n".join(f' "{dep}",' for dep in dev_dependencies) + + content = ( + "[build-system]\n" + 'requires = ["setuptools>=64", "wheel"]\n' + 'build-backend = "setuptools.build_meta"\n' + "\n" + "[project]\n" + 'name = "nvidia-llmc"\n' + 'version = "0.1.0"\n' + 'description = "llmc: standalone LLM compiler — ' + 'automatic model optimization and deployment for LLM inference"\n' + 'readme = "README.md"\n' + 'license = {text = "Apache-2.0"}\n' + 'requires-python = ">=3.10"\n' + "dependencies = [\n" + f"{deps_lines}\n" + "]\n" + "\n" + "[project.optional-dependencies]\n" + "dev = [\n" + f"{dev_deps_lines}\n" + "]\n" + "\n" + "[tool.setuptools.packages.find]\n" + 'include = ["llmc*"]\n' + "\n" + "[tool.pytest.ini_options]\n" + 'testpaths = ["tests"]\n' + "markers = [\n" + ' "threadleak(enabled): configure thread-leak checks (inert in standalone tests)",\n' + "]\n" + ) + + with open(os.path.join(output_dir, "pyproject.toml"), "w") as f: + f.write(content) + + +def create_standalone_package(output_dir: str) -> None: + """Create the standalone llmc package at the given output directory. + + Safe to run against an existing git repository: only the managed paths + (source, tests, and packaging files) are deleted and regenerated. The .git + directory and any repo-specific files (e.g., .github/) are preserved. + After running, ``git add -A && git commit`` captures all changes. + """ + if not os.path.isdir(AUTO_DEPLOY_SRC): + print(f"ERROR: auto_deploy source not found at {AUTO_DEPLOY_SRC}", file=sys.stderr) + sys.exit(1) + + os.makedirs(output_dir, exist_ok=True) + + # Clean only the paths this script manages, preserving .git and other repo files + for name in _MANAGED_PATHS: + target = os.path.join(output_dir, name) + if os.path.isdir(target): + shutil.rmtree(target) + elif os.path.isfile(target): + os.remove(target) + + print(f"Creating standalone package at: {output_dir}") + + # 1. Copy auto_deploy source as top-level `llmc/` package. No import + # rewriting is needed: in-package imports are relative (enforced by + # the auto-deploy-import-discipline pre-commit hook). + ad_dst = os.path.join(output_dir, "llmc") + count = _copy_tree(AUTO_DEPLOY_SRC, ad_dst) + print(f" Copied {count} source files to llmc/") + + # 2. Copy and rewrite tests (tests use absolute self-imports by design). + test_count = _copy_tests(output_dir) + rewrite_count = _rewrite_imports_in_dir(os.path.join(output_dir, "tests")) + print(f" Copied {test_count} test files to tests/ ({rewrite_count} import rewrites)") + + # 2b. Copy the Tier-1 e2e harness into runners/ (build_and_run_llmc_trtllm.py + # + model_registry) and rewrite its imports auto_deploy -> llmc. YAML is + # left untouched. + runner_count = _copy_runners(output_dir) + runner_rewrites = _rewrite_imports_in_dir( + os.path.join(output_dir, "runners"), + optional_trtllm_guards=False, + ) + print( + f" Copied {runner_count} runner files to runners/trtllm/ ({runner_rewrites} import rewrites)" + ) + + # 3. Resolve dependencies and create pyproject.toml + pinned = _read_pinned_versions(TRTLLM_REQUIREMENTS) + dev_pinned = _read_pinned_versions(TRTLLM_DEV_REQUIREMENTS) + # Merge: dev_pinned has the same packages as pinned plus test-only packages + all_pinned = {**pinned, **dev_pinned} + dependencies = _resolve_dependencies(STANDALONE_DEPS, pinned) + dev_dependencies = _resolve_dependencies(DEV_DEPS, all_pinned) + _create_pyproject_toml(output_dir, dependencies, dev_dependencies) + print(f" Created pyproject.toml ({len(dependencies)} deps + {len(dev_dependencies)} dev deps)") + + # 4. Generate standalone LICENSE (only vendored projects in auto_deploy) + generate_license(output_dir) + print(f" Generated LICENSE ({len(VENDORED_PROJECTS)} vendored projects)") + + # 5. Generate ATTRIBUTIONS-Python.md (direct dependency licenses) + generate_attributions(output_dir, dependencies) + print(f" Generated ATTRIBUTIONS-Python.md ({len(dependencies)} direct deps)") + + # 6. Copy README + if os.path.exists(LLMC_README): + shutil.copy2(LLMC_README, os.path.join(output_dir, "README.md")) + print(" Copied README.md") + + # 7. Copy CONTRIBUTING.md + if os.path.exists(LLMC_CONTRIBUTING): + shutil.copy2(LLMC_CONTRIBUTING, os.path.join(output_dir, "CONTRIBUTING.md")) + print(" Copied CONTRIBUTING.md") + + # 8. Copy .gitignore + if os.path.exists(TRTLLM_GITIGNORE): + shutil.copy2(TRTLLM_GITIGNORE, os.path.join(output_dir, ".gitignore")) + print(" Copied .gitignore") + + # 9. Copy .editorconfig + if os.path.exists(TRTLLM_EDITORCONFIG): + shutil.copy2(TRTLLM_EDITORCONFIG, os.path.join(output_dir, ".editorconfig")) + print(" Copied .editorconfig") + + # 10. Copy OSS compliance files (CODE_OF_CONDUCT, SECURITY) + for src, name in ( + (TRTLLM_CODE_OF_CONDUCT, "CODE_OF_CONDUCT.md"), + (TRTLLM_SECURITY, "SECURITY.md"), + ): + if os.path.exists(src): + shutil.copy2(src, os.path.join(output_dir, name)) + print(f" Copied {name}") + + print(f"\nStandalone package created at: {output_dir}") + print("\nTo install:") + print(f" cd {output_dir}") + print(" uv venv .venv --python 3.12") + print(" source .venv/bin/activate") + print(" uv pip install -e '.[dev]'") + print("\nTo run tests: pytest tests/") + print( + 'To verify: python -c "from llmc._compat import TRTLLM_AVAILABLE; print(TRTLLM_AVAILABLE)"' + ) + print( + "To run e2e: python runners/trtllm/build_and_run_llmc_trtllm.py " + "--model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --use-registry (needs tensorrt-llm installed)" + ) -def main() -> None: - target = Path(__file__).resolve().parents[1] / "paragraf" / "create_standalone_package.py" - sys.path.insert(0, str(target.parent)) - runpy.run_path(str(target), run_name="__main__") +def main(): + parser = argparse.ArgumentParser( + description="Create a standalone llmc package from TensorRT-LLM source.", + ) + parser.add_argument( + "--output-dir", + default=os.path.join(REPO_ROOT, "build", "llmc_standalone"), + help="Output directory for the standalone package (default: build/llmc_standalone)", + ) + args = parser.parse_args() + create_standalone_package(os.path.abspath(args.output_dir)) if __name__ == "__main__": diff --git a/examples/auto_deploy/model_registry/configs/qwen3.5_moe_400b.yaml b/examples/auto_deploy/model_registry/configs/qwen3.5_moe_400b.yaml index 687fb88c8181..e8fc2f04f468 100644 --- a/examples/auto_deploy/model_registry/configs/qwen3.5_moe_400b.yaml +++ b/examples/auto_deploy/model_registry/configs/qwen3.5_moe_400b.yaml @@ -36,6 +36,8 @@ transforms: apply_sharding_hints: enabled: true allreduce_strategy: SYMM_MEM + # Shared expert is excluded from sharding for performance purpose + shard_layers: ["moe", "delta", "mha"] simple_shard_filter: "lm_head" multi_stream_moe: stage: compile diff --git a/examples/auto_deploy/paragraf/create_standalone_package.py b/examples/auto_deploy/paragraf/create_standalone_package.py deleted file mode 100644 index d2e75e5cbf39..000000000000 --- a/examples/auto_deploy/paragraf/create_standalone_package.py +++ /dev/null @@ -1,1250 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Create a standalone paragraf package from the TensorRT-LLM source tree. - -This script copies the ``tensorrt_llm/_torch/auto_deploy`` source tree and -tests into a standalone pip-installable package. The output directory can -be pushed directly to the read-only standalone repository. - -Usage: - python create_standalone_package.py [--output-dir /path/to/output] - -The generated package uses ``paragraf`` as the canonical top-level Python package -name. During the compatibility window, ``llmc`` remains an import alias and -``nvidia-llmc`` remains the distribution name: - - from paragraf._compat import TRTLLM_AVAILABLE - from paragraf.custom_ops.attention_interface import SequenceInfo - -The ``auto_deploy`` source tree itself is copied verbatim — internal imports -must already be relative (enforced by the -``auto-deploy-import-discipline`` pre-commit hook), so no source rewriting -is required. Test files use absolute ``tensorrt_llm._torch.auto_deploy`` -imports by design and ARE rewritten to ``paragraf`` on copy. -""" - -import argparse -import os -import re -import shutil -import subprocess -import sys -import textwrap - -from _license_data import VENDORED_PROJECTS, generate_attributions, generate_license - -# --------------------------------------------------------------------------- -# Path constants -# --------------------------------------------------------------------------- -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", "..")) -AUTO_DEPLOY_SRC = os.path.join(REPO_ROOT, "tensorrt_llm", "_torch", "auto_deploy") -TRTLLM_REQUIREMENTS = os.path.join(REPO_ROOT, "requirements.txt") -TRTLLM_DEV_REQUIREMENTS = os.path.join(REPO_ROOT, "requirements-dev.txt") -TRTLLM_LICENSE = os.path.join(REPO_ROOT, "LICENSE") -TRTLLM_GITIGNORE = os.path.join(REPO_ROOT, ".gitignore") -TRTLLM_EDITORCONFIG = os.path.join(REPO_ROOT, ".editorconfig") -TRTLLM_CODE_OF_CONDUCT = os.path.join(REPO_ROOT, "CODE_OF_CONDUCT.md") -TRTLLM_SECURITY = os.path.join(REPO_ROOT, "SECURITY.md") -TRTLLM_ATTRIBUTIONS_PYTHON = os.path.join(REPO_ROOT, "ATTRIBUTIONS-Python.md") -PARAGRAF_README = os.path.join(SCRIPT_DIR, "README.md") -PARAGRAF_CONTRIBUTING = os.path.join(SCRIPT_DIR, "CONTRIBUTING.md") - -# Test source directories -AD_TESTS_DIR = os.path.join(REPO_ROOT, "tests", "unittest", "auto_deploy") -AD_TORCH_TESTS_DIR = os.path.join(REPO_ROOT, "tests", "unittest", "_torch", "auto_deploy") -AD_INTEGRATION_TESTS_DIR = os.path.join(REPO_ROOT, "tests", "integration", "defs") - -# Example/e2e harness sources (Tier-1 e2e: build_and_run_ad.py + model registry). -# These ship with the package so the standalone install can run e2e models via -# the same entrypoint as TRT-LLM. Python is rewritten auto_deploy -> paragraf; the -# model_registry YAML is data and copied verbatim. -AD_EXAMPLES_SRC = os.path.join(REPO_ROOT, "examples", "auto_deploy") -# Source filename -> destination filename in the standalone package. The e2e -# entrypoint is renamed to make its scope explicit in the paragraf distribution. -EXAMPLE_FILES = {"build_and_run_ad.py": "build_and_run_paragraf_trtllm.py"} -EXAMPLE_DIRS = ["model_registry"] -LEGACY_RUNNER_NAME = "build_and_run_llmc_trtllm.py" -LEGACY_RUNNER_WRAPPER = textwrap.dedent("""\ - #!/usr/bin/env python3 - # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - if __package__: - from .build_and_run_paragraf_trtllm import * - else: - from build_and_run_paragraf_trtllm import * - - if __name__ == "__main__": - main() - """) - -# --------------------------------------------------------------------------- -# Configuration -# --------------------------------------------------------------------------- -COPY_EXTENSIONS = {".py", ".yaml", ".yml", ".json", ".txt", ".md"} -EXCLUDE_PATTERNS = {"__pycache__", ".pyc", ".pyo"} - -# Standalone runtime dependencies (version pins pulled from TRT-LLM requirements.txt) -STANDALONE_DEPS = [ - "torch", - "transformers", - "pydantic", - "pydantic-settings", - "triton", - "flashinfer-python", - "safetensors", - "accelerate", - "huggingface-hub", - "omegaconf", - "pyyaml", - "numpy", - "pillow", - "einops", - # Transitive deps that may not be pulled in by all installers/platforms - "six", - "importlib-metadata", - "werkzeug", - "StrEnum", - "graphviz", -] - -# Dev/test dependencies (version pins pulled from TRT-LLM requirements-dev.txt) -DEV_DEPS = [ - "pytest", - "pytest-timeout", - "pytest-xdist", - "pytest-cov", - "pytest-mock", - "pytest-asyncio", - "parameterized", - "cloudpickle", - "mpi4py", - "openai", - "requests", - "scipy", -] - -# These tests validate the generator and its output. They belong to the source -# repository rather than the generated package. -SOURCE_ONLY_TEST_DIRS = {"standalone"} - -# Source test names retain their AutoDeploy identity inside TensorRT-LLM. Rename -# them only in the generated repository, where they exercise Paragraf through -# the optional TensorRT-LLM integration. -PARAGRAF_TRTLLM_TEST_RENAMES = { - "test_llm_api_autodeploy.py": "test_llm_api_paragraf_trtllm.py", - "test_ad_disagg.py": "test_paragraf_trtllm_disagg.py", - "test_ad_disagg_trtllm_serve.py": "test_paragraf_trtllm_disagg_serve.py", - "test_ad_guided_decoding.py": "test_paragraf_trtllm_guided_decoding.py", - "test_ad_speculative_decoding.py": "test_paragraf_trtllm_speculative_decoding.py", - "test_ad_dist_strategies.py": "test_paragraf_trtllm_dist_strategies.py", - "test_ad_allreduce_strategies.py": "test_paragraf_trtllm_allreduce_strategies.py", - "test_ad_build_small_multi.py": "test_paragraf_trtllm_build_small_multi.py", - "test_ad_moe_op.py": "test_paragraf_trtllm_moe_op.py", - "test_ad_executor_swa_eviction.py": "test_paragraf_trtllm_executor_swa_eviction.py", - "test_create_ad_executor.py": "test_create_paragraf_trtllm_executor.py", - "test_ad_build_small_single.py": "test_paragraf_trtllm_build_small_single.py", - "test_ad_guided_decoding_regex.py": "test_paragraf_trtllm_guided_decoding_regex.py", - "test_ad_trtllm_bench.py": "test_paragraf_trtllm_bench.py", - "test_ad_trtllm_sampler.py": "test_paragraf_trtllm_sampler.py", - "test_ad_trtllm_serve.py": "test_paragraf_trtllm_serve.py", -} -SOURCE_TEST_NAMES_BY_GENERATED_NAME = { - generated_name: source_name - for source_name, generated_name in PARAGRAF_TRTLLM_TEST_RENAMES.items() -} - -# Tests in this set are copied, but only collected when the optional TensorRT-LLM -# wheel is enabled through TRTLLM_REDIRECT_AD_TO_PARAGRAF. Keeping this explicit -# also covers indirect dependencies where a rewritten ``paragraf`` import loads -# a module that depends on TensorRT-LLM. -OPTIONAL_TRTLLM_TEST_FILES = { - # TRT-LLM kernel tests - "test_trtllm_moe.py", - "test_trtllm_attention_op.py", - "test_fla_cached_gated_delta_rule.py", - "test_flashinfer_trtllm_mla_op.py", - "test_trtllm_mla_op.py", - "test_fuse_trtllm_attention_quant_fp8.py", - "test_fuse_relu2_quant_nvfp4.py", - "test_moe_fusion.py", - "test_trtllm_gen_diag.py", - # Standalone flashinfer ROPE path has a known BF16 strided-interleaved mismatch. - "test_rope_op_variants.py", - # QKV fusion → trtllm cache insertion (TRT-LLM attention backend only) - "test_gemm_fusion_trtllm.py", - # Require TRT-LLM LlmArgs / runtime - "test_eagle.py", - "test_modeling_nemotron_h.py", - "test_example_configs.py", - "test_hybrid_patches.py", - "test_captured_graph.py", - # Require external scripts (build_and_run_ad.py) - "test_llama4_vlm_patch.py", - "test_mistral3_patches.py", - # Require TRT-LLM test utils not in standalone - "test_ad_moe_op.py", - "test_triton_moe.py", - # Require onnx (optional dep) - "test_export_fp8_linear_to_onnx.py", - # Depend on TRT-LLM mamba/fla kernels (relative imports beyond auto_deploy) - "test_mamba_rms_norm.py", - "test_triton_rms_norm.py", - "test_fuse_rmsnorm.py", - "test_fused_add_rms_norm.py", - "test_fuse_l2norm.py", - # Require TRT-LLM KVCacheManager or runtime - "test_kv_cache.py", - "test_torch_gated_delta_rule_cache.py", - "test_gated_delta_rule_cache.py", - "test_kv_cache_transformers.py", - # trtllm attention backend (insert_cached_attention backend=trtllm) not available standalone - "test_kv_cache_trtllm_multipool.py", - # Require TRT-LLM CUDA causal conv / mamba kernels (ops not registered standalone) - "test_cuda_causal_conv_cached_op.py", - "test_triton_causal_conv_cached_op.py", - "test_triton_mamba_cached_op.py", - "test_flashinfer_mamba_cached_op.py", - # Require TRT-LLM custom ops (dsv3_router_gemm_op, noaux_tc_op, etc.) - "test_deepseek_custom.py", - "test_glm4_moe_modeling.py", - "test_glm4_moe_lite_modeling.py", - "test_glm_moe_dsa_modeling.py", - # Full-model tests hit standalone-incompatible HF cache behavior. - "test_granite_moe_hybrid_modeling.py", - # Imports triton_kernels, which is not a standalone dependency. - "test_mxfp4_moe_layout.py", - # Require TRT-LLM distributed ops (trtllm_dist_all_gather) - "test_gather_logits_before_lm_head.py", - # Multimodal processors depend on TensorRT-LLM multimodal request types. - "test_gemma4_modeling.py", - "test_qwen3_5_moe.py", - # Hardware-specific (requires H100+ shared memory) - "test_triton_mla_op.py", - # Require TRT-LLM ops (noaux_tc_op) — split from test_export.py - "test_export_glm4_moe_lite.py", - # fuse_fp8_linear / fuse_nvfp4_linear / fuse_finegrained_fp8_linear transforms - # live in fuse_quant.py which imports tensorrt_llm.quantization.utils.fp8_utils; - # the module is silently skipped in standalone so the transforms aren't registered. - "test_quant_fusion.py", - # Imports utils.util.skip_pre_blackwell (not shipped in standalone) and exercises - # fuse_finegrained_fp8_swiglu which depends on TRT-LLM runtime. - "test_finegrained_fp8_swiglu.py", - # Exercise trtllm-gen MXFP4 MoE kernels (Blackwell-only) and import the - # prepare_trtllm_gen_moe_mxfp4_weights / utils.util helpers not in standalone. - "test_fuse_mxfp4_moe.py", - "test_trtllm_quant_mxfp4_trtllm_gen_moe.py", -} - -# Multi-GPU tests known to run without the optional TensorRT-LLM wheel. Other -# AutoDeploy multi-GPU tests are still copied, but receive the optional-wheel -# collection guard. -PURE_STANDALONE_MULTIGPU_TEST_FILES = { - "custom_ops/test_dist.py", - "custom_ops/test_sharded_rmsnorm.py", - "smoke/test_ad_build_small_multi.py", - "transformations/library/test_apply_sharding_hints.py", - "transformations/library/test_bmm_sharding.py", - "transformations/library/test_ep_sharding.py", - "transformations/library/test_rmsnorm_sharding.py", - "transformations/library/test_sharding_num_correctness.py", - "transformations/library/test_step3p7_sharding_ir.py", - "transformations/library/test_tp_sharding.py", -} - -# AutoDeploy integration files selected by the same classifier used by CI. -AUTODEPLOY_TEST_RE = re.compile(r"auto_?deploy|_ad_", re.IGNORECASE) -TEST_FILE_RE = re.compile(r"(?:^test_.*|.*_test)\.py$") - -# Support files needed by the selected integration tests. The source integration -# conftest is intentionally not copied because it pulls in the complete TRT-LLM -# CI harness; a focused replacement is generated below. -INTEGRATION_SUPPORT_FILES = ( - "__init__.py", - "common.py", - "trt_test_alternative.py", - "accuracy/__init__.py", - "accuracy/accuracy_core.py", - "accuracy/video_mme.py", - "disaggregated/disagg_test_utils.py", -) -INTEGRATION_SUPPORT_DIRS = ("accuracy/references",) - -# Additional source-tree test helpers imported by copied AutoDeploy tests. -TORCH_TEST_SUPPORT_FILES = ( - "tests/unittest/_torch/__init__.py", - "tests/unittest/_torch/helpers.py", -) - -# Import path rewrite: old -> new (applied to test files only). -_IMPORT_REWRITE = "tensorrt_llm._torch.auto_deploy" -_IMPORT_TARGET = "paragraf" -_BUILD_AND_RUN_AD_IMPORT = "from build_and_run_ad import ExperimentConfig, main" -_TRTLLM_IMPORT_RE = re.compile( - r"(?m)^(?:from|import) " - r"(?:tensorrt_llm(?:\.|\b)|paragraf\.models\.custom\.modeling_gpt_oss(?:\.|\b))" -) -_PARAGRAF_OPTIONAL_TRTLLM_GUARD = """ -_trtllm_environ = __import__("os").environ -_trtllm_redirect_value = _trtllm_environ.get("TRTLLM_REDIRECT_AD_TO_PARAGRAF") -if _trtllm_redirect_value is None: - _trtllm_redirect_value = _trtllm_environ.get("TRTLLM_REDIRECT_AD_TO_LLMC", "") -_trtllm_redirect_value = _trtllm_redirect_value.lower() -if _trtllm_redirect_value not in {"1", "true", "yes", "on"}: - pytest.skip( - "Paragraf optional TRT-LLM tests require TRTLLM_REDIRECT_AD_TO_PARAGRAF=true", - allow_module_level=True, - ) -pytest.importorskip("tensorrt_llm")""" -_PARAGRAF_TRTLLM_RUNNER_IMPORT = ( - "from runners.trtllm.build_and_run_paragraf_trtllm import ExperimentConfig, main" -) - -# Paths that the script owns and regenerates on every run. -# Everything else in the output directory (e.g., .git/, .github/) is preserved -# and owned by the standalone repo itself. -_MANAGED_PATHS = [ - "paragraf", - # Remove the package directory produced before the Paragraf rename. - "llmc", - "tests", - # Remove the source-shaped test-data path produced by older generators. - "examples/auto_deploy", - "runners", - "pyproject.toml", - "README.md", - "LICENSE", - "ATTRIBUTIONS-Python.md", - "CONTRIBUTING.md", - ".gitignore", - ".editorconfig", - "CODE_OF_CONDUCT.md", - "SECURITY.md", - "ATTRIBUTIONS-Python.md", -] - - -# --------------------------------------------------------------------------- -# Helper functions -# --------------------------------------------------------------------------- -def _should_copy(filepath: str) -> bool: - for pattern in EXCLUDE_PATTERNS: - if pattern in filepath: - return False - basename = os.path.basename(filepath) - if ".bak." in basename or basename.endswith("~"): - return False - _, ext = os.path.splitext(filepath) - return ext in COPY_EXTENSIONS - - -def _tracked_files_under(directory: str) -> list[str]: - """Return tracked files below a source directory, with an archive fallback.""" - relative_directory = os.path.relpath(directory, REPO_ROOT) - try: - result = subprocess.run( - ["git", "-C", REPO_ROOT, "ls-files", "--", relative_directory], - check=True, - capture_output=True, - text=True, - ) - except (FileNotFoundError, subprocess.CalledProcessError): - discovered_files = [] - for root, dirs, files in os.walk(directory): - dirs[:] = [directory_name for directory_name in dirs if directory_name != "__pycache__"] - discovered_files.extend(os.path.join(root, filename) for filename in files) - return discovered_files - - return [os.path.join(REPO_ROOT, path) for path in result.stdout.splitlines()] - - -def _copy_file(src_path: str, dst_path: str) -> int: - if not os.path.isfile(src_path) or not _should_copy(src_path): - return 0 - os.makedirs(os.path.dirname(dst_path), exist_ok=True) - shutil.copy2(src_path, dst_path) - return 1 - - -def _copy_tracked_tree(src_dir: str, dst_dir: str) -> int: - """Copy tracked files from a tree, or all files in a source archive.""" - count = 0 - for src_path in _tracked_files_under(src_dir): - rel_path = os.path.relpath(src_path, src_dir) - count += _copy_file(src_path, os.path.join(dst_dir, rel_path)) - return count - - -def _copy_tree(src_dir: str, dst_dir: str) -> int: - """Copy files from src_dir to dst_dir, preserving directory structure.""" - count = 0 - for root, dirs, files in os.walk(src_dir): - dirs[:] = [d for d in dirs if d != "__pycache__"] - for filename in files: - src_path = os.path.join(root, filename) - if not _should_copy(src_path): - continue - rel_path = os.path.relpath(src_path, src_dir) - dst_path = os.path.join(dst_dir, rel_path) - os.makedirs(os.path.dirname(dst_path), exist_ok=True) - shutil.copy2(src_path, dst_path) - count += 1 - return count - - -def _rewrite_generated_test_layout(filepath: str, content: str) -> str: - """Rewrite TensorRT-LLM source-tree paths in selected generated tests.""" - - def replace_required(old: str, new: str) -> None: - nonlocal content - if old not in content: - raise ValueError(f"Expected standalone layout pattern not found in {filepath}: {old}") - content = content.replace(old, new) - - def substitute_required(pattern: str, replacement: str) -> None: - nonlocal content - content, count = re.subn(pattern, replacement, content) - if count == 0: - raise ValueError( - f"Expected standalone layout pattern not found in {filepath}: {pattern}" - ) - - filename = os.path.basename(filepath) - model_registry_pattern = ( - r"""["']examples["']\s*/\s*["']auto_deploy["']\s*/\s*""" - r"""["']model_registry["']""" - ) - runner_model_registry = '"runners" / "trtllm" / "model_registry"' - - if filename == "test_llm_api_paragraf_trtllm.py": - substitute_required(model_registry_pattern, runner_model_registry) - for config_name in ("nano_v3.yaml", "super_v3.yaml"): - substitute_required( - rf"""["']examples["']\s*/\s*["']auto_deploy["']\s*/\s*["']{config_name}["']""", - f'{runner_model_registry} / "configs" / "{config_name}"', - ) - elif filename == "test_mrope_delta_cache.py": - replace_required( - "return Path(__file__).resolve().parents[6]", - "return Path(__file__).resolve().parents[4]", - ) - replace_required( - '_repo_root() / "tensorrt_llm" / "_torch" / "auto_deploy" / "config"', - '_repo_root() / "paragraf" / "config"', - ) - substitute_required(model_registry_pattern, runner_model_registry) - elif filename == "test_example_configs.py": - replace_required( - '_AD_EXAMPLES_DIR = _REPO_ROOT / "examples" / "auto_deploy"', - '_AD_EXAMPLES_DIR = _REPO_ROOT / "runners" / "trtllm" / "model_registry"', - ) - - return content - - -def _rewrite_imports_in_file( - filepath: str, - *, - optional_trtllm_guards: bool = True, - force_optional_trtllm_guard: bool = False, -) -> int: - """Rewrite imports in a copied test file for standalone mode. - - Source files inside ``tensorrt_llm/_torch/auto_deploy`` already use - relative imports (enforced by the ``auto-deploy-import-discipline`` - pre-commit hook), so no rewriting is needed for them. Tests, however, - are written against the canonical absolute path - ``tensorrt_llm._torch.auto_deploy`` and need to be rewritten to - ``paragraf``. Cross-package types (e.g. ``KvCacheConfig``, - ``ActivationType``) are sourced via ``..._torch.auto_deploy._compat``, - so the primary rewrite handles them too. - - Returns the number of line-level changes made. - """ - with open(filepath) as f: - content = f.read() - - original = content - content = content.replace(_IMPORT_REWRITE, _IMPORT_TARGET) - content = content.replace("from auto_deploy.", "from ") - content = re.sub( - r"(?m)^from (test_[A-Za-z0-9_]+|disagg_test_utils) import ", - r"from .\1 import ", - content, - ) - content = content.replace( - 'pytest_plugins = ["disagg_test_utils"]', - 'pytest_plugins = ["integration.defs.disaggregated.disagg_test_utils"]', - ) - content = content.replace( - "_REPO_ROOT = pathlib.Path(__file__).resolve().parents[5]", - "_REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]", - ) - content = _rewrite_generated_test_layout(filepath, content) - - def ensure_imports(before_pos: int, *imports: str) -> None: - nonlocal content - prefix = content[:before_pos] - missing_imports = [ - import_name for import_name in imports if f"import {import_name}\n" not in prefix - ] - if not missing_imports: - return - first_import = re.search(r"(?m)^(?:import|from) ", content) - if first_import is None: - raise ValueError(f"No import block found in {filepath}") - content = ( - content[: first_import.start()] - + "\n".join(f"import {import_name}" for import_name in missing_imports) - + "\n" - + content[first_import.start() :] - ) - - def insert_optional_trtllm_guard() -> None: - nonlocal content - if _PARAGRAF_OPTIONAL_TRTLLM_GUARD in content: - return - pytest_import = re.search(r"(?m)^import pytest\n", content) - if pytest_import is None: - raise ValueError(f"No pytest import found in {filepath}") - content = ( - content[: pytest_import.end()] - + _PARAGRAF_OPTIONAL_TRTLLM_GUARD - + "\n" - + content[pytest_import.end() :] - ) - - if optional_trtllm_guards and _BUILD_AND_RUN_AD_IMPORT in content: - build_import_pos = content.index(_BUILD_AND_RUN_AD_IMPORT) - ensure_imports(build_import_pos, "os", "pytest") - insert_optional_trtllm_guard() - content = content.replace(_BUILD_AND_RUN_AD_IMPORT, _PARAGRAF_TRTLLM_RUNNER_IMPORT) - elif optional_trtllm_guards and force_optional_trtllm_guard: - ensure_imports(len(content), "os", "pytest") - insert_optional_trtllm_guard() - elif optional_trtllm_guards: - trtllm_import = _TRTLLM_IMPORT_RE.search(content) - if trtllm_import is not None: - ensure_imports(trtllm_import.start(), "os", "pytest") - insert_optional_trtllm_guard() - - replacements = sum(1 for a, b in zip(original, content) if a != b) # rough count - if content != original: - with open(filepath, "w") as f: - f.write(content) - # Count actual line-level changes - replacements = sum(1 for a, b in zip(original.splitlines(), content.splitlines()) if a != b) - - return replacements - - -def _requires_optional_trtllm_guard(filepath: str, tests_dir: str) -> bool: - relative_path = os.path.relpath(filepath, tests_dir).replace("\\", "/") - if not TEST_FILE_RE.fullmatch(os.path.basename(filepath)): - return False - generated_basename = os.path.basename(filepath) - source_basename = SOURCE_TEST_NAMES_BY_GENERATED_NAME.get( - generated_basename, generated_basename - ) - path_parts = relative_path.split("/") - if relative_path.startswith("integration/"): - return True - if "shim" in path_parts: - return True - if source_basename in OPTIONAL_TRTLLM_TEST_FILES: - return True - if relative_path.startswith("multigpu/"): - multigpu_parts = relative_path.removeprefix("multigpu/").split("/") - multigpu_parts[-1] = source_basename - multigpu_path = "/".join(multigpu_parts) - return multigpu_path not in PURE_STANDALONE_MULTIGPU_TEST_FILES - return False - - -def _rewrite_imports_in_dir(directory: str, *, optional_trtllm_guards: bool = True) -> int: - """Rewrite imports in all .py files in a directory tree.""" - total = 0 - for root, _, files in os.walk(directory): - for filename in files: - if filename.endswith(".py"): - total += _rewrite_imports_in_file( - os.path.join(root, filename), - optional_trtllm_guards=optional_trtllm_guards, - force_optional_trtllm_guard=( - optional_trtllm_guards - and _requires_optional_trtllm_guard(os.path.join(root, filename), directory) - ), - ) - return total - - -def _read_pinned_versions(req_file: str) -> dict: - """Read requirements.txt and extract package->version-spec mapping.""" - versions = {} - if not os.path.exists(req_file): - return versions - with open(req_file) as f: - for line in f: - line = line.strip() - if not line or line.startswith("#") or line.startswith("-"): - continue - line = line.split("#")[0].strip() - # Handle semicolons (environment markers like ; python_version >= "3.10") - line = line.split(";")[0].strip() - match = re.match(r"^([a-zA-Z0-9_-]+(?:\[[^\]]+\])?)(.*)", line) - if match: - pkg_name = match.group(1).split("[")[0].lower() - version_spec = match.group(2).strip() - if version_spec: - versions[pkg_name] = version_spec - return versions - - -def _resolve_dependencies(dep_names: list, pinned: dict) -> list: - """Resolve dependency list by adding version pins from requirements.""" - resolved = [] - for name in dep_names: - extras = "" - if "[" in name: - base, extras_part = name.split("[", 1) - extras = f"[{extras_part}" - else: - base = name - version = pinned.get(base.lower(), "") - resolved.append(f"{base}{extras}{version}") - return resolved - - -# --------------------------------------------------------------------------- -# Test copying -# --------------------------------------------------------------------------- -def _copy_tests(output_dir: str) -> int: - """Copy auto_deploy test files to the standalone package tests/ directory.""" - tests_dst = os.path.join(output_dir, "tests") - count = 0 - - # Copy every tracked legacy AutoDeploy unit-test file except the tests of - # this generator itself. Optional TensorRT-LLM dependencies are handled by - # collection guards after the files are copied. - for src_path in _tracked_files_under(AD_TESTS_DIR): - rel_path = os.path.relpath(src_path, AD_TESTS_DIR) - if any(part in SOURCE_ONLY_TEST_DIRS for part in rel_path.split(os.sep)): - continue - generated_name = PARAGRAF_TRTLLM_TEST_RENAMES.get( - os.path.basename(rel_path), os.path.basename(rel_path) - ) - generated_rel_path = os.path.join(os.path.dirname(rel_path), generated_name) - count += _copy_file(src_path, os.path.join(tests_dst, generated_rel_path)) - - # Copy every tracked test from the newer unit-test tree. This avoids an - # allowlist that silently misses tests added alongside new AutoDeploy code. - for src_path in _tracked_files_under(AD_TORCH_TESTS_DIR): - rel_path = os.path.relpath(src_path, AD_TORCH_TESTS_DIR) - path_parts = rel_path.split(os.sep) - if path_parts[0] == "unit": - path_parts = path_parts[1:] - generated_rel_path = os.path.join(*path_parts) - generated_path = os.path.join(tests_dst, generated_rel_path) - if os.path.exists(generated_path): - raise FileExistsError(f"Generated test path collision for {src_path}: {generated_path}") - count += _copy_file( - src_path, - generated_path, - ) - - # The CI classifier also finds a small number of AutoDeploy integration - # tests outside the unit-test roots. Copy those tests and their focused - # support modules without bringing in the complete TensorRT-LLM CI suite. - integration_tests_root = os.path.join(REPO_ROOT, "tests", "integration") - for src_path in _tracked_files_under(AD_INTEGRATION_TESTS_DIR): - rel_from_tests = os.path.relpath(src_path, integration_tests_root) - if not TEST_FILE_RE.fullmatch(os.path.basename(src_path)): - continue - if not AUTODEPLOY_TEST_RE.search(rel_from_tests.replace("\\", "/")): - continue - generated_name = PARAGRAF_TRTLLM_TEST_RENAMES.get( - os.path.basename(rel_from_tests), os.path.basename(rel_from_tests) - ) - generated_rel_path = os.path.join(os.path.dirname(rel_from_tests), generated_name) - count += _copy_file( - src_path, - os.path.join(tests_dst, "integration", generated_rel_path), - ) - - for rel_path in INTEGRATION_SUPPORT_FILES: - count += _copy_file( - os.path.join(AD_INTEGRATION_TESTS_DIR, rel_path), - os.path.join(tests_dst, "integration", "defs", rel_path), - ) - for rel_path in INTEGRATION_SUPPORT_DIRS: - count += _copy_tracked_tree( - os.path.join(AD_INTEGRATION_TESTS_DIR, rel_path), - os.path.join(tests_dst, "integration", "defs", rel_path), - ) - - for rel_path in TORCH_TEST_SUPPORT_FILES: - count += _copy_file( - os.path.join(REPO_ROOT, rel_path), - os.path.join(tests_dst, os.path.relpath(rel_path, "tests/unittest")), - ) - - # Create conftest.py for test discovery and imports - _create_test_package_init_files(tests_dst) - _create_test_conftest(tests_dst) - _create_integration_conftest(tests_dst) - - # Create a stub for test_common.llm_data (used by some model tests) - _create_test_common_stub(tests_dst) - _create_test_utils_stub(tests_dst) - - return count - - -def _create_test_package_init_files(tests_dir: str) -> None: - """Give copied test directories stable package-qualified module names.""" - content = ( - "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION" - " & AFFILIATES. All rights reserved.\n" - "# SPDX-License-Identifier: Apache-2.0\n" - ) - for root, dirs, _ in os.walk(tests_dir): - dirs[:] = [directory for directory in dirs if directory != "__pycache__"] - if root == tests_dir: - continue - init_path = os.path.join(root, "__init__.py") - if not os.path.exists(init_path): - with open(init_path, "w") as f: - f.write(content) - - -def _create_test_conftest(tests_dir: str) -> None: - """Create a conftest.py that configures the test environment for standalone mode.""" - content = textwrap.dedent("""\ - # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - \"\"\"Conftest for standalone auto_deploy tests.\"\"\" - import importlib.util - import os - import sys - from pathlib import Path - - import pytest - - _trtllm_redirect_value = os.environ.get("TRTLLM_REDIRECT_AD_TO_PARAGRAF") - if _trtllm_redirect_value is None: - _trtllm_redirect_value = os.environ.get("TRTLLM_REDIRECT_AD_TO_LLMC", "") - _allow_trtllm_redirect = _trtllm_redirect_value.lower() in { - "1", "true", "yes", "on" - } - _trtllm_spec = importlib.util.find_spec("tensorrt_llm") - if _trtllm_spec is not None and not _allow_trtllm_redirect: - raise RuntimeError( - "Standalone paragraf tests must not be able to import tensorrt_llm; " - "set TRTLLM_REDIRECT_AD_TO_PARAGRAF=true only for optional TRT-LLM tests; " - f"found {getattr(_trtllm_spec, 'origin', None)!r}" - ) - - _tests_dir = os.path.dirname(__file__) - _package_root = os.path.dirname(_tests_dir) - _integration_tests_dir = os.path.join(_tests_dir, "integration") - - # Add generated package/test roots to the Python path so tests can import - # local paragraf, runners, integration helpers, and _utils_test even - # under safe-path settings. - sys.path.insert(0, _package_root) - sys.path.insert(0, _tests_dir) - sys.path.insert(0, _integration_tests_dir) - sys.path.insert(0, os.path.join(_tests_dir, "_utils_test")) - - - @pytest.fixture(scope="module") - def llm_root(): - env_root = os.environ.get("LLM_ROOT") - if env_root: - return Path(env_root) - return Path(_package_root) - """) - with open(os.path.join(tests_dir, "conftest.py"), "w") as f: - f.write(content) - - -def _create_integration_conftest(tests_dir: str) -> None: - """Create the focused helpers needed by copied AutoDeploy integration tests.""" - defs_dir = os.path.join(tests_dir, "integration", "defs") - os.makedirs(defs_dir, exist_ok=True) - content = textwrap.dedent("""\ - # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 - - import os - from pathlib import Path - - import pytest - import torch - - - def get_llm_root(): - return str(Path(__file__).resolve().parents[3]) - - - def llm_models_root(): - models_root = os.environ.get("LLM_MODELS_ROOT") - if not models_root: - pytest.skip("LLM_MODELS_ROOT is required for AutoDeploy integration tests") - return models_root - - - def get_sm_version(): - if not torch.cuda.is_available(): - return 0 - major, minor = torch.cuda.get_device_capability(0) - return major * 10 + minor - - - def get_device_count(): - return torch.cuda.device_count() - - - def get_device_memory(): - if not torch.cuda.is_available(): - return 0 - return torch.cuda.get_device_properties(0).total_memory // (1024 * 1024) - - - def check_device_contain(keyword_list): - if not torch.cuda.is_available(): - return False - device_name = torch.cuda.get_device_name(0) - return any(keyword in device_name for keyword in keyword_list) - - - skip_pre_ada = pytest.mark.skipif( - get_sm_version() < 89, - reason="This test is not supported in pre-Ada architecture", - ) - skip_pre_hopper = pytest.mark.skipif( - get_sm_version() < 90, - reason="This test is not supported in pre-Hopper architecture", - ) - skip_pre_blackwell = pytest.mark.skipif( - get_sm_version() < 100, - reason="This test is not supported in pre-Blackwell architecture", - ) - - - @pytest.fixture(autouse=True) - def _apply_resource_markers(request): - device_marker = request.node.get_closest_marker("skip_less_device") - if device_marker and get_device_count() < device_marker.args[0]: - pytest.skip(f"Test requires {device_marker.args[0]} GPUs") - - for memory_marker in request.node.iter_markers("skip_less_device_memory"): - if get_device_memory() < memory_marker.args[0]: - pytest.skip(f"Test requires {memory_marker.args[0]} MiB of GPU memory") - """) - with open(os.path.join(defs_dir, "conftest.py"), "w") as f: - f.write(content) - - -def _create_test_common_stub(tests_dir: str) -> None: - """Create a stub for test_common.llm_data (provides HF model path resolution). - - In standalone mode, tests that need local model weights will be skipped - unless LLM_MODELS_ROOT is set. - """ - stub_dir = os.path.join(tests_dir, "test_common") - os.makedirs(stub_dir, exist_ok=True) - - with open(os.path.join(stub_dir, "__init__.py"), "w") as f: - f.write( - "# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION" - " & AFFILIATES. All rights reserved.\n" - "# SPDX-License-Identifier: Apache-2.0\n" - ) - - content = textwrap.dedent("""\ - # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - \"\"\"Stub for test_common.llm_data in standalone mode.\"\"\" - import os - from pathlib import Path - from unittest.mock import patch - - LLM_MODELS_ROOT = os.environ.get("LLM_MODELS_ROOT") - - - def llm_models_root(): - return Path(LLM_MODELS_ROOT) if LLM_MODELS_ROOT else None - - - def hf_id_to_local_model_dir(hf_id: str): - root = llm_models_root() - if root is None: - return hf_id # Fall back to HF hub download - # Try direct match - candidate = root / hf_id.split("/")[-1] - if candidate.exists(): - return str(candidate) - return hf_id - - - def with_mocked_hf_download_for_single_gpu(func): - return func # No-op in standalone mode - """) - with open(os.path.join(stub_dir, "llm_data.py"), "w") as f: - f.write(content) - - -def _create_test_utils_stub(tests_dir: str) -> None: - """Create minimal TensorRT-LLM unittest utility shims used by copied tests.""" - utils_dir = os.path.join(tests_dir, "utils") - os.makedirs(utils_dir, exist_ok=True) - - with open(os.path.join(utils_dir, "__init__.py"), "w") as f: - f.write( - "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION" - " & AFFILIATES. All rights reserved.\n" - "# SPDX-License-Identifier: Apache-2.0\n" - ) - - content = textwrap.dedent("""\ - # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - \"\"\"Minimal unittest utility shims for standalone Paragraf tests.\"\"\" - - import pytest - import torch - - - def _sm_version() -> int: - if not torch.cuda.is_available(): - return 0 - major, minor = torch.cuda.get_device_capability(0) - return major * 10 + minor - - - skip_pre_hopper = pytest.mark.skipif( - _sm_version() < 90, - reason="This test is not supported in pre-Hopper architecture", - ) - skip_no_hopper = pytest.mark.skipif( - _sm_version() != 90, - reason="This test is only supported in Hopper architecture", - ) - skip_pre_blackwell = pytest.mark.skipif( - _sm_version() < 100, - reason="This test is not supported in pre-Blackwell architecture", - ) - """) - with open(os.path.join(utils_dir, "util.py"), "w") as f: - f.write(content) - - cpp_paths_content = textwrap.dedent("""\ - # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 - - from pathlib import Path - - import pytest - - - @pytest.fixture(scope="module") - def llm_root(): - return Path(__file__).resolve().parents[2] - """) - with open(os.path.join(utils_dir, "cpp_paths.py"), "w") as f: - f.write(cpp_paths_content) - - llm_data_content = textwrap.dedent("""\ - # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 - - from test_common.llm_data import llm_models_root - - __all__ = ["llm_models_root"] - """) - with open(os.path.join(utils_dir, "llm_data.py"), "w") as f: - f.write(llm_data_content) - - -# --------------------------------------------------------------------------- -# Example / e2e harness copying -# --------------------------------------------------------------------------- -def _copy_runners(output_dir: str) -> int: - """Copy the Tier-1 e2e harness into the standalone package under ``runners/trtllm/``. - - ``examples/auto_deploy/build_and_run_ad.py`` is copied to - ``runners/trtllm/build_and_run_paragraf_trtllm.py`` (see ``EXAMPLE_FILES``) together - with its sibling ``model_registry/``. Because the script resolves the registry - relative to its own location (``Path(__file__).parent / "model_registry"``), - the rename + relocation are safe and ``--use-registry`` keeps working. - ``.py`` files get the usual ``auto_deploy -> paragraf`` import rewrite (applied by - the caller); the ``model_registry`` YAML is data, copied verbatim. - """ - runners_dst = os.path.join(output_dir, "runners", "trtllm") - count = 0 - for src_name, dst_name in EXAMPLE_FILES.items(): - src = os.path.join(AD_EXAMPLES_SRC, src_name) - if os.path.isfile(src): - os.makedirs(runners_dst, exist_ok=True) - shutil.copy2(src, os.path.join(runners_dst, dst_name)) - count += 1 - for dname in EXAMPLE_DIRS: - src = os.path.join(AD_EXAMPLES_SRC, dname) - if os.path.isdir(src): - count += _copy_tree(src, os.path.join(runners_dst, dname)) - if os.path.isdir(runners_dst): - with open(os.path.join(runners_dst, LEGACY_RUNNER_NAME), "w") as f: - f.write(LEGACY_RUNNER_WRAPPER) - count += 1 - return count - - -# --------------------------------------------------------------------------- -# Package generation -# --------------------------------------------------------------------------- -def _create_pyproject_toml(output_dir: str, dependencies: list, dev_dependencies: list) -> None: - """Create a pyproject.toml for the standalone package.""" - deps_lines = "\n".join(f' "{dep}",' for dep in dependencies) - dev_deps_lines = "\n".join(f' "{dep}",' for dep in dev_dependencies) - - content = ( - "[build-system]\n" - 'requires = ["setuptools>=64", "wheel"]\n' - 'build-backend = "setuptools.build_meta"\n' - "\n" - "[project]\n" - 'name = "nvidia-llmc"\n' - 'version = "0.1.0"\n' - 'description = "paragraf: standalone LLM compiler — ' - 'automatic model optimization and deployment for LLM inference"\n' - 'readme = "README.md"\n' - 'license = {text = "Apache-2.0"}\n' - 'requires-python = ">=3.10"\n' - "dependencies = [\n" - f"{deps_lines}\n" - "]\n" - "\n" - "[project.optional-dependencies]\n" - 'trtllm = ["tensorrt-llm"]\n' - "dev = [\n" - f"{dev_deps_lines}\n" - "]\n" - "\n" - "[tool.setuptools.packages.find]\n" - 'include = ["paragraf*", "llmc"]\n' - "\n" - "[tool.pytest.ini_options]\n" - 'testpaths = ["tests"]\n' - "markers = [\n" - ' "threadleak(enabled): configure thread-leak checks (inert in standalone tests)",\n' - ' "skip_less_device(count): require at least count GPUs",\n' - ' "skip_less_device_memory(mib): require at least mib MiB on one GPU",\n' - "]\n" - ) - - with open(os.path.join(output_dir, "pyproject.toml"), "w") as f: - f.write(content) - - -def create_standalone_package(output_dir: str) -> None: - """Create the standalone paragraf package at the given output directory. - - Safe to run against an existing git repository: only the managed paths - (source, tests, and packaging files) are deleted and regenerated. The .git - directory and any repo-specific files (e.g., .github/) are preserved. - After running, ``git add -A && git commit`` captures all changes. - """ - if not os.path.isdir(AUTO_DEPLOY_SRC): - print(f"ERROR: auto_deploy source not found at {AUTO_DEPLOY_SRC}", file=sys.stderr) - sys.exit(1) - - os.makedirs(output_dir, exist_ok=True) - - # Clean only the paths this script manages, preserving .git and other repo files - for name in _MANAGED_PATHS: - target = os.path.join(output_dir, name) - if os.path.islink(target): - os.remove(target) - elif os.path.isdir(target): - shutil.rmtree(target) - elif os.path.isfile(target): - os.remove(target) - - print(f"Creating standalone package at: {output_dir}") - - # 1. Copy auto_deploy source as top-level `paragraf/` package. No import - # rewriting is needed: in-package imports are relative (enforced by - # the auto-deploy-import-discipline pre-commit hook). - ad_dst = os.path.join(output_dir, "paragraf") - count = _copy_tree(AUTO_DEPLOY_SRC, ad_dst) - print(f" Copied {count} source files to paragraf/") - - legacy_dst = os.path.join(output_dir, "llmc") - os.symlink("paragraf", legacy_dst, target_is_directory=True) - print(" Created legacy llmc -> paragraf package alias") - - # 2. Copy and rewrite tests (tests use absolute self-imports by design). - test_count = _copy_tests(output_dir) - rewrite_count = _rewrite_imports_in_dir(os.path.join(output_dir, "tests")) - print(f" Copied {test_count} test/support files ({rewrite_count} import rewrites)") - - # 2b. Copy the Tier-1 e2e harness into runners/ (build_and_run_paragraf_trtllm.py - # + model_registry) and rewrite its imports auto_deploy -> paragraf. YAML is - # left untouched. - runner_count = _copy_runners(output_dir) - runner_rewrites = _rewrite_imports_in_dir( - os.path.join(output_dir, "runners"), - optional_trtllm_guards=False, - ) - print( - f" Copied {runner_count} runner files to runners/trtllm/ ({runner_rewrites} import rewrites)" - ) - - # 3. Resolve dependencies and create pyproject.toml - pinned = _read_pinned_versions(TRTLLM_REQUIREMENTS) - dev_pinned = _read_pinned_versions(TRTLLM_DEV_REQUIREMENTS) - # Merge: dev_pinned has the same packages as pinned plus test-only packages - all_pinned = {**pinned, **dev_pinned} - dependencies = _resolve_dependencies(STANDALONE_DEPS, pinned) - dev_dependencies = _resolve_dependencies(DEV_DEPS, all_pinned) - _create_pyproject_toml(output_dir, dependencies, dev_dependencies) - print(f" Created pyproject.toml ({len(dependencies)} deps + {len(dev_dependencies)} dev deps)") - - # 4. Generate standalone LICENSE (only vendored projects in auto_deploy) - generate_license(output_dir) - print(f" Generated LICENSE ({len(VENDORED_PROJECTS)} vendored projects)") - - # 5. Generate ATTRIBUTIONS-Python.md (direct dependency licenses) - generate_attributions(output_dir, dependencies) - print(f" Generated ATTRIBUTIONS-Python.md ({len(dependencies)} direct deps)") - - # 6. Copy README - if os.path.exists(PARAGRAF_README): - shutil.copy2(PARAGRAF_README, os.path.join(output_dir, "README.md")) - print(" Copied README.md") - - # 7. Copy CONTRIBUTING.md - if os.path.exists(PARAGRAF_CONTRIBUTING): - shutil.copy2(PARAGRAF_CONTRIBUTING, os.path.join(output_dir, "CONTRIBUTING.md")) - print(" Copied CONTRIBUTING.md") - - # 8. Copy .gitignore - if os.path.exists(TRTLLM_GITIGNORE): - shutil.copy2(TRTLLM_GITIGNORE, os.path.join(output_dir, ".gitignore")) - print(" Copied .gitignore") - - # 9. Copy .editorconfig - if os.path.exists(TRTLLM_EDITORCONFIG): - shutil.copy2(TRTLLM_EDITORCONFIG, os.path.join(output_dir, ".editorconfig")) - print(" Copied .editorconfig") - - # 10. Copy OSS compliance files (CODE_OF_CONDUCT, SECURITY) - for src, name in ( - (TRTLLM_CODE_OF_CONDUCT, "CODE_OF_CONDUCT.md"), - (TRTLLM_SECURITY, "SECURITY.md"), - ): - if os.path.exists(src): - shutil.copy2(src, os.path.join(output_dir, name)) - print(f" Copied {name}") - - print(f"\nStandalone package created at: {output_dir}") - print("\nTo install:") - print(f" cd {output_dir}") - print(" uv venv .venv --python 3.12") - print(" source .venv/bin/activate") - print(" uv pip install -e '.[dev]'") - print("\nTo run tests: pytest tests/") - print("To run optional TensorRT-LLM tests:") - print(" uv pip install -e '.[dev,trtllm]'") - print(" TRTLLM_REDIRECT_AD_TO_PARAGRAF=true pytest tests/") - print( - 'To verify: python -c "from paragraf._compat import TRTLLM_AVAILABLE; print(TRTLLM_AVAILABLE)"' - ) - print( - "To run e2e: python runners/trtllm/build_and_run_paragraf_trtllm.py " - "--model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --use-registry (needs tensorrt-llm installed)" - ) - - -def main(): - parser = argparse.ArgumentParser( - description="Create a standalone paragraf package from TensorRT-LLM source.", - ) - parser.add_argument( - "--output-dir", - default=os.path.join(REPO_ROOT, "build", "paragraf_standalone"), - help="Output directory for the standalone package (default: build/paragraf_standalone)", - ) - args = parser.parse_args() - create_standalone_package(os.path.abspath(args.output_dir)) - - -if __name__ == "__main__": - main() diff --git a/examples/bindings/executor/README.md b/examples/bindings/executor/README.md new file mode 100644 index 000000000000..df44568fd4f1 --- /dev/null +++ b/examples/bindings/executor/README.md @@ -0,0 +1,76 @@ +# Python Bindings Example + +This example shows how to use the python bindings interface to generate tokens +using a TensorRT engine. + +## Setup + +Build a TensorRT engine for one of the supported TensorRT LLM model following +instructions in the corresponding `examples` folder. + +## Usage + +### Basic example + +Run `example_basic.py`, passing in the directory where the TensorRT engine was generated. For example: + +``` +cd examples/bindings +python3 example_basic.py --model_path=../llama/tmp/7B/trt_engines/fp16/1-gpu/ +``` + +### Debug example + +This example shows how you can define which engine IO tensors should be kept or dumped to numpy files. +Run `example_debug.py`, passing in the directory where the TensorRT engine was generated. For example: + +``` +cd examples/bindings +python3 example_debug.py --model_path=../llama/tmp/7B/trt_engines/fp16/1-gpu/ +``` + +### Advanced example + +This example shows how you can use the python bindings to generate tokens for a larger number of requests concurrently and demonstrate how tokens can be returned in a streaming fashion. + +The full list of supported input parameters can be obtained with: +``` +pytho3 example_advanced.py -h +``` + +For example, assuming a CSV file named `input_tokens.csv` exist which contains the following input tokens: +``` +1, 2, 3, 4, 5, 6 +1, 2, 3, 4 +1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +``` +one can generate output tokens for those 3 prompts with: +``` +python3 example_advanced.py --model_path --input_tokens_csv_file input_tokens.csv +``` +Upon successful completion, the output tokens will be written to file `output_tokens.csv`. + +### Multi-GPU Example + +To run the two examples for models requiring more than one gpu, you can run the example with MPI. + +For example, the basic example can be run as follows: +``` +mpirun -n 4 --allow-run-as-root python3 example_basic.py --model_path=../llama/tmp/7B/trt_engines/fp16/4gpu_tp4_pp1/ +``` + +The advanced example can also be run using the ORCHESTRATOR mode, where the additional processes needed for multi-GPU runs will automatically be spawned. +This can be done by running: +``` +python3 example_advanced.py --model_path=../llama/tmp/7B/trt_engines/fp16/4gpu_tp4_pp1/ --use_orchestrator_mode +``` + +### Logits post processor example + +This example shows how to generate JSON structured output using LogitsPostProcessor API. + +``` +python3 example_logits_processor.py -t -e --batch_size 8 +``` + +LogitsPostProcessorBatched, which fuses logits processing for all samples in a batch into a single callback, is enabled by `--lpp_batched` diff --git a/examples/bindings/executor/example_advanced.py b/examples/bindings/executor/example_advanced.py new file mode 100644 index 000000000000..25f063a855e2 --- /dev/null +++ b/examples/bindings/executor/example_advanced.py @@ -0,0 +1,166 @@ +import argparse +import csv +import datetime +from pathlib import Path + +import tensorrt_llm + +trtllm_package_dir = Path(tensorrt_llm.__file__).parent +executor_worker_path = trtllm_package_dir / 'bin' / 'executorWorker' + +import tensorrt_llm.bindings.executor as trtllm + + +# Read input tokens from csv file +def read_input_tokens(input_tokens_csv_file: str) -> list[int]: + + input_tokens = [] + with open(input_tokens_csv_file, mode='r') as file: + csvFile = csv.reader(file) + for lines in csvFile: + input_tokens.append([int(item) for item in lines]) + return input_tokens + + +# Prepare and enqueue the requests +def enqueue_requests(args: argparse.Namespace, + executor: trtllm.Executor) -> None: + + output_config = trtllm.OutputConfig() + output_config.exclude_input_from_output = args.exclude_input_from_output + sampling_config = trtllm.SamplingConfig(args.beam_width) + input_tokens = read_input_tokens(args.input_tokens_csv_file) + + request_ids = [] + for tokens in input_tokens: + req = trtllm.Request(input_token_ids=tokens, + max_tokens=args.max_tokens, + streaming=args.streaming, + sampling_config=sampling_config, + output_config=output_config) + req_id = executor.enqueue_request(req) + request_ids.append(req_id) + + return request_ids + + +# Wait for responses and store output tokens +def wait_for_responses(args: argparse.Namespace, request_ids: list[int], + executor: trtllm.Executor) -> dict[dict[list[int]]]: + + output_tokens = { + req_id: { + beam: [] + for beam in range(args.beam_width) + } + for req_id in request_ids + } + num_finished = 0 + iter = 0 + while (num_finished < len(request_ids) and iter < args.timeout_ms): + responses = executor.await_responses( + datetime.timedelta(milliseconds=args.timeout_ms)) + for response in responses: + req_id = response.request_id + if not response.has_error(): + result = response.result + num_finished += 1 if result.is_final else 0 + for beam, outTokens in enumerate(result.output_token_ids): + output_tokens[req_id][beam].extend(outTokens) + else: + raise RuntimeError( + str(req_id) + " encountered error:" + response.error_msg) + + return output_tokens + + +# Write the output tokens to file +def write_output_tokens(output_tokens_csv_file: str, request_ids: list[int], + output_tokens: dict[dict[list[int]]], + beam_width: int) -> None: + + with open(output_tokens_csv_file, 'w') as csvfile: + + writer = csv.writer(csvfile) + for req_id in request_ids: + out_tokens = output_tokens[req_id] + for beam in range(args.beam_width): + beam_tokens = out_tokens[beam] + writer.writerow(beam_tokens) + + print("Output tokens written to:", output_tokens_csv_file) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Executor Bindings Example") + parser.add_argument("--model_path", + type=str, + required=True, + help="Directory containing model engine") + parser.add_argument("--input_tokens_csv_file", + type=str, + required=True, + help="CSV file containing the input tokens") + parser.add_argument("--output_tokens_csv_file", + type=str, + required=False, + default="output_tokens.csv", + help="CSV file where to write output tokens") + parser.add_argument("--beam_width", + type=int, + required=False, + default=1, + help="The beam width") + parser.add_argument("--streaming", + default=False, + action="store_true", + help="Operate in streaming mode") + + parser.add_argument("--use_orchestrator_mode", + default=False, + action="store_true", + help="Operate in orchestrator mode for multi-GPU runs") + + parser.add_argument( + "--exclude_input_from_output", + default=False, + action="store_true", + help= + "Exclude input token when writing output tokens. Only has effect for streaming=False since in streaming mode, input tokens are never included in output." + ) + parser.add_argument("--max_tokens", + type=int, + required=False, + default=10, + help="The max number of tokens to be generated") + parser.add_argument( + "--timeout_ms", + type=int, + required=False, + default=10000, + help="The maximum time to wait for all responses, in milliseconds") + + args = parser.parse_args() + executor_config = trtllm.ExecutorConfig(args.beam_width) + + if args.use_orchestrator_mode: + orchestrator_config = trtllm.OrchestratorConfig( + True, str(executor_worker_path)) + executor_config.parallel_config = trtllm.ParallelConfig( + trtllm.CommunicationType.MPI, trtllm.CommunicationMode.ORCHESTRATOR, + None, None, orchestrator_config) + + # Create the executor. + executor = trtllm.Executor(args.model_path, trtllm.ModelType.DECODER_ONLY, + executor_config) + + if executor.can_enqueue_requests(): + # Enqueue the requests + request_ids = enqueue_requests(args, executor) + + # Wait for the responses + output_tokens = wait_for_responses(args, request_ids, executor) + + # Write the output tokens + write_output_tokens(args.output_tokens_csv_file, request_ids, + output_tokens, args.beam_width) diff --git a/examples/bindings/executor/example_basic.py b/examples/bindings/executor/example_basic.py new file mode 100644 index 000000000000..3c71bde594e1 --- /dev/null +++ b/examples/bindings/executor/example_basic.py @@ -0,0 +1,34 @@ +import argparse + +import tensorrt_llm.bindings.executor as trtllm + +# This example hows to use the python bindings to create an executor, enqueue a +# request, and get the generated tokens. + +# First, follow the steps in README.md to generate the engines. + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Executor Bindings Example") + parser.add_argument("--model_path", + type=str, + required=True, + help="Directory containing model engine") + args = parser.parse_args() + + # Create the executor. + executor = trtllm.Executor(args.model_path, trtllm.ModelType.DECODER_ONLY, + trtllm.ExecutorConfig(1)) + + if executor.can_enqueue_requests(): + # Create the request. + request = trtllm.Request(input_token_ids=[1, 2, 3, 4], max_tokens=10) + + # Enqueue the request. + request_id = executor.enqueue_request(request) + + # Wait for the new tokens. + responses = executor.await_responses(request_id) + output_tokens = responses[0].result.output_token_ids + + # Print tokens. + print(output_tokens) diff --git a/examples/bindings/executor/example_debug.py b/examples/bindings/executor/example_debug.py new file mode 100644 index 000000000000..f7c0669b1254 --- /dev/null +++ b/examples/bindings/executor/example_debug.py @@ -0,0 +1,67 @@ +import argparse +import pathlib as pl + +import numpy as np + +import tensorrt_llm.bindings.executor as trtllm + +# This example hows to use the python bindings to create an executor, enqueue a +# request, and get the generated tokens. + +# First, follow the steps in README.md to generate the engines. + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Executor Bindings Example") + parser.add_argument("--model_path", + type=str, + required=True, + help="Directory containing model engine") + parser.add_argument("--dump_tensors", + action="store_true", + help="Dump debug tensors to files") + args = parser.parse_args() + + max_tokens = 2 + + # Select which tensors should be kept or dumped + debug_config = trtllm.DebugConfig( + debug_tensor_names=["sequence_length"], + debug_tensors_max_iterations=0 if args.dump_tensors else max_tokens) + + # Create the executor. + executor = trtllm.Executor( + args.model_path, trtllm.ModelType.DECODER_ONLY, + trtllm.ExecutorConfig(1, debug_config=debug_config)) + + if executor.can_enqueue_requests(): + # Create the request. + request = trtllm.Request(input_token_ids=[1, 2, 3, 4], + max_tokens=max_tokens) + + # Enqueue the request. + request_id = executor.enqueue_request(request) + + # Wait for the new tokens. + responses = executor.await_responses(request_id) + output_tokens = responses[0].result.output_token_ids + + # Print tokens. + print(output_tokens) + + if args.dump_tensors: + print("debug tensors from files:") + debug_dir = pl.Path("/tmp/tllm_debug/PP_1/TP_1") + if debug_dir.is_dir(): + for iter_dir in [x for x in debug_dir.iterdir() if x.is_dir()]: + print(iter_dir.name) + for file in [x for x in iter_dir.iterdir() if x.is_file()]: + print(file.name, np.load(file)) + else: + print("debug dir not found") + else: + print("debug tensors from queue:") + debug_tensors = executor.get_latest_debug_tensors() + for debug_iter in debug_tensors: + print(f"iteration {debug_iter.iter}") + for [name, tensor] in debug_iter.debug_tensors.items(): + print(name, tensor) diff --git a/examples/bindings/executor/example_logits_processor.py b/examples/bindings/executor/example_logits_processor.py new file mode 100644 index 000000000000..6cb1a751a6da --- /dev/null +++ b/examples/bindings/executor/example_logits_processor.py @@ -0,0 +1,212 @@ +import argparse +import datetime +import typing as _tp + +import torch as _tor +from lmformatenforcer import (JsonSchemaParser, TokenEnforcer, + TokenEnforcerTokenizerData) +from pydantic import BaseModel +from transformers import AutoTokenizer + +import tensorrt_llm.bindings.executor as trtllm + + +def _build_regular_tokens_list( + tokenizer) -> _tp.List[_tp.Tuple[int, str, bool]]: + token_0 = [tokenizer.encode("0")[-1]] + regular_tokens = [] + vocab_size = tokenizer.vocab_size + for token_idx in range(vocab_size): + if token_idx in tokenizer.all_special_ids: + continue + # We prepend token 0 and skip the first letter of the result to get a space if the token is a start word. + tensor_after_0 = _tor.tensor(token_0 + [token_idx], dtype=_tor.long) + decoded_after_0 = tokenizer.decode(tensor_after_0)[1:] + decoded_regular = tokenizer.decode(token_0) + is_word_start_token = len(decoded_after_0) > len(decoded_regular) + regular_tokens.append((token_idx, decoded_after_0, is_word_start_token)) + return regular_tokens + + +def build_token_enforcer(tokenizer, character_level_parser): + """ + Build logits processor for feeding it into generate function (use_py_session should be True) + """ + regular_tokens = _build_regular_tokens_list(tokenizer) + + def _decode(tokens: _tp.List[int]) -> str: + tensor = _tor.tensor(tokens, dtype=_tor.long) + return tokenizer.decode(tensor) + + tokenizer_data = TokenEnforcerTokenizerData(regular_tokens, _decode, + tokenizer.eos_token_id) + return TokenEnforcer(tokenizer_data, character_level_parser) + + +# Prepare and enqueue the requests +def enqueue_requests(args: argparse.Namespace, + executor: trtllm.Executor) -> None: + + sampling_config = trtllm.SamplingConfig(args.beam_width) + + request_ids = [] + for iter_id in range(args.batch_size): + # Create the request. + request = trtllm.Request(input_token_ids=prompt, + max_tokens=25, + end_id=tokenizer.eos_token_id, + sampling_config=sampling_config, + client_id=iter_id % 2) + request.logits_post_processor_name = request.BATCHED_POST_PROCESSOR_NAME if args.lpp_batched else "my_logits_pp" + + # Enqueue the request. + req_id = executor.enqueue_request(request) + request_ids.append(req_id) + + return request_ids + + +# Wait for responses and store output tokens +def wait_for_responses(args: argparse.Namespace, request_ids: list[int], + executor: trtllm.Executor) -> dict[dict[list[int]]]: + + output_tokens = { + req_id: { + beam: [] + for beam in range(args.beam_width) + } + for req_id in request_ids + } + num_finished = 0 + iter = 0 + while (num_finished < len(request_ids) and iter < args.timeout_ms): + responses = executor.await_responses( + datetime.timedelta(milliseconds=args.timeout_ms)) + for response in responses: + req_id = response.request_id + if not response.has_error(): + result = response.result + num_finished += 1 if result.is_final else 0 + for beam, outTokens in enumerate(result.output_token_ids): + output_tokens[req_id][beam].extend(outTokens) + else: + raise RuntimeError( + str(req_id) + " encountered error:" + response.error_msg) + + return output_tokens + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Executor Bindings Example") + parser.add_argument("--tokenizer_path", + "-t", + type=str, + required=True, + help="Directory containing model tokenizer") + parser.add_argument("--engine_path", + "-e", + type=str, + required=True, + help="Directory containing model engine") + parser.add_argument("--beam_width", + type=int, + required=False, + default=1, + help="The beam width") + parser.add_argument("--batch_size", + type=int, + required=False, + default=1, + help="The batch size") + parser.add_argument( + "--timeout_ms", + type=int, + required=False, + default=10000, + help="The maximum time to wait for all responses, in milliseconds") + parser.add_argument("--lpp_batched", + action="store_true", + default=False, + help="Enable batched logits post processor") + + args = parser.parse_args() + + tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_path) + + class AnswerFormat(BaseModel): + last_name: str + year_of_birth: int + + parser = JsonSchemaParser(AnswerFormat.model_json_schema()) + token_enforcer = build_token_enforcer(tokenizer, parser) + + def get_allowed_tokens(ids, client_id): + if client_id is None or client_id == 0: return [42] + + def _trim(ids): + return [x for x in ids if x != tokenizer.eos_token_id] + + allowed = token_enforcer.get_allowed_tokens(_trim(ids[0])) + return allowed + + def logits_post_processor(req_id: int, logits: _tor.Tensor, + ids: _tp.List[_tp.List[int]], stream_ptr: int, + client_id: _tp.Optional[int]): + mask = _tor.full_like(logits, fill_value=float("-inf"), device="cpu") + allowed = get_allowed_tokens(ids, client_id) + mask[:, :, allowed] = 0 + + with _tor.cuda.stream(_tor.cuda.ExternalStream(stream_ptr)): + mask = mask.to(logits.device, non_blocking=True) + logits += mask + + def logits_post_processor_batched( + req_ids_batch: _tp.List[int], logits_batch: _tp.List[_tor.Tensor], + ids_batch: _tp.List[_tp.List[_tp.List[int]]], stream_ptr, + client_ids_batch: _tp.List[_tp.Optional[int]]): + masks = [] + for req_id, logits, ids, client_id in zip(req_ids_batch, logits_batch, + ids_batch, client_ids_batch): + del req_id + mask = _tor.full_like(logits, + fill_value=float("-inf"), + device="cpu") + allowed = get_allowed_tokens(ids, client_id) + mask[:, :, allowed] = 0 + masks.append(mask) + + with _tor.cuda.stream(_tor.cuda.ExternalStream(stream_ptr)): + for logits, mask in zip(logits_batch, masks): + logits += mask.to(logits.device, non_blocking=True) + + # Create the executor. + executor_config = trtllm.ExecutorConfig(args.beam_width) + logits_proc_config = trtllm.LogitsPostProcessorConfig() + if not args.lpp_batched: + logits_proc_config.processor_map = { + "my_logits_pp": logits_post_processor + } + else: + logits_proc_config.processor_batched = logits_post_processor_batched + executor_config.logits_post_processor_config = logits_proc_config + executor = trtllm.Executor(args.engine_path, trtllm.ModelType.DECODER_ONLY, + executor_config) + + input = "Please give me information about Michael Jordan. You MUST answer using the following json schema: " + prompt = tokenizer.encode(input) + print(f"Input text: {input}\n") + + if executor.can_enqueue_requests(): + request_ids = enqueue_requests(args, executor) + output_tokens = wait_for_responses(args, request_ids, executor) + + # Print output + for req_id in request_ids: + for beam_id in range(args.beam_width): + result = tokenizer.decode( + output_tokens[req_id][beam_id][len(prompt):]) + generated_tokens = len( + output_tokens[req_id][beam_id]) - len(prompt) + print( + f"Request {req_id} Beam {beam_id} ({generated_tokens} tokens): {result}" + ) diff --git a/examples/configs/curated/deepseek-v4-pro-latency.yaml b/examples/configs/curated/deepseek-v4-pro-latency.yaml deleted file mode 100644 index fa477ffa41aa..000000000000 --- a/examples/configs/curated/deepseek-v4-pro-latency.yaml +++ /dev/null @@ -1,45 +0,0 @@ -cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - enable_padding: true -disable_overlap_scheduler: false -enable_attention_dp: false -enable_lm_head_tp_in_adp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - tokens_per_block: 128 -max_batch_size: 128 -max_num_tokens: 8448 -max_seq_len: 9256 -moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true -moe_expert_parallel_size: 8 -num_postprocess_workers: 4 -pipeline_parallel_size: 1 -print_iter_log: true -speculative_config: - decoding_type: MTP - max_draft_len: 3 -stream_interval: 100 -tensor_parallel_size: 8 diff --git a/examples/configs/curated/deepseek-v4-pro-throughput.yaml b/examples/configs/curated/deepseek-v4-pro-throughput.yaml deleted file mode 100644 index c4fb8453466d..000000000000 --- a/examples/configs/curated/deepseek-v4-pro-throughput.yaml +++ /dev/null @@ -1,35 +0,0 @@ -attention_dp_config: - enable_balance: true -cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - enable_padding: true -disable_overlap_scheduler: false -enable_attention_dp: true -enable_lm_head_tp_in_adp: true -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - tokens_per_block: 128 -max_batch_size: 32 -max_num_tokens: 8448 -max_seq_len: 9256 -moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true -moe_expert_parallel_size: 8 -num_postprocess_workers: 4 -pipeline_parallel_size: 1 -print_iter_log: true -speculative_config: - decoding_type: MTP - max_draft_len: 1 -stream_interval: 100 -tensor_parallel_size: 8 diff --git a/examples/configs/curated/lookup.yaml b/examples/configs/curated/lookup.yaml index e813d6f8f0fe..e03f118f916c 100644 --- a/examples/configs/curated/lookup.yaml +++ b/examples/configs/curated/lookup.yaml @@ -86,13 +86,3 @@ config_path: examples/configs/curated/minimax-m3-throughput.yaml scenario: Max Throughput gpu_compatibility: "GB200" -- model: deepseek-ai/DeepSeek-V4-Pro - arch: DeepseekV4ForCausalLM - config_path: examples/configs/curated/deepseek-v4-pro-latency.yaml - scenario: Min Latency - gpu_compatibility: "B200" -- model: deepseek-ai/DeepSeek-V4-Pro - arch: DeepseekV4ForCausalLM - config_path: examples/configs/curated/deepseek-v4-pro-throughput.yaml - scenario: Max Throughput - gpu_compatibility: "B200" diff --git a/examples/configs/database/database.py b/examples/configs/database/database.py index af33a4dab41b..3dcdd13c0300 100644 --- a/examples/configs/database/database.py +++ b/examples/configs/database/database.py @@ -13,22 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. + import logging -import re -from collections import defaultdict from pathlib import Path -from typing import Any, Iterator, Literal +from typing import Any, Dict, Iterator, List, Tuple import yaml -from pydantic import ( - BaseModel, - ConfigDict, - Field, - PositiveInt, - RootModel, - field_validator, - model_validator, -) +from pydantic import BaseModel, Field, RootModel, field_validator logger = logging.getLogger(__name__) @@ -39,22 +30,11 @@ LOW_LATENCY_CONCURRENCY_THRESHOLD = 8 HIGH_THROUGHPUT_CONCURRENCY_THRESHOLD = 32 KEY_PROFILES = {"Min Latency", "Balanced", "Max Throughput"} -PROFILE_DISPLAY_NAMES = { - "latency": "Min Latency", - "balanced": "Balanced", - "throughput": "Max Throughput", -} -PROFILE_ORDER = {profile: idx for idx, profile in enumerate(PROFILE_DISPLAY_NAMES)} -VALIDATED_COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") -VALIDATED_VERSION_PATTERN = re.compile(r"^[0-9A-Za-z][0-9A-Za-z._+-]*$") -Profile = Literal["latency", "balanced", "throughput"] class CuratedRecipe(BaseModel): """A curated (hand-tuned) recipe entry.""" - model_config = ConfigDict(extra="forbid") - model: str = Field(description="HuggingFace model ID") arch: str = Field(description="Model architecture class name") config_path: str = Field(description="Relative path to YAML config") @@ -71,7 +51,7 @@ def _validate_config_path(cls, v: str) -> str: return v -class CuratedRecipeList(RootModel[list[CuratedRecipe]]): +class CuratedRecipeList(RootModel[List[CuratedRecipe]]): """Validated list of curated recipe entries.""" @classmethod @@ -95,58 +75,15 @@ def __len__(self) -> int: class Recipe(BaseModel): """Recipe record for scenario list.""" - model_config = ConfigDict(extra="forbid") - - model: str = Field(min_length=1, description="Model name") - arch: str = Field(min_length=1, description="Model architecture class name") - gpu: str = Field(min_length=1, description="GPU name") - isl: PositiveInt = Field(description="Input sequence length") - osl: PositiveInt = Field(description="Output sequence length") - concurrency: PositiveInt = Field(description="Concurrency") - config_path: str = Field(min_length=1, description="Configuration path") - num_gpus: PositiveInt = Field(description="Number of GPUs") - profile: Profile | None = Field( - default=None, - description="Profile discriminator used only when the workload key has multiple configs", - ) - validated_trtllm_commit: str | None = Field( - default=None, - description="Full TensorRT-LLM commit SHA against which the recipe was validated", - ) - validated_trtllm_version: str | None = Field( - default=None, - description="TensorRT-LLM release version reported by the validation source", - ) - - @field_validator("validated_trtllm_commit") - @classmethod - def _validate_commit(cls, value: str | None) -> str | None: - if value is None: - return None - normalized = value.strip().lower() - if not VALIDATED_COMMIT_PATTERN.fullmatch(normalized): - raise ValueError("validated_trtllm_commit must be a full 40-character Git SHA") - return normalized - - @field_validator("validated_trtllm_version") - @classmethod - def _validate_version(cls, value: str | None) -> str | None: - if value is None: - return None - normalized = value.strip() - if not VALIDATED_VERSION_PATTERN.fullmatch(normalized): - raise ValueError("validated_trtllm_version must be a release-tag-safe version") - return normalized - - @model_validator(mode="after") - def _validate_provenance_pair(self) -> "Recipe": - if bool(self.validated_trtllm_commit) != bool(self.validated_trtllm_version): - raise ValueError( - "validated_trtllm_commit and validated_trtllm_version must be provided together" - ) - return self - - def load_config(self) -> dict[str, Any]: + model: str = Field(description="Model name") + gpu: str = Field(description="GPU name") + isl: int = Field(description="Input sequence length") + osl: int = Field(description="Output sequence length") + concurrency: int = Field(description="Concurrency") + config_path: str = Field(description="Configuration path") + num_gpus: int = Field(description="Number of GPUs") + + def load_config(self) -> Dict[str, Any]: """Load and return the YAML config at config_path.""" config_relative_path = Path(self.config_path) # Ensure config path is within the repo root @@ -159,41 +96,7 @@ def load_config(self) -> dict[str, Any]: return yaml.safe_load(f) -class RecipeList(RootModel[list[Recipe]]): - @model_validator(mode="after") - def _validate_conflict_profiles(self) -> "RecipeList": - groups = defaultdict(list) - for recipe in self.root: - key = ( - recipe.model, - recipe.gpu, - recipe.num_gpus, - recipe.isl, - recipe.osl, - recipe.concurrency, - ) - groups[key].append(recipe) - - required_profiles = {"latency", "throughput"} - for key, recipes in groups.items(): - profiles = [recipe.profile for recipe in recipes] - if len(recipes) == 1: - if profiles[0] is not None: - raise ValueError(f"profile is only allowed for conflicting workload key {key}") - continue - profile_set = set(profiles) - if ( - None in profile_set - or len(profile_set) != len(profiles) - or not required_profiles.issubset(profile_set) - ): - raise ValueError( - "conflicting workload key " - f"{key} must have exactly one latency and throughput profile, " - "with an optional balanced profile" - ) - return self - +class RecipeList(RootModel[List[Recipe]]): @classmethod def from_yaml(cls, yaml_path: Path) -> "RecipeList": """Load and validate recipe list from YAML file.""" @@ -229,15 +132,12 @@ def assign_profile(num_recipes: int, idx: int, concurrency: int) -> str: return "High Throughput" -def select_key_recipes(recipes: list[Recipe]) -> list[tuple[Recipe, str]]: +def select_key_recipes(recipes: List[Recipe]) -> List[Tuple[Recipe, str]]: """Select key recipes (min latency, balanced, max throughput) from a list of recipes.""" if not recipes: return [] - sorted_recipes = sorted( - recipes, - key=lambda r: (r.concurrency, PROFILE_ORDER.get(r.profile, -1)), - ) + sorted_recipes = sorted(recipes, key=lambda r: r.concurrency) n = len(sorted_recipes) result = [] diff --git a/examples/configs/database/lookup.yaml b/examples/configs/database/lookup.yaml index d773fa361596..523cda14c412 100644 --- a/examples/configs/database/lookup.yaml +++ b/examples/configs/database/lookup.yaml @@ -558,8 +558,6 @@ concurrency: 4 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -584,8 +582,6 @@ concurrency: 32 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -594,8 +590,6 @@ concurrency: 64 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -604,8 +598,6 @@ concurrency: 128 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -614,8 +606,6 @@ concurrency: 256 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -760,8 +750,6 @@ concurrency: 4 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -794,8 +782,6 @@ concurrency: 64 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -804,8 +790,6 @@ concurrency: 128 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -814,8 +798,6 @@ concurrency: 256 config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml num_gpus: 8 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-v2 arch: DeepseekV3ForCausalLM gpu: B200_NVL @@ -1584,83 +1566,3 @@ concurrency: 1536 config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc1536.yaml num_gpus: 8 -- model: nvidia/DeepSeek-R1-0528-FP4-v2 - arch: DeepseekV3ForCausalLM - gpu: B200_NVL - isl: 1024 - osl: 1024 - concurrency: 4 - config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml - num_gpus: 4 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 -- model: nvidia/DeepSeek-R1-0528-FP4-v2 - arch: DeepseekV3ForCausalLM - gpu: B200_NVL - isl: 1024 - osl: 1024 - concurrency: 8 - config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml - num_gpus: 4 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 -- model: nvidia/DeepSeek-R1-0528-FP4-v2 - arch: DeepseekV3ForCausalLM - gpu: B200_NVL - isl: 1024 - osl: 1024 - concurrency: 16 - config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml - num_gpus: 4 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 -- model: nvidia/DeepSeek-R1-0528-FP4-v2 - arch: DeepseekV3ForCausalLM - gpu: B200_NVL - isl: 1024 - osl: 1024 - concurrency: 256 - config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml - num_gpus: 4 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 -- model: nvidia/DeepSeek-R1-0528-FP4-v2 - arch: DeepseekV3ForCausalLM - gpu: B200_NVL - isl: 8192 - osl: 1024 - concurrency: 4 - config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml - num_gpus: 4 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 -- model: nvidia/DeepSeek-R1-0528-FP4-v2 - arch: DeepseekV3ForCausalLM - gpu: B200_NVL - isl: 8192 - osl: 1024 - concurrency: 8 - config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml - num_gpus: 4 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 -- model: nvidia/DeepSeek-R1-0528-FP4-v2 - arch: DeepseekV3ForCausalLM - gpu: B200_NVL - isl: 8192 - osl: 1024 - concurrency: 16 - config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml - num_gpus: 4 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 -- model: nvidia/DeepSeek-R1-0528-FP4-v2 - arch: DeepseekV3ForCausalLM - gpu: B200_NVL - isl: 8192 - osl: 1024 - concurrency: 256 - config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml - num_gpus: 4 - validated_trtllm_commit: 93cb6518b6d6dbd6095748189e626db731f44545 - validated_trtllm_version: 1.3.0rc14 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml deleted file mode 100644 index c9986a111b5b..000000000000 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml +++ /dev/null @@ -1,14 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 1 -trust_remote_code: true -max_seq_len: 8192 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml deleted file mode 100644 index bb43a3699209..000000000000 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml +++ /dev/null @@ -1,24 +0,0 @@ -max_batch_size: 64 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -speculative_config: - decoding_type: MTP - max_draft_len: 1 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -max_num_tokens: 1216 -max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml deleted file mode 100644 index ade4e5b1de21..000000000000 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml +++ /dev/null @@ -1,19 +0,0 @@ -max_batch_size: 4 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 -stream_interval: 10 -moe_config: - backend: TRTLLM -speculative_config: - decoding_type: MTP - max_draft_len: 3 -tensor_parallel_size: 4 -moe_expert_parallel_size: 1 -trust_remote_code: true -max_num_tokens: 1152 -max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml deleted file mode 100644 index 7f49059c11b1..000000000000 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml +++ /dev/null @@ -1,19 +0,0 @@ -max_batch_size: 8 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 -stream_interval: 10 -moe_config: - backend: TRTLLM -speculative_config: - decoding_type: MTP - max_draft_len: 3 -tensor_parallel_size: 4 -moe_expert_parallel_size: 1 -trust_remote_code: true -max_num_tokens: 1152 -max_seq_len: 2304 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml index 3db82c4881df..ea4b03e17984 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml @@ -1,19 +1,24 @@ -max_batch_size: 128 +max_batch_size: 512 cuda_graph_config: enable_padding: true - max_batch_size: 128 + max_batch_size: 32 +enable_attention_dp: true print_iter_log: true kv_cache_config: dtype: fp8 free_gpu_memory_fraction: 0.8 stream_interval: 10 moe_config: - backend: TRTLLM + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 speculative_config: decoding_type: MTP - max_draft_len: 3 + max_draft_len: 1 tensor_parallel_size: 8 -moe_expert_parallel_size: 1 +moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 1600 -max_seq_len: 2304 +max_num_tokens: 3072 +max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml index 9bc761a79868..9031655d28fd 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml @@ -1,6 +1,7 @@ +max_batch_size: 512 cuda_graph_config: enable_padding: true - max_batch_size: 64 + max_batch_size: 256 enable_attention_dp: true print_iter_log: true kv_cache_config: @@ -13,7 +14,11 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +speculative_config: + decoding_type: MTP + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_seq_len: 8192 +max_num_tokens: 2112 +max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml index 03c9e6b27901..5d7f74990a3a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -max_batch_size: 32 +max_batch_size: 512 cuda_graph_config: enable_padding: true max_batch_size: 32 @@ -15,5 +15,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 1216 -max_seq_len: 2304 +max_num_tokens: 3136 +max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml index cf0bb6a44862..68eab71800d9 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -max_batch_size: 4 +max_batch_size: 512 cuda_graph_config: enable_padding: true max_batch_size: 4 @@ -13,7 +13,7 @@ speculative_config: decoding_type: MTP max_draft_len: 3 tensor_parallel_size: 8 -moe_expert_parallel_size: 1 +moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 1152 -max_seq_len: 2304 +max_num_tokens: 3136 +max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml index 3348357986a6..c239238e8f0d 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml @@ -1,7 +1,7 @@ -max_batch_size: 64 cuda_graph_config: enable_padding: true - max_batch_size: 64 + max_batch_size: 16 +enable_attention_dp: true print_iter_log: true kv_cache_config: dtype: fp8 @@ -9,11 +9,14 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 speculative_config: decoding_type: MTP - max_draft_len: 3 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 1344 -max_seq_len: 2304 +max_seq_len: 2068 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml deleted file mode 100644 index 283c8ccd18a3..000000000000 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml +++ /dev/null @@ -1,19 +0,0 @@ -max_batch_size: 16 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 -stream_interval: 10 -moe_config: - backend: TRTLLM -speculative_config: - decoding_type: MTP - max_draft_len: 3 -tensor_parallel_size: 4 -moe_expert_parallel_size: 1 -trust_remote_code: true -max_num_tokens: 8320 -max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml deleted file mode 100644 index aa75a4c8d4c5..000000000000 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml +++ /dev/null @@ -1,24 +0,0 @@ -max_batch_size: 64 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -speculative_config: - decoding_type: MTP - max_draft_len: 1 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -max_num_tokens: 8384 -max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml deleted file mode 100644 index 594aae2eaba4..000000000000 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml +++ /dev/null @@ -1,19 +0,0 @@ -max_batch_size: 4 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 -stream_interval: 10 -moe_config: - backend: TRTLLM -speculative_config: - decoding_type: MTP - max_draft_len: 3 -tensor_parallel_size: 4 -moe_expert_parallel_size: 1 -trust_remote_code: true -max_num_tokens: 8320 -max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml deleted file mode 100644 index 2dbc0435be33..000000000000 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml +++ /dev/null @@ -1,19 +0,0 @@ -max_batch_size: 8 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 -stream_interval: 10 -moe_config: - backend: TRTLLM -speculative_config: - decoding_type: MTP - max_draft_len: 3 -tensor_parallel_size: 4 -moe_expert_parallel_size: 1 -trust_remote_code: true -max_num_tokens: 8320 -max_seq_len: 9472 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml index 143fe4bba30b..baaa644ca539 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml @@ -1,4 +1,4 @@ -max_batch_size: 32 +max_batch_size: 256 cuda_graph_config: enable_padding: true max_batch_size: 32 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8320 -max_seq_len: 9472 +max_num_tokens: 8768 +max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml index 251ed48facac..93df61d6fa56 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml @@ -1,4 +1,4 @@ -max_batch_size: 64 +max_batch_size: 256 cuda_graph_config: enable_padding: true max_batch_size: 64 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8384 -max_seq_len: 9472 +max_num_tokens: 8768 +max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml index 97ce51217d82..6a6b9fb25c63 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -max_batch_size: 4 +max_batch_size: 512 cuda_graph_config: enable_padding: true max_batch_size: 4 @@ -13,7 +13,7 @@ speculative_config: decoding_type: MTP max_draft_len: 3 tensor_parallel_size: 8 -moe_expert_parallel_size: 1 +moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8320 -max_seq_len: 9472 +max_num_tokens: 10304 +max_seq_len: 9416 diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml index 6729a6fa5c6d..b39f07478407 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -max_batch_size: 16 +max_batch_size: 256 cuda_graph_config: enable_padding: true max_batch_size: 16 @@ -20,5 +20,5 @@ speculative_config: tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true -max_num_tokens: 8320 -max_seq_len: 9472 +max_num_tokens: 8768 +max_seq_len: 9416 diff --git a/examples/constraints.txt b/examples/constraints.txt index 01ee827c13ea..4141980256e5 100644 --- a/examples/constraints.txt +++ b/examples/constraints.txt @@ -1,3 +1,3 @@ -tensorrt_llm==1.3.0rc23 +tensorrt_llm==1.3.0rc21 evaluate~=0.4.1 rouge_score~=0.1.2 diff --git a/examples/cpp/executor/CMakeLists.txt b/examples/cpp/executor/CMakeLists.txt new file mode 100644 index 000000000000..b448667e8d75 --- /dev/null +++ b/examples/cpp/executor/CMakeLists.txt @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. cmake needs this line + +cmake_minimum_required(VERSION 3.27) + +set(TRTLLM_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../..") +list(APPEND CMAKE_MODULE_PATH "${TRTLLM_DIR}/cpp/cmake/modules") + +if(NOT TRTLLM_BUILD_DIR) + set(TRTLLM_BUILD_DIR "${TRTLLM_DIR}/cpp/build") +endif() +set(TRTLLM_LIB_PATH "${TRTLLM_BUILD_DIR}/tensorrt_llm/libtensorrt_llm.so") +if(NOT EXISTS ${TRTLLM_LIB_PATH}) + message(FATAL_ERROR "Cannot find ${TRTLLM_LIB_PATH}") +endif() + +set(TRTLLM_PLUGIN_PATH + "${TRTLLM_BUILD_DIR}/tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so" +) +set(TRTLLM_INCLUDE_DIR "${TRTLLM_DIR}/cpp/include") + +option( + ENABLE_MULTI_DEVICE + "Enable multi device/instance examples building (requires MPI headers/libs)" + ON) + +# Determine CXX11 ABI compatibility +execute_process( + COMMAND bash -c "nm -f posix -D ${TRTLLM_LIB_PATH} | grep __cxx11" + RESULT_VARIABLE GLIB_CXX11_FOUND + OUTPUT_QUIET) +if(GLIB_CXX11_FOUND EQUAL 0) + set(USE_CXX11_ABI 1) +else() + set(USE_CXX11_ABI 0) +endif() +message(STATUS "Use CXX11 ABI: ${USE_CXX11_ABI}") +add_compile_options("-D_GLIBCXX_USE_CXX11_ABI=${USE_CXX11_ABI}") + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED TRUE) +set(CMAKE_VERBOSE_MAKEFILE 1) + +# Define project name +project(executorExamples) + +# Compile options $ ? +if(ENABLE_MULTI_DEVICE) + set(EMD 1) +else() + set(EMD 0) +endif() +set(CMAKE_CXX_FLAGS "-Wall -pthread -lstdc++ -DENABLE_MULTI_DEVICE=${EMD} ") +set(CMAKE_CXX_FLAGS_RELEASE "-O3") +set(CMAKE_BUILD_TYPE release) + +find_package(CUDAToolkit REQUIRED COMPONENTS cuda_driver cudart_static nvml) +message(STATUS "CUDA library status:") +message(STATUS " version: ${CUDAToolkit_VERSION}") +message(STATUS " libraries: ${CUDAToolkit_LIBRARY_DIR}") +message(STATUS " include path: ${CUDAToolkit_INCLUDE_DIRS}") + +# TRT dependencies +find_package(TensorRT 10 REQUIRED) + +if(${CUDAToolkit_VERSION} VERSION_GREATER_EQUAL "11") + add_definitions("-DENABLE_BF16") + message( + STATUS + "CUDA_VERSION ${CUDA_VERSION} is greater or equal than 11.0, enable -DENABLE_BF16 flag" + ) +endif() + +if(${CUDAToolkit_VERSION} VERSION_GREATER_EQUAL "11.8") + add_definitions("-DENABLE_FP8") + message( + STATUS + "CUDA_VERSION ${CUDA_VERSION} is greater or equal than 11.8, enable -DENABLE_FP8 flag" + ) +endif() + +add_subdirectory(${TRTLLM_DIR}/3rdparty 3rdparty) +FetchContent_MakeAvailable(cxxopts) + +# tensorrt_llm shared lib +add_library(tensorrt_llm SHARED IMPORTED) +set_property(TARGET tensorrt_llm PROPERTY IMPORTED_LOCATION ${TRTLLM_LIB_PATH}) +set_property( + TARGET tensorrt_llm PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES + CUDA::cuda_driver CUDA::cudart_static CUDA::nvml) + +# nvinfer_plugin_tensorrt_llm shared lib +add_library(nvinfer_plugin_tensorrt_llm SHARED IMPORTED) +set_property(TARGET nvinfer_plugin_tensorrt_llm PROPERTY IMPORTED_LOCATION + ${TRTLLM_PLUGIN_PATH}) +set_property(TARGET nvinfer_plugin_tensorrt_llm + PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES tensorrt_llm) + +include_directories(${TRTLLM_INCLUDE_DIR} ${CUDAToolkit_INCLUDE_DIRS}) + +# Basic +add_executable(executorExampleBasic executorExampleBasic.cpp) +target_link_libraries(executorExampleBasic nvinfer_plugin_tensorrt_llm) + +add_executable(executorExampleDebug executorExampleDebug.cpp) +target_link_libraries(executorExampleDebug nvinfer_plugin_tensorrt_llm) + +add_executable(executorExampleKvEvents executorExampleKvEvents.cpp) +target_link_libraries(executorExampleKvEvents nvinfer_plugin_tensorrt_llm + cxxopts::cxxopts) + +add_executable(executorExampleLogitsProcessor + executorExampleLogitsProcessor.cpp) +target_link_libraries(executorExampleLogitsProcessor + nvinfer_plugin_tensorrt_llm) + +# Advanced +if(NOT TARGET cxxopts::cxxopts) + add_subdirectory(${CMAKE_BINARY_DIR}/_deps/cxxopts-src + ${CMAKE_CURRENT_BINARY_DIR}/cxxopts) +endif() + +add_executable(executorExampleAdvanced executorExampleAdvanced.cpp) +target_link_libraries(executorExampleAdvanced nvinfer_plugin_tensorrt_llm + cxxopts::cxxopts) + +# MultiInstance +if(ENABLE_MULTI_DEVICE) + find_package(MPI REQUIRED) + message(STATUS "Using MPI_C_INCLUDE_DIRS: ${MPI_C_INCLUDE_DIRS}") + message(STATUS "Using MPI_C_LIBRARIES: ${MPI_C_LIBRARIES}") + include_directories(${MPI_C_INCLUDE_DIRS}) + + add_executable(executorExampleAdvancedMultiInstances + executorExampleAdvancedMultiInstances.cpp) + target_link_libraries( + executorExampleAdvancedMultiInstances nvinfer_plugin_tensorrt_llm + cxxopts::cxxopts ${MPI_C_LIBRARIES}) + + # FastLogits + add_executable(executorExampleFastLogits executorExampleFastLogits.cpp) + target_link_libraries(executorExampleFastLogits nvinfer_plugin_tensorrt_llm + cxxopts::cxxopts ${MPI_C_LIBRARIES}) + + add_executable(executorExampleDisaggregated executorExampleDisaggregated.cpp) + target_link_libraries( + executorExampleDisaggregated nvinfer_plugin_tensorrt_llm cxxopts::cxxopts + ${MPI_C_LIBRARIES}) +endif() diff --git a/examples/cpp/executor/README.md b/examples/cpp/executor/README.md new file mode 100644 index 000000000000..597a38effe01 --- /dev/null +++ b/examples/cpp/executor/README.md @@ -0,0 +1,135 @@ +# Executor API examples + +This directory contains several examples that demonstrate how to use the `Executor` API: +- The example defined in `executorExampleBasic.cpp` shows how you can generate output tokens for a single prompt in only a few lines of code. +- The example defined in `executorExampleAdvanced.cpp` supports more options such as providing an arbitrary number of input requests with arbitrary tokens per request and running in streaming mode. +- The example defined in `executorExampleLogitsProcessor.cpp` shows how to use `LogitsPostProcessor` to control output tokens. +- The example defined in `executorExampleFastLogits.cpp` shows how to use `ExternalDraftTokensConfig` for speculative decoding and optionally use the fast logits feature. +- The example defined in `executorExampleKvEvents.cpp` shows how to use the KV cache event API. +- The example defined in `executorExampleDisaggregated.cpp` shows how to use the disaggregated executor API. + +## Building the examples + +To build the examples, you first need to build the TensorRT LLM C++ shared libraries (`libtensorrt_llm.so` and `libnvinfer_plugin_tensorrt_llm.so`) using the [`build_wheel.py`](source:scripts/build_wheel.py) script. Alternatively, if you have already build the TensorRT LLM libraries, you can modify the provided `CMakeLists.txt` such that the `libtensorrt_llm.so` and `libnvinfer_plugin_tensorrt_llm.so` are imported properly. + +Once the TensorRT LLM libraries are built, you can run + +``` +mkdir build +cd build +cmake .. +make -j +``` +from the `./examples/cpp/executor/` folder to build the basic and advanced examples. + +## Preparing the TensorRT LLM engine(s) + +Before you run the examples, please make sure that you have already built engine(s) using the TensorRT LLM API. + +Use `trtllm-build` to build the TRT-LLM engine. + +## Running the examples + +### executorExampleBasic + +From the `examples/cpp/executor/build` folder, you can get run the `executorExampleBasic` example with: + +``` +./executorExampleBasic +``` +where `` is the path to the directly containing the TensorRT engine files. + +### executorExampleDebug + +This example shows how you can define which engine IO tensors should be dumped to numpy files. +From the `examples/cpp/executor/build` folder, you can get run the `executorExampleDebug` example with: + +``` +./executorExampleDebug +``` +where `` is the path to the directly containing the TensorRT engine files. + +### executorExampleAdvanced + +From the `examples/cpp/executor/build` folder, you can also run the `executorExampleAdvanced` example. To get the full list of supported input arguments, type + +``` +./executorExampleAdvanced -h +``` + +For example, you can run: + +``` +./executorExampleAdvanced --engine_dir --input_tokens_csv_file ../inputTokens.csv +``` + +to run with the provided dummy input tokens from `inputTokens.csv`. Upon successful completion, you should see the following in the logs: +``` +[TensorRT-LLM][INFO] Creating request with 6 input tokens +[TensorRT-LLM][INFO] Creating request with 4 input tokens +[TensorRT-LLM][INFO] Creating request with 10 input tokens +[TensorRT-LLM][INFO] Got 20 tokens for beam 0 for requestId 3 +[TensorRT-LLM][INFO] Request id 3 is completed. +[TensorRT-LLM][INFO] Got 14 tokens for beam 0 for requestId 2 +[TensorRT-LLM][INFO] Request id 2 is completed. +[TensorRT-LLM][INFO] Got 16 tokens for beam 0 for requestId 1 +[TensorRT-LLM][INFO] Request id 1 is completed. +[TensorRT-LLM][INFO] Writing output tokens to outputTokens.csv +[TensorRT-LLM][INFO] Exiting. +``` + +#### Multi-GPU run + +To run the `executorExampleAdvanced` on models that require multiple GPUs, you can run the example using MPI as follows: + +``` +mpirun -n --allow-run-as-root ./executorExampleAdvanced --engine_dir --input_tokens_csv_file ../inputTokens.csv +``` +where `` must equal to `tp*pp` for the TensorRT engine. By default GPU device IDs `[0...(num_ranks-1)]` will be used. + +Alternatively, it's also possible to run multi-GPU model by using the so-called `Orchestrator` communication mode, where the `Executor` instance will automatically spawn additional processes to run the model on multiple GPUs. To use the `Orchestrator` communication mode, you can run the example with: + +``` +./executorExampleAdvanced --engine_dir --input_tokens_csv_file ../inputTokens.csv --use_orchestrator_mode --worker_executable_path +``` +where `` is the absolute path to the stand-alone executor worker executable, located at`cpp/build/tensorrt_llm/executor_worker/executorWorker` by default. + + +### executorExampleFastLogits + +To run the `executorExampleFastLogits`, you need two GPUs (one for the draft model and one for the target model). You can run it as follows: + +``` +mpirun -n 3 --allow-run-as-root ./executorExampleFastLogits --engine_dir --draft_engine_dir --num_draft_tokens=3 +``` + +The examples uses 3 MPI ranks (one for the orchestrator, one for the draft model and one for the target model). + +Use `--fast_logits=false` to disable the fast logits feature. + +### executorExampleKvEvents + +From the `examples/cpp/executor/build` folder, you can get run the `executorExampleKvEvents` example with: + +``` +./executorExampleKvEvents --engine_dir +``` +where `` is the path to the directly containing the TensorRT engine files. + +This example shows how the KV Cache Event API can be used to reconstruct the state of TRT-LLM's internal radix tree. This can be used in applications such as smart routing to route requests between multiple executor instances to maximize KV Cache reuse. Events are emitted when blocks are stored, removed, or updated in the radix tree. + +### executorExampleDisaggregated + +From the `examples/cpp/executor/build` folder, you can also run the `executorExampleDisaggregated` example. To get the full list of supported input arguments, type +``` +./executorExampleDisaggregated -h +``` +Note setting `TRTLLM_USE_UCX_KVCACHE=1` is required to run disaggregated executor. +For example, you can run : +``` +export TRTLLM_USE_UCX_KVCACHE=1 + +mpirun -n --allow-run-as-root --oversubscribe ./executorExampleDisaggregated --context_engine_dir --context_rank_size --generation_engine_dir --generation_rank_size --input_tokens_csv_file ../inputTokens.csv + +``` +where `` must equal to `tp*pp` for the context engine, and `` must equal to `tp*pp` for the generation engine,the context engine and generation engine can be heterogeneous in parallelism. `` must equal to `++1`, the additional rank is used as orchestrator process. diff --git a/examples/cpp/executor/executorExampleAdvanced.cpp b/examples/cpp/executor/executorExampleAdvanced.cpp new file mode 100644 index 000000000000..e2fdf489f63d --- /dev/null +++ b/examples/cpp/executor/executorExampleAdvanced.cpp @@ -0,0 +1,368 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include + +namespace tle = tensorrt_llm::executor; + +namespace fs = std::filesystem; + +struct RuntimeOptions +{ + std::string trtEnginePath; + std::string inputTokensCsvFile; + std::string outputTokensCsvFile; + + bool streaming; + bool excludeInputFromOutput; + tle::SizeType32 maxNewTokens; + tle::SizeType32 beamWidth; + std::optional numReturnSequences; + tle::SizeType32 timeoutMs; + + bool useOrchestratorMode; + std::string workerExecutablePath; +}; + +// Utility function to parse input arguments +RuntimeOptions parseArgs(int argc, char* argv[]); + +// Function that enqueues requests +std::vector enqueueRequests(RuntimeOptions const& runtimeOpts, tle::Executor& executor); + +// Function that waits for responses and stores output tokens +std::unordered_map waitForResponses( + RuntimeOptions const& runtimeOpts, std::vector const& requestIds, tle::Executor& executor); + +// Utility function to read input tokens from csv file +std::vector readInputTokens(std::string const& path); + +// Utility function to write output tokens from csv file +void writeOutputTokens(std::string const& path, std::vector& requestIds, + std::unordered_map const& outputTokens, tle::SizeType32 beamWidth); + +tle::SizeType32 getNumSequencesPerRequest(RuntimeOptions const& runtimeOpts); + +// Main +int main(int argc, char* argv[]) +{ + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + auto runtimeOpts = parseArgs(argc, argv); + + // Create the executor for this engine + auto executorConfig = tle::ExecutorConfig(runtimeOpts.beamWidth); + + if (runtimeOpts.useOrchestratorMode) + { + auto orchestratorConfig = tle::OrchestratorConfig(true, runtimeOpts.workerExecutablePath); + auto parallelConfig = tle::ParallelConfig(tle::CommunicationType::kMPI, tle::CommunicationMode::kORCHESTRATOR, + std::nullopt, std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + } + + auto executor = tle::Executor(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); + + if (executor.canEnqueueRequests()) + { + // Create the requests + auto requestIds = enqueueRequests(runtimeOpts, executor); + + // Wait for responses and store output tokens + auto outputTokens = waitForResponses(runtimeOpts, requestIds, executor); + + // Write output tokens csv file + TLLM_LOG_INFO("Writing output tokens to %s", runtimeOpts.outputTokensCsvFile.c_str()); + auto numSequences = getNumSequencesPerRequest(runtimeOpts); + writeOutputTokens(runtimeOpts.outputTokensCsvFile, requestIds, outputTokens, numSequences); + } + TLLM_LOG_INFO("Exiting."); + return 0; +} + +RuntimeOptions parseArgs(int argc, char* argv[]) +{ + RuntimeOptions runtimeOpts; + + cxxopts::Options options(argv[0], "Example that demonstrates how to use the Executor API"); + options.add_options()("h,help", "Print usage"); + options.add_options()("engine_dir", "Directory that store the engines.", cxxopts::value()); + options.add_options()("beam_width", "The beam width", cxxopts::value()->default_value("1")); + options.add_options()( + "num_return_sequences", "The number of return sequences per request.", cxxopts::value>()); + options.add_options()("streaming", "Operate in streaming mode", cxxopts::value()->default_value("false")); + options.add_options()("exclude_input_from_output", + "Exclude input tokens when writing output tokens. Only has effect for streaming = false. For streaming = true, " + "output tokens are not included.", + cxxopts::value()->default_value("false")); + options.add_options()( + "max_new_tokens", "The maximum number of tokens to generate", cxxopts::value()->default_value("10")); + options.add_options()( + "input_tokens_csv_file", "Path to a csv file that contains input tokens", cxxopts::value()); + options.add_options()("output_tokens_csv_file", "Path to a csv file that will contain the output tokens", + cxxopts::value()->default_value("outputTokens.csv")); + options.add_options()("timeout_ms", "The maximum time to wait for all responses, in milliseconds.", + cxxopts::value()->default_value("10000")); + options.add_options()("use_orchestrator_mode", "Use orchestrator communication mode.", + cxxopts::value()->default_value("false")); + options.add_options()("worker_executable_path", "The location of the worker executable.", + cxxopts::value()->default_value("")); + + auto parsedOptions = options.parse(argc, argv); + + // Argument: help + if (parsedOptions.count("help")) + { + TLLM_LOG_ERROR(options.help()); + exit(0); + } + + // Argument: Engine directory + if (!parsedOptions.count("engine_dir")) + { + TLLM_LOG_ERROR(options.help()); + TLLM_LOG_ERROR("Please specify engine directory."); + exit(1); + } + runtimeOpts.trtEnginePath = parsedOptions["engine_dir"].as(); + if (!fs::exists(runtimeOpts.trtEnginePath) || !fs::is_directory(runtimeOpts.trtEnginePath)) + { + TLLM_LOG_ERROR("Engine directory doesn't exist."); + exit(1); + } + + // Argument: Input tokens csv file + if (!parsedOptions.count("input_tokens_csv_file")) + { + TLLM_LOG_ERROR(options.help()); + TLLM_LOG_ERROR("Please specify input_tokens_csv_file"); + exit(1); + } + runtimeOpts.inputTokensCsvFile = parsedOptions["input_tokens_csv_file"].as(); + runtimeOpts.streaming = parsedOptions["streaming"].as(); + runtimeOpts.excludeInputFromOutput = parsedOptions["exclude_input_from_output"].as(); + runtimeOpts.maxNewTokens = parsedOptions["max_new_tokens"].as(); + runtimeOpts.beamWidth = parsedOptions["beam_width"].as(); + if (parsedOptions.count("num_return_sequences") > 0) + { + runtimeOpts.numReturnSequences = parsedOptions["num_return_sequences"].as>(); + } + runtimeOpts.timeoutMs = parsedOptions["timeout_ms"].as(); + runtimeOpts.outputTokensCsvFile = parsedOptions["output_tokens_csv_file"].as(); + + runtimeOpts.useOrchestratorMode = parsedOptions["use_orchestrator_mode"].as(); + runtimeOpts.workerExecutablePath = parsedOptions["worker_executable_path"].as(); + + return runtimeOpts; +} + +std::vector enqueueRequests(RuntimeOptions const& runtimeOpts, tle::Executor& executor) +{ + tle::OutputConfig outputConfig; + outputConfig.excludeInputFromOutput = runtimeOpts.excludeInputFromOutput; + tle::SamplingConfig samplingConfig(runtimeOpts.beamWidth); + if (runtimeOpts.numReturnSequences && runtimeOpts.beamWidth == 1) + { + samplingConfig.setTopP(0.9); + } + samplingConfig.setNumReturnSequences(runtimeOpts.numReturnSequences); + + TLLM_LOG_INFO("Reading input tokens from %s", runtimeOpts.inputTokensCsvFile.c_str()); + auto inputTokens = readInputTokens(runtimeOpts.inputTokensCsvFile); + TLLM_LOG_INFO("Number of requests: %d", inputTokens.size()); + + std::vector requests; + for (auto& tokens : inputTokens) + { + TLLM_LOG_INFO("Creating request with %d input tokens", tokens.size()); + requests.emplace_back( + std::move(tokens), runtimeOpts.maxNewTokens, runtimeOpts.streaming, samplingConfig, outputConfig); + } + + // Enqueue the requests + auto requestIds = executor.enqueueRequests(std::move(requests)); + + return requestIds; +} + +std::unordered_map waitForResponses( + RuntimeOptions const& runtimeOpts, std::vector const& requestIds, tle::Executor& executor) +{ + // Map that will be used to store output tokens for requests + std::unordered_map outputTokens; + auto numSequences = getNumSequencesPerRequest(runtimeOpts); + for (auto requestId : requestIds) + { + outputTokens[requestId] = tle::BeamTokens(numSequences); + } + + tle::SizeType32 numFinished{0}; + tle::SizeType32 iter{0}; + + // Get the new tokens for each request + while (numFinished < static_cast(requestIds.size()) && iter < runtimeOpts.timeoutMs) + { + std::chrono::milliseconds waitTime(1); + // Wait for any response + auto responses = executor.awaitResponses(waitTime); + + auto insertResponseTokens + = [&outputTokens](tle::IdType requestId, tle::SizeType32 seqIdx, tle::VecTokens const& respTokens) + { + TLLM_LOG_INFO("Got %d tokens for seqIdx %d for requestId %d", respTokens.size(), seqIdx, requestId); + + // Store the output tokens for that request id + auto& outTokens = outputTokens.at(requestId).at(seqIdx); + outTokens.insert(outTokens.end(), std::make_move_iterator(respTokens.begin()), + std::make_move_iterator(respTokens.end())); + }; + + // Loop over the responses + for (auto const& response : responses) + { + auto requestId = response.getRequestId(); + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + if (runtimeOpts.beamWidth > 1) + { + for (tle::SizeType32 beam = 0; beam < numSequences; ++beam) + { + insertResponseTokens(requestId, beam, result.outputTokenIds.at(beam)); + } + } + else + { + insertResponseTokens(requestId, result.sequenceIndex, result.outputTokenIds.at(0)); + } + if (result.isFinal) + { + TLLM_LOG_INFO("Request id %lu is completed.", requestId); + } + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + if (response.getErrorMsg() != err) + { + TLLM_THROW("Request id %lu encountered error: %s", requestId, response.getErrorMsg().c_str()); + } + } + } + ++iter; + } + if (iter == runtimeOpts.timeoutMs) + { + TLLM_THROW("Timeout exceeded."); + } + + return outputTokens; +} + +std::vector readInputTokens(std::string const& path) +{ + std::vector data; + std::ifstream file(path); + + if (!file.is_open()) + { + auto const err = std::string{"Failed to open file: "} + path; + TLLM_LOG_ERROR(err); + TLLM_THROW(err); + } + + std::string line; + while (std::getline(file, line)) + { + std::vector row; + std::stringstream ss(line); + std::string token; + + while (std::getline(ss, token, ',')) + { + try + { + row.push_back(std::stoi(token)); + } + catch (std::invalid_argument const& e) + { + TLLM_LOG_ERROR("Invalid argument: %s", e.what()); + } + catch (std::out_of_range const& e) + { + TLLM_LOG_ERROR("Out of range: %s", e.what()); + } + } + + data.push_back(row); + } + + file.close(); + return data; +} + +void writeOutputTokens(std::string const& path, std::vector& requestIds, + std::unordered_map const& outputTokens, tle::SizeType32 beamWidth) +{ + std::ofstream file(path); + + if (!file.is_open()) + { + TLLM_LOG_ERROR("Failed to open file %s", path.c_str()); + return; + } + + for (auto requestId : requestIds) + { + auto const& outTokens = outputTokens.at(requestId); + for (tle::SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto const& beamTokens = outTokens.at(beam); + for (size_t i = 0; i < beamTokens.size(); ++i) + { + file << beamTokens[i]; + if (i < beamTokens.size() - 1) + { + file << ", "; + } + } + file << "\n"; + } + } + + file.close(); +} + +tle::SizeType32 getNumSequencesPerRequest(RuntimeOptions const& runtimeOpts) +{ + auto numReturnSequences = runtimeOpts.numReturnSequences.value_or(runtimeOpts.beamWidth); + return runtimeOpts.beamWidth > 1 ? std::min(numReturnSequences, runtimeOpts.beamWidth) : numReturnSequences; +} diff --git a/examples/cpp/executor/executorExampleAdvancedMultiInstances.cpp b/examples/cpp/executor/executorExampleAdvancedMultiInstances.cpp new file mode 100644 index 000000000000..f967661ccd27 --- /dev/null +++ b/examples/cpp/executor/executorExampleAdvancedMultiInstances.cpp @@ -0,0 +1,381 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include + +namespace tle = tensorrt_llm::executor; + +namespace fs = std::filesystem; + +struct RuntimeOptions +{ + std::string trtEnginePath; + std::string inputTokensCsvFile; + std::string outputTokensCsvFile; + + bool streaming; + bool excludeInputFromOutput; + tle::SizeType32 maxNewTokens; + tle::SizeType32 beamWidth; + tle::SizeType32 timeoutMs; + + bool useOrchestratorMode; + std::string workerExecutablePath; + bool spawnProcesses; +}; + +// Utility function to parse input arguments +RuntimeOptions parseArgs(int argc, char* argv[]); + +// Function that enqueues requests +std::vector> enqueueRequests( + RuntimeOptions const& runtimeOpts, std::deque& executors); + +// Function that waits for responses and stores output tokens +std::map, tle::BeamTokens> waitForResponses(RuntimeOptions const& runtimeOpts, + std::vector> const& instanceRequestIds, std::deque& executors); + +// Utility function to read input tokens from csv file +std::vector readInputTokens(std::string const& path); + +// Utility function to write output tokens from csv file +void writeOutputTokens(std::string const& path, std::vector>& requestIds, + std::map, tle::BeamTokens> const& outputTokens, tle::SizeType32 beamWidth); + +// Main +int main(int argc, char* argv[]) +{ + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + auto runtimeOpts = parseArgs(argc, argv); + + // Create the executor for this engine + auto executorConfig = tle::ExecutorConfig(runtimeOpts.beamWidth); + + tle::KvCacheConfig kvCacheConfig{false, 10000}; + executorConfig.setKvCacheConfig(kvCacheConfig); + + bool isOrchestrator = true; + if (!runtimeOpts.spawnProcesses) + { + tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE); + int myRank = tensorrt_llm::mpi::MpiComm::world().getRank(); + isOrchestrator = (myRank == 0); + } + + auto orchestratorConfig = tle::OrchestratorConfig( + isOrchestrator, runtimeOpts.workerExecutablePath, nullptr, runtimeOpts.spawnProcesses); + auto parallelConfig = tle::ParallelConfig(tle::CommunicationType::kMPI, tle::CommunicationMode::kORCHESTRATOR, + std::nullopt, std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + int numInstances = 3; + if (!runtimeOpts.spawnProcesses) + { + // Keep one rank for orchestrator + numInstances = tensorrt_llm::mpi::MpiComm::world().getSize() - 1; + } + std::deque executors; + for (int instanceId = 0; instanceId < numInstances; ++instanceId) + { + auto executorConfigTmp = executorConfig; + // Set the rank id participating in each model instance + if (!runtimeOpts.spawnProcesses) + { + parallelConfig.setParticipantIds({instanceId + 1}); + } + executorConfigTmp.setParallelConfig(parallelConfig); + executors.emplace_back(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfigTmp); + } + + // Only orchestrator rank (rank 0) will enter + if (isOrchestrator) + { + // Create the requests + auto instanceRequestIds = enqueueRequests(runtimeOpts, executors); + + // Wait for responses and store output tokens + auto outputTokens = waitForResponses(runtimeOpts, instanceRequestIds, executors); + + // Write output tokens csv file + TLLM_LOG_INFO("Writing output tokens to %s", runtimeOpts.outputTokensCsvFile.c_str()); + writeOutputTokens(runtimeOpts.outputTokensCsvFile, instanceRequestIds, outputTokens, runtimeOpts.beamWidth); + } + TLLM_LOG_INFO("Exiting."); + return 0; +} + +RuntimeOptions parseArgs(int argc, char* argv[]) +{ + RuntimeOptions runtimeOpts; + + cxxopts::Options options(argv[0], "Example that demonstrates how to use the Executor API"); + options.add_options()("h,help", "Print usage"); + options.add_options()("engine_dir", "Directory that store the engines.", cxxopts::value()); + options.add_options()("beam_width", "The beam width", cxxopts::value()->default_value("1")); + options.add_options()("streaming", "Operate in streaming mode", cxxopts::value()->default_value("false")); + options.add_options()("exclude_input_from_output", + "Exclude input tokens when writing output tokens. Only has effect for streaming = false. For streaming = true, " + "output tokens are not included.", + cxxopts::value()->default_value("false")); + options.add_options()( + "max_new_tokens", "The maximum number of tokens to generate", cxxopts::value()->default_value("10")); + options.add_options()( + "input_tokens_csv_file", "Path to a csv file that contains input tokens", cxxopts::value()); + options.add_options()("output_tokens_csv_file", "Path to a csv file that will contain the output tokens", + cxxopts::value()->default_value("outputTokens.csv")); + options.add_options()("timeout_ms", "The maximum time to wait for all responses, in milliseconds.", + cxxopts::value()->default_value("10000")); + options.add_options()("worker_executable_path", "The location of the worker executable.", + cxxopts::value()->default_value("")); + options.add_options()("spawn_processes", + "Flag that controls if MPI_Comm_spawn should be used to spawn worker processes, or if they have been launched " + "with mpi already.", + cxxopts::value()->default_value("true")); + + auto parsedOptions = options.parse(argc, argv); + + // Argument: help + if (parsedOptions.count("help")) + { + TLLM_LOG_ERROR(options.help()); + exit(0); + } + + // Argument: Engine directory + if (!parsedOptions.count("engine_dir")) + { + TLLM_LOG_ERROR(options.help()); + TLLM_LOG_ERROR("Please specify engine directory."); + exit(1); + } + runtimeOpts.trtEnginePath = parsedOptions["engine_dir"].as(); + if (!fs::exists(runtimeOpts.trtEnginePath) || !fs::is_directory(runtimeOpts.trtEnginePath)) + { + TLLM_LOG_ERROR("Engine directory doesn't exist."); + exit(1); + } + + // Argument: Input tokens csv file + if (!parsedOptions.count("input_tokens_csv_file")) + { + TLLM_LOG_ERROR(options.help()); + TLLM_LOG_ERROR("Please specify input_tokens_csv_file"); + exit(1); + } + runtimeOpts.inputTokensCsvFile = parsedOptions["input_tokens_csv_file"].as(); + runtimeOpts.streaming = parsedOptions["streaming"].as(); + runtimeOpts.excludeInputFromOutput = parsedOptions["exclude_input_from_output"].as(); + runtimeOpts.maxNewTokens = parsedOptions["max_new_tokens"].as(); + runtimeOpts.beamWidth = parsedOptions["beam_width"].as(); + runtimeOpts.timeoutMs = parsedOptions["timeout_ms"].as(); + runtimeOpts.outputTokensCsvFile = parsedOptions["output_tokens_csv_file"].as(); + + runtimeOpts.workerExecutablePath = parsedOptions["worker_executable_path"].as(); + runtimeOpts.spawnProcesses = parsedOptions["spawn_processes"].as(); + + return runtimeOpts; +} + +std::vector> enqueueRequests( + RuntimeOptions const& runtimeOpts, std::deque& executors) +{ + tle::OutputConfig outputConfig; + outputConfig.excludeInputFromOutput = runtimeOpts.excludeInputFromOutput; + tle::SamplingConfig samplingConfig(runtimeOpts.beamWidth); + + TLLM_LOG_INFO("Reading input tokens from %s", runtimeOpts.inputTokensCsvFile.c_str()); + auto inputTokens = readInputTokens(runtimeOpts.inputTokensCsvFile); + TLLM_LOG_INFO("Number of requests: %d", inputTokens.size()); + + std::vector requests; + for (auto& tokens : inputTokens) + { + TLLM_LOG_INFO("Creating request with %d input tokens", tokens.size()); + requests.emplace_back( + std::move(tokens), runtimeOpts.maxNewTokens, runtimeOpts.streaming, samplingConfig, outputConfig); + } + + // Enqueue the requests + // Round robin over instances + std::vector> instanceRequestIds; + for (size_t req = 0; req < requests.size(); ++req) + { + auto instanceId = req % executors.size(); + TLLM_LOG_INFO("Enqueuing request %d for instance %d", req, instanceId); + auto requestId = executors.at(instanceId).enqueueRequest(requests[req]); + instanceRequestIds.emplace_back(instanceId, requestId); + } + TLLM_LOG_INFO("Enqueued %d requests", instanceRequestIds.size()); + return instanceRequestIds; +} + +std::map, tle::BeamTokens> waitForResponses(RuntimeOptions const& runtimeOpts, + std::vector> const& instanceRequestIds, std::deque& executors) +{ + // Map that will be used to store output tokens for requests + int numRequests = 0; + std::map, tle::BeamTokens> outputTokens; + for (auto instanceRequestId : instanceRequestIds) + { + outputTokens[instanceRequestId] = tle::BeamTokens(runtimeOpts.beamWidth); + numRequests++; + } + + tle::SizeType32 numFinished{0}; + tle::SizeType32 iter{0}; + + // Get the new tokens for each request + while (numFinished < numRequests && iter < runtimeOpts.timeoutMs) + { + std::chrono::milliseconds waitTime(1); + for (size_t instanceId = 0; instanceId < executors.size(); ++instanceId) + { + // Wait for any response for given instance + auto responses = executors.at(instanceId).awaitResponses(waitTime); + // Loop over the responses + for (auto const& response : responses) + { + auto requestId = response.getRequestId(); + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + TLLM_LOG_INFO("Number of finished requests: %d", numFinished); + + for (tle::SizeType32 beam = 0; beam < runtimeOpts.beamWidth; ++beam) + { + auto& respTokens = result.outputTokenIds.at(beam); + + TLLM_LOG_INFO("Got %d tokens for beam %d for requestId %d", respTokens.size(), beam, requestId); + + // Store the output tokens for that request id + auto& outTokens = outputTokens.at(std::make_pair(instanceId, requestId)).at(beam); + outTokens.insert(outTokens.end(), std::make_move_iterator(respTokens.begin()), + std::make_move_iterator(respTokens.end())); + } + if (result.isFinal) + { + TLLM_LOG_INFO("Request id %lu is completed.", requestId); + } + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "ReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + if (response.getErrorMsg() != err) + { + TLLM_THROW("Request id %lu encountered error: %s", requestId, response.getErrorMsg().c_str()); + } + } + } + } + ++iter; + } + if (iter == runtimeOpts.timeoutMs) + { + TLLM_THROW("Timeout exceeded."); + } + + return outputTokens; +} + +std::vector readInputTokens(std::string const& path) +{ + std::vector data; + std::ifstream file(path); + + if (!file.is_open()) + { + auto const err = std::string{"Failed to open file: "} + path; + TLLM_LOG_ERROR(err); + TLLM_THROW(err); + } + + std::string line; + while (std::getline(file, line)) + { + std::vector row; + std::stringstream ss(line); + std::string token; + + while (std::getline(ss, token, ',')) + { + try + { + row.push_back(std::stoi(token)); + } + catch (std::invalid_argument const& e) + { + TLLM_LOG_ERROR("Invalid argument: %s", e.what()); + } + catch (std::out_of_range const& e) + { + TLLM_LOG_ERROR("Out of range: %s", e.what()); + } + } + + data.push_back(row); + } + + file.close(); + return data; +} + +void writeOutputTokens(std::string const& path, std::vector>& instanceRequestIds, + std::map, tle::BeamTokens> const& outputTokens, tle::SizeType32 beamWidth) +{ + std::ofstream file(path); + + if (!file.is_open()) + { + TLLM_LOG_ERROR("Failed to open file %s", path.c_str()); + return; + } + + for (auto instanceRequestId : instanceRequestIds) + { + auto const& outTokens = outputTokens.at(instanceRequestId); + for (tle::SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto const& beamTokens = outTokens.at(beam); + for (size_t i = 0; i < beamTokens.size(); ++i) + { + file << beamTokens[i]; + if (i < beamTokens.size() - 1) + { + file << ", "; + } + } + file << "\n"; + } + } + + file.close(); +} diff --git a/examples/cpp/executor/executorExampleBasic.cpp b/examples/cpp/executor/executorExampleBasic.cpp new file mode 100644 index 000000000000..b3ae3328392c --- /dev/null +++ b/examples/cpp/executor/executorExampleBasic.cpp @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" + +namespace tlc = tensorrt_llm::common; +namespace tle = tensorrt_llm::executor; + +int main(int argc, char* argv[]) +{ + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + if (argc != 2) + { + TLLM_LOG_ERROR("Usage: %s ", argv[0]); + return 1; + } + + // Create the executor for this engine + tle::SizeType32 beamWidth = 1; + auto executorConfig = tle::ExecutorConfig(beamWidth); + auto trtEnginePath = argv[1]; + auto executor = tle::Executor(trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + tle::SizeType32 maxNewTokens = 5; + tle::VecTokens inputTokens{1, 2, 3, 4}; + auto request = tle::Request(inputTokens, maxNewTokens); + + // Enqueue the request + auto requestId = executor.enqueueRequest(request); + + // Wait for the response + auto responses = executor.awaitResponses(requestId); + + // Get outputTokens + auto outputTokens = responses.at(0).getResult().outputTokenIds.at(beamWidth - 1); + + TLLM_LOG_INFO("Output tokens: %s", tlc::vec2str(outputTokens).c_str()); + + return 0; +} diff --git a/examples/cpp/executor/executorExampleDebug.cpp b/examples/cpp/executor/executorExampleDebug.cpp new file mode 100644 index 000000000000..d0af1a8140b5 --- /dev/null +++ b/examples/cpp/executor/executorExampleDebug.cpp @@ -0,0 +1,65 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" + +namespace tlc = tensorrt_llm::common; +namespace tle = tensorrt_llm::executor; + +int main(int argc, char* argv[]) +{ + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + if (argc != 2) + { + TLLM_LOG_ERROR("Usage: %s ", argv[0]); + return 1; + } + + // Create the executor for this engine + tle::SizeType32 beamWidth = 1; + auto executorConfig = tle::ExecutorConfig(beamWidth); + // Select which tensors should be dumped + auto debugConfig = tle::DebugConfig(); + debugConfig.setDebugTensorNames({"host_request_types"}); + executorConfig.setDebugConfig(debugConfig); + + auto trtEnginePath = argv[1]; + auto executor = tle::Executor(trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + tle::SizeType32 maxNewTokens = 2; + tle::VecTokens inputTokens{1, 2, 3, 4}; + auto request = tle::Request(inputTokens, maxNewTokens); + + // Enqueue the request + auto requestId = executor.enqueueRequest(request); + + // Wait for the response + auto responses = executor.awaitResponses(requestId); + + // Get outputTokens + auto outputTokens = responses.at(0).getResult().outputTokenIds.at(beamWidth - 1); + + TLLM_LOG_INFO("Output tokens: %s", tlc::vec2str(outputTokens).c_str()); + + return 0; +} diff --git a/examples/cpp/executor/executorExampleDisaggregated.cpp b/examples/cpp/executor/executorExampleDisaggregated.cpp new file mode 100644 index 000000000000..ef9ff85c5e3b --- /dev/null +++ b/examples/cpp/executor/executorExampleDisaggregated.cpp @@ -0,0 +1,441 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/executor/types.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" + +#include + +namespace tle = tensorrt_llm::executor; + +namespace fs = std::filesystem; + +struct RuntimeOptions +{ + std::string trtContextEnginePath; + std::string trtGenerationEnginePath; + std::string inputTokensCsvFile; + std::string outputTokensCsvFile; + + bool streaming; + bool excludeInputFromOutput; + int contextRankSize; + int generationRankSize; + tle::SizeType32 maxNewTokens; + tle::SizeType32 beamWidth; + std::optional numReturnSequences; + tle::SizeType32 timeoutMs; +}; + +RuntimeOptions parseArgs(int argc, char* argv[]); + +// Function that enqueues requests into context executor and generation executor +std::unordered_map enqueueRequests( + RuntimeOptions const& runtimeOpts, tle::Executor& contextExecutor, tle::Executor& generationExecutor); + +// Function that waits for gen responses and stores output tokens +std::unordered_map waitForGenResponses(RuntimeOptions const& runtimeOpts, + std::unordered_map const& genRequestIdToContextRequestId, + tle::Executor& generationExecutor); + +// Utility function to read input tokens from csv file +std::vector readInputTokens(std::string const& path); + +// Utility function to write output tokens from csv file +void writeOutputTokens(std::string const& path, + std::unordered_map& genRequestIdToContextRequestId, + std::unordered_map const& outputTokens, tle::SizeType32 beamWidth); + +int main(int argc, char* argv[]) +{ + + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + auto runtimeOpts = parseArgs(argc, argv); + TLLM_CHECK_WITH_INFO(runtimeOpts.beamWidth == 1, "Only support beamWidth =1"); + TLLM_CHECK_WITH_INFO( + runtimeOpts.numReturnSequences.has_value() == false || runtimeOpts.numReturnSequences.value() == 1, + "Only support numReturnSequences =1"); + // Create the executor for this engine + auto contextExecutorConfig = tle::ExecutorConfig(runtimeOpts.beamWidth); + auto generationExecutorConfig = tle::ExecutorConfig(runtimeOpts.beamWidth); + bool isOrchestrator = (tensorrt_llm::mpi::MpiComm::world().getRank() == 0); + auto orchestratorConfig = tle::OrchestratorConfig(isOrchestrator, "", nullptr, false); + int contextRankSize = runtimeOpts.contextRankSize; + int generationRankSize = runtimeOpts.generationRankSize; + TLLM_CHECK_WITH_INFO(tensorrt_llm::mpi::MpiComm::world().getSize() >= contextRankSize + generationRankSize + 1, + " MPI should launch at least [contextRankSize+generationRankSize+1]: %d processes", + contextRankSize + generationRankSize + 1); + int deviceCount = -1; + TLLM_CHECK(cudaGetDeviceCount(&deviceCount) == cudaSuccess); + + std::vector contextRankIds(contextRankSize); + std::vector contextDeviceIds(contextRankSize); + std::vector generationRankIds(generationRankSize); + std::vector generationDeviceIds(generationRankSize); + for (int i = 0; i < contextRankSize; i++) + { + contextRankIds[i] = i + 1; + contextDeviceIds[i] = i % deviceCount; + TLLM_LOG_INFO("context Rank %d on device %d", contextRankIds[i], contextDeviceIds[i]); + } + tle::ParallelConfig contextParallelConfig{tensorrt_llm::executor::CommunicationType::kMPI, + tensorrt_llm::executor::CommunicationMode::kORCHESTRATOR, contextDeviceIds, contextRankIds, orchestratorConfig}; + + for (int i = 0; i < generationRankSize; i++) + { + generationRankIds[i] = i + 1 + contextRankSize; + generationDeviceIds[i] = (i + contextRankSize) % deviceCount; + TLLM_LOG_INFO("generation Rank %d on device %d", generationRankIds[i], generationDeviceIds[i]); + } + tle::ParallelConfig generationParallelConfig{tensorrt_llm::executor::CommunicationType::kMPI, + tensorrt_llm::executor::CommunicationMode::kORCHESTRATOR, generationDeviceIds, generationRankIds, + orchestratorConfig}; + + contextExecutorConfig.setParallelConfig(contextParallelConfig); + generationExecutorConfig.setParallelConfig(generationParallelConfig); + + auto contextExecutor + = tle::Executor(runtimeOpts.trtContextEnginePath, tle::ModelType::kDECODER_ONLY, contextExecutorConfig); + auto generationExecutor + = tle::Executor(runtimeOpts.trtGenerationEnginePath, tle::ModelType::kDECODER_ONLY, generationExecutorConfig); + tensorrt_llm::mpi::MpiComm::world().barrier(); + + if (tensorrt_llm::mpi::MpiComm::world().getRank() == 0) + { + + TLLM_CHECK_WITH_INFO(contextExecutor.canEnqueueRequests(), "contextExecutor can't enqueue requests"); + TLLM_CHECK_WITH_INFO(generationExecutor.canEnqueueRequests(), "generationExecutor can't enqueue requests"); + auto genRequestIdsToContextRequestIds = enqueueRequests(runtimeOpts, contextExecutor, generationExecutor); + auto outputTokens = waitForGenResponses(runtimeOpts, genRequestIdsToContextRequestIds, generationExecutor); + TLLM_LOG_INFO("Writing output tokens to %s", runtimeOpts.outputTokensCsvFile.c_str()); + writeOutputTokens( + runtimeOpts.outputTokensCsvFile, genRequestIdsToContextRequestIds, outputTokens, runtimeOpts.beamWidth); + } + tensorrt_llm::mpi::MpiComm::world().barrier(); + + TLLM_LOG_INFO("Exiting."); + return 0; +} + +RuntimeOptions parseArgs(int argc, char* argv[]) +{ + + RuntimeOptions runtimeOpts; + + cxxopts::Options options(argv[0], "Example that demonstrates how to use the Executor Disaggregated API"); + options.add_options()("h,help", "Print usage"); + options.add_options()( + "context_engine_dir", "Directory that store the context engine.", cxxopts::value()); + options.add_options()( + "generation_engine_dir", "Directory that store the generation engine.", cxxopts::value()); + options.add_options()( + "context_rank_size", "The number of ranks for the context engine", cxxopts::value()->default_value("1")); + options.add_options()("generation_rank_size", "The number of ranks for the generation engine", + cxxopts::value()->default_value("1")); + options.add_options()("beam_width", "The beam width", cxxopts::value()->default_value("1")); + options.add_options()( + "num_return_sequences", "The number of return sequences per request.", cxxopts::value>()); + options.add_options()("streaming", "Operate in streaming mode", cxxopts::value()->default_value("false")); + options.add_options()("exclude_input_from_output", + "Exclude input tokens when writing output tokens. Only has effect for streaming = false. For streaming = true, " + "output tokens are not included.", + cxxopts::value()->default_value("false")); + options.add_options()( + "max_new_tokens", "The maximum number of tokens to generate", cxxopts::value()->default_value("10")); + options.add_options()( + "input_tokens_csv_file", "Path to a csv file that contains input tokens", cxxopts::value()); + options.add_options()("output_tokens_csv_file", "Path to a csv file that will contain the output tokens", + cxxopts::value()->default_value("outputTokens.csv")); + options.add_options()("timeout_ms", "The maximum time to wait for all responses, in milliseconds.", + cxxopts::value()->default_value("10000")); + + auto parsedOptions = options.parse(argc, argv); + + // Argument: help + if (parsedOptions.count("help")) + { + TLLM_LOG_ERROR(options.help()); + exit(0); + } + + runtimeOpts.trtContextEnginePath = parsedOptions["context_engine_dir"].as(); + if (!fs::exists(runtimeOpts.trtContextEnginePath) || !fs::is_directory(runtimeOpts.trtContextEnginePath)) + { + TLLM_LOG_ERROR("Context engine directory doesn't exist."); + exit(1); + } + + runtimeOpts.trtGenerationEnginePath = parsedOptions["generation_engine_dir"].as(); + if (!fs::exists(runtimeOpts.trtGenerationEnginePath) || !fs::is_directory(runtimeOpts.trtGenerationEnginePath)) + { + TLLM_LOG_ERROR("Generation engine directory doesn't exist."); + exit(1); + } + // Argument: Input tokens csv file + if (!parsedOptions.count("input_tokens_csv_file")) + { + TLLM_LOG_ERROR(options.help()); + TLLM_LOG_ERROR("Please specify input_tokens_csv_file"); + exit(1); + } + + runtimeOpts.inputTokensCsvFile = parsedOptions["input_tokens_csv_file"].as(); + runtimeOpts.streaming = parsedOptions["streaming"].as(); + runtimeOpts.excludeInputFromOutput = parsedOptions["exclude_input_from_output"].as(); + runtimeOpts.maxNewTokens = parsedOptions["max_new_tokens"].as(); + runtimeOpts.beamWidth = parsedOptions["beam_width"].as(); + runtimeOpts.contextRankSize = parsedOptions["context_rank_size"].as(); + runtimeOpts.generationRankSize = parsedOptions["generation_rank_size"].as(); + if (parsedOptions.count("num_return_sequences") > 0) + { + runtimeOpts.numReturnSequences = parsedOptions["num_return_sequences"].as>(); + } + runtimeOpts.timeoutMs = parsedOptions["timeout_ms"].as(); + runtimeOpts.outputTokensCsvFile = parsedOptions["output_tokens_csv_file"].as(); + + return runtimeOpts; +} + +std::unordered_map enqueueRequests( + RuntimeOptions const& runtimeOpts, tle::Executor& contextExecutor, tle::Executor& generationExecutor) +{ + + tle::OutputConfig outputConfig; + outputConfig.excludeInputFromOutput = runtimeOpts.excludeInputFromOutput; + tle::SamplingConfig samplingConfig(runtimeOpts.beamWidth); + std::unordered_map genRequestIdToContextRequestId; + if (runtimeOpts.numReturnSequences && runtimeOpts.beamWidth == 1) + { + samplingConfig.setTopP(0.9); + } + samplingConfig.setNumReturnSequences(runtimeOpts.numReturnSequences); + + TLLM_LOG_INFO("Reading input tokens from %s", runtimeOpts.inputTokensCsvFile.c_str()); + auto inputTokens = readInputTokens(runtimeOpts.inputTokensCsvFile); + TLLM_LOG_INFO("Number of requests: %d", inputTokens.size()); + + std::vector requests; + for (auto& tokens : inputTokens) + { + TLLM_LOG_INFO("Creating request with %d input tokens", tokens.size()); + requests.emplace_back( + std::move(tokens), runtimeOpts.maxNewTokens, runtimeOpts.streaming, samplingConfig, outputConfig); + requests.back().setRequestType(tensorrt_llm::executor::RequestType::REQUEST_TYPE_CONTEXT_ONLY); + } + + auto contextRequestIds = contextExecutor.enqueueRequests(requests); + + for (size_t i = 0; i < requests.size(); i++) + { + + TLLM_LOG_INFO("waiting response for Context request id: %lu,", contextRequestIds[i]); + auto response = contextExecutor.awaitResponses(contextRequestIds[i]); + TLLM_LOG_INFO("response received for Context request id: %lu", contextRequestIds[i]); + TLLM_CHECK(response.size() == 1); + TLLM_CHECK(response.back().getResult().contextPhaseParams.has_value()); + requests.at(i).setContextPhaseParams(response.back().getResult().contextPhaseParams.value()); + requests.at(i).setRequestType(tensorrt_llm::executor::RequestType::REQUEST_TYPE_GENERATION_ONLY); + auto genRequestId = generationExecutor.enqueueRequest(requests.at(i)); + genRequestIdToContextRequestId[genRequestId] = contextRequestIds[i]; + + TLLM_LOG_INFO("enqueuing generation request for Context request id: %lu, generation request id: %lu", + contextRequestIds[i], genRequestId); + } + + return genRequestIdToContextRequestId; +} + +std::unordered_map waitForGenResponses(RuntimeOptions const& runtimeOpts, + std::unordered_map const& genRequestIdToContextRequestId, + tle::Executor& generationExecutor) +{ + + // Map that will be used to store output tokens for requests + std::unordered_map outputTokens; + std::vector contextRequestIds{}; + std::vector genRequestIds{}; + for (auto const& [key, value] : genRequestIdToContextRequestId) + { + genRequestIds.push_back(key); + contextRequestIds.push_back(value); + } + for (auto contextRequestId : contextRequestIds) + { + outputTokens[contextRequestId] = tle::BeamTokens(runtimeOpts.beamWidth); + } + + tle::SizeType32 numFinished{0}; + tle::SizeType32 iter{0}; + + // Get the new tokens for each request + while (numFinished < static_cast(genRequestIds.size()) && iter < runtimeOpts.timeoutMs) + { + std::chrono::milliseconds waitTime(1); + // Wait for any response + auto responses = generationExecutor.awaitResponses(waitTime); + + auto insertResponseTokens = [&outputTokens, &genRequestIdToContextRequestId](tle::IdType genRequestId, + tle::SizeType32 seqIdx, tle::VecTokens const& respTokens) + { + TLLM_LOG_INFO("Got %d tokens for seqIdx %d for genRequestId %d,contextRequestId %d", respTokens.size(), + seqIdx, genRequestId, genRequestIdToContextRequestId.at(genRequestId)); + + // Store the output tokens for that request id + auto& outTokens = outputTokens.at(genRequestIdToContextRequestId.at(genRequestId)).at(seqIdx); + outTokens.insert(outTokens.end(), std::make_move_iterator(respTokens.begin()), + std::make_move_iterator(respTokens.end())); + }; + + // Loop over the responses + for (auto const& response : responses) + { + auto genRequestId = response.getRequestId(); + if (!response.hasError()) + { + auto result = response.getResult(); + numFinished += result.isFinal; + if (runtimeOpts.beamWidth > 1) + { + for (tle::SizeType32 beam = 0; beam < runtimeOpts.beamWidth; ++beam) + { + insertResponseTokens(genRequestId, beam, result.outputTokenIds.at(beam)); + } + } + else + { + insertResponseTokens(genRequestId, result.sequenceIndex, result.outputTokenIds.at(0)); + } + if (result.isFinal) + { + TLLM_LOG_INFO("genRequest id %lu ,contextRequestId %lu is completed.", genRequestId, + genRequestIdToContextRequestId.at(genRequestId)); + } + } + else + { + // Allow response with error only if awaitResponse processed a terminated request id + std::string err = "genReqId " + std::to_string(response.getRequestId()) + + " has already been processed and was terminated."; + if (response.getErrorMsg() != err) + { + TLLM_THROW("GenRequest id %lu encountered error: %s", genRequestId, response.getErrorMsg().c_str()); + } + } + } + ++iter; + } + if (iter == runtimeOpts.timeoutMs) + { + TLLM_THROW("Timeout exceeded."); + } + + return outputTokens; +} + +std::vector readInputTokens(std::string const& path) +{ + std::vector data; + std::ifstream file(path); + + if (!file.is_open()) + { + auto const err = std::string{"Failed to open file: "} + path; + TLLM_LOG_ERROR(err); + TLLM_THROW(err); + } + + std::string line; + while (std::getline(file, line)) + { + std::vector row; + std::stringstream ss(line); + std::string token; + + while (std::getline(ss, token, ',')) + { + try + { + row.push_back(std::stoi(token)); + } + catch (std::invalid_argument const& e) + { + TLLM_LOG_ERROR("Invalid argument: %s", e.what()); + } + catch (std::out_of_range const& e) + { + TLLM_LOG_ERROR("Out of range: %s", e.what()); + } + } + + data.push_back(row); + } + + file.close(); + return data; +} + +void writeOutputTokens(std::string const& path, + std::unordered_map& genRequestIdToContextRequestId, + std::unordered_map const& outputTokens, tle::SizeType32 beamWidth) +{ + std::ofstream file(path); + + if (!file.is_open()) + { + TLLM_LOG_ERROR("Failed to open file %s", path.c_str()); + return; + } + std::vector requestIds; + for (auto const& [key, value] : genRequestIdToContextRequestId) + { + requestIds.push_back(value); + } + std::sort(requestIds.begin(), requestIds.end()); + + for (auto requestId : requestIds) + { + auto const& outTokens = outputTokens.at(requestId); + for (tle::SizeType32 beam = 0; beam < beamWidth; ++beam) + { + auto const& beamTokens = outTokens.at(beam); + for (size_t i = 0; i < beamTokens.size(); ++i) + { + file << beamTokens[i]; + if (i < beamTokens.size() - 1) + { + file << ", "; + } + } + file << "\n"; + } + } + + file.close(); +} diff --git a/examples/cpp/executor/executorExampleFastLogits.cpp b/examples/cpp/executor/executorExampleFastLogits.cpp new file mode 100644 index 000000000000..3611a1bb73d7 --- /dev/null +++ b/examples/cpp/executor/executorExampleFastLogits.cpp @@ -0,0 +1,264 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include "tensorrt_llm/runtime/utils/mpiUtils.h" +#include + +namespace tlc = tensorrt_llm::common; +namespace tle = tensorrt_llm::executor; + +namespace fs = std::filesystem; + +struct RuntimeOptions +{ + std::string trtDraftEnginePath; + std::string trtEnginePath; + bool fastLogits; + tle::SizeType32 numDraftTokens; +}; + +// Utility function to parse input arguments +RuntimeOptions parseArgs(int argc, char* argv[]); + +// Runs a draft request +tle::Result executeDraftRequest(tle::Executor& executor, RuntimeOptions const& runtimeOpts); + +// Runs a target request +tle::Result executeTargetRequest( + tle::Executor& executor, tle::Result const& draftResult, RuntimeOptions const& runtimeOpts); + +// Main +int main(int argc, char* argv[]) +{ + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + auto runtimeOpts = parseArgs(argc, argv); + + // Create the executor for this engine + auto executorConfig = tle::ExecutorConfig(); + + tensorrt_llm::mpi::initialize(tensorrt_llm::mpi::MpiThreadSupport::THREAD_MULTIPLE); + int const myRank = tensorrt_llm::mpi::MpiComm::world().getRank(); + bool const isOrchestrator = (myRank == 0); + + auto kvCacheConfig = tle::KvCacheConfig(true /* enableBlockReuse */); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto orchestratorConfig + = tle::OrchestratorConfig(isOrchestrator, "" /* workerExecutablePath */, nullptr, false /* spawnPrcesses */); + auto parallelConfig = tle::ParallelConfig(tle::CommunicationType::kMPI, tle::CommunicationMode::kORCHESTRATOR, + std::nullopt, std::nullopt, orchestratorConfig); + executorConfig.setParallelConfig(parallelConfig); + + auto specDecConfig = tle::SpeculativeDecodingConfig(runtimeOpts.fastLogits); + executorConfig.setSpecDecConfig(specDecConfig); + + std::unique_ptr draftExecutor; + std::unique_ptr targetExecutor; + + if (isOrchestrator) + { + auto executorConfigDraft = executorConfig; + parallelConfig.setParticipantIds({1}); + executorConfigDraft.setParallelConfig(parallelConfig); + + draftExecutor = std::make_unique( + runtimeOpts.trtDraftEnginePath, tle::ModelType::kDECODER_ONLY, executorConfigDraft); + + parallelConfig.setParticipantIds({2}); + executorConfig.setParallelConfig(parallelConfig); + + targetExecutor + = std::make_unique(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); + } + else if (myRank == 1) // draft model process + { + parallelConfig.setParticipantIds({1}); + parallelConfig.setDeviceIds({0}); + executorConfig.setParallelConfig(parallelConfig); + draftExecutor = std::make_unique( + runtimeOpts.trtDraftEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); + } + else if (myRank == 2) // target model process + { + parallelConfig.setParticipantIds({2}); + parallelConfig.setDeviceIds({1}); + executorConfig.setParallelConfig(parallelConfig); + targetExecutor + = std::make_unique(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); + ; + } + + // Only orchestrator rank (rank 0) will enter + if (isOrchestrator) + { + auto draftResult = executeDraftRequest(*draftExecutor, runtimeOpts); + + executeTargetRequest(*targetExecutor, draftResult, runtimeOpts); + } + TLLM_LOG_INFO("Exiting."); + return 0; +} + +RuntimeOptions parseArgs(int argc, char* argv[]) +{ + RuntimeOptions runtimeOpts; + + cxxopts::Options options(argv[0], "Example that demonstrates how to use the Executor API"); + options.add_options()("h,help", "Print usage"); + options.add_options()("engine_dir", "Directory that store the engine.", cxxopts::value()); + options.add_options()("draft_engine_dir", "Directory that store the draft engine.", cxxopts::value()); + options.add_options()( + "fast_logits", "Use speculative decoding fast logits feature", cxxopts::value()->default_value("true")); + options.add_options()( + "num_draft_tokens", "Number of draft tokens to use", cxxopts::value()->default_value("5")); + + auto parsedOptions = options.parse(argc, argv); + + // Argument: help + if (parsedOptions.count("help")) + { + TLLM_LOG_ERROR(options.help()); + exit(0); + } + + // Argument: Engine directory + if (!parsedOptions.count("engine_dir")) + { + TLLM_LOG_ERROR(options.help()); + TLLM_LOG_ERROR("Please specify engine directory."); + exit(1); + } + runtimeOpts.trtEnginePath = parsedOptions["engine_dir"].as(); + if (!fs::exists(runtimeOpts.trtEnginePath) || !fs::is_directory(runtimeOpts.trtEnginePath)) + { + TLLM_LOG_ERROR("Engine directory doesn't exist."); + exit(1); + } + + // Argument: Draft engine directory + if (!parsedOptions.count("draft_engine_dir")) + { + TLLM_LOG_ERROR(options.help()); + TLLM_LOG_ERROR("Please specify draft engine directory."); + exit(1); + } + runtimeOpts.trtDraftEnginePath = parsedOptions["draft_engine_dir"].as(); + if (!fs::exists(runtimeOpts.trtDraftEnginePath) || !fs::is_directory(runtimeOpts.trtDraftEnginePath)) + { + TLLM_LOG_ERROR("Draft engine directory doesn't exist."); + exit(1); + } + + runtimeOpts.fastLogits = parsedOptions["fast_logits"].as(); + runtimeOpts.numDraftTokens = parsedOptions["num_draft_tokens"].as(); + + return runtimeOpts; +} + +tle::Result executeDraftRequest(tle::Executor& executor, RuntimeOptions const& runtimeOpts) +{ + tle::OutputConfig outputConfig; + outputConfig.returnGenerationLogits = true; + + // Create the request + tle::SizeType32 maxNewTokens = runtimeOpts.numDraftTokens; + tle::VecTokens inputTokens{1, 2, 3, 4}; + + tle::Request request{std::move(inputTokens), maxNewTokens}; + request.setOutputConfig(outputConfig); + + // Enqueue the request + auto requestId = executor.enqueueRequest(std::move(request)); + + // Wait for the response + auto responses = executor.awaitResponses(requestId); + + if (responses.at(0).hasError()) + { + TLLM_LOG_ERROR(responses.at(0).getErrorMsg()); + exit(1); + } + + auto outputTokens = responses.at(0).getResult().outputTokenIds.at(0); + + TLLM_LOG_INFO("[DRAFT] Output tokens: %s", tlc::vec2str(outputTokens).c_str()); + + return responses.at(0).getResult(); +} + +tle::Result executeTargetRequest( + tle::Executor& executor, tle::Result const& draftResult, RuntimeOptions const& runtimeOpts) +{ + // Create the request + tle::SizeType32 maxNewTokens = runtimeOpts.numDraftTokens + 1; + tle::VecTokens inputTokens{1, 2, 3, 4}; + + tle::Request request{std::move(inputTokens), maxNewTokens}; + + tle::VecTokens const& outputTokenIds = draftResult.outputTokenIds.at(0); + tle::VecTokens draftTokens(outputTokenIds.end() - runtimeOpts.numDraftTokens, outputTokenIds.end()); + TLLM_LOG_INFO("[DRAFT] Draft tokens: %s", tlc::vec2str(draftTokens).c_str()); + + tle::Tensor logitsTensor; + + if (runtimeOpts.fastLogits) + { + auto const& logitsInfo = draftResult.specDecFastLogitsInfo.value(); + logitsTensor = logitsInfo.toTensor(); + } + else + { + auto generationLogits = draftResult.generationLogits.value(); + auto logitsShape = generationLogits.getShape(); + TLLM_CHECK(logitsShape[0] == 1); + logitsTensor = tle::Tensor::cpu(generationLogits.getDataType(), {logitsShape[1], logitsShape[2]}); + std::memcpy(logitsTensor.getData(), generationLogits.getData(), generationLogits.getSizeInBytes()); + } + + tle::ExternalDraftTokensConfig draftTokensConfig( + std::move(draftTokens), logitsTensor, std::nullopt /* acceptance threshold */, runtimeOpts.fastLogits); + request.setExternalDraftTokensConfig(draftTokensConfig); + + // Enqueue the request + auto requestId = executor.enqueueRequest(std::move(request)); + + // Wait for the response + auto responses = executor.awaitResponses(requestId); + + if (responses.at(0).hasError()) + { + TLLM_LOG_ERROR(responses.at(0).getErrorMsg()); + exit(1); + } + + auto outputTokens = responses.at(0).getResult().outputTokenIds.at(0); + + TLLM_LOG_INFO("[TARGET] Output tokens: %s", tlc::vec2str(outputTokens).c_str()); + + return responses.at(0).getResult(); +} diff --git a/examples/cpp/executor/executorExampleKvEvents.cpp b/examples/cpp/executor/executorExampleKvEvents.cpp new file mode 100644 index 000000000000..ea1923294382 --- /dev/null +++ b/examples/cpp/executor/executorExampleKvEvents.cpp @@ -0,0 +1,341 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include + +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" +#include + +namespace tlc = tensorrt_llm::common; +namespace tle = tensorrt_llm::executor; +namespace fs = std::filesystem; + +struct RuntimeOptions +{ + std::string trtEnginePath; + tle::SizeType32 numSysPrompts; + + tle::SizeType32 sysPromptTokens; + tle::SizeType32 contextTokens; + + tle::SizeType32 maxTokensMean; + tle::SizeType32 maxTokensStddev; + + tle::SizeType32 numRequests; + + size_t hostCacheSize; + size_t maxTokensInPagedKvCache; +}; + +struct KVCacheBlock +{ + KVCacheBlock(size_t hash, int cacheLevel, int priority, std::optional loraId = std::nullopt, + std::shared_ptr prevBlock = nullptr, std::optional cacheSalt = std::nullopt); + + size_t hash; + int cacheLevel; + int priority; + + std::optional loraId; + std::optional cacheSalt; + + std::shared_ptr prevBlock; + std::unordered_map> nextBlocks; +}; + +class RadixTree +{ +public: + explicit RadixTree(tle::Executor& executor); + // Check the executor for new events. + void pollEvents(); + +private: + std::shared_ptr mCacheEventManager; + // The root block of the radix tree + std::shared_ptr root; + // A table mapping block hashes to their pointers + std::unordered_map> blockTable; + // Event counter + size_t eventCounter; +}; + +// Utility function to parse input arguments +RuntimeOptions parseArgs(int argc, char* argv[]); + +// Create a tle::Request +tle::Request makeRequest(int sysPromptTokens, int contextTokens, std::uniform_int_distribution sysPromptSelector, + std::normal_distribution maxNumTokensSelector); + +std::default_random_engine gen; + +int main(int argc, char* argv[]) +{ + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + auto runtimeOpts = parseArgs(argc, argv); + + // Create the executor for this engine + auto executorConfig = tle::ExecutorConfig(1); // Beam width 1 is required for cache block reuse + auto kvCacheConfig = tle::KvCacheConfig(true, + runtimeOpts.maxTokensInPagedKvCache ? std::optional(runtimeOpts.maxTokensInPagedKvCache) + : std::nullopt); // Enable cache block reuse + kvCacheConfig.setHostCacheSize(runtimeOpts.hostCacheSize); + kvCacheConfig.setEventBufferMaxSize(32768); + executorConfig.setKvCacheConfig(kvCacheConfig); + + auto executor = tle::Executor(runtimeOpts.trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); + + auto radixTree = RadixTree(executor); + + auto activeRequests = runtimeOpts.numRequests; + + std::uniform_int_distribution sysPromptSelector( + 1, runtimeOpts.numSysPrompts); // Select a system prompt between 1 and `runtimeOpts.numSysPrompts` + std::normal_distribution maxNumTokensSelector(runtimeOpts.maxTokensMean, runtimeOpts.maxTokensStddev); + + // Create and enqueue the requests + for (int i = 0; i < runtimeOpts.numRequests; i++) + { + std::ignore = executor.enqueueRequest(makeRequest( + runtimeOpts.sysPromptTokens, runtimeOpts.contextTokens, sysPromptSelector, maxNumTokensSelector)); + } + + while (activeRequests > 0) + { + auto responses = executor.awaitResponses(std::chrono::milliseconds(20)); + for (auto const& response : responses) + { + if (response.getResult().isFinal) + activeRequests--; + } + // Only call pollEvents once every 20ms. Events are only added to the queue once per iteration, so no need to + // poll faster than this. + radixTree.pollEvents(); + } + + return 0; +} + +RuntimeOptions parseArgs(int argc, char* argv[]) +{ + RuntimeOptions runtimeOpts; + + cxxopts::Options options(argv[0], "Example that demonstrates how to use the ExecutorKVCacheManager API"); + options.add_options()("h,help", "Print usage"); + options.add_options()("engine_dir", "Directory that store the engines.", cxxopts::value()); + options.add_options()("num_sys_prompts", "Amount of unique simulated system prompts to use", + cxxopts::value()->default_value("10")); + options.add_options()( + "sys_prompt_tokens", "Size of the simulated system prompts", cxxopts::value()->default_value("256")); + options.add_options()("context_tokens", "Amount of varying context tokens coming after the system prompts", + cxxopts::value()->default_value("128")); + options.add_options()( + "max_tokens_mean", "Mean number of max output tokens", cxxopts::value()->default_value("128")); + options.add_options()( + "max_tokens_stddev", "Standard deviation of max output tokens", cxxopts::value()->default_value("32")); + options.add_options()( + "num_requests", "Amount of requests to send to the engine", cxxopts::value()->default_value("100")); + options.add_options()("host_cache_size", "Size of the KV Cache in host memory in bytes", + cxxopts::value()->default_value("0")); + options.add_options()("max_tokens_in_paged_kv_cache", "Amount of tokens in the kv cache", + cxxopts::value()->default_value("0")); + + auto parsedOptions = options.parse(argc, argv); + + // Argument: help + if (parsedOptions.count("help")) + { + TLLM_LOG_ERROR(options.help()); + exit(0); + } + + // Argument: Engine directory + if (!parsedOptions.count("engine_dir")) + { + TLLM_LOG_ERROR(options.help()); + TLLM_LOG_ERROR("Please specify engine directory."); + exit(1); + } + runtimeOpts.trtEnginePath = parsedOptions["engine_dir"].as(); + if (!fs::exists(runtimeOpts.trtEnginePath) || !fs::is_directory(runtimeOpts.trtEnginePath)) + { + TLLM_LOG_ERROR("Engine directory doesn't exist."); + exit(1); + } + + runtimeOpts.numSysPrompts = parsedOptions["num_sys_prompts"].as(); + runtimeOpts.sysPromptTokens = parsedOptions["sys_prompt_tokens"].as(); + runtimeOpts.contextTokens = parsedOptions["context_tokens"].as(); + runtimeOpts.maxTokensMean = parsedOptions["max_tokens_mean"].as(); + runtimeOpts.maxTokensStddev = parsedOptions["max_tokens_stddev"].as(); + runtimeOpts.numRequests = parsedOptions["num_requests"].as(); + runtimeOpts.hostCacheSize = parsedOptions["host_cache_size"].as(); + runtimeOpts.maxTokensInPagedKvCache = parsedOptions["max_tokens_in_paged_kv_cache"].as(); + + return runtimeOpts; +} + +KVCacheBlock::KVCacheBlock(size_t hash, int cacheLevel, int priority, std::optional loraId, + std::shared_ptr prevBlock, std::optional cacheSalt) + : hash{hash} + , cacheLevel{cacheLevel} + , priority{priority} + , loraId{loraId} + , cacheSalt{std::move(cacheSalt)} + , prevBlock{prevBlock} + , nextBlocks{} +{ +} + +RadixTree::RadixTree(tle::Executor& executor) + : mCacheEventManager(*executor.getKVCacheEventManager()) + , eventCounter{1} +{ + // Use id=-1 for the root block. Doesn't matter what exact id is used, just that it is unique. + root = std::make_shared(-1, -1, -1); + blockTable[-1] = root; + + // Wait for the `CREATED` event to be emitted. + while (true) + { + auto events = mCacheEventManager->getLatestEvents(); + if (events.size() == 1) + { + auto const& eventData = std::get(events.front().data); + TLLM_LOG_INFO("Event ID %d: KV Cache Manager initialized with blocks per level of: %s", + events.front().eventId, tlc::vec2str(eventData.numBlocksPerCacheLevel).c_str()); + break; + } + } +}; + +void RadixTree::pollEvents() +{ + auto events = mCacheEventManager->getLatestEvents(std::chrono::milliseconds(20)); + for (tle::KVCacheEvent const& event : events) + { + TLLM_CHECK(event.eventId == eventCounter++); + if (std::holds_alternative(event.data)) + { + // Blocks have been stored into the radix tree + auto const& eventData = std::get(event.data); + auto prevBlock = blockTable[eventData.parentHash.value_or(-1)]; + + // This block should be in the tree + TLLM_CHECK(blockTable.find(prevBlock->hash) != blockTable.end()); + + for (auto& block : eventData.blocks) + { + + TLLM_LOG_INFO("Event ID %d: Block %04x was inserted into the radix tree with parent %04x.", + event.eventId, block.blockHash, prevBlock->hash); + + // This block shouldn't already exist in the tree, and should have tokens associated with it + TLLM_CHECK(blockTable.find(block.blockHash) == blockTable.end()); + TLLM_CHECK(block.tokens.size() > 0); + + auto thisBlock = std::make_shared( + block.blockHash, block.cacheLevel, block.priority, block.loraId, prevBlock, block.cacheSalt); + + blockTable[block.blockHash] = thisBlock; + // Link the parent to the new block + prevBlock->nextBlocks[block.blockHash] = thisBlock; + + prevBlock = thisBlock; + } + } + else if (std::holds_alternative(event.data)) + { + auto const& eventData = std::get(event.data); + + for (auto const& hash : eventData.blockHashes) + { + + TLLM_LOG_INFO("Event ID %d: Block %04x was removed from the radix tree.", event.eventId, hash); + + // This block should exist in the tree + TLLM_CHECK(blockTable.find(hash) != blockTable.end()); + + auto& block = blockTable[hash]; + + // Check that the block has no children, and that the parent has the block listed as a child + TLLM_CHECK(block->nextBlocks.size() == 0); + TLLM_CHECK(block->prevBlock->nextBlocks.find(block->hash) != block->prevBlock->nextBlocks.end()); + + // Remove the block from it's parent, and remove the entry in the block table + block->prevBlock->nextBlocks.erase(block->hash); + blockTable.erase(hash); + } + } + else if (std::holds_alternative(event.data)) + { + auto const& eventData = std::get(event.data); + + if (eventData.priority.has_value()) + { + // The block priority was updated + TLLM_LOG_INFO("Event ID %d: Block %04x priority was changed from %d to %d", event.eventId, + eventData.blockHash, eventData.priority->oldValue, eventData.priority->newValue); + + TLLM_CHECK(blockTable[eventData.blockHash]->priority == eventData.priority->oldValue); + blockTable[eventData.blockHash]->priority = eventData.priority->newValue; + } + + if (eventData.cacheLevel.has_value()) + { + // The block cache level was updated + TLLM_LOG_INFO("Event ID %d: Block %04x cache level was changed from %d to %d", event.eventId, + eventData.blockHash, eventData.cacheLevel->oldValue, eventData.cacheLevel->newValue); + + TLLM_CHECK(blockTable[eventData.blockHash]->cacheLevel == eventData.cacheLevel->oldValue); + blockTable[eventData.blockHash]->cacheLevel = eventData.cacheLevel->newValue; + } + } + else + { + TLLM_LOG_ERROR("Unsupported event type. This shouldn't happen!"); + } + } +} + +tle::Request makeRequest(int sysPromptTokens, int contextTokens, std::uniform_int_distribution sysPromptSelector, + std::normal_distribution maxNumTokensSelector) +{ + int sysPromptVersion = sysPromptSelector(gen); + tle::VecTokens inputTokens; + + // Add `sysPromptTokens` tokens. Add the version to the token ids to create a unique system prompt + for (int i = 0; i < sysPromptTokens; i++) + { + inputTokens.emplace_back(sysPromptVersion + i); + } + // Add random context tokens + for (int i = 0; i < contextTokens; i++) + { + inputTokens.emplace_back(rand() % 1000); + } + + return tle::Request(inputTokens, maxNumTokensSelector(gen)); +} diff --git a/examples/cpp/executor/executorExampleLogitsProcessor.cpp b/examples/cpp/executor/executorExampleLogitsProcessor.cpp new file mode 100644 index 000000000000..0913b77b1775 --- /dev/null +++ b/examples/cpp/executor/executorExampleLogitsProcessor.cpp @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/common/logger.h" +#include "tensorrt_llm/executor/executor.h" +#include "tensorrt_llm/plugins/api/tllmPlugin.h" + +namespace tlc = tensorrt_llm::common; +namespace tle = tensorrt_llm::executor; + +int main(int argc, char* argv[]) +{ + // Register the TRT-LLM plugins + initTrtLlmPlugins(); + + if (argc != 2) + { + TLLM_LOG_ERROR("Usage: %s ", argv[0]); + return 1; + } + + int constexpr sentinels[] = {42, 29}; + int step = 0; + + auto logitsPostProcessorFn + = [&step, &sentinels](tle::IdType reqId, tle::Tensor& logits, tle::BeamTokens const& tokens, + tle::StreamPtr const& streamPtr, std::optional clientId) + { + auto logitsDataType = logits.getDataType(); + auto logitsCpu = tensorrt_llm::executor::Tensor::cpu(logitsDataType, logits.getShape()); + auto* dataPtr = logitsCpu.getData(); + auto* dataPtrFloat = static_cast(dataPtr); + for (size_t i = 0; i < logitsCpu.getSize(); ++i) + { + dataPtrFloat[i] = -1.0e20; + } + dataPtrFloat[sentinels[step]] = 0.0f; + + logits.setFrom(logitsCpu, streamPtr); + step = (1 - step); + }; + + std::string logitsPostProcessorName = "MyLogitsPP"; + + // Create the executor for this engine + tle::SizeType32 beamWidth = 1; + auto executorConfig = tle::ExecutorConfig(beamWidth); + + auto logitsProcConfig = tle::LogitsPostProcessorConfig(); + logitsProcConfig.setProcessorMap(std::unordered_map{ + {logitsPostProcessorName, logitsPostProcessorFn}}); + executorConfig.setLogitsPostProcessorConfig(logitsProcConfig); + + auto trtEnginePath = argv[1]; + auto executor = tle::Executor(trtEnginePath, tle::ModelType::kDECODER_ONLY, executorConfig); + + // Create the request + tle::SizeType32 maxNewTokens = 5; + tle::VecTokens inputTokens{1, 2, 3, 4}; + auto request = tle::Request(inputTokens, maxNewTokens); + request.setLogitsPostProcessorName(logitsPostProcessorName); + + // Enqueue the request + auto requestId = executor.enqueueRequest(std::move(request)); + + // Wait for the response + auto responses = executor.awaitResponses(requestId); + + // Get outputTokens + auto outputTokens = responses.at(0).getResult().outputTokenIds.at(beamWidth - 1); + + TLLM_LOG_INFO("Output tokens: %s", tlc::vec2str(outputTokens).c_str()); + + return 0; +} diff --git a/examples/cpp/executor/inputTokens.csv b/examples/cpp/executor/inputTokens.csv new file mode 100644 index 000000000000..4cb3974a91b5 --- /dev/null +++ b/examples/cpp/executor/inputTokens.csv @@ -0,0 +1,3 @@ +1, 2, 3, 4, 5, 6 +1, 2, 3, 4 +1, 2, 3, 4, 5, 6, 7, 8, 9, 10 diff --git a/examples/cpp_library/CMakeLists.txt b/examples/cpp_library/CMakeLists.txt new file mode 100644 index 000000000000..c60ff48d4cde --- /dev/null +++ b/examples/cpp_library/CMakeLists.txt @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. cmake needs this line +cmake_minimum_required(VERSION 3.1) +# cmake_minimum_required(VERSION 2.8) + +# Enable C++11 +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED TRUE) + +# Define project name +set(TARGET_NAME trt_llm_plugins_cpp_load_example) +project(${TARGET_NAME}) + +set(CMAKE_VERBOSE_MAKEFILE 1) + +# Compile options +set(CMAKE_C_FLAGS "-Wall -pthread ") +set(CMAKE_C_FLAGS_DEBUG "-g -O0") +set(CMAKE_C_FLAGS_RELEASE "-O2") +set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -lstdc++") +set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) +set(CMAKE_CXX_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE}) + +set(CMAKE_BUILD_TYPE release) +# set(CMAKE_BUILD_TYPE debug) + +find_package(CUDA REQUIRED) +message(STATUS "CUDA library status:") +message(STATUS " config: ${CUDA_DIR}") +message(STATUS " version: ${CUDA_VERSION}") +message(STATUS " libraries: ${CUDA_LIBRARIES}") +message(STATUS " include path: ${CUDA_INCLUDE_DIRS}") + +# Declare the executable target built from your sources +add_executable(${TARGET_NAME} main.cpp) + +# Link your application with CUDA libraries +target_link_libraries(${TARGET_NAME} LINK_PRIVATE ${CUDA_LIBRARIES}) +target_link_libraries(${TARGET_NAME} LINK_PRIVATE cudnn) +target_link_libraries(${TARGET_NAME} LINK_PRIVATE nvinfer) +target_link_libraries(${TARGET_NAME} LINK_PRIVATE nvinfer_plugin_tensorrt_llm) + +target_include_directories(${TARGET_NAME} PUBLIC /usr/local/cuda/include) diff --git a/examples/cpp_library/build.sh b/examples/cpp_library/build.sh new file mode 100755 index 000000000000..a384e5cad4a0 --- /dev/null +++ b/examples/cpp_library/build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +BUILD_DIR="build" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +rm -rf ${BUILD_DIR} && mkdir -p ${BUILD_DIR} + +pushd ${BUILD_DIR} + +cmake \ + -DCMAKE_BUILD_TYPE=Release \ + .. + +make -j"$(grep -c ^processor /proc/cpuinfo)" + +export LD_LIBRARY_PATH="${SCRIPT_DIR}:${LD_LIBRARY_PATH}" + +# Test Lib +echo +echo "--------------------------------------------------------------------" +./trt_llm_plugins_cpp_load_example +echo "--------------------------------------------------------------------" +echo + +popd diff --git a/examples/cpp_library/main.cpp b/examples/cpp_library/main.cpp new file mode 100644 index 000000000000..7613a75a140c --- /dev/null +++ b/examples/cpp_library/main.cpp @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "tensorrt_llm_libutils.h" + +int main(int argc, char* argv[]) +{ + class TRTLogger : public nvinfer1::ILogger + { + public: + void log(nvinfer1::ILogger::Severity severity, char const* msg) noexcept override + { + if (severity <= nvinfer1::ILogger::Severity::kERROR) + std::cerr << "[TensorRT LLM ERR]: " << msg << std::endl; + else if (severity == nvinfer1::ILogger::Severity::kWARNING) + std::cerr << "[TensorRT LLM WARNING]: " << msg << std::endl; + else + std::cout << "[TensorRT LLM LOG]: " << msg << std::endl; + } + }; + + TRTLogger* trtLogger = new TRTLogger(); + + std::string libname = "libtensorrt_llm_plugin.so"; + + /* =============== initLibNvInferPlugins =============== */ + + typedef bool (*initLibNvInferPlugins_sig)(void*, void const*); + + auto initLibNvInferPlugins = getTrtLLMFunction( + /*libFileSoName=*/libname, + /*symbol=*/"initLibNvInferPlugins"); + + std::cout << std::endl; + + std::string libNamespace = "tensorrt_llm"; + char const* libNamespace_cstr = libNamespace.data(); + + bool status1 = initLibNvInferPlugins(trtLogger, libNamespace_cstr); + std::cout << "Success Status: " << status1 << std::endl << std::endl; + + bool status2 = initLibNvInferPlugins(trtLogger, libNamespace_cstr); + std::cout << "Success Status: " << status2 << std::endl; + + /* =============== getInferLibVersion =============== */ + + std::cout << std::endl; + std::cout << "--------------------------------------------------------------------" << std::endl; + + typedef int32_t (*getInferLibVersion_sig)(); + + auto getInferLibVersion = getTrtLLMFunction( + /*libFileSoName=*/libname, + /*symbol=*/"getInferLibVersion"); + + std::cout << std::endl; + + int32_t version = getInferLibVersion(); + std::cout << "Version: " << version << std::endl; + + return 0; +} diff --git a/examples/cpp_library/tensorrt_llm_libutils.h b/examples/cpp_library/tensorrt_llm_libutils.h new file mode 100644 index 000000000000..aa60444eefa3 --- /dev/null +++ b/examples/cpp_library/tensorrt_llm_libutils.h @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#if !defined(_WIN32) +#include +#endif // !defined(_WIN32) +#include +#include +#include + +#include "NvInfer.h" + +template +tSymbolSignature getTrtLLMFunction(std::string libFileSoName, std::string symbol) +{ +#if !defined(_WIN32) + std::cout << "Trying to load " << libFileSoName << " ..." << std::endl; + + // 1. Defining a handle to the library + void* handle = dlopen(libFileSoName.c_str(), RTLD_LAZY | RTLD_GLOBAL); + + // 2. Check for errors + char const* dl_error1 = dlerror(); + if (!handle) + { + throw std::runtime_error("Cannot open library: " + std::string(dl_error1)); + } + + // 3. Load actual queried `symbol` + std::cout << "Loading symbol `" << symbol << "` ..." << std::endl; + + tSymbolSignature symbolFctn = nullptr; + *(void**) (&symbolFctn) = dlsym(handle, symbol.c_str()); + + // 4. Check for errors + char const* dl_error2 = dlerror(); + if (dl_error2) + { + dlclose(handle); + throw std::runtime_error("Cannot load symbol '" + symbol + "': " + std::string(dl_error2)); + } + + return symbolFctn; +#else // on windows + throw std::runtime_error( + "`tSymbolSignature getTrtLLMFunction(std::string, std::string)` is not implemented on Windows."); + return nullptr; +#endif // !defined(_WIN32) +} diff --git a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm index d73430ba5820..275016c255dd 100644 --- a/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm +++ b/examples/disaggregated/slurm/benchmark/disaggr_torch.slurm @@ -101,7 +101,7 @@ elif [ -d "${trtllm_repo}" ]; then if [ "${build_wheel}" = "true" ]; then echo "Building TensorRT-LLM wheel on one node..." - build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --use_ccache --clean" + build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --clean" if [ -n "${cuda_architectures:-}" ]; then build_command="${build_command} --cuda_architectures \"${cuda_architectures}\"" fi diff --git a/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm b/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm index 07015e6473a0..8835399d6c50 100644 --- a/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm +++ b/examples/disaggregated/slurm/benchmark/disaggr_torch_dwdp.slurm @@ -101,7 +101,7 @@ elif [ -d "${trtllm_repo}" ]; then if [ "${build_wheel}" = "true" ]; then echo "Building TensorRT-LLM wheel on one node..." - build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --use_ccache --clean" + build_command="python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --clean" if [ -n "${cuda_architectures:-}" ]; then build_command="${build_command} --cuda_architectures \"${cuda_architectures}\"" fi diff --git a/examples/disaggregated/slurm/cache_transceiver_test/README.md b/examples/disaggregated/slurm/cache_transceiver_test/README.md index e99c57dc4fe3..14619e4d387c 100644 --- a/examples/disaggregated/slurm/cache_transceiver_test/README.md +++ b/examples/disaggregated/slurm/cache_transceiver_test/README.md @@ -64,7 +64,7 @@ resolved_config.json # the validated config the job ran with logs/ctt-.log # batch-level log (stdout+stderr merged) logs/install.log # TensorRT-LLM install log logs/sweep__rank*.log # per-rank logs: transfer START/DONE+verify, UCX_PROTO_INFO -csv//ctx|gen/_*_{send,recv}.csv # C++ transceiver timing (Bandwidth(Gbps)); renamed to _*_{send,recv}__c.csv per combination +csv//ctx|gen/rank_*_{send,recv}.csv # C++ transceiver timing (Bandwidth(Gbps)) csv//ctx/py_*_*.csv # Python transceiver perf log (throughput_mbs) status/sweep_.jsonl # PASS / MISMATCH / TRANSFER_ERROR / TIMEOUT results.json # full results, grouped per combination (longest req_len) @@ -96,7 +96,7 @@ deliverable for tuning your cluster. | Transceiver | Env enabling timing | File | Column (native) | |---|---|---|---| -| C++ (UCX/NIXL) | `TRTLLM_KVCACHE_TIME_OUTPUT_PATH` (set by the driver) | `_*_recv.csv` (renamed `_*_recv__c.csv` per combination) | `Bandwidth(Gbps)` | +| C++ (UCX/NIXL) | `TRTLLM_KVCACHE_TIME_OUTPUT_PATH` (set by the driver) | `rank_*_recv.csv` | `Bandwidth(Gbps)` | | Python (NIXL) | `TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1`, `TLLM_KV_TRANSFER_PERF_LOG_FILE` (set by the driver) | `py_*_*.csv` | `throughput_mbs` (MiB/s) | `report.py` normalizes both to **per-GPU GB/s** (bytes, ÷1e9): C++ diff --git a/examples/disaggregated/slurm/cache_transceiver_test/report.py b/examples/disaggregated/slurm/cache_transceiver_test/report.py index 00ecab6b4e49..70d2658eab6b 100644 --- a/examples/disaggregated/slurm/cache_transceiver_test/report.py +++ b/examples/disaggregated/slurm/cache_transceiver_test/report.py @@ -184,7 +184,7 @@ def _mean(values): def _parse_cpp_recv_csvs(csv_dir): - """Return {rid: [per_rank_GBps, ...]} from C++ *_recv.csv files. + """Return {rid: [per_rank_GBps, ...]} from C++ rank_*_recv.csv files. Each rank writes one row per request with a repeating Bandwidth(Gbps) column per transmission; we take the mean transmission bandwidth as that rank's @@ -194,10 +194,9 @@ def _parse_cpp_recv_csvs(csv_dir): per-rank rates with unequal durations would overstate the real throughput. """ per_rid = {} # rid -> list[per-rank mean bw] - # C++ writes "__recv.csv" (instanceId is a runtime UUID); - # the driver renames each combination's *_recv.csv to *_recv__cl
  • .csv + # The driver renames each combination's rank_*_recv.csv to rank_*_recv__cl
  • .csv # (and the un-renamed name may exist for the last iteration). Match both. - for path in glob.glob(os.path.join(csv_dir, "*_recv*.csv")): + for path in glob.glob(os.path.join(csv_dir, "rank_*_recv*.csv")): with open(path) as f: reader = csv.reader(f) header = next(reader, None) diff --git a/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py b/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py index 138a43c96063..6a6775d5c08e 100644 --- a/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py +++ b/examples/disaggregated/slurm/cache_transceiver_test/run_cache_transceiver_test.py @@ -24,7 +24,7 @@ deterministic, rank-specific pattern, sends it, and the gen side verifies the received blocks regenerate to the same pattern. Bandwidth is emitted by the transceivers themselves into per-rank CSVs (parsed later by report.py): - C++ -> TRTLLM_KVCACHE_TIME_OUTPUT_PATH (_*_send.csv / _*_recv.csv) + C++ -> TRTLLM_KVCACHE_TIME_OUTPUT_PATH (rank_*_send.csv / rank_*_recv.csv) Py -> TLLM_KV_TRANSFER_PERF_LOG_FILE (py_*_*.csv, throughput_mbs) This driver mirrors the single-process test (tests/unittest/others/ @@ -437,19 +437,15 @@ def _preserve_cpp_csvs(csv_dir, ci, rank): request lengths of a combination and appends a row per request, so we move the whole combination's output aside (rid encodes req_len). - C++ names files "__.csv" (instanceId is a runtime - UUID). Each rank touches ONLY files carrying its own "__.csv" - suffix: all ranks share `csv_dir`, so matching a broader pattern would race - -- multiple ranks renaming the same file, leaving some with - FileNotFoundError, crashing those ranks and deadlocking the rest on the next - case's collective KVCacheManager allreduce. + Each rank touches ONLY its own files: all ranks share `csv_dir`, so a glob + over `rank_*` would race -- multiple ranks renaming the same file, leaving + some with FileNotFoundError, crashing those ranks and deadlocking the rest on + the next case's collective KVCacheManager allreduce. """ for tag in ("send", "recv"): - suffix = f"_{rank}_{tag}.csv" - for name in os.listdir(csv_dir): - if name.endswith(suffix) and "__c" not in name: - base = name[: -len(".csv")] - os.replace(os.path.join(csv_dir, name), os.path.join(csv_dir, f"{base}__c{ci}.csv")) + path = os.path.join(csv_dir, f"rank_{rank}_{tag}.csv") + if os.path.exists(path): + os.replace(path, os.path.join(csv_dir, f"rank_{rank}_{tag}__c{ci}.csv")) def main(): diff --git a/examples/dora/README.md b/examples/dora/README.md new file mode 100644 index 000000000000..21f70768a257 --- /dev/null +++ b/examples/dora/README.md @@ -0,0 +1,65 @@ +# DoRA + +This document shows how to run a model using DoRA adapters. +DoRA is a PEFT strategy extending LoRA. It is fully supported in the Huggingface `peft` library. For a more detailed description please refer to the DoRA [paper](https://arxiv.org/abs/2402.09353) or official [repo](https://github.com/NVlabs/DoRA). + +## Support Matrix + * FP16/BF16 (over arbitrary precision of the base model). + * Supports adapters from Huggingface `peft` or from the official NVlabs [checkpoints](https://huggingface.co/sliuau/DoRA-weights). + * Multiple adapters (+ mixed LoRA/DoRA setups). + * inflight loading of new adapters to a preloaded base model. + * C++ and python runtime. + * Tensor parallelism and Pipeline parallelism. + +## Usage +Using DoRA is almost exactly the same as using LoRA in TRTLLM, with an additional preprocessing step. +While the official DoRA paper describes the magnitude normalization as part of the execution flow, it can be performed once beforehand to boost inference performance. + +Start by obtaining a local copy of your desired DoRA adapter **and** your base model. We'll use the official NVlabs checkpoint for LLaMA3-8B as an example: + +``` bash +git clone https://huggingface.co/sliuau/DoRA-weights +git clone https://huggingface.co/meta-llama/Meta-Llama-3-8B +``` + +Next, use the [normalize_weights.py](./normalize_weights.py) script to normalize the DoRA magnitude vectors in the adapter checkpoint. +The script requires access to both the local adapter weights and the local base model weights: + +``` bash +export NORMALIZED_DORA_ADAPTER=path/to/normalized/adapter/ckpt + +python ./normalize_weights.py -i DoRA-weights/llama_dora_commonsense_checkpoints/LLama3-8B/dora_r32 -b Meta-Llama-3-8B -o $NORMALIZED_DORA_ADAPTER +``` + +The script will create a new adapter checkpoint, with normalized DoRA vectors, in the provided path. + +Now we may convert our Llama checkpoint and build our TRT engine as described in the Llama [examples](../models/core/llama/README.md). When doing so, ensure you pass `--dora_plugin=enable` to the `trtllm-build` command, as well as enabling the lora plugin: + +``` bash +export CHECKPOINT_DIR=path/to/trtllm/ckpt +export ENGIRE_DIR=path/to/trtllm/engine + +python ../models/core/llama/convert_checkpoint.py --model_dir Meta-Llama-3-8B \ + --output_dir $CHECKPOINT_DIR \ + --dtype float16 + +trtllm-build --checkpoint_dir $CHECKPOINT_DIR \ + --output_dir $ENGINE_DIR \ + --gemm_plugin=auto \ + --lora_plugin=auto \ + --dora_plugin=enable \ + --lora_dir $NORMALIZED_DORA_ADAPTER +``` + +If you wish, you may provide additional LoRA / DoRA adapters to `trtllm-build`. + +**NOTE**: if you omit `--dora_plugin=enable`, you will not receive any warning even if you provide a DoRA adapter to `--lora_dir`. In such a case the DoRA magnitudes will simply be ignored during inference and you may receive wrong output. + +Proceed to execute the engine as you would a normal LoRA engine: + +``` bash +python ../run.py --engine_dir $ENGINE_DIR --tokenizer_dir Meta-Llama-3-8B --lora_task_uids 0 --max_output_len 32 --input_text ... +``` + +## Usage with Triton Server +Using DoRA over Triton is the same as using LoRA, but before using [hf_lora_convert.py](../hf_lora_convert.py), make sure you call [normalize_weights.py](./normalize_weights.py) and use the resulting normalized adapter. diff --git a/examples/dora/normalize_weights.py b/examples/dora/normalize_weights.py new file mode 100755 index 000000000000..40d5ae172431 --- /dev/null +++ b/examples/dora/normalize_weights.py @@ -0,0 +1,289 @@ +#! /usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This script applies preprocessing to the DoRA magnitude vector to speed up inference. +DoRA applies columnwise normalization and scaling to the LoRA output. +By applying the normalization to the scaling vector, we can skip calculating the normalization vector at inference time. +""" +import abc +import enum +import json +from pathlib import Path + +import numpy as np +import safetensors.torch as st +import torch + +StateDict = dict[str, torch.Tensor] + +PEFT_MODULE_PREFIX = "base_model.model." +PEFT_MODULE_SUFFIXES = [".lora_A.weight", ".lora_B.weight"] +DORA_VECTOR_SUFFIXES = [ + ".lora_magnitude_vector", # HF peft + ".weight_m_wdecomp.weight" # NVLabs +] + + +class HFWeightsReader(abc.ABC): + + @abc.abstractmethod + def __init__(self, model_dir: str) -> None: + ... + + @abc.abstractmethod + def get_weight(self, weight_name: str) -> torch.Tensor: + ... + + @abc.abstractmethod + def get_all(self) -> StateDict: + ... + + +class HFSafeTensorsReader(HFWeightsReader): + + def __init__(self, model_dir: str) -> None: + self.model_dir = Path(model_dir) + + self._fds = [ + st.safe_open(f, framework="torch") + for f in self.model_dir.glob("*.safetensors") + ] + + self._weight_to_fd = {} + for f in self._fds: + for k in f.keys(): + self._weight_to_fd[k] = f + + def get_weight(self, weight_name: str) -> torch.Tensor: + return self._weight_to_fd[weight_name].get_tensor(weight_name) + + def get_all(self) -> StateDict: + return {k: self.get_weight(k) for k in self._weight_to_fd.keys()} + + +class HFBinWeightsReader(HFWeightsReader): + + def __init__(self, model_dir: str) -> None: + self.model_dir = Path(model_dir) + + self._weights = {} + + for f in self.model_dir.glob("*.bin"): + self._weights.update( + torch.load(f, weights_only=True, mmap=True, map_location="cpu")) + + def get_weight(self, weight_name: str) -> torch.Tensor: + weight_name = f"{weight_name}.weight" + return self._weights[weight_name] + + def get_all(self) -> StateDict: + return self._weights + + +class WeightsFormat(enum.Enum): + BINARY = enum.auto() + SAFETENSORS = enum.auto() + UNKNOWN = enum.auto() + + def __str__(self) -> str: + if self == self.BINARY: + return "bin" + elif self == self.SAFETENSORS: + return "safetensors" + return "unknown" + + +def deduce_weights_format(model_dir: str) -> WeightsFormat: + model_dir_p = Path(model_dir) + + if any(model_dir_p.glob("*.safetensors")): + return WeightsFormat.SAFETENSORS + elif any(model_dir_p.glob("*.bin")): + return WeightsFormat.BINARY + return WeightsFormat.UNKNOWN + + +def get_weights_reader(model_dir: str) -> HFWeightsReader: + model_dir_p = Path(model_dir) + + if not model_dir_p.is_dir(): + raise ValueError( + f"{model_dir} is not a valid model directory: not found") + + weights_format = deduce_weights_format(model_dir) + + if weights_format == WeightsFormat.SAFETENSORS: + return HFSafeTensorsReader(model_dir) + elif weights_format == WeightsFormat.BINARY: + return HFBinWeightsReader(model_dir) + else: + raise ValueError( + f"{model_dir} does not contain .safetensors or .bin weights") + + +def normalize_hf_peft_module_name(module_name: str) -> str: + """ + Remove parts of the module name in the peft adapter to derive the module name of the base model. + """ + + if module_name.startswith(PEFT_MODULE_PREFIX): + module_name = module_name[len(PEFT_MODULE_PREFIX):] + + for suffix in PEFT_MODULE_SUFFIXES + DORA_VECTOR_SUFFIXES: + if module_name.endswith(suffix): + module_name = module_name[:-len(suffix)] + + return module_name + + +def get_peft_module_names(base_module_name: str) -> tuple[str, ...]: + """ + Convert the name of a base module to the names of its LoRA A and LoRA B weights. + """ + return tuple([ + f"{PEFT_MODULE_PREFIX}{base_module_name}{suffix}" + for suffix in PEFT_MODULE_SUFFIXES + ]) + + +def get_dora_magnitude_names(base_module_name: str) -> tuple[str, ...]: + """ + Convert the name of a base module to the potential names of its DoRA magnitude vectors. + """ + return tuple([ + f"{PEFT_MODULE_PREFIX}{base_module_name}{suffix}" + for suffix in DORA_VECTOR_SUFFIXES + ]) + + +def normalize_dora_vector(W: torch.Tensor, A: torch.Tensor, B: torch.Tensor, + mag: torch.Tensor, scale: float) -> torch.Tensor: + return mag / torch.linalg.norm(W + scale * B @ A, dim=1).to(W.dtype) + + +def normalize_dora_scales(lora_sd: StateDict, + weights_reader: HFWeightsReader, + alpha: float, + use_rslora: bool, + strip: bool = False) -> StateDict: + out_sd = {} + + while lora_sd: + # take some lora weight name + module_name = next(iter(lora_sd.keys())) + base_module_name = normalize_hf_peft_module_name(module_name) + A_name, B_name = get_peft_module_names(base_module_name) + magnitude_names = get_dora_magnitude_names(base_module_name) + + if module_name not in [A_name, B_name] + list(magnitude_names): + raise ValueError(f"Encountered unknown weight: {module_name}") + + # get lora weights + A = lora_sd.pop(A_name) + B = lora_sd.pop(B_name) + for name in magnitude_names: + if name in lora_sd: + mag_name = name + mag = lora_sd.pop(mag_name).view(-1) + break + else: + mag_name = "" + mag = None + + out_sd[A_name] = A.contiguous() + out_sd[B_name] = B.contiguous() + + if mag is not None and not strip: + # get base weight and normalize + W = weights_reader.get_weight(base_module_name + ".weight") + + adapter_size = A.size(0) + + if use_rslora: + scale = alpha / np.sqrt(adapter_size) + else: + scale = alpha / adapter_size + + mag = normalize_dora_vector(W, A, B, mag, scale) + out_sd[mag_name] = mag.contiguous() + + return out_sd + + +def save_state_dict(out_file: str, sd: StateDict) -> None: + out_path = Path(out_file) + + if out_path.suffix == ".safetensors": + st.save_file(sd, out_path) + elif out_path.suffix == ".bin": + torch.save(sd, out_path) + else: + raise ValueError(f"Unregornized weights format: {out_path.suffix}") + + +def normalize_peft_ckpt(model_dir: str, + base_model_dir: str, + out_dir: str, + strip: bool = False) -> None: + out_path = Path(out_dir) + out_path.mkdir(parents=True, exist_ok=True) + + weights_reader = get_weights_reader(base_model_dir) + lora_sd = get_weights_reader(model_dir).get_all() + + adapter_config_path = Path(f"{model_dir}/adapter_config.json") + with adapter_config_path.open() as f: + adapter_config = json.load(f) + + alpha = adapter_config["lora_alpha"] + use_rslora = adapter_config.get("use_rslora", False) + + new_sd = normalize_dora_scales(lora_sd, weights_reader, alpha, use_rslora, + strip) + + with (out_path / "adapter_config.json").open("w") as f: + json.dump(adapter_config, f) + + weights_format = deduce_weights_format(model_dir) + save_state_dict( + (out_path / f"adapter_model.{str(weights_format)}").as_posix(), new_sd) + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument( + '--out-dir', + '-o', + type=Path, + help='path to output adapter weights with normalized DoRA vectors', + required=True) + parser.add_argument('--in-dir', + '-i', + type=Path, + help='path to input lora checkpoint file', + required=True) + parser.add_argument("--base-model", + "-b", + help="Path to base model", + required=True) + parser.add_argument("--strip", + action="store_true", + help="remove DoRA vectors entirely") + + args = parser.parse_args() + + normalize_peft_ckpt(args.in_dir, args.base_model, args.out_dir, args.strip) diff --git a/examples/eval_long_context.py b/examples/eval_long_context.py new file mode 100644 index 000000000000..90b7ef2dd270 --- /dev/null +++ b/examples/eval_long_context.py @@ -0,0 +1,331 @@ +# MIT License + +# Copyright (c) 2023 OpenBMB + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# reference: https://github.com/OpenBMB/InfiniteBench/blob/main/src/eval_yarn_mistral.py + +import argparse +import ast +import json +from pathlib import Path + +import torch +from infinitebench.compute_scores import compute_scores +from infinitebench.eval_utils import (DATA_NAME_TO_MAX_NEW_TOKENS, + create_prompt, dump_jsonl, get_answer, + load_data) +from utils import (DEFAULT_HF_MODEL_DIRS, DEFAULT_PROMPT_TEMPLATES, + add_common_args, load_tokenizer, read_model_name) + +import tensorrt_llm +import tensorrt_llm.profiler as profiler +from tensorrt_llm.logger import logger +from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelRunner + +if PYTHON_BINDINGS: + from tensorrt_llm.runtime import ModelRunnerCpp + +MAX_POSITION_ID = 128 * 1024 # Determined by the model +TRUNCATE_LEN = 128 * 1024 + + +def parse_arguments(args=None): + parser = argparse.ArgumentParser() + parser.add_argument('--batch_size', type=int, default=1) + parser.add_argument('--max_input_length', type=int, default=923) + parser.add_argument('--output_log_probs_npy', + type=str, + help='Numpy file where the log_probs are stored', + default=None) + + parser.add_argument('--output_cum_log_probs_npy', + type=str, + help='Numpy file where the cum_log_probs are stored', + default=None) + + parser.add_argument( + "--task", + type=str, + choices=['passkey', 'kv_retrieval'], + required=True, + help= + "Which task to use. Note that \"all\" can only be used in `compute_scores.py`.", # noqa + ) + parser.add_argument('--data_dir', + type=str, + default='./', + help="The directory of data.") + parser.add_argument("--output_dir", + type=str, + default=None, + help="Where to dump the prediction results.") # noqa + parser.add_argument( + "--start_idx", + type=int, + default=0, + help= + "The index of the first example to infer on. This is used if you want to evaluate on a (contiguous) subset of the data." + ) # noqa + parser.add_argument( + "--stop_idx", + type=int, + help= + "The index of the last example to infer on. This is used if you want to evaluate on a (contiguous) subset of the data. Defaults to the length of dataset." + ) # noqa + parser.add_argument('--tensorrt_llm_accuracy_threshold', + type=float, + default=99) + parser = add_common_args(parser) + + return parser.parse_args(args=args) + + +def parse_input(tokenizer, + input_text=None, + prompt_template=None, + add_special_tokens=True, + max_input_length=923, + pad_id=None, + num_prepend_vtokens=[], + model_name=None, + model_version=None): + if pad_id is None: + pad_id = tokenizer.pad_token_id + + batch_input_ids = [] + for curr_text in input_text: + if prompt_template is not None: + curr_text = prompt_template.format(input_text=curr_text) + input_ids = tokenizer.encode(curr_text, + add_special_tokens=add_special_tokens, + truncation=True, + max_length=max_input_length) + batch_input_ids.append(input_ids) + + if num_prepend_vtokens: + assert len(num_prepend_vtokens) == len(batch_input_ids) + base_vocab_size = tokenizer.vocab_size - len( + tokenizer.special_tokens_map.get('additional_special_tokens', [])) + for i, length in enumerate(num_prepend_vtokens): + batch_input_ids[i] = list( + range(base_vocab_size, + base_vocab_size + length)) + batch_input_ids[i] + + if 'GLM' in model_name and model_version == 'glm': + for ids in batch_input_ids: + ids.append(tokenizer.sop_token_id) + + batch_input_ids = [ + torch.tensor(x, dtype=torch.int32) for x in batch_input_ids + ] + return batch_input_ids + + +def main(args): + # model_name = "yarn-mistral" + runtime_rank = tensorrt_llm.mpi_rank() + logger.set_level(args.log_level) + + print(json.dumps(vars(args), indent=4)) + data_name = args.task + + # Model + max_tokens = DATA_NAME_TO_MAX_NEW_TOKENS[data_name] + + model_name, model_version = read_model_name(args.engine_dir) + if args.tokenizer_dir is None: + logger.warning( + "tokenizer_dir is not specified. Try to infer from model_name, but this may be incorrect." + ) + args.tokenizer_dir = DEFAULT_HF_MODEL_DIRS[model_name] + + tokenizer, pad_id, end_id = load_tokenizer( + tokenizer_dir=args.tokenizer_dir, + vocab_file=args.vocab_file, + model_name=model_name, + model_version=model_version, + tokenizer_type=args.tokenizer_type, + ) + + if not PYTHON_BINDINGS and not args.use_py_session: + logger.warning( + "Python bindings of C++ session is unavailable, fallback to Python session." + ) + args.use_py_session = True + if args.debug_mode and not args.use_py_session: + logger.warning( + "Debug mode is not supported in C++ session for now, fallback to Python session." + ) + args.use_py_session = True + runner_cls = ModelRunner if args.use_py_session else ModelRunnerCpp + runner_kwargs = dict( + engine_dir=args.engine_dir, + lora_dir=args.lora_dir, + rank=runtime_rank, + debug_mode=args.debug_mode, + lora_ckpt_source=args.lora_ckpt_source, + gpu_weights_percent=args.gpu_weights_percent, + ) + if args.medusa_choices is not None: + args.medusa_choices = ast.literal_eval(args.medusa_choices) + assert args.temperature == 1.0, "Medusa should use temperature == 1.0" + assert args.num_beams == 1, "Medusa should use num_beams == 1" + runner_kwargs.update(medusa_choices=args.medusa_choices) + if not args.use_py_session: + runner_kwargs.update( + max_batch_size=args.batch_size, + max_input_len=args.max_input_length, + max_output_len=max_tokens, + max_beam_width=args.num_beams, + max_attention_window_size=args.max_attention_window_size, + sink_token_length=args.sink_token_length, + max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, + kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, + kv_cache_free_gpu_memory_fraction=args. + kv_cache_free_gpu_memory_fraction, + enable_chunked_context=args.enable_chunked_context, + ) + runner = runner_cls.from_dir(**runner_kwargs) + + # Data + examples = load_data(data_name, data_dir=args.data_dir) + if args.stop_idx is None: + args.stop_idx = len(examples) + + output_path = None + if runtime_rank == 0: + if args.output_dir is not None: + result_dir = Path(args.output_dir, model_name) + result_dir.mkdir(exist_ok=True, parents=True) + + if args.stop_idx is None: + output_path = (result_dir / f"preds_{data_name}.jsonl") + else: + output_path = ( + result_dir / + f"preds_{data_name}_{args.start_idx}-{args.stop_idx}.jsonl" # noqa + ) + + prompt_template = None + if args.use_prompt_template and model_name in DEFAULT_PROMPT_TEMPLATES: + prompt_template = DEFAULT_PROMPT_TEMPLATES[model_name] + + if runtime_rank == 0: + preds = [] + logger.info("==== Evaluation ====") + logger.info(f"# examples: {len(examples)}") + logger.info(f"Start index: {args.start_idx}") + logger.info(f"Stop index: {args.stop_idx}") + logger.info(f"Max tokens: {max_tokens}") + assert args.batch_size == 1 + profiler.start('Evaluation') + for i in range(args.start_idx, args.stop_idx): + eg = examples[i] + input_text = [create_prompt(eg, data_name, args.data_dir)] + batch_input_ids = parse_input( + tokenizer=tokenizer, + input_text=input_text, + prompt_template=prompt_template, + add_special_tokens=args.add_special_tokens, + max_input_length=args.max_input_length, + pad_id=pad_id, + num_prepend_vtokens=args.num_prepend_vtokens, + model_name=model_name, + model_version=model_version) + input_lengths = [x.size(0) for x in batch_input_ids] + + if runtime_rank == 0: + logger.debug(f"====== Example {i} ======") + logger.debug(f"input_lengths: {input_lengths}") + logger.debug(f"input_text: {input_text}") + logger.debug(f"answer: {get_answer(eg, data_name)}") + outputs = runner.generate( + batch_input_ids, + max_new_tokens=max_tokens, + max_attention_window_size=args.max_attention_window_size, + sink_token_length=args.sink_token_length, + end_id=end_id, + pad_id=pad_id, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + num_beams=args.num_beams, + length_penalty=args.length_penalty, + early_stopping=args.early_stopping, + beam_width_array=args.beam_width_array, + repetition_penalty=args.repetition_penalty, + presence_penalty=args.presence_penalty, + frequency_penalty=args.frequency_penalty, + prompt_ignore_length=args.prompt_ignore_length, + # stop_words_list=stop_words_list, + # bad_words_list=bad_words_list, + output_cum_log_probs=(args.output_cum_log_probs_npy != None), + output_log_probs=(args.output_log_probs_npy != None), + lora_uids=args.lora_task_uids, + prompt_table=args.prompt_table_path, + prompt_tasks=args.prompt_tasks, + streaming=args.streaming, + output_sequence_lengths=True, + return_dict=True, + medusa_choices=args.medusa_choices) + torch.cuda.synchronize() + if runtime_rank == 0: + output_ids = outputs['output_ids'] + output_beams_list = [ + tokenizer.batch_decode(output_ids[batch_idx, :, + input_lengths[batch_idx]:], + skip_special_tokens=True) + for batch_idx in range(args.batch_size) + ] + + logger.debug(f"preds: {output_beams_list[0]}") + preds.append({ + "id": i, + "prediction": output_beams_list[0][0], + "ground_truth": get_answer(eg, data_name), + "input_lengths": input_lengths, + }) + if output_path is not None: + dump_jsonl(preds, output_path) + profiler.stop('Evaluation') + + if runtime_rank == 0: + logger.info( + f'Evaluation takes: {profiler.elapsed_time_in_sec("Evaluation")} sec.' + ) + logger.info("Compute the score") + acc = compute_scores(preds, args.task) * 100 + logger.info(f"{args.task} accuracy: {acc:.2f} ({len(preds)})") + + if args.tensorrt_llm_accuracy_threshold is not None: + assert acc >= args.tensorrt_llm_accuracy_threshold, f"acc ({acc}) < tensorrt_llm_accuracy_threshold ({args.tensorrt_llm_accuracy_threshold})" + + +if __name__ == "__main__": + args = parse_arguments() + main(args) diff --git a/examples/generate_checkpoint_config.py b/examples/generate_checkpoint_config.py new file mode 100644 index 000000000000..a11104eeba68 --- /dev/null +++ b/examples/generate_checkpoint_config.py @@ -0,0 +1,149 @@ +import argparse +import json +import os + +from tensorrt_llm.quantization import KV_CACHE_QUANT_ALGO_LIST, QUANT_ALGO_LIST + + +def parse_arguments(): + parser = argparse.ArgumentParser() + + parser.add_argument( + '--output_path', + type=str, + default='config.json', + help='The path to save the TensorRT LLM checkpoint config.json file') + parser.add_argument('--architecture', type=str, default='GPTForCausalLM') + parser.add_argument('--dtype', + type=str, + default='float16', + choices=['float32', 'bfloat16', 'float16']) + parser.add_argument('--vocab_size', type=int, default=32000) + parser.add_argument('--max_position_embeddings', type=int, default=1024) + parser.add_argument('--hidden_size', type=int, default=768) + parser.add_argument('--intermediate_size', type=int, default=None) + parser.add_argument('--num_hidden_layers', type=int, default=12) + parser.add_argument('--num_attention_heads', type=int, default=12) + parser.add_argument('--num_key_value_heads', type=int, default=None) + parser.add_argument('--hidden_act', type=str, default='gelu') + parser.add_argument('--norm_epsilon', type=float, default=1e-5) + parser.add_argument('--position_embedding_type', + type=str, + default='learned_absolute') + parser.add_argument( + '--use_parallel_embedding', + action='store_true', + default=False, + help= + 'By default embedding parallelism is disabled. By setting this flag, embedding parallelism is enabled' + ) + parser.add_argument( + '--embedding_sharding_dim', + type=int, + default=0, + choices=[0, 1], + help= + 'By default the embedding lookup table is sharded along vocab dimension (embedding_sharding_dim=0). ' + 'To shard it along hidden dimension, set embedding_sharding_dim=1' + 'Note: embedding sharing is only enabled when embedding_sharding_dim = 0' + ) + + parser.add_argument('--tp_size', + type=int, + default=1, + help='N-way tensor parallelism size') + parser.add_argument('--pp_size', + type=int, + default=1, + help='N-way pipeline parallelism size') + + parser.add_argument('--quant_algo', + type=str, + default=None, + choices=[None] + QUANT_ALGO_LIST) + parser.add_argument('--kv_cache_quant_algo', + type=str, + default=None, + choices=[None] + KV_CACHE_QUANT_ALGO_LIST) + parser.add_argument('--group_size', type=int, default=64) + parser.add_argument('--smoothquant_val', type=float, default=None) + parser.add_argument('--has_zero_point', default=False, action='store_true') + parser.add_argument('--pre_quant_scale', default=False, action='store_true') + parser.add_argument('--exclude_modules', nargs='+', default=None) + + parser.add_argument('--bias', default=False, action='store_true') + parser.add_argument('--apply_query_key_layer_scaling', + default=False, + action='store_true') + parser.add_argument('--rotary_pct', type=float, default=1.0) + parser.add_argument('--rotary_base', type=float, default=10000.0) + parser.add_argument('--rotary_scaling', nargs=2, type=str, default=None) + + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_arguments() + world_size = args.tp_size * args.pp_size + + assert args.output_path.endswith('.json') + output_dir = os.path.dirname(args.output_path) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir) + + config = { + 'architecture': args.architecture, + 'dtype': args.dtype, + 'vocab_size': args.vocab_size, + 'max_position_embeddings': args.max_position_embeddings, + 'hidden_size': args.hidden_size, + 'intermediate_size': args.intermediate_size, + 'num_hidden_layers': args.num_hidden_layers, + 'num_attention_heads': args.num_attention_heads, + 'num_key_value_heads': args.num_key_value_heads, + 'hidden_act': args.hidden_act, + 'norm_epsilon': args.norm_epsilon, + 'position_embedding_type': args.position_embedding_type, + 'use_parallel_embedding': args.use_parallel_embedding, + 'embedding_sharding_dim': args.embedding_sharding_dim, + 'quantization': { + 'quant_algo': args.quant_algo, + 'kv_cache_quant_algo': args.kv_cache_quant_algo, + 'exclude_modules': args.exclude_modules, + }, + 'mapping': { + 'world_size': world_size, + 'tp_size': args.tp_size, + 'pp_size': args.pp_size, + }, + 'bias': args.bias, + 'apply_query_key_layer_scaling': args.apply_query_key_layer_scaling, + 'rotary_pct': args.rotary_pct, + 'rotary_base': args.rotary_base, + 'rotary_scaling': args.rotary_scaling, + } + + if args.intermediate_size is None: + config['intermediate_size'] = args.hidden_size * 4 + if args.num_key_value_heads is None: + config['num_key_value_heads'] = args.num_attention_heads + + if args.quant_algo is not None: + if 'AWQ' in args.quant_algo or 'GPTQ' in args.quant_algo: + config['quantization'].update({ + 'group_size': + args.group_size, + 'has_zero_point': + args.has_zero_point, + 'pre_quant_scale': + args.pre_quant_scale, + }) + if 'SQ' in args.quant_algo: + config['quantization'].update({ + 'smoothquant_val': + args.smoothquant_val, + }) + + with open(args.output_path, 'w') as f: + json.dump(config, f, indent=4) diff --git a/examples/generate_xgrammar_tokenizer_info.py b/examples/generate_xgrammar_tokenizer_info.py new file mode 100644 index 000000000000..67e05eb9c968 --- /dev/null +++ b/examples/generate_xgrammar_tokenizer_info.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +from pathlib import Path + +from transformers import AutoTokenizer + +from tensorrt_llm.llmapi.tokenizer import _xgrammar_tokenizer_info + + +def generate_xgrammar_tokenizer_info(args): + + tokenizer = AutoTokenizer.from_pretrained(str(args.model_dir)) + tokenizer_info = _xgrammar_tokenizer_info(tokenizer) + + os.makedirs(args.output_dir, exist_ok=True) + with open(str(args.output_dir / "xgrammar_tokenizer_info.json"), 'w') as f: + json.dump(tokenizer_info, f) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--model_dir', + type=Path, + default=None, + required=True, + help="HF model directory") + parser.add_argument( + '--output_dir', + type=Path, + default=None, + required=True, + help="File path to save xgrammar's info. in json format") + args = parser.parse_args() + generate_xgrammar_tokenizer_info(args) diff --git a/examples/hf_lora_convert.py b/examples/hf_lora_convert.py new file mode 100755 index 000000000000..019d78a48563 --- /dev/null +++ b/examples/hf_lora_convert.py @@ -0,0 +1,266 @@ +#! /usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import datetime +import json +import logging +import re +from collections import defaultdict +from pathlib import Path + +import numpy as np +import torch + +from tensorrt_llm._utils import str_dtype_to_torch, torch_to_numpy +from tensorrt_llm.lora_manager import LoraManager +from tensorrt_llm.models.convert_utils import get_model_path, load_state_dict + +log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s" +logging.basicConfig(format=log_format) +LOGGER = logging.getLogger(__name__) + + +def save_val(val, dir, key, tp_num=None, write_npy=False): + ext = "npy" if write_npy else "bin" + suffix = ext if tp_num is None else f"{tp_num}.{ext}" + if write_npy: + np.save(dir / f"model.{key}.{suffix}", val) + else: + val.tofile(dir / f"model.{key}.{suffix}") + + +def get_all_lora_weights(lora_weights): + all_weights = defaultdict(lambda: defaultdict(dict)) + pattern = re.compile( + r'(.*\.layers\.([0-9]+)\.(self_attn|mlp)\.([a-z_]+))\.(?:lora_(?:(A|B)\.weight|(magnitude)_vector)|weight_(m_wdecomp).weight).*' + ) + moe_pattern = re.compile( + r'(.*\.layers\.([0-9]+)\.(block_sparse_moe)\.((experts)\.([0-9]+)\.|)([a-zA-Z0-9_]+))\.(?:lora_(?:(A|B)\.weight|(magnitude)_vector)|weight_(m_wdecomp).weight).*' + ) + for key, weights in lora_weights.items(): + m = pattern.match(key) + m_moe = moe_pattern.match(key) + if m: + layer_idx = int(m.group(2)) + hf_module = m.group(4) + inout = m.group(5) + dora_magnitude = m.group(6) or m.group(7) + + if inout: + inout = "in" if inout == "A" else "out" + all_weights[layer_idx][hf_module][inout] = weights + elif dora_magnitude: + LOGGER.warning( + "Detected DoRA magnitude vector, make sure it was preprocessed and normalized using the proper base model weights" + ) + all_weights[layer_idx][hf_module]["magnitude"] = weights.view( + -1) + + elif m_moe: + layer_idx = int(m_moe.group(2)) + hf_module = m_moe.group(7) + inout = m_moe.group(8) + dora_magnitude = m_moe.group(9) or m.group(10) + + if inout: + inout = "in" if inout == "A" else "out" + all_weights[layer_idx][hf_module][inout] = weights + elif dora_magnitude: + LOGGER.warning( + "Detected DoRA magnitude vector, make sure it was preprocessed and normalized using the proper base model weights" + ) + all_weights[layer_idx][hf_module]["magnitude"] = weights.view( + -1) + else: + print(f"no match {key}") + continue + return all_weights + + +def preprocess_lora_weights(lora_model): + # Swap weights of gate_up_proj + for key, value in lora_model.items(): + if "gate_up_proj.lora_B.weight" in key: + print("Swap {}".format(key)) + original_weights = value.contiguous().clone() + half_split = original_weights.shape[0] // 2 + first_half = original_weights[:half_split, :] + second_half = original_weights[half_split:, :] + value = torch.cat((second_half, first_half), dim=0) + lora_model[key] = value + return lora_model + + +hf_modules_to_trtllm_modules = { + "q_proj": "attn_q", + "v_proj": "attn_v", + "k_proj": "attn_k", + "qkv_proj": "attn_qkv", + "query_key_value": "attn_qkv", + "o_proj": "attn_dense", + "dense": "attn_dense", + "gate_proj": "mlp_h_to_4h", + "down_proj": "mlp_4h_to_h", + "up_proj": "mlp_gate", + "gate_up_proj": "mlp_h_to_4h", + "c_fc": "mlp_h_to_4h", + "c_proj": "mlp_4h_to_h", + "w1": "moe_h_to_4h", + "w2": "moe_4h_to_h", + "w3": "moe_gate", + "gate": "moe_router", +} # lora modules on llama +hf_modules_to_module_id = { + k: LoraManager.LORA_MODULE_IDS[v] + for k, v in hf_modules_to_trtllm_modules.items() +} + + +def convert_hf_model(model_dir, dtype, out_dir): + saved_dir = Path(out_dir) + saved_dir.mkdir(parents=True, exist_ok=True) + with open(f"{model_dir}/adapter_config.json", "r") as f: + config = json.load(f) + + alpha = config.get("lora_alpha") + use_rslora = config.get("use_rslora", False) + + lora_model = load_state_dict(get_model_path(model_dir, "adapter_model")) + lora_model = preprocess_lora_weights(lora_model) + all_weights = get_all_lora_weights(lora_model) + converted_weights = [] + converted_config = [] + + def derive_adapter_size(inout_weight: torch.Tensor) -> int: + assert len(inout_weight.shape) == 2 + dim0, dim1 = inout_weight.shape + # assume the hidden dim is the larger of the 2 + adapter_size = min(dim0, dim1) + return adapter_size + + def derive_weights_scale(adapter_size: int, alpha: float, + use_rslora: bool) -> float: + if use_rslora: + return alpha / np.sqrt(adapter_size) + return alpha / adapter_size + + for layer_idx, layer_weights in all_weights.items(): + for hf_module, module_weights in layer_weights.items(): + in_weights = module_weights['in'] + out_weights = module_weights['out'] + magnitude = module_weights.get("magnitude", None) + is_dora = magnitude is not None + + processed_weights = [] + + assert len(in_weights.shape) == 2 + assert len(out_weights.shape) == 2 + assert not is_dora or len(magnitude.shape) == 1 + + adapter_size = derive_adapter_size(in_weights) + assert adapter_size == derive_adapter_size( + out_weights), "adapter size of A mismatches adapter size of B" + scale = derive_weights_scale(adapter_size, alpha, use_rslora) + + for w, inout in ((in_weights, "in"), (out_weights, "out")): + dim0 = w.shape[0] + dim1 = w.shape[1] + # in_weights should have shape [adaper_size, hidden] + if dim1 < dim0 and inout == "in": + w = w.transpose(1, 0) + # out_weights should have shape [hidden, adapter_size] + elif dim0 < dim1 and inout == "out": + w = w.transpose(1, 0) + if inout == "out": + w = w * scale + w = w.contiguous().flatten().to(dtype=str_dtype_to_torch(dtype)) + processed_weights.append(w) + + if is_dora: + processed_weights.append(magnitude.contiguous().flatten().to( + dtype=str_dtype_to_torch(dtype))) + + processed_weights = torch.concatenate(processed_weights).flatten() + converted_weights.append(processed_weights) + converted_config.append([ + hf_modules_to_module_id[hf_module], layer_idx, adapter_size, + 1 if is_dora else 0 + ]) + max_row_size = 0 + for t in converted_weights: + max_row_size = max(max_row_size, t.shape[0]) + for i in range(len(converted_weights)): + converted_weights[i] = torch.nn.functional.pad( + converted_weights[i], + (0, max_row_size - converted_weights[i].shape[0])).unsqueeze(0) + converted_weights = torch_to_numpy( + torch.concatenate( + converted_weights, + dim=0).unsqueeze(0).to(dtype=str_dtype_to_torch(dtype)).cpu()) + converted_config = torch.tensor(converted_config, + dtype=torch.int32, + device='cpu').unsqueeze(0).numpy() + + save_val(converted_weights, + saved_dir, + "lora_weights", + tp_num=None, + write_npy=True) + save_val(converted_config, + saved_dir, + "lora_config", + tp_num=None, + write_npy=True) + + +def main(args): + start_time = datetime.datetime.now() + convert_hf_model(args.in_file, args.storage_type, args.out_dir) + + LOGGER.info("Spent %s (h:m:s) to convert the prompt model", + datetime.datetime.now() - start_time) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + '--out-dir', + '-o', + type=Path, + help='path to output embedding table file in the .npy format', + required=True) + parser.add_argument('--in-file', + '-i', + type=Path, + help='path to input lora checkpoint file', + required=True) + parser.add_argument("--verbose", + action="store_true", + help="Provide verbose messages") + parser.add_argument("--storage-type", + type=str, + default="float16", + choices=["float32", "float16", "bfloat16"]) + args = parser.parse_args() + + LOGGER.setLevel(logging.DEBUG if args.verbose else logging.INFO) + + print("\n=============== Argument ===============") + for key in vars(args): + print(f"{key}: {vars(args)[key]}") + print("========================================") + + main(args) diff --git a/examples/language_adapter/README.md b/examples/language_adapter/README.md new file mode 100755 index 000000000000..8487c8ab42a0 --- /dev/null +++ b/examples/language_adapter/README.md @@ -0,0 +1,99 @@ +# Language-Adapter + +This document shows how to build and run a model with Language-Adapter plugin in TensorRT LLM on NVIDIA GPUs. + +## Overview +The concept of Language Adapter during inference time was introduced in [MAD-X: An Adapter-Based Framework for Multi-Task Cross-Lingual Transfer +](https://arxiv.org/pdf/2005.00052): +> we can simply replace a language-specific adapter trained for English with a language-specific adapter trained for Quechua at inference time. + +The implementation is done with MOE plugin with static expert selection passed during runtime as a parameter in request. + +For instance, encoder-decoder model may leverage language adapter for language-specific translation tasks when each of the language-adapter is trained for a specific language, this language adapter plugin achieves the language switching within one session only by passing in the `language_task_uid` to the plugin. + +The model checkpoint here is not publicly available. Please leverage `layers/language_adapter.py` in your own model. + +### Engine Preparation (convert and build) +``` +MODEL_DIR="dummy_model" # model not publicly available +INFERENCE_PRECISION="float16" +TP_SIZE=1 +PP_SIZE=1 +WORLD_SIZE=1 +MODEL_TYPE=language_adapter +MODEL_NAME=$MODEL_TYPE +CKPT_DIR=/scratch/tmp/trt_models/${MODEL_NAME}/${WORLD_SIZE}-gpu/${INFERENCE_PRECISION} +ENGINE_DIR=/scratch/tmp/trt_engines/${MODEL_NAME}/${WORLD_SIZE}-gpu/${INFERENCE_PRECISION} + +max_beam=5 +max_batch=32 +max_input_len=1024 +max_output_len=1024 + +python ../enc_dec/convert_checkpoint.py --model_type ${MODEL_TYPE} \ + --model_dir ${MODEL_DIR} \ + --output_dir $CKPT_DIR \ + --tp_size ${TP_SIZE} \ + --pp_size ${PP_SIZE} \ + --dtype ${INFERENCE_PRECISION} \ + --workers 1 + +trtllm-build --checkpoint_dir $CKPT_DIR/encoder \ + --output_dir $ENGINE_DIR/encoder \ + --paged_kv_cache disable \ + --moe_plugin auto \ + --bert_attention_plugin ${INFERENCE_PRECISION} \ + --gpt_attention_plugin ${INFERENCE_PRECISION} \ + --gemm_plugin ${INFERENCE_PRECISION} \ + --remove_input_padding enable \ + --max_input_len ${max_input_len} \ + --max_beam_width ${max_beam} \ + --max_batch_size ${max_batch} + +trtllm-build --checkpoint_dir $CKPT_DIR/decoder \ + --output_dir $ENGINE_DIR/decoder \ + --paged_kv_cache enable \ + --moe_plugin auto \ + --bert_attention_plugin ${INFERENCE_PRECISION} \ + --gpt_attention_plugin ${INFERENCE_PRECISION} \ + --gemm_plugin ${INFERENCE_PRECISION} \ + --remove_input_padding enable \ + --max_input_len 1 \ + --max_beam_width ${max_beam} \ + --max_batch_size ${max_batch} \ + --max_seq_len ${max_output_len} +``` + +### CPP runtime +A list `language_task_uids` that includes the language_task_uid for each input prompt is required: +``` +# translate 2 sentence, 1 to France (language_task_uid=3) 1 to Spanish (language_task_uid=2). +# language_task_uids = [3, 2] + +TEXT="Where is the nearest restaurant? Wikipedia is a free online encyclopedia written and maintained by a community of volunteers (called Wikis) through open collaboration and the use of MediaWiki, a wiki-based editing system." + +python3 ../run.py --engine_dir $ENGINE_DIR --tokenizer_type "language_adapter" --max_input_length 512 --max_output_len 512 --num_beams 1 --input_file input_ids.npy --tokenizer_dir $MODEL_DIR --language_task_uids 3 2 + +# Input [Text 0]: "" +# Output [Text 0 Beam 0]: "Où se trouve le restaurant le plus proche ? Wikipédia est une encyclopédie en ligne gratuite écrite et maintenue par une communauté de bénévoles (appelés Wikis) grâce à une collaboration ouverte et à l'utilisation de MediaWiki, un système d'édition basé sur wiki." +# Input [Text 1]: "" +# Output [Text 1 Beam 0]: "¿Dónde está el restaurante más cercano? Wikipedia es una enciclopedia en línea gratuita escrita y mantenida por una comunidad de voluntarios (llamada Wikis) a través de la colaboración abierta y el uso de MediaWiki, un sistema de edición basado en wiki." + +``` + +### Python runtime +Currently Python runtime does not support beam_width > 1. + +For Python runtime, full routing information of length [num_tokens, 1] is required for both encoder and decoder, which stacks routing information for each token in a batch of requests. +``` +# language_adapter_routing = get_language_adapter_routings(language_task_uid, input_ids) + +TEXT="Where is the nearest restaurant? Wikipedia is a free online encyclopedia written and maintained by a community of volunteers (called Wikis) through open collaboration and the use of MediaWiki, a wiki-based editing system." + +python3 ../enc_dec/run.py --engine_dir $ENGINE_DIR --engine_name ${MODEL_NAME} --model_name $MODEL_DIR --max_new_token=64 --num_beams=1 + +# in the run.py, 2 input prompts and 2 language task uids are provided. The two task uid represent the language of the input prompts to be translated to. + +# TRT-LLM output text: ['¿Dónde está el restaurante más cercano? Wikipedia es una enciclopedia en línea gratuita escrita y mantenida por una comunidad de voluntarios (llamada Wikis) a través de la colaboración abierta y el uso de MediaWiki, un sistema de edición basado en wiki.', "Où se trouve le restaurant le plus proche ? Wikipédia est une encyclopédie en ligne gratuite é +crite et maintenue par une communauté de bénévoles (appelés Wikis) grâce à une collaboration ouverte et à l'utilisation de MediaWiki, un système d'édition basé sur wiki."] +``` diff --git a/examples/layer_wise_benchmarks/sample_performance_alignment.sh b/examples/layer_wise_benchmarks/sample_performance_alignment.sh index 30140c280f9b..812fe46f0024 100755 --- a/examples/layer_wise_benchmarks/sample_performance_alignment.sh +++ b/examples/layer_wise_benchmarks/sample_performance_alignment.sh @@ -14,9 +14,8 @@ export TLLM_AUTOTUNER_CACHE_PATH="$PROFILE_DIR/sample_performance_alignment_cach mkdir -p -- "$PROFILE_DIR" mkdir -p -- "$(dirname -- "$TLLM_AUTOTUNER_CACHE_PATH")" -trtllm-bench \ - --model "$MODEL" \ - prepare-dataset \ +python3 ../../benchmarks/cpp/prepare_dataset.py \ + --tokenizer "$MODEL" \ --stdout \ --random-seed 42 \ token-norm-dist \ diff --git a/examples/llm-api/quickstart_advanced.py b/examples/llm-api/quickstart_advanced.py index d2acd490d44a..718d4a410e2c 100644 --- a/examples/llm-api/quickstart_advanced.py +++ b/examples/llm-api/quickstart_advanced.py @@ -310,9 +310,6 @@ def setup_llm(args, **kwargs): relaxed_topk=args.relaxed_topk, relaxed_delta=args.relaxed_delta, mtp_eagle_one_model=args.use_one_model, - use_dynamic_tree=args.use_dynamic_tree, - dynamic_tree_max_topK=args.dynamic_tree_max_topK, - max_total_draft_tokens=args.max_total_draft_tokens, speculative_model=args.model_dir) elif spec_decode_algo == "EAGLE3": spec_config = Eagle3DecodingConfig( diff --git a/examples/llm-eval/lm-eval-harness/README.md b/examples/llm-eval/lm-eval-harness/README.md new file mode 100644 index 000000000000..c3854654dd0d --- /dev/null +++ b/examples/llm-eval/lm-eval-harness/README.md @@ -0,0 +1,67 @@ +# Evaluation scripts for LLM tasks + +This folder includes code to use the [LM-Eval-Harness](https://github.com/EleutherAI/lm-evaluation-harness), a unified framework to test generative language models on a large number of different evaluation tasks. The supported eval tasks are [here](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks). + +The following instructions show how to evaluate TRT-LLM engines with the benchmark. + +## Instructions + +### TRT-LLM API + +Build the TRT-LLM engine using `trtllm-build`. + +Install the `lm_eval` package in the `requirements.txt` file in this folder. + +Run the evaluation script with the following command: + +```sh +python lm_eval_tensorrt_llm.py --model trt-llm \ + --model_args tokenizer=,model=,chunk_size= \ + --tasks +``` + +In the LM-Eval-Harness, model args are submitted as a comma-separated list of the form `arg=value`. The `trt-llm` model supports the following `model_args`: + +| Name | Description | Default Value | +|--------------------------|-------------------------------------------------------------------|----------------| +| tokenizer | directory containing the HF tokenizer. | | +| model | directory containing the TRTLLM engine or torch model. | | +| max_gen_toks | max number of tokens to generate (if not specified in gen_kwargs) | 256 | +| chunk_size | number of async requests to send at once to the engine | 200 | +| max_tokens_kv_cache | max tokens in paged KV cache | None | +| free_gpu_memory_fraction | KV cache free GPU memory fraction | 0.9 | +| trust_remote_code | trust remote code; use if necessary to set up the tokenizer | False | +| tp | tensor parallel size (for torch backend) | no. of workers | +| use_cuda_graph | enable CUDA graph | True | +| max_context_length | maximum context length for evaluation | None | +| moe_expert_parallel_size | expert parallel size for MoE models | None | +| moe_backend | backend for MoE models (e.g., "TRTLLM") | "TRTLLM" | + +### Torch backend + +Install the `lm_eval` package in the `requirements.txt` file in this folder. + +Run the evaluation script with the same command as above, but include `backend=torch` in the `model_args`. For example: + +```sh +python lm_eval_tensorrt_llm.py --model trt-llm \ + --model_args model=,backend=torch,chunk_size= \ + --tasks +``` + +### trtllm-serve + +Build the TRT-LLM engine using `trtllm-build` and deploy with `trtllm-serve`. + +Install the `lm_eval` package in the `requirements.txt` file in this folder. + +Run the evaluation script with the following command: + +```sh +python lm_eval_tensorrt_llm.py --model local-completions \ + --model_args base_url=http://${HOST_NAME}:8001/v1/completions,model=,tokenizer= \ + --tasks \ + --batch_size <#> +``` + +Because `trtllm-serve` is OpenAI API compatible, we can use the `local-completions` model built in to `lm_eval`, which supports [these model_args](https://github.com/EleutherAI/lm-evaluation-harness/blob/v0.4.7/lm_eval/models/openai_completions.py#L12). diff --git a/examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py b/examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py new file mode 100644 index 000000000000..1738242267d9 --- /dev/null +++ b/examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py @@ -0,0 +1,317 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import gc +import json +import logging +import os +import signal +import threading +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import torch +import torch.nn.functional as F +import transformers +from lm_eval.__main__ import cli_evaluate +from lm_eval.api.model import TemplateLM +from lm_eval.api.registry import register_model +from packaging.version import parse +from tqdm import tqdm + +import tensorrt_llm +from tensorrt_llm import LLM as TORCH_LLM +from tensorrt_llm._tensorrt_engine import LLM as TRT_LLM +from tensorrt_llm.bindings.executor import DecodingConfig +from tensorrt_llm.llmapi import KvCacheConfig as TRT_KvCacheConfig +from tensorrt_llm.llmapi import RequestOutput, SamplingParams +from tensorrt_llm.llmapi.llm_args import MoeConfig + +logger = logging.getLogger(__name__) + + +@register_model("trt-llm") +class TRTLLMEvalBase(TemplateLM): + + def __init__( + self, + model: str, + tokenizer: Optional[str] = None, + tp: int = 0, # tensor_parallel_size + max_gen_toks: int = 256, + chunk_size: int = 200, + max_tokens_kv_cache: Optional[int] = None, + free_gpu_memory_fraction: float = 0.9, + trust_remote_code: bool = False, + use_cuda_graph: bool = True, + backend: str = 'trt', + max_context_length: Optional[int] = None, + moe_expert_parallel_size: Optional[int] = None, + moe_backend: Optional[str] = "TRTLLM", + enable_chunked_prefill: bool = False, + max_num_tokens: Optional[int] = None, + **kwargs, + ): + # initialize TemplateLM, copied from TemplateAPI + super().__init__() + assert isinstance(model, str) + assert parse(tensorrt_llm.__version__) >= parse("0.15.0") + + self.max_gen_toks = max_gen_toks + self.chunk_size = chunk_size + self.backend = backend + self.max_context_length = max_context_length + self.moe_expert_parallel_size = moe_expert_parallel_size + self.moe_backend = moe_backend + trt_kv_cache_config = TRT_KvCacheConfig(enable_block_reuse=False) + trt_kv_cache_config.free_gpu_memory_fraction = free_gpu_memory_fraction + if max_tokens_kv_cache is not None: + trt_kv_cache_config.max_tokens = max_tokens_kv_cache + + if tokenizer is None: + # Assume the tokenizer is stored in the model_dir if not specified. + tokenizer = model + logger.info(f"Tokenizer: {tokenizer}") + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + tokenizer, trust_remote_code=trust_remote_code) + + if self.tokenizer.pad_token_id is None: + self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + + if self.backend == 'torch': + kwargs.pop('batch_size') + if tp < 1: + tp = torch.cuda.device_count() + + pytorch_config_params = { + 'cuda_graph_config': {} if use_cuda_graph else None, + "print_iter_log": False, + 'moe_config': MoeConfig(backend=self.moe_backend) + } + + # stop words not currently supported by torch backend + self.use_stop_words = False + + self.llm = TORCH_LLM( + model=model, + tensor_parallel_size=tp, + trust_remote_code=trust_remote_code, + enable_chunked_prefill=enable_chunked_prefill, + max_num_tokens=max_num_tokens, + **pytorch_config_params, + tokenizer=self.tokenizer, + kv_cache_config=trt_kv_cache_config, + moe_expert_parallel_size=self.moe_expert_parallel_size, + **kwargs) + logger.info("Loaded TRT-LLM Torch engine") + else: + with open(Path(model) / "config.json", "r") as engine_config_file: + engine_config = json.load(engine_config_file) + build_config = engine_config["build_config"] + world_size = (engine_config.get("pretrained_config", {}).get( + "mapping", {}).get("world_size", 1)) + if max_tokens_kv_cache is None: + max_tokens_kv_cache = build_config[ + "max_seq_len"] * build_config["max_batch_size"] + self.gather_context_logits = build_config.get( + "gather_context_logits", False) + + medusa_choices = kwargs[ + 'medusa_choices'] if 'medusa_choices' in kwargs else None + kwargs = {} + if medusa_choices is not None: + decoding_config = DecodingConfig() + decoding_config.medusa_choices = medusa_choices + kwargs["decoding_config"] = decoding_config + assert world_size == 1, "decoding_config does not support multi TP in HLAPI." + + self.llm = TRT_LLM(model=model, + tokenizer=self.tokenizer, + kv_cache_config=trt_kv_cache_config, + **kwargs) + self.max_length = build_config['max_seq_len'] - 1 + logger.info("Loaded TRT-LLM engine") + + @property + def eot_token_id(self) -> int: + return self.llm.tokenizer.eos_token_id + + def tok_encode(self, string, add_special_tokens=False, **kwargs): + return self.llm.tokenizer.encode(string, + add_special_tokens=add_special_tokens, + **kwargs) + + def _loglikelihood_tokens( + self, + requests: List[Any], + disable_tqdm: bool = False) -> List[Tuple[float, bool]]: + """Compute the log likelihood of the continuation given the context.""" + if self.backend == 'torch': + raise NotImplementedError( + 'Torch backend does not return context logits yet') + + num_r = len(requests) + desc = "Processing loglikelihood requests" + sampling_params = SamplingParams(max_tokens=1, + return_context_logits=True) + + # process requests + futures: Dict[int, RequestOutput] = {} + results = [] + for i, request in tqdm(enumerate(requests), + desc=desc, + total=num_r, + disable=disable_tqdm): + # asynchronously submit a chunk of requests ahead of time... + if i % self.chunk_size == 0: + for j in range(i, min(i + self.chunk_size, num_r)): + prompt_ids = requests[j][1] + requests[j][2] + futures[j] = self.llm.generate_async( + prompt_ids, sampling_params) + + # process the output of the request i + r_out: RequestOutput = futures.pop(i).result() + + # check continuation portion of the prompt + # NOTE: context_logits are offset by 1 since they predict future token + ctxlen = len(request[1]) + token_ids_cont = request[2] + logits_cont = r_out.context_logits[ctxlen - 1:-1] # [sl, vocab] + logprobs_cont = F.log_softmax(logits_cont, dim=-1) # [sl, vocab] + top_tokens_cont = logprobs_cont.argmax(dim=-1).tolist() # [sl] + + # compute logprob and check for greedy + logprob_sum = sum(logprobs_cont[list(range(len(logprobs_cont))), + token_ids_cont]).item() + is_greedy = top_tokens_cont == token_ids_cont + + results.append((logprob_sum, is_greedy)) + + # clear response + del r_out + + return results + + def loglikelihood_rolling(self, requests, disable_tqdm: bool = False): + raise NotImplementedError + + def generate_until(self, + requests: List[Any], + disable_tqdm: bool = False) -> List[str]: + # some book-keeping and parameters... + num_r = len(requests) + desc = "Processing generate requests" + + if self.max_context_length is not None: + """ + Create updated_requests to contain qualified requests with the context length <= max_context_length. + Unqualified requests cannot simply be dropped as lm-eval library requires the number of requests to be the same. + + Note: The final score will drop if disqualified requests exist. + """ + request_idx_to_replace = [] + qualified_requests = [] + updated_requests = [] + for i, request in enumerate(requests): + context, gen_kwargs = request.args + if len(self.tok_encode(context)) > self.max_context_length: + request_idx_to_replace.append(i) + else: + qualified_requests.append(request) + + assert len( + qualified_requests + ) > 1, "No requests with context length <= max_context_length. Cannot run the evaluation." + if len(request_idx_to_replace) > 0: + print( + f"Warning: {len(request_idx_to_replace)} requests with context length > max_context_length will be replaced. The final score will drop." + ) + + for i, request in enumerate(requests): + if i in request_idx_to_replace: + # Replace the requests with context length > max_context_length with the qualified requests + updated_requests.append( + qualified_requests[i % len(qualified_requests)]) + else: + updated_requests.append(request) + assert len( + updated_requests + ) == num_r, "Number of updated requests does not match the number of requests." + requests = updated_requests + + def _get_sp(gen_kwargs): + k_mapping = { + "temperature": "temperature", + "top_p": "top_p", + "max_gen_toks": "max_tokens", + "until": "stop", + } + kwargs_mapped = { + k_sp: gen_kwargs[k_gen] + for k_gen, k_sp in k_mapping.items() if k_gen in gen_kwargs + } + if "max_tokens" not in kwargs_mapped: + kwargs_mapped["max_tokens"] = self.max_gen_toks + return SamplingParams(**kwargs_mapped) + + # process requests + futures: Dict[int, RequestOutput] = {} + future_stop_words: Dict[int, RequestOutput] = {} + results = [] + for i, _ in tqdm(enumerate(requests), + desc=desc, + total=num_r, + disable=disable_tqdm): + # asynchronously submit a chunk of requests ahead of time... + if i % self.chunk_size == 0: + for j in range(i, min(i + self.chunk_size, num_r)): + context, gen_kwargs = requests[j].args + prompt_ids = self.tok_encode(context) + if self.max_context_length is not None: + assert len( + prompt_ids + ) <= self.max_context_length, f"Prompt length > {self.max_context_length}, {len(prompt_ids)}, should be filtered out." + kwargs_mapped = _get_sp(gen_kwargs) + futures[j] = self.llm.generate_async( + prompt_ids, kwargs_mapped) + del kwargs_mapped + future_stop_words[j] = gen_kwargs["until"] + + # process the output of the request i + r_out: RequestOutput = futures.pop(i).result() + stop_words = future_stop_words.pop(i) + txt = r_out.outputs[0].text + if stop_words: + for word in stop_words: + word_index = txt.find(word) + if word_index >= 0: + txt = txt[:word_index] + results.append(txt) + + return results + + +if __name__ == "__main__": + cli_evaluate() + # Force clean up the LLM instance and void hanging. + gc.collect() + + # Force terminate in case gc.collect() is not enough. + def _terminate(): + time.sleep(10) + os.kill(os.getpid(), signal.SIGTERM) + + termination_thread = threading.Thread(target=_terminate, daemon=True) + termination_thread.start() diff --git a/examples/llm-eval/lm-eval-harness/requirements.txt b/examples/llm-eval/lm-eval-harness/requirements.txt new file mode 100644 index 000000000000..44a3383f612d --- /dev/null +++ b/examples/llm-eval/lm-eval-harness/requirements.txt @@ -0,0 +1 @@ +lm_eval[api]==0.4.7 diff --git a/examples/mmlu.py b/examples/mmlu.py new file mode 100644 index 000000000000..82bb9a7ec9ad --- /dev/null +++ b/examples/mmlu.py @@ -0,0 +1,479 @@ +# SPDX-FileCopyrightText: Copyright (c) 2020 Dan Hendrycks +# SPDX-FileCopyrightText: Copyright (c) 2023 Deep Cognition and Language Research (DeCLaRe) Lab +# SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 and MIT +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Adapted from https://github.com/declare-lab/instruct-eval +Helper script to compare TRTLLM and HF models on the MMLU dataset. +Example usage: + mkdir data; wget https://people.eecs.berkeley.edu/~hendrycks/data.tar -O data/mmlu.tar + tar -xf data/mmlu.tar -C data && mv data/data data/mmlu + + python mmlu.py --hf_model_dir --engine_dir --test_trt_llm + python mmlu.py --hf_model_dir --engine_dir --test_hf +""" + +import argparse +import os +import random + +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +from tqdm import tqdm +from transformers import (AutoConfig, AutoModel, AutoModelForCausalLM, + AutoModelForSeq2SeqLM, AutoTokenizer, + GenerationConfig) +from utils import (add_common_args, load_tokenizer, prepare_enc_dec_inputs, + read_is_enc_dec, read_model_name) + +import tensorrt_llm +from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelRunner + +if PYTHON_BINDINGS: + from tensorrt_llm.runtime import ModelRunnerCpp + +os.environ["TOKENIZERS_PARALLELISM"] = "false" + +DTYPE_STR_MAPPING = { + "fp32": torch.float32, + "fp16": torch.float16, + "bf16": torch.bfloat16, + "float32": torch.float32, + "float16": torch.float16, + "bfloat16": torch.bfloat16, +} +RAND_SEED = 1234 + + +def get_choices(): + return ["A", "B", "C", "D"] + + +def get_subcategories(): + return { + "abstract_algebra": ["math"], + "anatomy": ["health"], + "astronomy": ["physics"], + "business_ethics": ["business"], + "clinical_knowledge": ["health"], + "college_biology": ["biology"], + "college_chemistry": ["chemistry"], + "college_computer_science": ["computer science"], + "college_mathematics": ["math"], + "college_medicine": ["health"], + "college_physics": ["physics"], + "computer_security": ["computer science"], + "conceptual_physics": ["physics"], + "econometrics": ["economics"], + "electrical_engineering": ["engineering"], + "elementary_mathematics": ["math"], + "formal_logic": ["philosophy"], + "global_facts": ["other"], + "high_school_biology": ["biology"], + "high_school_chemistry": ["chemistry"], + "high_school_computer_science": ["computer science"], + "high_school_european_history": ["history"], + "high_school_geography": ["geography"], + "high_school_government_and_politics": ["politics"], + "high_school_macroeconomics": ["economics"], + "high_school_mathematics": ["math"], + "high_school_microeconomics": ["economics"], + "high_school_physics": ["physics"], + "high_school_psychology": ["psychology"], + "high_school_statistics": ["math"], + "high_school_us_history": ["history"], + "high_school_world_history": ["history"], + "human_aging": ["health"], + "human_sexuality": ["culture"], + "international_law": ["law"], + "jurisprudence": ["law"], + "logical_fallacies": ["philosophy"], + "machine_learning": ["computer science"], + "management": ["business"], + "marketing": ["business"], + "medical_genetics": ["health"], + "miscellaneous": ["other"], + "moral_disputes": ["philosophy"], + "moral_scenarios": ["philosophy"], + "nutrition": ["health"], + "philosophy": ["philosophy"], + "prehistory": ["history"], + "professional_accounting": ["other"], + "professional_law": ["law"], + "professional_medicine": ["health"], + "professional_psychology": ["psychology"], + "public_relations": ["politics"], + "security_studies": ["politics"], + "sociology": ["culture"], + "us_foreign_policy": ["politics"], + "virology": ["health"], + "world_religions": ["philosophy"], + } + + +def get_categories(): + return { + "STEM": [ + "physics", + "chemistry", + "biology", + "computer science", + "math", + "engineering", + ], + "humanities": ["history", "philosophy", "law"], + "social sciences": [ + "politics", + "culture", + "economics", + "geography", + "psychology", + ], + "other (business, health, misc.)": ["other", "business", "health"], + } + + +def format_subject(subject): + line = subject.split("_") + s = "" + for entry in line: + s += " " + entry + return s + + +def format_example(df, idx, include_answer=True): + prompt = df.iloc[idx, 0] + k = df.shape[1] - 2 + for j in range(k): + prompt += "\n{}. {}".format(get_choices()[j], df.iloc[idx, j + 1]) + prompt += "\nAnswer:" + if include_answer: + prompt += " {}\n\n".format(df.iloc[idx, k + 1]) + return prompt + + +def gen_prompt(train_df, subject, k=-1): + prompt = "The following are multiple choice questions (with answers) about {}.\n\n".format( + format_subject(subject)) + if k == -1: + k = train_df.shape[0] + for i in range(k): + prompt += format_example(train_df, i) + return prompt + + +def evaluate(args, subject, pipeline, dev_df, test_df): + rank = tensorrt_llm.mpi_rank() + cors = [] + all_probs = [] + for i in range(test_df.shape[0]): + if i >= args.max_ite: + break + # get prompt and make sure it fits + k = args.ntrain + prompt_end = format_example(test_df, i, include_answer=False) + train_prompt = gen_prompt(dev_df, subject, k) + prompt = train_prompt + prompt_end + + while not pipeline.check_valid_length(prompt) and k > 0: + k -= 1 + train_prompt = gen_prompt(dev_df, subject, k) + prompt = train_prompt + prompt_end + + label = test_df.iloc[i, test_df.shape[1] - 1] + pred = pipeline(prompt) + + if rank == 0: + probs = [0 for _ in get_choices()] + cor = pred.strip().startswith(label) + cors.append(cor) + all_probs.append(probs) + + if rank == 0: + acc = np.mean(cors) + cors = np.array(cors) + + all_probs = np.array(all_probs) + print("Average accuracy {:.3f} - {}".format(acc, subject)) + + return cors, acc, all_probs + else: + return None, 0, None + + +def get_tokenizer(ckpt_path, max_seq_len): + print(f"Initializing tokenizer from {ckpt_path}") + tokenizer = AutoTokenizer.from_pretrained( + ckpt_path, + model_max_length=max_seq_len, + padding_side="left", + trust_remote_code=True, + ) + tokenizer.pad_token = tokenizer.eos_token + + return tokenizer + + +class Pipeline: + + def __init__(self, tokenizer, model, model_name, pad_id, end_id, + max_attention_window_size, is_enc_dec, hf_model_dir, + engine_dir): + self.tokenizer = tokenizer + self.model = model + self.model_name = model_name + self.pad_id = pad_id + self.end_id = end_id + self.max_attention_window_size = max_attention_window_size + self.output_len = 2 + self.is_enc_dec = is_enc_dec + self.decoder_start_token_id = None + self.engine_dir = engine_dir + if self.is_enc_dec: + self.decoder_start_token_id = AutoConfig.from_pretrained( + hf_model_dir).decoder_start_token_id + + def __call__(self, prompt): + rank = tensorrt_llm.mpi_rank() + # Run the model in batch size 1 and beam size 1 + inputs = self.tokenizer.encode(prompt, return_tensors="pt").squeeze(0) + batch_input_ids = [inputs] + + # For multi-choice tasks like MMLU, we don't need to adjust following parameters + output_len = self.output_len + top_k = 1 + top_p = 0.0 + + input_lengths = [x.size(0) for x in batch_input_ids] + + with torch.no_grad(): + if isinstance(self.model, nn.Module): + # Left padding for HF + max_length = max(input_lengths) + paddings = [ + torch.ones(max_length - l, dtype=torch.int32) * self.pad_id + for l in input_lengths + ] + batch_input_ids = [ + torch.cat([pad, x]) + for x, pad in zip(batch_input_ids, paddings) + ] + batch_input_ids = torch.stack(batch_input_ids) + batch_input_ids = batch_input_ids.cuda() + if self.is_enc_dec: + batch_decoder_input_ids = torch.IntTensor( + [[self.decoder_start_token_id]]).to('cuda') + batch_decoder_input_ids = batch_decoder_input_ids.repeat( + (batch_input_ids.shape[0], 1)) + + with torch.no_grad(): + # Use default temperature and top_k + outputs = self.model.generate( + batch_input_ids, + max_new_tokens=output_len, + top_k=top_k, + decoder_input_ids=batch_decoder_input_ids + if self.is_enc_dec else None) + if not self.is_enc_dec: + output_ids = outputs[0, input_lengths[0]:] + else: + output_ids = outputs[0] + + elif isinstance(self.model, ModelRunnerCpp) or isinstance( + self.model, ModelRunner): + if self.is_enc_dec: + encoder_input_ids, encoder_input_features, encoder_output_lengths, decoder_input_ids = prepare_enc_dec_inputs( + batch_input_ids, self.model_name, self.engine_dir, None) + + outputs = self.model.generate( + batch_input_ids=decoder_input_ids + if self.is_enc_dec else batch_input_ids, + encoder_input_ids=encoder_input_ids + if self.is_enc_dec else None, + encoder_input_features=encoder_input_features + if self.is_enc_dec else None, + encoder_output_lengths=encoder_output_lengths + if self.is_enc_dec else None, + max_new_tokens=output_len, + max_attention_window_size=self.max_attention_window_size, + end_id=self.end_id, + pad_id=self.pad_id, + top_k=top_k, + top_p=top_p, + ) + torch.cuda.synchronize() + if rank == 0: + if not self.is_enc_dec: + output_ids = outputs[0, 0, input_lengths[0]:] + else: + output_ids = outputs[0, 0] + if rank == 0: + return self.tokenizer.decode(output_ids, skip_special_tokens=True) + else: + return None + + def check_valid_length(self, prompt): + if isinstance(self.model, nn.Module): + return True + input_len = len(self.tokenizer.encode(prompt)) + return input_len <= self.model.max_input_len and input_len + self.output_len <= self.model.max_seq_len + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--data_dir", + type=str, + default="data/mmlu", + help=("Path to the data directory. If not available, " + "download https://people.eecs.berkeley.edu/~hendrycks/data.tar"), + ) + parser.add_argument("--ntrain", type=int, default=5) + parser.add_argument("--max_input_length", type=int, default=2048) + parser.add_argument("--test_trt_llm", action="store_true") + parser.add_argument("--test_hf", action="store_true") + parser.add_argument('--check_accuracy', action='store_true') + parser.add_argument('--accuracy_threshold', type=float, default=30) + parser.add_argument('--max_ite', type=int, default=10000000) + parser = add_common_args(parser) + + args = parser.parse_args() + + return args + + +def main(): + args = parse_args() + if args.tokenizer_dir is None: + args.tokenizer_dir = args.hf_model_dir + random.seed(RAND_SEED) + np.random.seed(RAND_SEED) + runtime_rank = tensorrt_llm.mpi_rank() + + os.path.dirname(os.path.abspath(__file__)) + data_fullpath = os.path.join(args.data_dir, "test") + + subjects = sorted([ + f.split("_test.csv")[0] for f in os.listdir(data_fullpath) + if "_test.csv" in f + ]) + + all_cors = [] + subcat_cors = { + subcat: [] + for subcat_lists in get_subcategories().values() + for subcat in subcat_lists + } + cat_cors = {cat: [] for cat in get_categories()} + + # different handling if encoder-decoder models + is_enc_dec = read_is_enc_dec( + args.engine_dir if not args.test_hf else args.hf_model_dir, + args.test_hf) + + model_name, model_version = read_model_name( + (args.engine_dir if not is_enc_dec else os.path.join( + args.engine_dir, 'encoder')) + if not args.test_hf else args.hf_model_dir, args.test_hf) + + tokenizer, pad_id, end_id = load_tokenizer( + tokenizer_dir=args.tokenizer_dir, + vocab_file=args.vocab_file, + model_name=model_name, + model_version=model_version, + ) + + if args.test_trt_llm: + assert not args.test_hf, "Cannot test both TRT-LLM and HF" + runner_cls = ModelRunner if not PYTHON_BINDINGS else ModelRunnerCpp + runner_kwargs = {} + if PYTHON_BINDINGS: + runner_kwargs.update(max_beam_width=1) + runner_kwargs.update( + is_enc_dec=is_enc_dec, + max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, + kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, + kv_cache_free_gpu_memory_fraction=args. + kv_cache_free_gpu_memory_fraction, + cross_kv_cache_fraction=args.cross_kv_cache_fraction + if is_enc_dec else None, + enable_chunked_context=args.enable_chunked_context, + multi_block_mode=args.multi_block_mode) + model = runner_cls.from_dir(engine_dir=args.engine_dir, + rank=runtime_rank, + **runner_kwargs) + else: + assert args.test_hf, "Must test either TRT-LLM or HF" + if 'GLM' in model_name and model_version == 'glm': + auto_model_cls = AutoModelForSeq2SeqLM + elif 'GLM' in model_name and model_version == 'chatglm': + auto_model_cls = AutoModel + elif is_enc_dec: + auto_model_cls = AutoModelForSeq2SeqLM + else: + auto_model_cls = AutoModelForCausalLM + model = auto_model_cls.from_pretrained( + args.hf_model_dir, + trust_remote_code=True, + dtype=DTYPE_STR_MAPPING[args.hf_data_type], + device_map="auto" if args.hf_device_map_auto else None, + ) + if not args.hf_device_map_auto: + model.cuda() + if model_name == "qwen": + model.generation_config = GenerationConfig.from_pretrained( + args.hf_model_dir, trust_remote_code=True) + + pipeline = Pipeline(tokenizer, model, model_name, pad_id, end_id, + args.max_attention_window_size, is_enc_dec, + args.hf_model_dir, args.engine_dir) + + for subject in tqdm(subjects): + dev_df = pd.read_csv(os.path.join(args.data_dir, "dev", + subject + "_dev.csv"), + header=None)[:args.ntrain] + test_df = pd.read_csv(os.path.join(args.data_dir, "test", + subject + "_test.csv"), + header=None) + + cors, acc, probs = evaluate(args, subject, pipeline, dev_df, test_df) + subcats = get_subcategories()[subject] + for subcat in subcats: + subcat_cors[subcat].append(cors) + for key in get_categories().keys(): + if subcat in get_categories()[key]: + cat_cors[key].append(cors) + all_cors.append(cors) + + if runtime_rank == 0: + for subcat in subcat_cors: + acc = np.mean(np.concatenate(subcat_cors[subcat])) * 100 + print(f"Average accuracy {acc:.2f} - {subcat}") + + for cat in cat_cors: + acc = np.mean(np.concatenate(cat_cors[cat])) * 100 + print(f"Average accuracy {acc:.2f} - {cat}") + + weighted_acc = np.mean(np.concatenate(all_cors)) * 100 + print(f"MMLU weighted average accuracy: {weighted_acc:.2f}") + + if args.check_accuracy: + assert weighted_acc >= args.accuracy_threshold, f"Expected accuracy >= {args.accuracy_threshold} while got {weighted_acc}" + return weighted_acc + + +if __name__ == "__main__": + main() diff --git a/examples/models/contrib/arctic/README.md b/examples/models/contrib/arctic/README.md new file mode 100644 index 000000000000..d346a3644517 --- /dev/null +++ b/examples/models/contrib/arctic/README.md @@ -0,0 +1,94 @@ +# Arctic + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document shows how to build and run a [Arctic](https://huggingface.co/Snowflake/snowflake-arctic-instruct) model in TensorRT-LLM. + +The TensorRT LLM Arctic implementation is based on the LLaMA model, with Mixture of Experts (MoE) enabled. The implementation can +be found in [llama/model.py](../../../../tensorrt_llm/models/llama/model.py). +See the LLaMA example [`examples/models/core/llama`](../../core/llama) for details. + +- [Arctic](#arctic) + - [Download model checkpoints](#download-model-checkpoints) + - [TensorRT LLM workflow](#tensorrt-llm-workflow) + - [Apply FP8 PTQ](#apply-fp8-ptq) + - [Build TensorRT engine](#build-tensorrt-engine) + - [Run Engine](#run-engine) + - [OOTB](#ootb) + +## Download model checkpoints + +First, download the HuggingFace BF16 checkpoints of Arctic model. + +**CAVEAT: this model is a pretty large Mixture-of-Experts (MoE) model, which has nearly 500B parameters and requires around 900GB disk space for storage. Please make sure you have enough space before proceeding.** + +```bash +HF_MODEL="arctic" +git clone https://huggingface.co/Snowflake/snowflake-arctic-instruct tmp/hf_checkpoints/${HF_MODEL} + +``` + +## TensorRT LLM workflow +Next, we use the general quantization script `quantize.py` to convert the checkpoints in FP8, and build the model with `trtllm-build` on multi-GPUs. In the example below, we use Tensor Parallelism (TP) across 8 GPUs. + +**Note: for such large model, it is deemed necessary to apply Post-Training Quantization (PTQ) methods on the model weights to deploy it on a cluster node, e.g., 8xH100 GPUs. In this example, we demonstrate the FP8 quantization workflow, which is supported on Hopper-and-next GPU architectures. For instructions of other PTQ methods other than FP8, please refer to the LLaMA or Mixtral examples.** + + +Set environment variables and necessary directory: + +```bash +PREC_RAW="bfloat16" +PREC_QUANT="fp8" +TP=8 +ENGINE="${HF_MODEL}_${PREC_QUANT}_tp${TP}" + +mkdir -p tmp/trt_engines +``` + +### Apply FP8 PTQ + +Notes: +- currently quantize.py does not support for Expert Parallelism (EP) mode yet. User should use `../../core/llama/convert_checkpoint.py` and specify `--moe_ep_size 1` instead, if needed. +- TensorRT LLM uses static quantization methods, which is expected to be faster at runtime as compared to dynamic quantization methods. This comes at a cost of an offline calibration step during quantization. `batch_size` and `calib_size` can be adjusted to shorten the calibration time. Please refer to ../quantization/README.md for explanation. +- **due to the large model size and the calibration step (which has to load the HuggingFace model and run forward passes), it is likely that you will need more number of GPUs during quantization step than the number of GPUs for engine building and final deployment. For example, using 16xH100 or 8xH200 for quantization & 8xH100 for deployment.** + +```bash +python ../../../quantization/quantize.py --model_dir tmp/hf_checkpoints/${HF_MODEL} \ + --dtype ${PREC_RAW} \ + --qformat ${PREC_QUANT} \ + --kv_cache_dtype ${PREC_QUANT} \ + --output_dir tmp/tllm_checkpoints/${ENGINE} \ + --batch_size 1 \ + --calib_size 128 \ + --tp_size ${TP} |& tee tmp/trt_engines/${ENGINE}_quantize.log + +``` + +### Build TensorRT engine +```bash +# Enable fp8 context fmha to get further acceleration by setting `--use_fp8_context_fmha enable` +# Use --workers to enable parallel build +trtllm-build --checkpoint_dir ./tmp/tllm_checkpoints/${ENGINE} \ + --output_dir ./tmp/trt_engines/${ENGINE} \ + --gpt_attention_plugin ${PREC_RAW} \ + --gemm_plugin ${PREC_RAW} \ + --workers ${TP} |& tee tmp/trt_engines/${ENGINE}_build.log +``` + +### Run Engine +Test your engine with the [run.py](../../../run.py) script: + +```bash +mpirun -n ${TP} --allow-run-as-root python ../../../run.py --engine_dir ./tmp/trt_engines/${ENGINE} --tokenizer_dir tmp/hf_checkpoints/${HF_MODEL} --max_output_len 20 --input_text "The future of AI is" |& tee tmp/trt_engines/${ENGINE}_run.log +``` + +For more examples see [`examples/models/core/llama/README.md`](../../core/llama/README.md) + + +### OOTB + +Arctic supports OOTB operation without the plugin, however this comes at a significant performance cost. Users should prefer using the plugin path whenever possible. diff --git a/examples/models/contrib/blip2/README.md b/examples/models/contrib/blip2/README.md new file mode 100644 index 000000000000..0e80ed4fd81b --- /dev/null +++ b/examples/models/contrib/blip2/README.md @@ -0,0 +1,4 @@ +This example has been moved to [`../multimodal`](../../../multimodal) and merged with other multimodal examples. +Please follow [`../multimodal/README.md`](../../../multimodal/README.md) for new instructions. + +**NOTICE:** This folder will be removed in v1.0 release. diff --git a/examples/models/contrib/chatglm-6b/README.md b/examples/models/contrib/chatglm-6b/README.md new file mode 100644 index 000000000000..00a8c2424665 --- /dev/null +++ b/examples/models/contrib/chatglm-6b/README.md @@ -0,0 +1,112 @@ +# ChatGLM + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document explains how to build the [ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b) models using TensorRT LLM and run on a single GPU, a single node with multiple GPUs or multiple nodes with multiple GPUs. + +- [ChatGLM](#chatglm) + - [Overview](#overview) + - [Support Matrix](#support-matrix) + - [Model comparison](#model-comparison) + - [Tokenizer and special tokens comparison](#tokenizer-and-special-tokens-comparison) + - [Usage](#usage) + - [1. Download repo and weights from HuggingFace Transformers](#1-download-repo-and-weights-from-huggingface-transformers) + - [2. Convert weights from HF Transformers to TensorRT LLM format](#2-convert-weights-from-hf-transformers-to-tensorrt-llm-format) + - [3. Build TensorRT engine(s)](#3-build-tensorrt-engines) + - [Enable plugins](#enable-plugins) + - [In-flight batching](#in-flight-batching) + - [4. Run inference](#4-run-inference) + - [Single node, single GPU](#single-node-single-gpu) + - [Single node, multi GPU](#single-node-multi-gpu) + - [5. Run summarization task](#5-run-summarization-task) + - [Weight Only quantization](#weight-only-quantization) + - [Smooth Quantization (SQ)](#smooth-quantization-sq) + - [Activation-aware Weight Quantization (AWQ)](#activation-aware-weight-quantization-awq) + - [FP8 Quantization](#fp8-quantization) + - [Benchmark](#benchmark) + + +## Overview + +The TensorRT LLM ChatGLM implementation can be found in [`tensorrt_llm/models/chatglm/model.py`](../../tensorrt_llm/models/chatglm/model.py). +The TensorRT LLM ChatGLM example code is located in [`examples/models/contrib/chatglm-6b`](./). There is one main file: + +* [`examples/models/core/glm-4-9b/convert_checkpoint.py`](../../../glm-4-9b/convert_checkpoint.py) to convert a checkpoint from the [HuggingFace (HF) Transformers](https://github.com/huggingface/transformers) format to the TensorRT LLM format. + +In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: + +* [`../../../run.py`](../../../run.py) to run the inference on an input text; +* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +## Support Matrix + +| Model Name | FP16 | FMHA | WO | SQ | AWQ | FP8 | TP | PP | ST | C++ | benchmark | IFB | +| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :-------: | :---: | +| chatglm_6b | Y | | Y | | | | Y | | Y | Y | Y | Y | +| glm_10b | Y | Y | Y | | Y | Y | Y | | Y | Y | Y | Y | + +* Model Name: the name of the model, the same as the name on HuggingFace +* FMHA: Fused MultiHead Attention (see introduction below) +* WO: Weight Only Quantization (int8 / int4) +* SQ: Smooth Quantization (int8) +* AWQ: Activation Aware Weight Quantization (int4) +* FP8: FP8 Quantization +* TP: Tensor Parallel +* PP: Pipeline Parallel +* ST: Strongly Typed +* C++: C++ Runtime +* benchmark: benchmark by python / C++ Runtime +* IFB: In-flight Batching (see introduction below) + +## Model comparison + +| Name | nL | nAH | nKH | nHW | nH | nF | nMSL | nV | bP2D | bBQKV | bBDense | Comments | +| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :----: | :---: | :---: | :-----: | :----------------------------------------------------------------- | +| chatglm_6b | 28 | 32 | 32 | 128 | 4096 | 16384 | 2048 | 130528 | Y | Y | Y | | +| glm_10b | 48 | 64 | 32 | 64 | 4096 | 16384 | 1024 | 50304 | Y | Y | Y | | + +* nL: number of layers +* nAH: number of attention heads +* nKH: number of kv heads (less than nAH if multi_query_attention is used) +* nHW: head width +* nH: hidden size +* nF: FFN hidden size +* nMSL: max sequence length (input + output) +* nV: vocabulary size +* bP2D: use position_encoding_2d (Y: Yes, N: No) +* bBQKV: use bias for QKV multiplication in self-attention +* bBDense: use bias for Dense multiplication in self-attention + +## Tokenizer and special tokens comparison + +| Name | Tokenizer | bos | eos | pad | cls | startofpiece | endofpiece | mask | smask | gmask | +| :--------------: | :--------------: | :----: | :----: | :---: | :---: | :----------: | :--------: | :----: | :---: | :----: | +| chatglm_6b | ChatGLMTokenizer | 130004 | 130005 | 3 | | 130004 | 130005 | 130000 | | 130001 | +| glm_10b | GLMGPT2Tokenizer | 50257 | 50256 | 50256 | 50259 | 50257 | 50258 | 50260 | 50264 | 50263 | + +## Usage + +The next section describe how to build the engine and run the inference demo. + +### 1. Download repo and weights from HuggingFace Transformers + +```bash +pip install -r requirements.txt +apt-get update +apt-get install git-lfs +rm -rf chatglm* + +# clone one or more models we want to build +git clone https://huggingface.co/THUDM/chatglm-6b chatglm_6b +git clone https://huggingface.co/THUDM/glm-10b glm_10b + +# replace tokenization file if using transformers-4.36.1 for model ChatGLM-6B (this might be needless in the future) +cp chatglm_6b/tokenization_chatglm.py chatglm_6b/tokenization_chatglm.py-backup +cp tokenization_chatglm.py chatglm_6b +``` + +For more example codes, please refer to the [examples/models/core/glm-4-9b/README.md](../../../glm-4-9b/README.md). diff --git a/examples/models/contrib/chatglm-6b/requirements.txt b/examples/models/contrib/chatglm-6b/requirements.txt new file mode 100644 index 000000000000..cdc65bf2bb38 --- /dev/null +++ b/examples/models/contrib/chatglm-6b/requirements.txt @@ -0,0 +1,8 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +datasets==3.1.0 +evaluate +protobuf +rouge_score +sentencepiece +tiktoken diff --git a/examples/models/contrib/chatglm-6b/tokenization_chatglm.py b/examples/models/contrib/chatglm-6b/tokenization_chatglm.py new file mode 100755 index 000000000000..8ae124909e98 --- /dev/null +++ b/examples/models/contrib/chatglm-6b/tokenization_chatglm.py @@ -0,0 +1,467 @@ +"""Tokenization classes for ChatGLM.""" +import os +from typing import Dict, List, Optional, Union + +import numpy as np +import sentencepiece as spm +from transformers.tokenization_utils import PreTrainedTokenizer +from transformers.tokenization_utils_base import BatchEncoding, EncodedInput +from transformers.utils import PaddingStrategy, logging + +logger = logging.get_logger(__name__) + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "THUDM/chatglm-6b": 2048, +} + + +class TextTokenizer: + + def __init__(self, model_path): + self.sp = spm.SentencePieceProcessor() + self.sp.Load(model_path) + self.num_tokens = self.sp.vocab_size() + + def encode(self, text): + return self.sp.EncodeAsIds(text) + + def decode(self, ids: List[int]): + return self.sp.DecodeIds(ids) + + def tokenize(self, text): + return self.sp.EncodeAsPieces(text) + + def convert_tokens_to_string(self, tokens): + return self.sp.DecodePieces(tokens) + + def convert_tokens_to_ids(self, tokens): + return [self.sp.PieceToId(token) for token in tokens] + + def convert_token_to_id(self, token): + return self.sp.PieceToId(token) + + def convert_id_to_token(self, idx): + return self.sp.IdToPiece(idx) + + def __len__(self): + return self.num_tokens + + +class SPTokenizer: + + def __init__( + self, + vocab_file, + num_image_tokens=20000, + max_blank_length=80, + byte_fallback=True, + ): + assert vocab_file is not None + self.vocab_file = vocab_file + self.num_image_tokens = num_image_tokens + self.special_tokens = [ + "[MASK]", "[gMASK]", "[sMASK]", "", "", "", + "", "" + ] + self.max_blank_length = max_blank_length + self.byte_fallback = byte_fallback + self.text_tokenizer = TextTokenizer(vocab_file) + + def _get_text_tokenizer(self): + return self.text_tokenizer + + @staticmethod + def get_blank_token(length: int): + assert length >= 2 + return f"<|blank_{length}|>" + + @staticmethod + def get_tab_token(): + return f"<|tab|>" + + @property + def num_text_tokens(self): + return self.text_tokenizer.num_tokens + + @property + def num_tokens(self): + return self.num_image_tokens + self.num_text_tokens + + @staticmethod + def _encode_whitespaces(text: str, max_len: int = 80): + text = text.replace("\t", SPTokenizer.get_tab_token()) + for i in range(max_len, 1, -1): + text = text.replace(" " * i, SPTokenizer.get_blank_token(i)) + return text + + def _preprocess(self, text: str, linebreak=True, whitespaces=True): + if linebreak: + text = text.replace("\n", "") + if whitespaces: + text = self._encode_whitespaces(text, max_len=self.max_blank_length) + return text + + def encode(self, + text: str, + linebreak=True, + whitespaces=True, + add_dummy_prefix=True) -> List[int]: + """ + @param text: Text to encode. + @param linebreak: Whether to encode newline (\n) in text. + @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding. + @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text. + @param add_dummy_prefix: Whether to add dummy blank space in the beginning. + """ + text = self._preprocess(text, linebreak, whitespaces) + if not add_dummy_prefix: + text = "" + text + tmp = self._get_text_tokenizer().encode(text) + tokens = [x + self.num_image_tokens for x in tmp] + return tokens if add_dummy_prefix else tokens[2:] + + def postprocess(self, text): + text = text.replace("", "\n") + text = text.replace(SPTokenizer.get_tab_token(), "\t") + for i in range(2, self.max_blank_length + 1): + text = text.replace(self.get_blank_token(i), " " * i) + return text + + def decode(self, text_ids: List[int]) -> str: + ids = [int(_id) - self.num_image_tokens for _id in text_ids] + ids = [_id for _id in ids if _id >= 0] + text = self._get_text_tokenizer().decode(ids) + text = self.postprocess(text) + return text + + def decode_tokens(self, tokens: List[str]) -> str: + text = self._get_text_tokenizer().convert_tokens_to_string(tokens) + text = self.postprocess(text) + return text + + def tokenize(self, + text: str, + linebreak=True, + whitespaces=True, + add_dummy_prefix=True) -> List[str]: + """ + @param text: Text to encode. + @param linebreak: Whether to encode newline (\n) in text. + @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding. + @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text. + @param add_dummy_prefix: Whether to add dummy blank space in the beginning. + """ + text = self._preprocess(text, linebreak, whitespaces) + if not add_dummy_prefix: + text = "" + text + tokens = self._get_text_tokenizer().tokenize(text) + return tokens if add_dummy_prefix else tokens[2:] + + def __getitem__(self, x: Union[int, str]): + if isinstance(x, int): + if x < self.num_image_tokens: + return "".format(x) + else: + return self.text_tokenizer.convert_id_to_token( + x - self.num_image_tokens) + elif isinstance(x, str): + if x.startswith("") and x[7:-1].isdigit(): + return int(x[7:-1]) + else: + return self.text_tokenizer.convert_token_to_id( + x) + self.num_image_tokens + else: + raise ValueError("The key should be str or int.") + + +class ChatGLMTokenizer(PreTrainedTokenizer): + """ + Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + """ + + vocab_files_names = {"vocab_file": "ice_text.model"} + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ["input_ids", "attention_mask", "position_ids"] + + def __init__(self, + vocab_file, + do_lower_case=False, + remove_space=False, + bos_token='', + eos_token='', + end_token='', + mask_token='[MASK]', + gmask_token='[gMASK]', + padding_side="left", + pad_token="", + unk_token="", + num_image_tokens=20000, + **kwargs) -> None: # Fix for new transformers + + self.do_lower_case = do_lower_case + self.remove_space = remove_space + self.vocab_file = vocab_file + + self.bos_token = bos_token + self.eos_token = eos_token + self.end_token = end_token + self.mask_token = mask_token + self.gmask_token = gmask_token + + self.sp_tokenizer = SPTokenizer(vocab_file, + num_image_tokens=num_image_tokens) + + super().__init__( + do_lower_case=do_lower_case, # Fix for new transformers + remove_space=remove_space, + padding_side=padding_side, + bos_token=bos_token, + eos_token=eos_token, + end_token=end_token, + mask_token=mask_token, + gmask_token=gmask_token, + pad_token=pad_token, + unk_token=unk_token, + num_image_tokens=num_image_tokens, + **kwargs) + """ Initialization """ + + @property + def gmask_token_id(self) -> Optional[int]: + if self.gmask_token is None: + return None + return self.convert_tokens_to_ids(self.gmask_token) + + @property + def end_token_id(self) -> Optional[int]: + """ + `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been + set. + """ + if self.end_token is None: + return None + return self.convert_tokens_to_ids(self.end_token) + + @property + def vocab_size(self): + """ Returns vocab size """ + return self.sp_tokenizer.num_tokens + + def get_vocab(self): + """ Returns vocab as a dict """ + vocab = { + self._convert_id_to_token(i): i + for i in range(self.vocab_size) + } + vocab.update(self.added_tokens_encoder) + return vocab + + def preprocess_text(self, inputs): + if self.remove_space: + outputs = " ".join(inputs.strip().split()) + else: + outputs = inputs + + if self.do_lower_case: + outputs = outputs.lower() + + return outputs + + def _tokenize(self, text, **kwargs): + """ Returns a tokenized string. """ + text = self.preprocess_text(text) + + seq = self.sp_tokenizer.tokenize(text) + + return seq + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + return self.sp_tokenizer.decode_tokens(tokens) + + def _decode(self, token_ids: Union[int, List[int]], **kwargs) -> str: + if isinstance(token_ids, int): + token_ids = [token_ids] + if len(token_ids) == 0: + return "" + if self.pad_token_id in token_ids: # remove pad + token_ids = list(filter((self.pad_token_id).__ne__, token_ids)) + return super()._decode(token_ids, **kwargs) + + def _convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + return self.sp_tokenizer[token] + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.sp_tokenizer[index] + + def save_vocabulary(self, save_directory, filename_prefix=None): + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + filename_prefix (`str`, *optional*): + An optional prefix to add to the named of the saved files. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if os.path.isdir(save_directory): + vocab_file = os.path.join(save_directory, + self.vocab_files_names["vocab_file"]) + else: + vocab_file = save_directory + + with open(self.vocab_file, 'rb') as fin: + proto_str = fin.read() + + with open(vocab_file, "wb") as writer: + writer.write(proto_str) + + return (vocab_file, ) + + def build_inputs_with_special_tokens( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A BERT sequence has the following format: + + - single sequence: `[CLS] X [SEP]` + - pair of sequences: `[CLS] A [SEP] B [SEP]` + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + gmask_id = self.sp_tokenizer[self.gmask_token] + eos_id = self.sp_tokenizer[self.eos_token] + token_ids_0 = token_ids_0 + [ + gmask_id, self.sp_tokenizer[self.bos_token] + ] + if token_ids_1 is not None: + token_ids_0 = token_ids_0 + token_ids_1 + [eos_id] + return token_ids_0 + + def _pad( + self, + encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], + max_length: Optional[int] = None, + padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + padding_side: str = "left", # Fix for new transformers + ) -> dict: + """ + Pad encoded inputs (on left/right and up to predefined length or max length in the batch) + + Args: + encoded_inputs: + Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). + max_length: maximum length of the returned list and optionally padding length (see below). + Will truncate by taking into account the special tokens. + padding_strategy: PaddingStrategy to use for padding. + + - PaddingStrategy.LONGEST Pad to the longest sequence in the batch + - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) + - PaddingStrategy.DO_NOT_PAD: Do not pad + The tokenizer padding sides are defined in self.padding_side: + + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability + `>= 7.5` (Volta). + return_attention_mask: + (optional) Set to False to avoid returning attention mask (default: set to model specifics) + """ + # Load from model defaults + bos_token_id = self.sp_tokenizer[self.bos_token] + mask_token_id = self.sp_tokenizer[self.mask_token] + gmask_token_id = self.sp_tokenizer[self.gmask_token] + assert self.padding_side == "left" + + required_input = encoded_inputs[self.model_input_names[0]] + seq_length = len(required_input) + + if padding_strategy == PaddingStrategy.LONGEST: + max_length = len(required_input) + + if max_length is not None and pad_to_multiple_of is not None and ( + max_length % pad_to_multiple_of != 0): + max_length = ( + (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( + required_input) != max_length + + # Initialize attention mask if not present. + if max_length is not None: + if "attention_mask" not in encoded_inputs: + if bos_token_id in required_input: + context_length = required_input.index(bos_token_id) + else: + context_length = seq_length + attention_mask = np.ones((1, seq_length, seq_length)) + attention_mask = np.tril(attention_mask) + attention_mask[:, :, :context_length] = 1 + attention_mask = np.bool_(attention_mask < 0.5) + encoded_inputs["attention_mask"] = attention_mask + + if "position_ids" not in encoded_inputs: + if bos_token_id in required_input: + context_length = required_input.index(bos_token_id) + else: + context_length = seq_length + position_ids = np.arange(seq_length, dtype=np.int64) + mask_token = mask_token_id if mask_token_id in required_input else gmask_token_id + if mask_token in required_input: + mask_position = required_input.index(mask_token) + position_ids[context_length:] = mask_position + block_position_ids = np.concatenate([ + np.zeros(context_length, dtype=np.int64), + np.arange(1, + seq_length - context_length + 1, + dtype=np.int64) + ]) + encoded_inputs["position_ids"] = np.stack( + [position_ids, block_position_ids], axis=0) + + if needs_to_be_padded: + difference = max_length - len(required_input) + + if "attention_mask" in encoded_inputs: + encoded_inputs["attention_mask"] = np.pad( + encoded_inputs["attention_mask"], + pad_width=[(0, 0), (difference, 0), (difference, 0)], + mode='constant', + constant_values=True) + if "token_type_ids" in encoded_inputs: + encoded_inputs["token_type_ids"] = [ + self.pad_token_type_id + ] * difference + encoded_inputs["token_type_ids"] + if "special_tokens_mask" in encoded_inputs: + encoded_inputs["special_tokens_mask"] = [ + 1 + ] * difference + encoded_inputs["special_tokens_mask"] + if "position_ids" in encoded_inputs: + encoded_inputs["position_ids"] = np.pad( + encoded_inputs["position_ids"], + pad_width=[(0, 0), (difference, 0)]) + encoded_inputs[self.model_input_names[ + 0]] = [self.pad_token_id] * difference + required_input + + return encoded_inputs diff --git a/examples/models/contrib/chatglm2-6b/README.md b/examples/models/contrib/chatglm2-6b/README.md new file mode 100644 index 000000000000..c25818e3f95c --- /dev/null +++ b/examples/models/contrib/chatglm2-6b/README.md @@ -0,0 +1,108 @@ +# ChatGLM + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document explains how to build the [ChatGLM2-6B](https://huggingface.co/THUDM/chatglm2-6b), [ChatGLM2-6B-32k](https://huggingface.co/THUDM/chatglm2-6b-32k) models using TensorRT LLM and run on a single GPU, a single node with multiple GPUs or multiple nodes with multiple GPUs. + +- [ChatGLM](#chatglm) + - [Overview](#overview) + - [Support Matrix](#support-matrix) + - [Model comparison](#model-comparison) + - [Tokenizer and special tokens comparison](#tokenizer-and-special-tokens-comparison) + - [Usage](#usage) + - [1. Download repo and weights from HuggingFace Transformers](#1-download-repo-and-weights-from-huggingface-transformers) + - [2. Convert weights from HF Transformers to TensorRT LLM format](#2-convert-weights-from-hf-transformers-to-tensorrt-llm-format) + - [3. Build TensorRT engine(s)](#3-build-tensorrt-engines) + - [Enable plugins](#enable-plugins) + - [In-flight batching](#in-flight-batching) + - [4. Run inference](#4-run-inference) + - [Single node, single GPU](#single-node-single-gpu) + - [Single node, multi GPU](#single-node-multi-gpu) + - [5. Run summarization task](#5-run-summarization-task) + - [Weight Only quantization](#weight-only-quantization) + - [Smooth Quantization (SQ)](#smooth-quantization-sq) + - [Activation-aware Weight Quantization (AWQ)](#activation-aware-weight-quantization-awq) + - [FP8 Quantization](#fp8-quantization) + - [Benchmark](#benchmark) + + +## Overview + +The TensorRT LLM ChatGLM implementation can be found in [`tensorrt_llm/models/chatglm/model.py`](../../tensorrt_llm/models/chatglm/model.py). +The TensorRT LLM ChatGLM example code is located in [`examples/models/contrib/chatglm2-6b`](./). There is one main file: + +* [`examples/models/core/glm-4-9b/convert_checkpoint.py`](../../../glm-4-9b/convert_checkpoint.py) to convert a checkpoint from the [HuggingFace (HF) Transformers](https://github.com/huggingface/transformers) format to the TensorRT LLM format. + +In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: + +* [`../../../run.py`](../../../run.py) to run the inference on an input text; +* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +## Support Matrix + +| Model Name | FP16 | FMHA | WO | SQ | AWQ | FP8 | TP | PP | ST | C++ | benchmark | IFB | +| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :-------: | :---: | +| chatglm2_6b | Y | Y | Y | Y | Y | Y | Y | | Y | Y | Y | Y | +| chatglm2_6b_32k | Y | Y | Y | | Y | Y | Y | | Y | Y | Y | Y | + +* Model Name: the name of the model, the same as the name on HuggingFace +* FMHA: Fused MultiHead Attention (see introduction below) +* WO: Weight Only Quantization (int8 / int4) +* SQ: Smooth Quantization (int8) +* AWQ: Activation Aware Weight Quantization (int4) +* FP8: FP8 Quantization +* TP: Tensor Parallel +* PP: Pipeline Parallel +* ST: Strongly Typed +* C++: C++ Runtime +* benchmark: benchmark by python / C++ Runtime +* IFB: In-flight Batching (see introduction below) + +## Model comparison + +| Name | nL | nAH | nKH | nHW | nH | nF | nMSL | nV | bP2D | bBQKV | bBDense | Comments | +| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :----: | :---: | :---: | :-----: | :----------------------------------------------------------------- | +| chatglm2_6b | 28 | 32 | 2 | 128 | 4096 | 13696 | 32768 | 65024 | N | Y | N | Multi_query_attention, RMSNorm rather than LayerNorm in chatglm_6b | +| chatglm2_6b_32k | 28 | 32 | 2 | 128 | 4096 | 13696 | 32768 | 65024 | N | Y | N | RoPE base=160000 rather than 10000 in chatglm2_6b | + +* nL: number of layers +* nAH: number of attention heads +* nKH: number of kv heads (less than nAH if multi_query_attention is used) +* nHW: head width +* nH: hidden size +* nF: FFN hidden size +* nMSL: max sequence length (input + output) +* nV: vocabulary size +* bP2D: use position_encoding_2d (Y: Yes, N: No) +* bBQKV: use bias for QKV multiplication in self-attention +* bBDense: use bias for Dense multiplication in self-attention + +## Tokenizer and special tokens comparison + +| Name | Tokenizer | bos | eos | pad | cls | startofpiece | endofpiece | mask | smask | gmask | +| :--------------: | :--------------: | :----: | :----: | :---: | :---: | :----------: | :--------: | :----: | :---: | :----: | +| chatglm2_6b | ChatGLMTokenizer | 1 | 2 | 0 | | | | | | | +| chatglm2_6b_32k | ChatGLMTokenizer | 1 | 2 | 0 | | | | | | | + +## Usage + +The next section describe how to build the engine and run the inference demo. + +### 1. Download repo and weights from HuggingFace Transformers + +```bash +pip install -r requirements.txt +apt-get update +apt-get install git-lfs +rm -rf chatglm* + +# clone one or more models we want to build +git clone https://huggingface.co/THUDM/chatglm2-6b chatglm2_6b +git clone https://huggingface.co/THUDM/chatglm2-6b-32k chatglm2_6b_32k +``` + +For more example codes, please refer to the [examples/models/core/glm-4-9b/README.md](../../../glm-4-9b/README.md). diff --git a/examples/models/contrib/chatglm2-6b/requirements.txt b/examples/models/contrib/chatglm2-6b/requirements.txt new file mode 100644 index 000000000000..cdc65bf2bb38 --- /dev/null +++ b/examples/models/contrib/chatglm2-6b/requirements.txt @@ -0,0 +1,8 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +datasets==3.1.0 +evaluate +protobuf +rouge_score +sentencepiece +tiktoken diff --git a/examples/models/contrib/chatglm2-6b/tokenization_chatglm.py b/examples/models/contrib/chatglm2-6b/tokenization_chatglm.py new file mode 100644 index 000000000000..dea5fdc7c7dc --- /dev/null +++ b/examples/models/contrib/chatglm2-6b/tokenization_chatglm.py @@ -0,0 +1,282 @@ +import os +from typing import Dict, List, Optional, Union + +from sentencepiece import SentencePieceProcessor +from transformers import PreTrainedTokenizer +from transformers.tokenization_utils_base import BatchEncoding, EncodedInput +from transformers.utils import PaddingStrategy + + +class SPTokenizer: + + def __init__(self, model_path: str): + # reload tokenizer + assert os.path.isfile(model_path), model_path + self.sp_model = SentencePieceProcessor(model_file=model_path) + + # BOS / EOS token IDs + self.n_words: int = self.sp_model.vocab_size() + self.bos_id: int = self.sp_model.bos_id() + self.eos_id: int = self.sp_model.eos_id() + self.pad_id: int = self.sp_model.unk_id() + assert self.sp_model.vocab_size() == self.sp_model.get_piece_size() + + special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + self.special_tokens = {} + self.index_special_tokens = {} + for token in special_tokens: + self.special_tokens[token] = self.n_words + self.index_special_tokens[self.n_words] = token + self.n_words += 1 + + def tokenize(self, s: str): + return self.sp_model.EncodeAsPieces(s) + + def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]: + assert type(s) is str + t = self.sp_model.encode(s) + if bos: + t = [self.bos_id] + t + if eos: + t = t + [self.eos_id] + return t + + def decode(self, t: List[int]) -> str: + return self.sp_model.decode(t) + + def decode_tokens(self, tokens: List[str]) -> str: + text = self.sp_model.DecodePieces(tokens) + return text + + def convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + if token in self.special_tokens: + return self.special_tokens[token] + return self.sp_model.PieceToId(token) + + def convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + if index in self.index_special_tokens or index in [ + self.eos_id, self.bos_id, self.pad_id + ] or index < 0: + return "" + return self.sp_model.IdToPiece(index) + + +class ChatGLMTokenizer(PreTrainedTokenizer): + vocab_files_names = {"vocab_file": "tokenizer.model"} + + model_input_names = ["input_ids", "attention_mask", "position_ids"] + + def __init__(self, + vocab_file, + padding_side="left", + clean_up_tokenization_spaces=False, + **kwargs): + self.name = "GLMTokenizer" + + self.vocab_file = vocab_file + self.tokenizer = SPTokenizer(vocab_file) + self.special_tokens = { + "": self.tokenizer.bos_id, + "": self.tokenizer.eos_id, + "": self.tokenizer.pad_id + } + super().__init__( + padding_side=padding_side, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs) + + def get_command(self, token): + if token in self.special_tokens: + return self.special_tokens[token] + assert token in self.tokenizer.special_tokens, f"{token} is not a special token for {self.name}" + return self.tokenizer.special_tokens[token] + + @property + def unk_token(self) -> str: + return "" + + @property + def pad_token(self) -> str: + return "" + + @property + def pad_token_id(self): + return self.get_command("") + + @property + def eos_token(self) -> str: + return "" + + @property + def eos_token_id(self): + return self.get_command("") + + @property + def vocab_size(self): + return self.tokenizer.n_words + + def get_vocab(self): + """ Returns vocab as a dict """ + vocab = { + self._convert_id_to_token(i): i + for i in range(self.vocab_size) + } + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text, **kwargs): + return self.tokenizer.tokenize(text) + + def _convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + return self.tokenizer.convert_token_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.tokenizer.convert_id_to_token(index) + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + return self.tokenizer.decode_tokens(tokens) + + def save_vocabulary(self, save_directory, filename_prefix=None): + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + filename_prefix (`str`, *optional*): + An optional prefix to add to the named of the saved files. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if os.path.isdir(save_directory): + vocab_file = os.path.join(save_directory, + self.vocab_files_names["vocab_file"]) + else: + vocab_file = save_directory + + with open(self.vocab_file, 'rb') as fin: + proto_str = fin.read() + + with open(vocab_file, "wb") as writer: + writer.write(proto_str) + + return (vocab_file, ) + + def get_prefix_tokens(self): + prefix_tokens = [self.get_command("[gMASK]"), self.get_command("sop")] + return prefix_tokens + + def build_prompt(self, query, history=None): + if history is None: + history = [] + prompt = "" + for i, (old_query, response) in enumerate(history): + prompt += "[Round {}]\n\n问：{}\n\n答：{}\n\n".format( + i + 1, old_query, response) + prompt += "[Round {}]\n\n问：{}\n\n答：".format(len(history) + 1, query) + return prompt + + def build_inputs_with_special_tokens( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A BERT sequence has the following format: + + - single sequence: `[CLS] X [SEP]` + - pair of sequences: `[CLS] A [SEP] B [SEP]` + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + prefix_tokens = self.get_prefix_tokens() + token_ids_0 = prefix_tokens + token_ids_0 + if token_ids_1 is not None: + token_ids_0 = token_ids_0 + token_ids_1 + [ + self.get_command("") + ] + return token_ids_0 + + def _pad( + self, + encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], + max_length: Optional[int] = None, + padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + padding_side: str = "left", # Fix for new transformers + ) -> dict: + """ + Pad encoded inputs (on left/right and up to predefined length or max length in the batch) + + Args: + encoded_inputs: + Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). + max_length: maximum length of the returned list and optionally padding length (see below). + Will truncate by taking into account the special tokens. + padding_strategy: PaddingStrategy to use for padding. + + - PaddingStrategy.LONGEST Pad to the longest sequence in the batch + - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) + - PaddingStrategy.DO_NOT_PAD: Do not pad + The tokenizer padding sides are defined in self.padding_side: + + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability + `>= 7.5` (Volta). + return_attention_mask: + (optional) Set to False to avoid returning attention mask (default: set to model specifics) + """ + # Load from model defaults + assert self.padding_side == "left" + + required_input = encoded_inputs[self.model_input_names[0]] + seq_length = len(required_input) + + if padding_strategy == PaddingStrategy.LONGEST: + max_length = len(required_input) + + if max_length is not None and pad_to_multiple_of is not None and ( + max_length % pad_to_multiple_of != 0): + max_length = ( + (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( + required_input) != max_length + + # Initialize attention mask if not present. + if "attention_mask" not in encoded_inputs: + encoded_inputs["attention_mask"] = [1] * seq_length + + if "position_ids" not in encoded_inputs: + encoded_inputs["position_ids"] = list(range(seq_length)) + + if needs_to_be_padded: + difference = max_length - len(required_input) + + if "attention_mask" in encoded_inputs: + encoded_inputs["attention_mask"] = [ + 0 + ] * difference + encoded_inputs["attention_mask"] + if "position_ids" in encoded_inputs: + encoded_inputs["position_ids"] = [ + 0 + ] * difference + encoded_inputs["position_ids"] + encoded_inputs[self.model_input_names[ + 0]] = [self.pad_token_id] * difference + required_input + + return encoded_inputs diff --git a/examples/models/contrib/chatglm3-6b-32k/README.md b/examples/models/contrib/chatglm3-6b-32k/README.md new file mode 100644 index 000000000000..0636cba34f9c --- /dev/null +++ b/examples/models/contrib/chatglm3-6b-32k/README.md @@ -0,0 +1,112 @@ +# ChatGLM + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document explains how to build the [ChatGLM3-6B](https://huggingface.co/THUDM/chatglm3-6b), [ChatGLM3-6B-Base](https://huggingface.co/THUDM/chatglm3-6b-base), [ChatGLM3-6B-32k](https://huggingface.co/THUDM/chatglm3-6b-32k) models using TensorRT LLM and run on a single GPU, a single node with multiple GPUs or multiple nodes with multiple GPUs. + +- [ChatGLM](#chatglm) + - [Overview](#overview) + - [Support Matrix](#support-matrix) + - [Model comparison](#model-comparison) + - [Tokenizer and special tokens comparison](#tokenizer-and-special-tokens-comparison) + - [Usage](#usage) + - [1. Download repo and weights from HuggingFace Transformers](#1-download-repo-and-weights-from-huggingface-transformers) + - [2. Convert weights from HF Transformers to TensorRT LLM format](#2-convert-weights-from-hf-transformers-to-tensorrt-llm-format) + - [3. Build TensorRT engine(s)](#3-build-tensorrt-engines) + - [Enable plugins](#enable-plugins) + - [In-flight batching](#in-flight-batching) + - [4. Run inference](#4-run-inference) + - [Single node, single GPU](#single-node-single-gpu) + - [Single node, multi GPU](#single-node-multi-gpu) + - [5. Run summarization task](#5-run-summarization-task) + - [Weight Only quantization](#weight-only-quantization) + - [Smooth Quantization (SQ)](#smooth-quantization-sq) + - [Activation-aware Weight Quantization (AWQ)](#activation-aware-weight-quantization-awq) + - [FP8 Quantization](#fp8-quantization) + - [Benchmark](#benchmark) + + +## Overview + +The TensorRT LLM ChatGLM implementation can be found in [`tensorrt_llm/models/chatglm/model.py`](../../tensorrt_llm/models/chatglm/model.py). +The TensorRT LLM ChatGLM example code is located in [`examples/models/contrib/chatglm3-6b-32k`](./). There is one main file: + +* [`examples/models/core/glm-4-9b/convert_checkpoint.py`](../../../glm-4-9b/convert_checkpoint.py) to convert a checkpoint from the [HuggingFace (HF) Transformers](https://github.com/huggingface/transformers) format to the TensorRT LLM format. + +In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: + +* [`../../../run.py`](../../../run.py) to run the inference on an input text; +* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +## Support Matrix + +| Model Name | FP16 | FMHA | WO | SQ | AWQ | FP8 | TP | PP | ST | C++ | benchmark | IFB | +| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :-------: | :---: | +| chatglm3_6b | Y | Y | Y | Y | Y | Y | Y | | Y | Y | Y | Y | +| chatglm3_6b_base | Y | Y | Y | Y | Y | Y | Y | | Y | Y | Y | Y | +| chatglm3_6b_32k | Y | Y | Y | Y | Y | Y | Y | | Y | Y | Y | Y | + +* Model Name: the name of the model, the same as the name on HuggingFace +* FMHA: Fused MultiHead Attention (see introduction below) +* WO: Weight Only Quantization (int8 / int4) +* SQ: Smooth Quantization (int8) +* AWQ: Activation Aware Weight Quantization (int4) +* FP8: FP8 Quantization +* TP: Tensor Parallel +* PP: Pipeline Parallel +* ST: Strongly Typed +* C++: C++ Runtime +* benchmark: benchmark by python / C++ Runtime +* IFB: In-flight Batching (see introduction below) + +## Model comparison + +| Name | nL | nAH | nKH | nHW | nH | nF | nMSL | nV | bP2D | bBQKV | bBDense | Comments | +| :--------------: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :----: | :---: | :---: | :-----: | :----------------------------------------------------------------- | +| chatglm3_6b | 28 | 32 | 2 | 128 | 4096 | 13696 | 8192 | 65024 | N | Y | N | Different in preprocess and postprocess than chatglm2_6b | +| chatglm3_6b_base | 28 | 32 | 2 | 128 | 4096 | 13696 | 32768 | 65024 | N | Y | N | | +| chatglm3_6b_32k | 28 | 32 | 2 | 128 | 4096 | 13696 | 32768 | 65024 | N | Y | N | RoPE base=500000 rather than 10000 in chatglm3_6b | + +* nL: number of layers +* nAH: number of attention heads +* nKH: number of kv heads (less than nAH if multi_query_attention is used) +* nHW: head width +* nH: hidden size +* nF: FFN hidden size +* nMSL: max sequence length (input + output) +* nV: vocabulary size +* bP2D: use position_encoding_2d (Y: Yes, N: No) +* bBQKV: use bias for QKV multiplication in self-attention +* bBDense: use bias for Dense multiplication in self-attention + +## Tokenizer and special tokens comparison + +| Name | Tokenizer | bos | eos | pad | cls | startofpiece | endofpiece | mask | smask | gmask | +| :--------------: | :--------------: | :----: | :----: | :---: | :---: | :----------: | :--------: | :----: | :---: | :----: | +| chatglm3_6b | ChatGLMTokenizer | 1 | 2 | 0 | | | | 130000 | | | +| chatglm3_6b_base | ChatGLMTokenizer | 1 | 2 | 0 | | | | 130000 | | | +| chatglm3_6b_32k | ChatGLMTokenizer | 1 | 2 | 0 | | | | 130000 | | | + +## Usage + +The next section describe how to build the engine and run the inference demo. + +### 1. Download repo and weights from HuggingFace Transformers + +```bash +pip install -r requirements.txt +apt-get update +apt-get install git-lfs +rm -rf chatglm* + +# clone one or more models we want to build +git clone https://huggingface.co/THUDM/chatglm3-6b chatglm3_6b +git clone https://huggingface.co/THUDM/chatglm3-6b-base chatglm3_6b_base +git clone https://huggingface.co/THUDM/chatglm3-6b-32k chatglm3_6b_32k +``` + +For more example codes, please refer to the [examples/models/core/glm-4-9b/README.md](../../../glm-4-9b/README.md). diff --git a/examples/models/contrib/chatglm3-6b-32k/requirements.txt b/examples/models/contrib/chatglm3-6b-32k/requirements.txt new file mode 100644 index 000000000000..cdc65bf2bb38 --- /dev/null +++ b/examples/models/contrib/chatglm3-6b-32k/requirements.txt @@ -0,0 +1,8 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +datasets==3.1.0 +evaluate +protobuf +rouge_score +sentencepiece +tiktoken diff --git a/examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py b/examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py new file mode 100644 index 000000000000..67a3d52442a6 --- /dev/null +++ b/examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py @@ -0,0 +1,313 @@ +import json +import os +from typing import Dict, List, Optional, Union + +from sentencepiece import SentencePieceProcessor +from transformers import PreTrainedTokenizer +from transformers.tokenization_utils_base import BatchEncoding, EncodedInput +from transformers.utils import PaddingStrategy + + +class SPTokenizer: + + def __init__(self, model_path: str): + # reload tokenizer + assert os.path.isfile(model_path), model_path + self.sp_model = SentencePieceProcessor(model_file=model_path) + + # BOS / EOS token IDs + self.n_words: int = self.sp_model.vocab_size() + self.bos_id: int = self.sp_model.bos_id() + self.eos_id: int = self.sp_model.eos_id() + self.pad_id: int = self.sp_model.unk_id() + assert self.sp_model.vocab_size() == self.sp_model.get_piece_size() + + special_tokens = [ + "[MASK]", "[gMASK]", "[sMASK]", "sop", "eop", "<|system|>", + "<|user|>", "<|assistant|>", "<|observation|>" + ] + self.special_tokens = {} + self.index_special_tokens = {} + for token in special_tokens: + self.special_tokens[token] = self.n_words + self.index_special_tokens[self.n_words] = token + self.n_words += 1 + + def tokenize(self, s: str): + return self.sp_model.EncodeAsPieces(s) + + def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]: + assert type(s) is str + t = self.sp_model.encode(s) + if bos: + t = [self.bos_id] + t + if eos: + t = t + [self.eos_id] + return t + + def decode(self, t: List[int]) -> str: + text, buffer = "", [] + for token in t: + if token in self.index_special_tokens: + if buffer: + text += self.sp_model.decode(buffer) + buffer = [] + text += self.index_special_tokens[token] + else: + buffer.append(token) + if buffer: + text += self.sp_model.decode(buffer) + return text + + def decode_tokens(self, tokens: List[str]) -> str: + text = self.sp_model.DecodePieces(tokens) + return text + + def convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + if token in self.special_tokens: + return self.special_tokens[token] + return self.sp_model.PieceToId(token) + + def convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + if index in self.index_special_tokens: + return self.index_special_tokens[index] + if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0: + return "" + return self.sp_model.IdToPiece(index) + + +class ChatGLMTokenizer(PreTrainedTokenizer): + vocab_files_names = {"vocab_file": "tokenizer.model"} + + model_input_names = ["input_ids", "attention_mask", "position_ids"] + + def __init__(self, + vocab_file, + padding_side="left", + clean_up_tokenization_spaces=False, + **kwargs): + self.name = "GLMTokenizer" + + self.vocab_file = vocab_file + self.tokenizer = SPTokenizer(vocab_file) + self.special_tokens = { + "": self.tokenizer.bos_id, + "": self.tokenizer.eos_id, + "": self.tokenizer.pad_id + } + super().__init__( + padding_side=padding_side, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs) + + def get_command(self, token): + if token in self.special_tokens: + return self.special_tokens[token] + assert token in self.tokenizer.special_tokens, f"{token} is not a special token for {self.name}" + return self.tokenizer.special_tokens[token] + + @property + def unk_token(self) -> str: + return "" + + @property + def pad_token(self) -> str: + return "" + + @property + def pad_token_id(self): + return self.get_command("") + + @property + def eos_token(self) -> str: + return "" + + @property + def eos_token_id(self): + return self.get_command("") + + @property + def vocab_size(self): + return self.tokenizer.n_words + + def get_vocab(self): + """ Returns vocab as a dict """ + vocab = { + self._convert_id_to_token(i): i + for i in range(self.vocab_size) + } + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text, **kwargs): + return self.tokenizer.tokenize(text) + + def _convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + return self.tokenizer.convert_token_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.tokenizer.convert_id_to_token(index) + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + return self.tokenizer.decode_tokens(tokens) + + def save_vocabulary(self, save_directory, filename_prefix=None): + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + filename_prefix (`str`, *optional*): + An optional prefix to add to the named of the saved files. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if os.path.isdir(save_directory): + vocab_file = os.path.join(save_directory, + self.vocab_files_names["vocab_file"]) + else: + vocab_file = save_directory + + with open(self.vocab_file, 'rb') as fin: + proto_bytes = fin.read() + + with open(vocab_file, "wb") as writer: + writer.write(proto_bytes) + + return (vocab_file, ) + + def get_prefix_tokens(self): + prefix_tokens = [self.get_command("[gMASK]"), self.get_command("sop")] + return prefix_tokens + + def build_single_message(self, role, metadata, message): + assert role in ["system", "user", "assistant", "observation"], role + role_tokens = [self.get_command(f"<|{role}|>") + ] + self.tokenizer.encode(f"{metadata}\n") + message_tokens = self.tokenizer.encode(message) + tokens = role_tokens + message_tokens + return tokens + + def build_chat_input(self, query, history=None, role="user"): + if history is None: + history = [] + input_ids = [] + for item in history: + content = item["content"] + if item["role"] == "system" and "tools" in item: + content = content + "\n" + json.dumps( + item["tools"], indent=4, ensure_ascii=False) + input_ids.extend( + self.build_single_message(item["role"], + item.get("metadata", ""), content)) + input_ids.extend(self.build_single_message(role, "", query)) + input_ids.extend([self.get_command("<|assistant|>")]) + return self.batch_encode_plus([input_ids], + return_tensors="pt", + is_split_into_words=True) + + def build_inputs_with_special_tokens( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A BERT sequence has the following format: + + - single sequence: `[CLS] X [SEP]` + - pair of sequences: `[CLS] A [SEP] B [SEP]` + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + prefix_tokens = self.get_prefix_tokens() + token_ids_0 = prefix_tokens + token_ids_0 + if token_ids_1 is not None: + token_ids_0 = token_ids_0 + token_ids_1 + [ + self.get_command("") + ] + return token_ids_0 + + def _pad( + self, + encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], + max_length: Optional[int] = None, + padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + padding_side: str = "left", # Fix for new transformers + ) -> dict: + """ + Pad encoded inputs (on left/right and up to predefined length or max length in the batch) + + Args: + encoded_inputs: + Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). + max_length: maximum length of the returned list and optionally padding length (see below). + Will truncate by taking into account the special tokens. + padding_strategy: PaddingStrategy to use for padding. + + - PaddingStrategy.LONGEST Pad to the longest sequence in the batch + - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) + - PaddingStrategy.DO_NOT_PAD: Do not pad + The tokenizer padding sides are defined in self.padding_side: + + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability + `>= 7.5` (Volta). + return_attention_mask: + (optional) Set to False to avoid returning attention mask (default: set to model specifics) + """ + # Load from model defaults + assert self.padding_side == "left" + + required_input = encoded_inputs[self.model_input_names[0]] + seq_length = len(required_input) + + if padding_strategy == PaddingStrategy.LONGEST: + max_length = len(required_input) + + if max_length is not None and pad_to_multiple_of is not None and ( + max_length % pad_to_multiple_of != 0): + max_length = ( + (max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len( + required_input) != max_length + + # Initialize attention mask if not present. + if "attention_mask" not in encoded_inputs: + encoded_inputs["attention_mask"] = [1] * seq_length + + if "position_ids" not in encoded_inputs: + encoded_inputs["position_ids"] = list(range(seq_length)) + + if needs_to_be_padded: + difference = max_length - len(required_input) + + if "attention_mask" in encoded_inputs: + encoded_inputs["attention_mask"] = [ + 0 + ] * difference + encoded_inputs["attention_mask"] + if "position_ids" in encoded_inputs: + encoded_inputs["position_ids"] = [ + 0 + ] * difference + encoded_inputs["position_ids"] + encoded_inputs[self.model_input_names[ + 0]] = [self.pad_token_id] * difference + required_input + + return encoded_inputs diff --git a/examples/models/contrib/internlm/.gitignore b/examples/models/contrib/internlm/.gitignore new file mode 100644 index 000000000000..7ce339719a3e --- /dev/null +++ b/examples/models/contrib/internlm/.gitignore @@ -0,0 +1,2 @@ +internlm* +tokenizer.model diff --git a/examples/models/contrib/internlm/README.md b/examples/models/contrib/internlm/README.md new file mode 100644 index 000000000000..f44180f01aad --- /dev/null +++ b/examples/models/contrib/internlm/README.md @@ -0,0 +1,320 @@ +# InternLM + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document shows how to build and run InternLM 7B / 20B models in TensorRT LLM on both single GPU, single node multi-GPU and multi-node multi-GPU. + +- [InternLM](#internlm) + - [Overview](#overview) + - [Support Matrix](#support-matrix) + - [Usage](#usage) + - [Build TensorRT engine(s)](#build-tensorrt-engines) + - [INT8 weight only + INT8 KV cache](#int8-weight-only--int8-kv-cache) + - [SmoothQuant](#smoothquant) + - [Run](#run) + - [Summarization using the InternLM model](#summarization-using-the-internlm-model) + +## Overview + +The TensorRT LLM InternLM implementation is based on the LLaMA model. The implementation can +be found in [tensorrt_llm/models/llama/model.py](../../../../tensorrt_llm/models/llama/model.py). +The TensorRT LLM InternLM example code lies in [`examples/models/contrib/internlm`](./): + +* [`convert_checkpoint.py`](../../core/llama/convert_checkpoint.py) converts the Huggingface Model of InternLM into TensorRT LLM checkpoint. +* [`convert_checkpoint.py`] to to convert a checkpoint from the [HuggingFace (HF) Transformers](https://github.com/huggingface/transformers) format to the TensorRT LLM format + +In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: + +* [`../../../run.py`](../../../run.py) to run the inference on an input text; +* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +## Support Matrix + * FP16 / BF16 + * INT8 & INT4 Weight-Only + * Smooth Quant + * INT8 KV Cache + * Tensor Parallel & Pipeline Parallel + +## Usage + +The TensorRT LLM InternLM example code locates at [examples/models/contrib/internlm](./). It takes HF weights as input, and builds the corresponding TensorRT engines. The number of TensorRT engines depends on the number of GPUs used to run inference. + +### Build TensorRT engine(s) + +Please install required packages first: + +```bash +pip install -r requirements.txt +``` + +TensorRT LLM InternLM builds TensorRT engine(s) from HF checkpoint. If no checkpoint directory is specified, TensorRT LLM will build engine(s) with dummy weights. + +InternLM has released several checkpoints of different size or capabilities under https://huggingface.co/internlm. Users can pick any one repository and follow instructions to prepare the checkpoint. + +Below examples use [internlm-chat-7b](https://huggingface.co/internlm/internlm-chat-7b) and [internlm-chat-20b](https://huggingface.co/internlm/internlm-chat-20b) and assume these repositories are cloned or linked under this directory, for example `./internlm-chat-7b/`. + +Normally `trtllm-build` only requires single GPU, but if you've already got all the GPUs needed for inference, you could enable parallel building to make the engine building process faster by adding `--workers` argument. Please note that currently `--workers` feature only supports single node. + +Here're some examples: + +```bash +# Build a single-GPU float16 engine from HF weights. +# gpt_attention_plugin is necessary in InternLM. +# Try use_gemm_plugin to prevent accuracy issue. +cd examples/models/core/llama + +# Convert the InternLM 7B model using a single GPU and FP16. +python convert_checkpoint.py --model_dir ./internlm-chat-7b/ \ + --dtype float16 \ + --output_dir ./internlm-chat-7b/trt_engines/fp16/1-gpu/ +# Note: setting `--dtype bfloat16` to use bfloat16 precision. + +# BUild the InternLM 7B model using a single GPU +trtllm-build --checkpoint_dir ./internlm-chat-7b/trt_engines/fp16/1-gpu/ \ + --output_dir ./engine_outputs \ + --gemm_plugin float16 + +# Convert the InternLM 7B model using a single GPU and apply INT8 weight-only quantization.. +python convert_checkpoint.py --model_dir ./internlm-chat-7b/ \ + --dtype float16 \ + --output_dir ./internlm-chat-7b/trt_engines/int8/1-gpu/ \ + --use_weight_only \ + --weight_only_precision int8 + +trtllm-build --checkpoint_dir ./internlm-chat-7b/trt_engines/int8/1-gpu/ \ + --output_dir ./engine_outputs \ + --gemm_plugin float16 + +# Note: setting `--weight_only_precision int4` to use INT4 weight-only quantization + +# Build InternLM 7B using 2-way tensor parallelism. +python convert_checkpoint.py --model_dir ./internlm-chat-7b/ \ + --dtype float16 \ + --output_dir ./internlm-chat-7b/trt_engines/fp16/2-gpu/ \ + --tp_size 2 + +trtllm-build --checkpoint_dir ./internlm-chat-7b/trt_engines/fp16/2-gpu/ \ + --output_dir ./engine_outputs \ + --gemm_plugin float16 + +# Build InternLM 20B using 2-way tensor parallelism. +python convert_checkpoint.py --model_dir ./internlm-chat-20b/ \ + --dtype bfloat16 \ + --output_dir ./internlm-chat-20b/trt_engines/bf16/2-gpu/ \ + --tp_size 2 --workers 2 + +trtllm-build --checkpoint_dir ./internlm-chat-7b/trt_engines/bf16/2-gpu/ \ + --output_dir ./engine_outputs \ + --gpt_attention_plugin bfloat16 \ + --gemm_plugin bfloat16 +``` + +#### INT8 weight only + INT8 KV cache + +For INT8 KV cache, [`convert_checkpoint.py`](./convert_checkpoint.py) features a +`--int8_kv_cache` option. Setting `--int8_kv_cache` will calibrate the model, +and then export the scaling factors needed for INT8 KV cache inference. + + +Example: + +```bash +cd examples/models/core/llama + +# For 7B models +python convert_checkpoint.py --model_dir ./internlm-chat-7b \ + --output_dir ./internlm-chat-7b/smooth_internlm/int8_kv_cache/ \ + --dtype float16 \ + --use_weight_only \ + --weight_only_precision int8 \ + --int8_kv_cache + +# Build 7B model with both INT8 weight-only and INT8 KV cache enabled +trtllm-build --checkpoint_dir ./internlm-chat-7b/smooth_internlm/int8_kv_cache/ \ + --output_dir ./engine_outputs \ + --gemm_plugin float16 \ +``` + + +```bash +cd examples/models/core/llama + +# For 20B models +python convert_checkpoint.py --model_dir ./internlm-chat-20b \ + --output_dir ./internlm-chat-20b/smooth_internlm/int8_kv_cache/ \ + --dtype float16 \ + --use_weight_only \ + --weight_only_precision int8 \ + --int8_kv_cache + +# Build 20B model with both INT8 weight-only and INT8 KV cache enabled +trtllm-build --checkpoint_dir ./internlm-chat-20b/smooth_internlm/int8_kv_cache/ \ + --output_dir ./engine_outputs \ + --gemm_plugin float16 \ +``` + + +Test with `../../../run.py` or `../../../summarize.py`: + +```bash +python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-7b/ \ + --engine_dir ./internlm-chat-7b/trt_engines/int8_kv_cache_weight_only/1-gpu + +python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-20b/ \ + --engine_dir ./internlm-chat-20b/trt_engines/int8_kv_cache_weight_only/1-gpu + +python ../../../summarize.py --test_trt_llm --test_hf \ + --hf_model_dir ./internlm-chat-7b \ + --data_type fp16 \ + --engine_dir ./internlm-chat-7b/trt_engines/int8_kv_cache_weight_only/1-gpu + +python ../../../summarize.py --test_trt_llm --test_hf \ + --hf_model_dir ./internlm-chat-20b \ + --data_type fp16 \ + --engine_dir ./internlm-chat-20b/trt_engines/int8_kv_cache_weight_only/1-gpu +``` + +#### SmoothQuant + +Unlike the FP16 build where the HF weights are processed and loaded into the TensorRT LLM directly, the SmoothQuant needs to load INT8 weights which should be pre-processed before building an engine. + +Example: +```bash +cd examples/models/core/llama + +# For 7B models +python convert_checkpoint.py --model_dir ./internlm-chat-7b --output_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ --dtype float16 --smoothquant 0.5 +# Build the engine +trtllm-build --checkpoint_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ \ + --output_dir ./engine_outputs \ + --gemm_plugin float16 + +# For 20B models +cd examples/models/core/llama + +python convert_checkpoint.py --model_dir ./internlm-chat-20b --output_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ --dtype float16 --smoothquant 0.5 +trtllm-build --checkpoint_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ \ + --output_dir ./engine_outputs \ + --gemm_plugin float16 +``` + +[`convert_checkpoint.py`](./convert_checkpoint.py) add new options for the support of INT8 inference of SmoothQuant models. + +`--smoothquant` is the starting point of INT8 inference. By default, it +will run the model in the _per-tensor_ mode. + +Then, you can add any combination of `--per-token` and `--per-channel` to get the corresponding behaviors. + +Examples of build invocations: + +```bash +# Build model for SmoothQuant in the _per_token_ + _per_channel_ mode +cd examples/models/core/llama + +# 7B model +python convert_checkpoint.py --model_dir ./internlm-chat-7b --output_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ --dtype float16 --smoothquant 0.5 --per_channel --per_token + +# 20B model +python convert_checkpoint.py --model_dir ./internlm-chat-20b --output_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ --dtype float16 --smoothquant 0.5 --per_channel --per_token +``` + + +Test with `../../../run.py` or `../../../summarize.py`: + +```bash +python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-7b/ \ + --engine_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ + +python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-20b/ \ + --engine_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ + +python ../../../summarize.py --test_trt_llm --test_hf \ + --hf_model_dir ./internlm-chat-7b \ + --data_type fp16 \ + --engine_dir ./internlm-chat-7b/smooth_internlm/sq0.5/ + +python ../../../summarize.py --test_trt_llm --test_hf \ + --hf_model_dir ./internlm-chat-20b \ + --data_type fp16 \ + --engine_dir ./internlm-chat-20b/smooth_internlm/sq0.5/ +``` + +### Run + +To run a TensorRT LLM InternLM model using the engines generated by `trtllm-build` + +```bash +# InternLM 7B with fp16 +python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-7b/ \ + --engine_dir=./internlm-chat-7b/trt_engines/fp16/1-gpu/ + +# InternLM 7B with bf16 +python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-7b/ \ + --engine_dir=./internlm-chat-7b/trt_engines/bf16/1-gpu/ + +# InternLM 7B with int8 weight only quantization +python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-7b/ \ + --engine_dir=./internlm-chat-7b/trt_engines/weight_only/1-gpu/ + +# InternLM 7B with fp16 and tensor parallelism +mpirun -n 2 --allow-run-as-root \ + python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-7b/ \ + --engine_dir=./internlm-chat-7b/trt_engines/fp16/2-gpu/ + +# InternLM 20B with fp16 and tensor parallelism and pipeline parallelism +mpirun -n 4 --allow-run-as-root \ + python ../../../run.py --max_output_len=120 \ + --input_text 'Tell me about yourself.' \ + --tokenizer_dir ./internlm-chat-7b/ \ + --engine_dir=./internlm-chat-7b/trt_engines/bf16/4-gpu/ +``` + +### Summarization using the InternLM model + +```bash +# Run summarization using the InternLM 7B model in FP16. +python ../../../summarize.py --test_trt_llm --test_hf \ + --hf_model_dir ./internlm-chat-7b/ \ + --data_type fp16 \ + --engine_dir ./engine_outputs + +# Run summarization using the InternLM 7B model quantized to INT8. +python ../../../summarize.py --test_trt_llm --test_hf \ + --hf_model_dir ./internlm-chat-7b/ \ + --data_type fp16 \ + --engine_dir ./engine_outputs + +# Run summarization using the InternLM 7B model in FP16 using two GPUs. +mpirun -n 2 --allow-run-as-root \ + python ../../../summarize.py --test_trt_llm --test_hf \ + --hf_model_dir ./internlm-chat-7b/ \ + --data_type fp16 \ + --engine_dir ./internlm-chat-7b/trt_engines/fp16/2-gpu/ + +# Run summarization using the InternLM 20B model in BF16 using 4 GPUs. +mpirun -n 4 --allow-run-as-root \ + python ../../../summarize.py --test_trt_llm --test_hf \ + --hf_model_dir ./internlm-chat-20b/ \ + --data_type bf16 \ + --engine_dir ./internlm-chat-20b/trt_engines/bf16/4-gpu/ +``` diff --git a/examples/models/contrib/internlm/requirements.txt b/examples/models/contrib/internlm/requirements.txt new file mode 100644 index 000000000000..d9354a133c65 --- /dev/null +++ b/examples/models/contrib/internlm/requirements.txt @@ -0,0 +1,6 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +datasets==3.1.0 +rouge_score +sentencepiece>=0.1.99 +evaluate diff --git a/examples/models/contrib/jais/README.md b/examples/models/contrib/jais/README.md new file mode 100644 index 000000000000..d15cfd116962 --- /dev/null +++ b/examples/models/contrib/jais/README.md @@ -0,0 +1,125 @@ +# Jais + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document elaborates how to build Jais model to runnable engines on multi-GPU node and perform a summarization task using these engines. + +Currently it has been tested on +- [Jais-13b-chat](https://huggingface.co/core42/jais-13b-chat) +- [Jais-30b-chat-v3](https://huggingface.co/core42/jais-30b-chat-v3) + + +- [Jais](#jais) + - [Overview](#overview) + - [Support Matrix](#support-matrix) + - [Usage](#usage) + - [Build TensorRT engine(s)](#build-tensorrt-engines) + - [Run inference](#run) + +## Overview + +The TensorRT LLM support for Jais is based on the GPT model, the implementation can be found in [tensorrt_llm/models/gpt/model.py](../../../../tensorrt_llm/models/gpt/model.py). Jais model resembles GPT very much except it uses alibi embedding, embedding scale, swiglu, and logits scale, we therefore reuse the [GPT example code](../../../gpt) for Jais, + +* [`convert_checkpoint.py`](../../../gpt/convert_checkpoint.py) to convert the Jais model into TensorRT LLM checkpoint format. + +In addition, there are two shared files in the parent folder [`examples`](../) for inference and evaluation: + +* [`../../../run.py`](../../../run.py) to run the inference on an input text; +* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +## Support Matrix +The tested configurations are: + * FP16 + * FP8 + * Inflight Batching + * Tensor Parallel + +## Usage + +This section gives a whole process where we convert HF models, build TensorRT LLM engines and ultimately perform summarization. + +### Build TensorRT engine(s) + +Run the following commands and TRT-LLM will first transforms a HF model into its own checkpoint format, then builds a TRT engine based on the checkpoint + +```bash +# single gpu, dtype float16 for jais-13b-chat +python3 ../../../gpt/convert_checkpoint.py --model_dir core42/jais-13b-chat \ + --dtype float16 \ + --output_dir jais-13b-chat/trt_ckpt/fp16/1-gpu + +# 2-way tensor parallelism for jais-30b-chat-v3 +python3 ../../../gpt/convert_checkpoint.py --model_dir core42/jais-30b-chat-v3 \ + --dtype float16 \ + --tp_size 2 \ + --output_dir jais-30b-chat-v3/trt_ckpt/fp16/2-gpu +``` + +```bash +# Build a single-GPU float16 engine from TensorRT LLM checkpoint for jais-13b-chat +# Enable the special TensorRT LLM GPT Attention plugin (--gpt_attention_plugin) to increase runtime performance. +# It is recommend to use --remove_input_padding along with --gpt_attention_plugin for better performance +trtllm-build --checkpoint_dir jais-13b-chat/trt_ckpt/fp16/1-gpu \ + --gpt_attention_plugin float16 \ + --remove_input_padding enable \ + --output_dir jais-13b-chat/trt_engines/fp16/1-gpu + +# Build 2-way tensor parallelism engines from TensorRT LLM checkpoint for jais-30b-chat-v3 +trtllm-build --checkpoint_dir jais-30b-chat-v3/trt_ckpt/fp16/2-gpu \ + --gpt_attention_plugin float16 \ + --remove_input_padding enable \ + --output_dir jais-30b-chat-v3/trt_engines/fp16/2-gpu +``` + + +### Run + +The [`../../../run.py`](../../../run.py) script can be used to run inference with the built engine(s). + +```bash +python3 ../../../run.py --engine_dir jais-13b-chat/trt_engines/fp16/1-gpu \ + --tokenizer_dir core42/jais-13b-chat \ + --max_output_len 10 +``` + +If the engines are run successfully, you will see output like: +``` +...... +Input [Text 0]: "Born in north-east France, Soyer trained as a" +Output [Text 0 Beam 0]: " chef in Paris before moving to England in 1816" +``` + +```bash +python3 ../../../run.py --engine_dir jais-13b-chat/trt_engines/fp16/1-gpu \ + --tokenizer_dir core42/jais-13b-chat \ + --max_output_len 8 \ + --input_text "ولد في 1304 ميلادياً ابن بطوطه, لقد ذهب" +``` + +If the engines are run successfully, you will see output like: +``` +..... +Input [Text 0]: "ولد في 1304 ميلادياً ابن بطوطه, لقد ذهب" +Output [Text 0 Beam 0]: " في جميع أنحاء العالم المعروف في ذلك الوقت" +``` + + +To run a 2 TP model you can do the following +```bash +mpirun -np 2 \ + python3 ../../../run.py --engine_dir jais-30b-chat-v3/trt_engines/fp16/2-gpu \ + --tokenizer_dir core42/jais-30b-chat-v3 \ + --max_output_len 30 +``` + +If the engines are run successfully, you will see output like: +``` +Input [Text 0]: "Born in north-east France, Soyer trained as a" +Output [Text 0 Beam 0]: " chef, working in a series of high-end establishments. + +Soyer's career took him to work in a number of establishments across Europe," +``` diff --git a/examples/models/contrib/jais/requirements.txt b/examples/models/contrib/jais/requirements.txt new file mode 100644 index 000000000000..592e01e5ba6d --- /dev/null +++ b/examples/models/contrib/jais/requirements.txt @@ -0,0 +1,6 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +datasets==3.1.0 +evaluate +rouge_score +SentencePiece>=0.1.99 diff --git a/examples/models/contrib/sdxl/README.md b/examples/models/contrib/sdxl/README.md new file mode 100644 index 000000000000..05d874f5f27a --- /dev/null +++ b/examples/models/contrib/sdxl/README.md @@ -0,0 +1,42 @@ +# Stable Diffusion XL + +This document showcases how to build and run the [Stable Diffusion XL (SDXL)](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) model on multiple GPUs using TensorRT-LLM. The community-contributed SDXL example in TRT-LLM is intended solely to showcase distributed inference for high-resolution use cases. For an optimized single-GPU setup in Stable Diffusion inference, please refer to the [TensorRT DemoDiffusion example](https://github.com/NVIDIA/TensorRT/tree/main/demo/Diffusion). + +The design of distributed parallel inference comes from the CVPR 2024 paper [DistriFusion](https://github.com/mit-han-lab/distrifuser) from [MIT HAN Lab](https://hanlab.mit.edu/). To simplify the implementation, all communications in this example are handled synchronously. + +## Usage + +### 1. Build TensorRT Engine + +```bash +# 1 gpu +python build_sdxl_unet.py --size 1024 + +# 2 gpus +mpirun -n 2 --allow-run-as-root python build_sdxl_unet.py --size 1024 +``` + +### 2. Generate images using the engine + + +```bash +# 1 gpu +python run_sdxl.py --size 1024 --prompt "flowers, rabbit" + +# 2 gpus +mpirun -n 2 --allow-run-as-root python run_sdxl.py --size 1024 --prompt "flowers, rabbit" +``` + +## Latency Benchmark +This benchmark is provided as reference points and should not be considered as the peak inference speed that can be delivered by TensorRT-LLM. + +| Framework | Resolution | n_gpu | A100 latency (s) | A100 speedup | H100 latency (s) | H100 speedup | +|:---------:|:----------:|:-----:|:---------------:|:-------------:|:---------------:|:-------------:| +| Torch | 1024x1024 | 1 | 6.280 | 1 | 5.820 | 1 | +| TRT-LLM | 1024x1024 | 2 | 2.803 | **2.24x** | 1.719 | **3.39x** | +| TRT-LLM | 1024x1024 | 4 | 2.962 | **2.12x** | 2.592 | **2.25x** | +| Torch | 2048x2048 | 1 | 27.865 | 1 | 18.330 | 1 | +| TRT-LLM | 2048x2048 | 2 | 13.152 | **2.12x** | 7.943 | **2.31x** | +| TRT-LLM | 2048x2048 | 4 | 9.781 | **2.85x** | 7.596 | **2.41x** | + +torch v2.5.0. TRT-LLM v0.15.0.dev2024102900, `--num-warmup-runs=5; --avg-runs=20`. All communications are synchronous. diff --git a/examples/models/contrib/sdxl/build_sdxl_unet.py b/examples/models/contrib/sdxl/build_sdxl_unet.py new file mode 100755 index 000000000000..e2893859f731 --- /dev/null +++ b/examples/models/contrib/sdxl/build_sdxl_unet.py @@ -0,0 +1,148 @@ +import argparse +import os + +import tensorrt as trt +import torch +from diffusers import DiffusionPipeline + +import tensorrt_llm +from tensorrt_llm.builder import Builder +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.models.unet.pp.unet_pp import DistriUNetPP +from tensorrt_llm.models.unet.unet_2d_condition import UNet2DConditionModel +from tensorrt_llm.models.unet.weights import load_from_hf_unet +from tensorrt_llm.network import net_guard + +parser = argparse.ArgumentParser(description='build the UNet TensorRT engine.') +parser.add_argument('--model_dir', + type=str, + default='stabilityai/stable-diffusion-xl-base-1.0') +parser.add_argument('--size', type=int, default=1024, help='image size') +parser.add_argument('--output_dir', + type=str, + default=None, + help='output directory') + +args = parser.parse_args() + +model_dir = args.model_dir +size = args.size +sample_size = size // 8 + +world_size = tensorrt_llm.mpi_world_size() +rank = tensorrt_llm.mpi_rank() +output_dir = f'sdxl_s{size}_w{world_size}' if args.output_dir is None else args.output_dir +if rank == 0 and not os.path.exists(output_dir): + os.makedirs(output_dir) + +device_per_batch = world_size // 2 if world_size > 1 else 1 +batch_group = 2 if world_size > 1 else 1 + +# Use tp_size to indicate the size of patch parallelism +# Use pp_size to indicate the size of batch parallelism +mapping = Mapping(world_size=world_size, + rank=rank, + tp_size=device_per_batch, + pp_size=batch_group) + +torch.cuda.set_device(tensorrt_llm.mpi_rank()) + +tensorrt_llm.logger.set_level('verbose') +builder = Builder() +builder_config = builder.create_builder_config( + name='UNet2DConditionModel', + precision='float16', + timing_cache='model.cache', + profiling_verbosity='detailed', + tensor_parallel=world_size, + precision_constraints= + None, # do not use obey or the precision error will be too large +) + +pipeline = DiffusionPipeline.from_pretrained(model_dir, + torch_dtype=torch.float16) +model = UNet2DConditionModel( + sample_size=sample_size, + in_channels=4, + out_channels=4, + center_input_sample=False, + flip_sin_to_cos=True, + freq_shift=0, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(320, 640, 1280), + layers_per_block=2, + downsample_padding=1, + mid_block_scale_factor=1.0, + act_fn="silu", + norm_num_groups=32, + norm_eps=1e-5, + cross_attention_dim=2048, + attention_head_dim=[5, 10, 20], + addition_embed_type="text_time", + addition_time_embed_dim=256, + projection_class_embeddings_input_dim=2816, + transformer_layers_per_block=[1, 2, 10], + use_linear_projection=True, + dtype=trt.float16, +) + +load_from_hf_unet(pipeline.unet, model) +model = DistriUNetPP(model, mapping) + +# Module -> Network +network = builder.create_network() +network.plugin_config.to_legacy_setting() +if mapping.world_size > 1: + network.plugin_config.set_nccl_plugin('float16') + +with net_guard(network): + # Prepare + network.set_named_parameters(model.named_parameters()) + + # Forward + sample = tensorrt_llm.Tensor( + name='sample', + dtype=trt.float16, + shape=[2, 4, sample_size, sample_size], + ) + timesteps = tensorrt_llm.Tensor( + name='timesteps', + dtype=trt.float16, + shape=[ + 1, + ], + ) + encoder_hidden_states = tensorrt_llm.Tensor( + name='encoder_hidden_states', + dtype=trt.float16, + shape=[2, 77, 2048], + ) + text_embeds = tensorrt_llm.Tensor( + name='text_embeds', + dtype=trt.float16, + shape=[2, 1280], + ) + time_ids = tensorrt_llm.Tensor( + name='time_ids', + dtype=trt.float16, + shape=[2, 6], + ) + + output = model(sample, timesteps, encoder_hidden_states, text_embeds, + time_ids) + + # Mark outputs + output_dtype = trt.float16 + output.mark_output('pred', output_dtype) + +# Network -> Engine +engine = builder.build_engine(network, builder_config) +assert engine is not None, 'Failed to build engine.' + +engine_name = f'sdxl_unet_s{size}_w{world_size}_r{rank}.engine' +engine_path = os.path.join(output_dir, engine_name) +with open(engine_path, 'wb') as f: + f.write(engine) +builder.save_config(builder_config, os.path.join(output_dir, 'config.json')) diff --git a/examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py b/examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py new file mode 100755 index 000000000000..018f3fad783f --- /dev/null +++ b/examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py @@ -0,0 +1,1365 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import json +import os +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import tensorrt as trt +import torch +from diffusers.image_processor import PipelineImageInput, VaeImageProcessor +from diffusers.loaders import (FromSingleFileMixin, IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin) +from diffusers.models import AutoencoderKL, UNet2DConditionModel +from diffusers.models.attention_processor import (AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor) +from diffusers.models.lora import adjust_lora_scale_text_encoder +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.pipelines.stable_diffusion_xl.pipeline_output import \ + StableDiffusionXLPipelineOutput +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import (USE_PEFT_BACKEND, deprecate, + is_invisible_watermark_available, + is_torch_xla_available, logging, + replace_example_docstring, scale_lora_layers, + unscale_lora_layers) +from diffusers.utils.torch_utils import randn_tensor +from transformers import (CLIPImageProcessor, CLIPTextModel, + CLIPTextModelWithProjection, CLIPTokenizer, + CLIPVisionModelWithProjection) + +import tensorrt_llm +from tensorrt_llm.runtime import Session, TensorInfo + +if is_invisible_watermark_available(): + from diffusers.pipelines.stable_diffusion_xl.watermark import \ + StableDiffusionXLWatermarker + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLPipeline + + >>> pipe = StableDiffusionXLPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), + keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + \ + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set( + inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionXLPipeline( + DiffusionPipeline, + FromSingleFileMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + In addition the pipeline inherits the following loading methods: + - *LoRA*: [`loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] + - *Ckpt*: [`loaders.FromSingleFileMixin.from_single_file`] + + as well as the following saving methods: + - *LoRA*: [`loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to + watermark output images. If not defined, it will default to True if the package is installed, otherwise no + watermarker will be used. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "image_encoder", + "feature_extractor", + ] + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "add_text_embeds", + "add_time_ids", + "negative_pooled_prompt_embeds", + "negative_add_time_ids", + ] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + ) + self.register_to_config( + force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2**(len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor) + + self.default_sample_size = self.unet.config.sample_size + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available( + ) + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + self.execution_device = torch.device('cpu') + self.engine = {} + + def to( + self, + torch_device: Optional[Union[str, torch.device]] = None, + torch_dtype: Optional[torch.dtype] = None, + silence_dtype_warnings: bool = False, + ): + super().to(torch_device) + if isinstance(torch_device, str): + torch_device = torch.device(torch_device) + self.execution_device = torch_device + return self + + def prepare(self, path, size): + self.unet.cpu() + torch.cuda.empty_cache() + + def trt_dtype_to_torch(dtype): + if dtype == trt.float16: + return torch.float16 + elif dtype == trt.float32: + return torch.float32 + elif dtype == trt.int32: + return torch.int32 + else: + raise TypeError("%s is not supported" % dtype) + + config_path = os.path.join(path, 'config.json') + with open(config_path, 'r') as f: + config = json.load(f) + config['builder_config']['precision'] + world_size = config['builder_config']['tensor_parallel'] + + runtime_world_size = tensorrt_llm.mpi_world_size() + assert world_size == runtime_world_size, f'Engine world size ({world_size}) != Runtime world size ({runtime_world_size})' + runtime_rank = tensorrt_llm.mpi_rank() if world_size > 1 else 0 + torch.cuda.set_device(runtime_rank) + + serialize_file = f'sdxl_unet_s{size}_w{world_size}_r{runtime_rank}.engine' + serialize_path = os.path.join(path, serialize_file) + self.stream = torch.cuda.current_stream().cuda_stream + print(f'Loading engine from {serialize_path}') + with open(serialize_path, 'rb') as f: + engine_buffer = f.read() + print(f'Creating session from engine') + self.session = Session.from_serialized_engine(engine_buffer) + + output_info = self.session.infer_shapes([ + TensorInfo('sample', trt.DataType.HALF, + [2, 4, size // 8, size // 8]), + TensorInfo('timesteps', trt.DataType.HALF, [ + 1, + ]), + TensorInfo('encoder_hidden_states', trt.DataType.HALF, + [2, 77, 2048]), + TensorInfo('text_embeds', trt.DataType.HALF, [2, 1280]), + TensorInfo('time_ids', trt.DataType.HALF, [2, 6]), + ]) + self.outputs = { + t.name: + torch.empty(tuple(t.shape), + dtype=trt_dtype_to_torch(t.dtype), + device='cuda') + for t in output_info + } + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_tiling + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = self.execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance( + self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, + lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, + lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2 + ] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ([ + self.text_encoder, self.text_encoder_2 + ] if self.text_encoder is not None else [self.text_encoder_2]) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: procecss multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, + text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, + padding="longest", + return_tensors="pt").input_ids + + if untruncated_ids.shape[ + -1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids): + removed_text = tokenizer.batch_decode( + untruncated_ids[:, tokenizer.model_max_length - 1:-1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}") + + prompt_embeds = text_encoder(text_input_ids.to(device), + output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like( + pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * \ + [negative_prompt] if isinstance( + negative_prompt, str) else negative_prompt + negative_prompt_2 = (batch_size * [negative_prompt_2] if isinstance( + negative_prompt_2, str) else negative_prompt_2) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}.") + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`.") + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip( + uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt( + negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[ + -2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, + dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, + device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, + device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, + seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to( + dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to( + dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat( + 1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view( + batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat( + 1, num_images_per_prompt).view(bs_embed * num_images_per_prompt, -1) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat( + 1, num_images_per_prompt).view(bs_embed * num_images_per_prompt, + -1) + + if self.text_encoder is not None: + if isinstance( + self, + StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance( + self, + StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, + return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, + dim=0) + + uncond_image_embeds = torch.zeros_like(image_embeds) + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set( + inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set( + inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError( + f"`height` and `width` have to be divisible by 8 but are {height} and {width}." + ) + + if callback_steps is not None and (not isinstance(callback_steps, int) + or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs + for k in callback_on_step_end_tensor_inputs): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two.") + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two.") + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) + and not isinstance(prompt, list)): + raise ValueError( + f"`prompt` has to be of type `str` or `list` but is {type(prompt)}" + ) + elif prompt_2 is not None and (not isinstance(prompt_2, str) + and not isinstance(prompt_2, list)): + raise ValueError( + f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}" + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}.") + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None): + shape = (batch_size, num_channels_latents, + height // self.vae_scale_factor, + width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, + generator=generator, + device=device, + dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def _get_add_time_ids(self, + original_size, + crops_coords_top_left, + target_size, + dtype, + text_encoder_projection_dim=None): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + + text_encoder_projection_dim) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_freeu + def enable_freeu(self, s1: float, s2: float, b1: float, b2: float): + r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497. + + The suffixes after the scaling factors represent the stages where they are being applied. + + Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values + that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. + + Args: + s1 (`float`): + Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to + mitigate "oversmoothing effect" in the enhanced denoising process. + s2 (`float`): + Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to + mitigate "oversmoothing effect" in the enhanced denoising process. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + if not hasattr(self, "unet"): + raise ValueError("The pipeline must have `unet` for using FreeU.") + self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_freeu + def disable_freeu(self): + """Disables the FreeU mechanism if enabled.""" + self.unet.disable_freeu() + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, + w, + embedding_dim=512, + dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def denoising_end(self): + return self._denoising_end + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, + List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._denoising_end = denoising_end + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self.execution_device + + # 3. Encode input prompt + lora_scale = (self.cross_attention_kwargs.get("scale", None) + if self.cross_attention_kwargs is not None else None) + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + if negative_original_size is not None and negative_target_size is not None: + negative_add_time_ids = self._get_add_time_ids( + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + else: + negative_add_time_ids = add_time_ids + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], + dim=0) + add_text_embeds = torch.cat( + [negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], + dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat( + batch_size * num_images_per_prompt, 1) + + if ip_adapter_image is not None: + image_embeds, negative_image_embeds = self.encode_image( + ip_adapter_image, device, num_images_per_prompt) + if self.do_classifier_free_guidance: + image_embeds = torch.cat([negative_image_embeds, image_embeds]) + image_embeds = image_embeds.to(device) + + # 8. Denoising loop + num_warmup_steps = max( + len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + # 8.1 Apply denoising_end + if (self.denoising_end is not None + and isinstance(self.denoising_end, float) + and self.denoising_end > 0 and self.denoising_end < 1): + discrete_timestep_cutoff = int( + round(self.scheduler.config.num_train_timesteps - + (self.denoising_end * + self.scheduler.config.num_train_timesteps))) + num_inference_steps = len( + list( + filter(lambda ts: ts >= discrete_timestep_cutoff, + timesteps))) + timesteps = timesteps[:num_inference_steps] + + # 9. Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - + 1).repeat( + batch_size * + num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, + embedding_dim=self.unet.config.time_cond_proj_dim).to( + device=device, dtype=latents.dtype) + + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat( + [latents] * + 2) if self.do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input( + latent_model_input, t) + + # predict the noise residual + added_cond_kwargs = { + "text_embeds": add_text_embeds, + "time_ids": add_time_ids + } + if ip_adapter_image is not None: + added_cond_kwargs["image_embeds"] = image_embeds + + t = t.to(latent_model_input.dtype) + feed_dict = { + 'sample': latent_model_input, + 'timesteps': t.unsqueeze(0), + 'encoder_hidden_states': prompt_embeds, + 'text_embeds': add_text_embeds, + 'time_ids': add_time_ids, + } + ok = self.session.run(feed_dict, self.outputs, self.stream) + assert ok, "Runtime execution failed" + noise_pred = self.outputs['pred'] + torch.cuda.synchronize() + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * \ + (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg( + noise_pred, + noise_pred_text, + guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, + t, + latents, + **extra_step_kwargs, + return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end( + self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop( + "prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop( + "negative_prompt_embeds", negative_prompt_embeds) + add_text_embeds = callback_outputs.pop( + "add_text_embeds", add_text_embeds) + negative_pooled_prompt_embeds = callback_outputs.pop( + "negative_pooled_prompt_embeds", + negative_pooled_prompt_embeds) + add_time_ids = callback_outputs.pop("add_time_ids", + add_time_ids) + negative_add_time_ids = callback_outputs.pop( + "negative_add_time_ids", negative_add_time_ids) + + # call the callback, if provided + if i == len(timesteps) - 1 or ( + (i + 1) > num_warmup_steps and + (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if XLA_AVAILABLE: + xm.mark_step() + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to( + next(iter(self.vae.post_quant_conv.parameters())).dtype) + + image = self.vae.decode(latents / self.vae.config.scaling_factor, + return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + + if not output_type == "latent": + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, + output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, ) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/examples/models/contrib/sdxl/run_sdxl.py b/examples/models/contrib/sdxl/run_sdxl.py new file mode 100755 index 000000000000..077b93f9ab7e --- /dev/null +++ b/examples/models/contrib/sdxl/run_sdxl.py @@ -0,0 +1,96 @@ +import argparse +import time + +import numpy as np +import torch +from pipeline_stable_diffusion_xl import StableDiffusionXLPipeline + +import tensorrt_llm + +world_size = tensorrt_llm.mpi_world_size() +rank = tensorrt_llm.mpi_rank() + + +def parseArgs(): + parser = argparse.ArgumentParser( + description='run SDXL with the UNet TensorRT engine.') + parser.add_argument('--model_dir', + type=str, + default='stabilityai/stable-diffusion-xl-base-1.0') + parser.add_argument('--size', type=int, default=1024) + parser.add_argument('--seed', type=int, default=233) + parser.add_argument('--num_inference_steps', type=int, default=50) + parser.add_argument( + '--prompt', + type=str, + default= + "masterpiece, gouache painting, 1girl, distant view, lone boat, willow trees" + ) + parser.add_argument('--engine_dir', + type=str, + default=None, + help='engine directory') + parser.add_argument('--num-warmup-runs', type=int, default=3) + parser.add_argument('--avg-runs', type=int, default=10) + parser.add_argument("--ignore_ratio", + type=float, + default=0.2, + help="Ignored ratio of the slowest and fastest steps") + parser.add_argument("--output", + type=str, + default="output.png", + help="Output file name") + return parser.parse_args() + + +if __name__ == "__main__": + args = parseArgs() + model_dir = args.model_dir + size = args.size + seed = args.seed + prompt = args.prompt + num_inference_steps = args.num_inference_steps + engine_dir = f'sdxl_s{size}_w{world_size}' if args.engine_dir is None else args.engine_dir + num_warmup_runs = args.num_warmup_runs + avg_runs = args.avg_runs + output_file = args.output + + pipeline = StableDiffusionXLPipeline.from_pretrained( + model_dir, + torch_dtype=torch.float16, + use_safetensors=True, + ) + pipeline.set_progress_bar_config(disable=rank != 0) + pipeline.prepare(engine_dir, size) + pipeline.to('cuda') + + # warm up + for i in range(num_warmup_runs): + image = pipeline( + num_inference_steps=num_inference_steps, + prompt=prompt, + generator=torch.Generator(device="cuda").manual_seed(seed), + height=size, + width=size).images[0] + + latency_list = [] + for i in range(avg_runs): + st = time.time() + image = pipeline( + num_inference_steps=num_inference_steps, + prompt=prompt, + generator=torch.Generator(device="cuda").manual_seed(seed), + height=size, + width=size, + ).images[0] + ed = time.time() + latency_list.append(ed - st) + + latency_list = sorted(latency_list) + ignored_count = int(args.ignore_ratio * len(latency_list) / 2) + if ignored_count > 0: + latency_list = latency_list[ignored_count:-ignored_count] + + if rank == 0: + print(f"Avg latency: {np.sum(latency_list) / len(latency_list):.5f} s") + image.save(output_file) diff --git a/examples/models/contrib/skywork/README.md b/examples/models/contrib/skywork/README.md new file mode 100644 index 000000000000..59b22a861987 --- /dev/null +++ b/examples/models/contrib/skywork/README.md @@ -0,0 +1,113 @@ +# Skywork + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document elaborates how to build the [Skywork](https://huggingface.co/Skywork/) model to runnable engines on single GPU node and perform a summarization task using these engines. + +## Overview +The TensorRT LLM Skywork implementation is based on the LLaMA model. The implementation can +be found in [tensorrt_llm/models/llama/model.py](../../../../tensorrt_llm/models/llama/model.py). +The TensorRT LLM Skywork example code lies in [`examples/models/contrib/skywork`](./): + +* [`convert_checkpoint.py`](../../core/llama/convert_checkpoint.py) converts the Huggingface Model of Skywork into TensorRT LLM checkpoint. + +In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: + +* [`../../../run.py`](../../../run.py) to run the inference on an input text; +* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +## Support Matrix + * FP16 & BF16 + +## Usage + +This section gives a whole process where we convert HF models, build TensorRT LLM engines and ultimately perform summarization. + +### 1. Clone Code and Weights from Huggingface + +To download checkpoints from HF, you need to have `git-lfs` installed in your machine: + +```bash +pip install -r requirements.txt && sudo apt-get install git-lfs +``` + +Then clone the HF repository with: + +```bash +# Skywork 13B Base Model +git clone https://huggingface.co/Skywork/Skywork-13B-base +``` + +### 2. Convert HF Model to TRT Checkpoint + +```bash +cd examples/models/core/llama + +# fp16 model +python3 convert_checkpoint.py --model_dir ./Skywork-13B-base \ + --dtype float16 \ + --output_dir ./skywork-13b-base/trt_ckpt/fp16 + +# bf16 model +python3 convert_checkpoint.py --model_dir ./Skywork-13B-base \ + --dtype bfloat16 \ + --output_dir ./skywork-13b-base/trt_ckpt/bf16 +``` + +### 3. Build TensorRT Engine(s) + +```bash +# fp16 +trtllm-build --checkpoint_dir ./skywork-13b-base/trt_ckpt/fp16 \ + --gemm_plugin float16 \ + --gpt_attention_plugin float16 \ + --context_fmha enable \ + --max_batch_size 32 \ + --max_input_len 512 \ + --max_seq_len 1024 \ + --output_dir ./skywork-13b-base/trt_engine/fp16 + +# bf16 +trtllm-build --checkpoint_dir ./skywork-13b-base/trt_ckpt/bf16 \ + --gemm_plugin bfloat16 \ + --gpt_attention_plugin bfloat16 \ + --context_fmha enable \ + --max_batch_size 32 \ + --max_input_len 512 \ + --max_seq_len 1024 \ + --output_dir ./skywork-13b-base/trt_engine/bf16 +``` + +### 4. Summarization using the Engines + +After building TRT engines, we can use them to perform various tasks. TensorRT LLM provides handy code to run summarization on [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset and get [ROUGE](https://en.wikipedia.org/wiki/ROUGE_(metric)) scores. The `ROUGE-1` score can be used to validate model implementations. + +```bash +# fp16 +python ../../../summarize.py --hf_model_dir ./Skywork-13B-base \ + --test_hf \ + --batch_size 32 \ + --max_input_length 512 \ + --output_len 512 \ + --test_trt_llm \ + --engine_dir ./skywork-13b-base/trt_engine/fp16 \ + --data_type fp16 \ + --check_accuracy \ + --tensorrt_llm_rouge1_threshold=14 + +# bf16 +python ../../../summarize.py --hf_model_dir ./Skywork-13B-base \ + --test_hf \ + --batch_size 32 \ + --max_input_length 512 \ + --output_len 512 \ + --test_trt_llm \ + --engine_dir ./skywork-13b-base/trt_engine/bf16 \ + --data_type bf16 \ + --check_accuracy \ + --tensorrt_llm_rouge1_threshold=14 +``` diff --git a/examples/models/contrib/skywork/requirements.txt b/examples/models/contrib/skywork/requirements.txt new file mode 100644 index 000000000000..88232baef811 --- /dev/null +++ b/examples/models/contrib/skywork/requirements.txt @@ -0,0 +1,6 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +datasets==3.1.0 +evaluate +rouge_score +sentencepiece>=0.1.99 diff --git a/examples/models/contrib/smaug/README.md b/examples/models/contrib/smaug/README.md new file mode 100644 index 000000000000..2d96da5854e1 --- /dev/null +++ b/examples/models/contrib/smaug/README.md @@ -0,0 +1,62 @@ +# Smaug + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document elaborates how to build the [Smaug-72B-v0.1](https://huggingface.co/abacusai/Smaug-72B-v0.1) model to runnable engines on multi-GPU node and perform a summarization task using these engines. + +## Overview + +The TensorRT LLM support for Smaug-72B-v0.1 is based on the LLaMA model, the implementation can be found in [tensorrt_llm/models/llama/model.py](../../../../tensorrt_llm/models/llama/model.py). Smaug model resembles LLaMA very much except it uses bias term in its attention module, we therefore reuse the [LLaMA example code](../../core/llama) for Smaug, + +* [`convert_checkpoint.py`](./convert_checkpoint.py) to convert the LLaMA model into TensorRT LLM checkpoint format. + +In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: + +* [`../../../run.py`](../../../run.py) to run the inference on an input text; +* [`../../../summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +## Support Matrix + +* FP16 + +## Usage + +This section gives a whole process where we convert HF models, build TensorRT LLM engines and ultimately perform summarization. + +### Build TensorRT engine(s) + +Run the following commands and TRT-LLM will first transforms a HF model into its own checkpoint format, then builds a TRT engine based on the checkpoint + +```bash +python ../../../llama/convert_checkpoint.py \ + --model_dir ./Smaug-72B-v0.1 \ + --output_dir ./tllm_checkpoint_8gpu_tp8 \ + --dtype float16 \ + --tp_size 8 + +trtllm-build --checkpoint_dir ./tllm_checkpoint_8gpu_tp8 \ + --output_dir ./Smaug_72B_tp8 \ + --gemm_plugin float16 \ + --gpt_attention_plugin float16 \ + --context_fmha=enable \ + --max_batch_size 64 \ + --remove_input_padding=enable +``` + +### Run Summarization + +After building TRT engine, we can use it to perform various tasks. TensorRT LLM provides handy code to run summarization on [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset and get [ROUGE](https://en.wikipedia.org/wiki/ROUGE_(metric)) scores. The `ROUGE-1` score can be used to validate model implementations. + +```bash +mpirun -n 8 -allow-run-as-root python ../../../summarize.py \ + --hf_model_dir ../Smaug-72B-v0.1 \ + --engine_dir ./Smaug_72B_tp8 \ + --data_type fp16 \ + --test_hf \ + --hf_device_map_auto \ + --test_trt_llm +``` diff --git a/examples/models/contrib/smaug/requirements.txt b/examples/models/contrib/smaug/requirements.txt new file mode 100644 index 000000000000..88232baef811 --- /dev/null +++ b/examples/models/contrib/smaug/requirements.txt @@ -0,0 +1,6 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +datasets==3.1.0 +evaluate +rouge_score +sentencepiece>=0.1.99 diff --git a/examples/models/core/deepseek_v3/README.md b/examples/models/core/deepseek_v3/README.md index 6f3af476866a..6c9774b83624 100644 --- a/examples/models/core/deepseek_v3/README.md +++ b/examples/models/core/deepseek_v3/README.md @@ -863,10 +863,10 @@ echo "All processes completed!" The converted checkpoint could be used as `` and consumed by other commands. ### KV Cache Reuse -KV cache reuse is supported for MLA on SM90, SM100, SM103, SM120 and SM121. It is enabled by default. Due to extra operations like memcpy and GEMMs, GPU memory consumption may be higher and the E2E performance may have regression in some cases. Users could pass `KvCacheConfig(enable_block_reuse=False)` to LLM API to disable it. +KV cache reuse is supported for MLA on SM90, SM100 and SM120. It is enabled by default. Due to extra operations like memcpy and GEMMs, GPU memory consumption may be higher and the E2E performance may have regression in some cases. Users could pass `KvCacheConfig(enable_block_reuse=False)` to LLM API to disable it. ### Chunked Prefill -Chunked Prefill is supported for MLA on SM90, SM100, SM103 and SM120. You should add `--enable_chunked_prefill` to enable it. The GPU memory consumption is highly correlated with `max_num_tokens` and `max_batch_size`. If encountering out-of-memory errors, you may make these values smaller. (`max_num_tokens` must be divisible by kv cache's `tokens_per_block`) +Chunked Prefill is supported for MLA only on SM90 and SM100 currently. You should add `--enable_chunked_prefill` to enable it. The GPU memory consumption is highly correlated with `max_num_tokens` and `max_batch_size`. If encountering out-of-memory errors, you may make these values smaller. (`max_num_tokens` must be divisible by kv cache's `tokens_per_block`) More specifically, we can imitate what we did in the [Quick Start](#quick-start): diff --git a/examples/models/core/granite/README.md b/examples/models/core/granite/README.md new file mode 100644 index 000000000000..b6cf34c50c5f --- /dev/null +++ b/examples/models/core/granite/README.md @@ -0,0 +1,89 @@ +# Granite + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document shows how to build and run a [Granite 3.0](https://huggingface.co/collections/ibm-granite/granite-30-language-models-66fdb59bbb54785c3512114f) model in TensorRT-LLM. + +The TensorRT LLM Granite implementation is based on the LLaMA model, with Mixture of Experts (MoE) enabled. The implementation can be found in [`llama/model.py`](../../../../tensorrt_llm/models/llama/model.py). See the LLaMA example [`examples/models/core/llama`](../llama) for details. + +- [Granite 3.0](#Granite) + - [Download model checkpoints](#download-model-checkpoints) + - [Convert weights from HF Transformers to TensorRT LLM format](#Convert-weights-from-HF-Transformers-to-TensorRT-LLM-format) + - [Build TensorRT engine](#build-tensorrt-engine) + - [Run Engine](#run-engine) + +## Download model checkpoints + +First, download the HuggingFace BF16 checkpoints of Granite 3.0 model. + +```bash +HF_MODEL="granite-3.0-8b-instruct" # or granite-3.0-3b-a800m-instruct +# clone the model we want to build +git clone https://huggingface.co/ibm-granite/${HF_MODEL} tmp/hf_checkpoints/${HF_MODEL} +``` + +## Convert weights from HF Transformers to TensorRT LLM format +Set environment variables and necessary directory: + +```bash +PREC_RAW="bfloat16" +TP=1 +mkdir -p tmp/trt_engines +``` + +### BF16 +Convert the weights using the `convert_checkpoint.py` script: + +```bash +ENGINE="${HF_MODEL}_${PREC_RAW}_tp${TP}" +export TRTLLM_DISABLE_UNIFIED_CONVERTER=1 # The current checkpoint conversion code requires legacy path +python3 ../llama/convert_checkpoint.py --model_dir tmp/hf_checkpoints/${HF_MODEL} \ + --output_dir tmp/tllm_checkpoints/${ENGINE} \ + --dtype ${PREC_RAW} \ + --tp_size ${TP} \ + --use_embedding_sharing + + +``` +### FP8 PTQ +Notes: +- Currently quantize.py does not support Expert Parallelism (EP) mode yet. User should use `../llama/convert_checkpoint.py` and specify `--moe_ep_size 1` instead, if needed. +- TensorRT LLM uses static quantization methods, which is expected to be faster at runtime as compared to dynamic quantization methods. This comes at a cost of an offline calibration step during quantization. `batch_size` and `calib_size` can be adjusted to shorten the calibration time. Please refer to `../../../quantization/README.md` for explanation. + +```bash +PREC_QUANT="fp8" +ENGINE="${HF_MODEL}_${PREC_QUANT}_tp${TP}" +python ../../../quantization/quantize.py --model_dir tmp/hf_checkpoints/${HF_MODEL} \ + --dtype ${PREC_RAW} \ + --qformat ${PREC_QUANT} \ + --kv_cache_dtype ${PREC_QUANT} \ + --output_dir tmp/tllm_checkpoints/${ENGINE} \ + --batch_size 1 \ + --calib_size 128 \ + --tp_size ${TP} + +``` + +## Build TensorRT engine +```bash +# Enable fp8 context fmha to get further acceleration by setting `--use_fp8_context_fmha enable` +# Use --workers to enable parallel build +trtllm-build --checkpoint_dir ./tmp/tllm_checkpoints/${ENGINE} \ + --output_dir ./tmp/trt_engines/${ENGINE} \ + --gpt_attention_plugin ${PREC_RAW} \ + --gemm_plugin ${PREC_RAW} \ + --workers ${TP} +``` + +## Run Engine +Test your engine with the [run.py](../../../run.py) script: + +```bash +mpirun -n ${TP} --allow-run-as-root python ../../../run.py --engine_dir ./tmp/trt_engines/${ENGINE} --tokenizer_dir tmp/hf_checkpoints/${HF_MODEL} --max_output_len 20 --input_text "The future of AI is" +``` + +For more usage examples see [`examples/models/core/llama/README.md`](../llama/README.md) diff --git a/examples/models/core/mixtral/README.md b/examples/models/core/mixtral/README.md new file mode 100644 index 000000000000..079753f88c92 --- /dev/null +++ b/examples/models/core/mixtral/README.md @@ -0,0 +1,219 @@ +# Mixtral + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document shows how to build and run a Mixtral model in TensorRT LLM on both single GPU, single node multi-GPU and +multi-node multi-GPU. Mixtral 8x22B is also supported and can be replace Mixtral 8x7B below as long as GPU memory is +sufficient. + +## Overview + +The TensorRT LLM Mixtral implementation is based on the LLaMA model, with Mixture of Experts enabled. The implementation can +be found in [tensorrt_llm/models/llama/model.py](../../../../tensorrt_llm/models/llama/model.py). +See the LLaMA example [`examples/models/core/llama`](../llama) for details. + +### Build TensorRT engine(s) + +#### Download Mixtral 8x7b weights +Get the weights by downloading from HF https://huggingface.co/mistralai/Mixtral-8x7B-v0.1. +See also https://huggingface.co/docs/transformers/main/en/model_doc/mixtral + +```bash +git lfs install +git clone https://huggingface.co/mistralai/Mixtral-8x7B-v0.1 +``` + +#### Download Mixtral 8x22b weights +Get the weights by downloading from HF https://huggingface.co/mistralai/Mixtral-8x22B-v0.1. +See also https://huggingface.co/docs/transformers/main/en/model_doc/mixtral + +```bash +git lfs install +git clone https://huggingface.co/mistralai/Mixtral-8x22B-v0.1 +``` + +We use the LLaMA `convert_checkpoint.py` script to convert and build the model. TensorRT LLM LLaMA builds TensorRT engine(s) from HF checkpoint provided by `--model_dir`. +If no checkpoint directory is specified, TensorRT LLM will build engine(s) with dummy weights. + +`trtllm-build` uses one GPU by default, but if you have already more GPUs available at build time, +you may enable parallel builds to make the engine building process faster by adding the `--workers` argument. + +Here are some examples: + +```bash +# Build Mixtral8x7B with pipeline parallelism +python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ + --output_dir ./tllm_checkpoint_mixtral_2gpu \ + --dtype float16 \ + --pp_size 2 +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ + --output_dir ./trt_engines/mixtral/pp2 \ + --gemm_plugin float16 + +``` + +```bash +# Build Mixtral8x7B with tensor parallelism +python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ + --output_dir ./tllm_checkpoint_mixtral_2gpu \ + --dtype float16 \ + --tp_size 2 \ + --moe_tp_size 2 +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ + --output_dir ./trt_engines/mixtral/tp2 \ + --gemm_plugin float16 + + +# Build Mixtral8x22B with tensor parallelism and expert parallelism +python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x22B-v0.1 \ + --output_dir ./tllm_checkpoint_mixtral_8gpu \ + --dtype float16 \ + --tp_size 8 \ + --moe_tp_size 2 \ + --moe_ep_size 4 +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_8gpu \ + --output_dir ./trt_engines/mixtral/tp2ep4 \ + --gemm_plugin float16 +``` + +Then, you can test your engine with the [run.py](../../../run.py) script: + +```bash +mpirun -n 2 python3 ../../../run.py --engine_dir ./trt_engines/mixtral/tp2 --tokenizer_dir ./Mixtral-8x7B-v0.1 --max_output_len 8 --input_text "I love french quiche" +``` + +For more examples see [`examples/models/core/llama/README.md`](../llama/README.md) + +### Parallelism Modes + +Mixture of Experts supports 3 parallelism modes, these are Expert Parallelism (EP), Tensor Parallelism (TP), and the hybrid of the two (TP+EP). + +In TP mode (default) expert weight matrices are sliced evenly between all GPUs, so that all GPUs work together to calculate the result for each expert. + +In EP mode each GPU is assigned a subset of the expert weights matrices, so each GPU works independently to calculate the result for its assigned experts. This may cause load balancing issues where some GPUs have more work than others, thus increasing latency. + +In TP+EP mode, both strategies are used simultaneously. This means each GPU handles a portion of the expert weights matrices (as in EP mode) and these weights are further sliced across multiple GPUs (as in TP mode). This hybrid approach aims to balance the workload more evenly across GPUs, enhancing efficiency and reducing the likelihood of bottlenecks associated with EP mode alone. + +You can enable Expert Parallel or hybrid parallel by setting `--moe_tp_size` and `--moe_ep_size` when calling `convert_coneckpoint.py`. If only `--moe_tp_size` is provided, TRT-LLM will use Tensor Parallel for the MoE model; if only `--moe_ep_size` is provided, TRT-LLM will use Expert Parallel; if both are provided, the hybrid parallel will be used. + +Be sure that the product of `moe_tp_size` and `moe_ep_size` should equal to `tp_size`, since the total number of MoE parallelism across all GPUs must match the total number of parallelism in other parts of the model. + +```bash +# Build Mixtral8x7B with Expert Parallelism +python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ + --output_dir ./tllm_checkpoint_mixtral_2gpu \ + --dtype float16 \ + --tp_size 2 \ + --moe_ep_size 2 +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ + --output_dir ./trt_engines/mixtral/ep2 \ + --gemm_plugin float16 + +# Build Mixtral8x7B with Expert Parallelism and Tensor Parallelism +python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ + --output_dir ./tllm_checkpoint_mixtral_4gpu \ + --dtype float16 \ + --tp_size 4 \ + --moe_tp_size 2 \ + --moe_ep_size 2 +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_4gpu \ + --output_dir ./trt_engines/mixtral/tp2ep2 \ + --gemm_plugin float16 +``` + +### Normalization Modes + +MOE Supports different normalization modes which influence how the scales are calculated for the final weighted sum in +of the different top-k values. + +- 0 (NONE) corresponds to: `scales = topk(softmax(routing values))` +- 1 (RENORM) corresponds to: `scales = softmax(topk(routing values))` +- 2 (SPARSE_MIXER) corresponds to: `scales = sparsemixer(routing values)` + +Mixtral uses `RENORM` mode, this is set as the default. To use a different mode use the `--moe_normalization_mode` flag. +See [tensorrt_llm/layers/moe.py](../../../../tensorrt_llm/layers/moe.py#L56) for available values + + +## Quantization + +### Weight-only Quantization + +Mixtral supports weight only quantization + +```bash +# Build Mixtral8x7B with weight only +python ../llama/convert_checkpoint.py --model_dir ./Mixtral-8x7B-v0.1 \ + --output_dir ./tllm_checkpoint_mixtral_2gpu \ + --dtype float16 \ + --tp_size 2 \ + --use_weight_only \ + --weight_only_precision int8 +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ + --output_dir ./trt_engines/mixtral/tp2 \ + --gemm_plugin float16 +``` + +### FP8 Post-Training Quantization + +Mixtral supports FP8 quantization, using Modelopt. See [`examples/models/core/llama/README.md`](../llama/README.md#fp8-post-training-quantization) for full details on installing Modelopt + +```bash +# Quantize HF Mixtral into FP8 and export trtllm checkpoint +python ../../../quantization/quantize.py --model_dir ./Mixtral-8x7B-v0.1 \ + --dtype float16 \ + --qformat fp8 \ + --kv_cache_dtype fp8 \ + --output_dir ./tllm_checkpoint_mixtral_2gpu \ + --calib_size 512 \ + --tp_size 2 + +# Build trtllm engines from the trtllm checkpoint +# Enable fp8 context fmha to get further acceleration by setting `--use_fp8_context_fmha enable` +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_2gpu \ + --output_dir ./engine_outputs \ + --workers 2 +``` + +### AWQ Quantization + +Mixtral supports AWQ quantization using [AutoAWQ](https://github.com/casper-hansen/AutoAWQ). + +```bash +# Convert AutoAWQ HF checkpoints into TRT-LLM checkpoint +python ../llama/convert_checkpoint.py --model_dir ./tmp/mixtral-8x7b-v0.1-AWQ/ \ + --output_dir ./tllm_checkpoint_mixtral_awq_1gpu + +# Build trtllm engines from the trtllm checkpoint +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_awq_1gpu \ + --output_dir ./engine_outputs +``` + +You may found `quant_algo = W4A16_GPTQ` in the configuration file of the converted checkpoints, and that's because AutoAWQ is using exactly the same components as GPTQ. + +### NVFP4 Post-Training Quantization + +Mixtral supports NVFP4 quantization. + +```bash +# Quantize HF Mixtral into FP8 and export trtllm checkpoint +python ../../../quantization/quantize.py --model_dir ./Mixtral-8x7B-v0.1 \ + --dtype float16 \ + --qformat nvfp4 \ + --kv_cache_dtype fp8 \ + --output_dir ./tllm_checkpoint_mixtral_nvfp4_1gpu \ + --calib_size 512 \ + --tp_size 1 + +# Build trtllm engines from the trtllm checkpoint +# Enable fp8 context fmha to get further acceleration by setting `--use_fp8_context_fmha enable` +trtllm-build --checkpoint_dir ./tllm_checkpoint_mixtral_nvfp4_1gpu \ + --output_dir ./engine_outputs +``` + +## OOTB + +Mixtral supports OOTB operation without the plugin, however this comes at a significant performance cost. Users should prefer using the plugin path whenever possible diff --git a/examples/models/core/mixtral/requirements.txt b/examples/models/core/mixtral/requirements.txt new file mode 100644 index 000000000000..c1b98e5d3f7a --- /dev/null +++ b/examples/models/core/mixtral/requirements.txt @@ -0,0 +1,4 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +transformers==4.56.0 +accelerate==0.25.0 diff --git a/examples/models/core/multimodal/README.md b/examples/models/core/multimodal/README.md index 924561126cfb..159e26d4c7d8 100644 --- a/examples/models/core/multimodal/README.md +++ b/examples/models/core/multimodal/README.md @@ -1,17 +1,1169 @@ # Multi-Modal -The engine-build multimodal workflow that used to live here -(`build_multimodal_engine.py` / `run.py` / `eval.py` on top of -`trtllm-build`) was removed together with the legacy TensorRT backend. +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. -Multimodal models are supported on the PyTorch backend. See: +This document shows how to run multimodal pipelines with TensorRT-LLM, e.g. from image+text input modalities to text output. -- [Supported models](https://nvidia.github.io/TensorRT-LLM/models/supported-models.html) -- [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) and the - [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) -- Multimodal serving examples under `examples/llm-api/` and - `examples/serve/` +Multimodal models' LLM part has an additional parameter `--max_multimodal_len` compared to LLM-only build commands. Under the hood, `max_multimodal_len` and `max_prompt_embedding_table_size` are effectively the same concept, i.e., prepended/concatenated embeddings (either multimodal feature embeddings or prompt tuning embeddings) to the LLM input embeddings. The multimodal features from the visual encoder of shape `[batch_size, num_visual_features, visual_hidden_dim]` is flattened as `[batch_size * num_visual_features, visual_hidden_dim]` and passed like a prompt embedding table. + +We first describe three runtime modes for running multimodal models and how to run each model on a single GPU. We then provide general guidelines on using tensor parallelism for the LLM part of the pipeline. + +- [Runtime Mode](#runtime-modes) +- [BLIP2](#blip2) +- [CogVLM](#cogvlm) +- [Deplot](#deplot) +- [Fuyu](#fuyu) +- [Gemma3](#gemma3) +- [InternLM-XComposer2](#internlm-xcomposer2) +- [InternVL2](#internvl2) +- [Kosmos-2](#kosmos-2) +- [LLaVA, LLaVa-NeXT, LLaVA-OneVision and VILA](#llava-llava-next-llava-onevision-and-vila) +- [MLLaMA](#mllama) +- [NeVA](#neva) +- [Nougat](#nougat) +- [Phi-3-vision](#phi-3-vision) +- [Phi-4-multimodal](#phi-4-multimodal) +- [Qwen2-VL](#qwen2-vl) +- [Qwen-Image-Bench Evaluator](#qwen-image-bench-evaluator) +- [Video NeVA](#video-neva) +- [Dataset Evaluation](#dataset-evaluation) +- [Enabling Tensor Parallelism for multi-GPU](#enabling-tensor-parallelism-for-multi-gpu) +- [Enabling Embedding Table Offloading](#enabling-embedding-table-offloading) + +## Runtime Modes +TensorRT LLM supports three runtime modes for running multimodal models. +- `cpp_llm_only` (default): vision engine runs in python runtime, LLM in pybind C++ runtime +- `python`: everything runs in python runtime +- `cpp`: everything runs in C++ runtime + +This can be specified by the `--session RUNTIME_MODE` argument in `run.py` (see instructions of each model below). +Not all models supports end-to-end `cpp` mode, the checked ones below are supported. See footnotes for reasons models are unsupported +- [ ] BLIP-2-T5 [^1] +- [x] BLIP-2-OPT +- [x] CogVLM +- [ ] Deplot [^1] +- [ ] Pix2Struct [^1] +- [x] Fuyu +- [x] InternVL2-2b +- [x] Kosmos-2 +- [x] LLaVA +- [ ] LLaVA-NeXT / OneVision [^2] +- [x] VILA [^3] +- [ ] Mllama [^1] +- [x] NeVA +- [ ] Nougat [^1] +- [ ] Phi-3-Vision [^2] +- [ ] Phi-4-multimodal +- [ ] Qwen2-VL [^4] +- [x] Video-NeVA + + +[^1]: Model uses cross attention to feed visiual features to LLM decoder, which is not supported +[^2]: Model requires post processing its encoder output features, which is not supported +[^3]: Currently C++ runtime only supports single image per request (VILA mode 2) +[^4]: Vision encoder requires additional inputs not supported by the C++ runtime + +## BLIP2 + +This BLIP section covers both BLIP2-OPT and BLIP2-T5, with minor changes needed when switching the LLM backbone. + +1. Download Huggingface weights and convert original checkpoint to TRT-LLM checkpoint format + following example in `examples/models/contrib/opt/README.md` and `examples/models/core/enc_dec/README.md`. + + ```bash + export MODEL_NAME="blip2-opt-2.7b" # options: blip2-opt-6.7b, blip2-flan-t5-xl, blip2-flan-t5-xxl + git clone https://huggingface.co/Salesforce/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` + + For BLIP2-OPT family, + ```bash + python ../../contrib/opt/convert_checkpoint.py --model_type blip2 \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --dtype float16 + ``` + + For BLIP2-T5 family, + ```bash + python ../enc_dec/convert_checkpoint.py --model_type blip2 \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/bfloat16 \ + --tp_size 1 \ + --pp_size 1 \ + --dtype bfloat16 + ``` + +2. Build TRT-LLM engine from TRT-LLM checkpoint + + For BLIP2-OPT family, + ```bash + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gemm_plugin float16 \ + --max_beam_width 1 \ + --max_batch_size 8 \ + --max_seq_len 1024 \ + --max_input_len 924 \ + --max_multimodal_len 256 # 8 (max_batch_size) * 32 (num_visual_features) + ``` + + For BLIP2-T5 family, + ```bash + trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/bfloat16/encoder \ + --output_dir tmp/trt_engines/${MODEL_NAME}/bfloat16/llm/encoder \ + --paged_kv_cache disable \ + --moe_plugin disable \ + --gemm_plugin bfloat16 \ + --bert_attention_plugin bfloat16 \ + --gpt_attention_plugin bfloat16 \ + --remove_input_padding enable \ + --context_fmha disable \ + --max_beam_width 1 \ + --max_batch_size 8 \ + --max_input_len 924 \ + --max_multimodal_len 256 # 8 (max_batch_size) * 32 (num_visual_features) + + trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/bfloat16/decoder \ + --output_dir tmp/trt_engines/${MODEL_NAME}/bfloat16/llm/decoder \ + --paged_kv_cache disable \ + --moe_plugin disable \ + --gemm_plugin bfloat16 \ + --bert_attention_plugin bfloat16 \ + --gpt_attention_plugin bfloat16 \ + --remove_input_padding enable \ + --context_fmha disable \ + --max_beam_width 1 \ + --max_batch_size 8 \ + --max_seq_len 1024 \ + --max_encoder_input_len 924 \ + --max_input_len 1 # Same command for decoder but don't set --max_multimodal_len + ``` + + **NOTE**: `max_multimodal_len = max_batch_size * num_visual_features`, so if you change max_batch_size, max multimodal length **MUST** be changed accordingly. + +3. Build TensorRT engines for vision encoders + + ```bash + python build_multimodal_engine.py --model_type blip2 --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/bfloat16/vision --max_batch_size 8 + ``` + + The built engines are located in `tmp/trt_engines/${MODEL_NAME}/bfloat16/vision` for BLIP2-T5, similarly for BLIP-OPT. + + To run the BLIP2 pipeline with batch size > 1, change `--max_batch_size` argument to `build_multimodal_engine.py` accordingly. + +4. Assemble everything into BLIP2 pipeline + + For BLIP2-OPT family, + ```bash + python run.py \ + --max_new_tokens 30 \ + --input_text "Question: which city is this? Answer:" \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu + ``` + + For BLIP2-T5 family, + ```bash + python run.py \ + --max_new_tokens 30 \ + --input_text "Question: which city is this? Answer:" \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/bfloat16 + ``` + +5. (Optional) INT8/INT4 weight-only quantization for OPT can be enabled using commands as follows (take `INT4` as an example, while `INT8` is the default precision for weight-only quantization): + ```bash + python ../../contrib/opt/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --dtype float16 \ + --output_dir tmp/trt_models/${MODEL_NAME}/int4_weightonly/1-gpu \ + --use_weight_only \ + --weight_only_precision int4 + + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/int4_weightonly/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/int4_weightonly/1-gpu/llm \ + --gemm_plugin float16 \ + --max_beam_width 1 \ + --max_batch_size 8 \ + --max_multimodal_len 256 \ + --max_input_len 924 \ + --max_seq_len 1024 + ``` + + The built OPT engines lie in `tmp/trt_engines/${MODEL_NAME}/int4_weightonly/1-gpu/llm`. + You should use this directory without the `llm` part as `--engine_dir` argument to `run.py` + + **NOTE:** INT8/INT4 option is not supported for BLIP2-T5, because quantization support has not been + added for encoder-decoder models yet. + +## CogVLM + +Currently, CogVLM only support bfloat16 precision. + +1. Download Huggingface weights + + ```bash + export MODEL_NAME="cogvlm-chat-hf" + git clone https://huggingface.co/THUDM/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + export TOKENIZER_NAME="vicuna-7b-v1.5" + git clone https://huggingface.co/lmsys/${TOKENIZER_NAME} tmp/hf_models/${TOKENIZER_NAME} + ``` + + Because currently onnx doesn't support `xops.memory_efficient_attention`, we need to modify some source code of the huggingface CogVLM. + ``` + cd tmp/hf_models/${MODEL_NAME} + sed -i '4s/.*//;40s/.*/ out = self.attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)).transpose(1, 2).contiguous()/;41s/.*//;42s/.*//' visual.py # It will replace memory_efficient_attention with some basic ops + ``` + +2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/cogvlm` + + CogVLM uses a Vit encoder as LLM encoder and a modified Llama as decoder. + + ```bash + python ../../contrib/cogvlm/convert_checkpoint.py --model_dir tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_models/${MODEL_NAME} --dtype bfloat16 --use_prompt_tuning + + trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME} \ + --output_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu/llm \ + --gemm_plugin bfloat16 \ + --gpt_attention_plugin bfloat16 \ + --remove_input_padding enable \ + --max_batch_size 48 \ + --max_input_len 2048 \ + --max_seq_len 3076 \ + --paged_kv_cache enable \ + --bert_attention_plugin disable \ + --moe_plugin disable \ + --max_multimodal_len 61440 # 48 (max_batch_size) * 1280 (max_num_visual_features) + ``` + +3. Generate TensorRT engines for visual components and combine everything into final pipeline. + + ```bash + python build_multimodal_engine.py --model_type cogvlm --model_path tmp/hf_models/${MODEL_NAME} --max_batch_size 48 --output_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu/vision + + python run.py \ + --max_new_tokens 1000 \ + --input_text " [INST] please describe this image in detail [/INST] " \ + --hf_model_dir tmp/hf_models/${TOKENIZER_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu \ + --batch_size 1 \ + --top_p 0.4 \ + --top_k 1 \ + --temperature 0.2 \ + --repetition_penalty 1.2 \ + --enable_context_fmha_fp32_acc + + CogVLM uses model_runner_cpp for its LLM decoder by default. To switch to model_runner, set `--session python` in the command mentioned above. + ``` + +## Deplot + +1. Download Huggingface weights and convert original checkpoint to TRT-LLM checkpoint format + following example in `examples/models/core/enc_dec/README.md`. + + ```bash + export MODEL_NAME="deplot" + git clone https://huggingface.co/google/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + + python ../enc_dec/convert_checkpoint.py --model_type pix2struct \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/float16 \ + --tp_size 1 \ + --pp_size 1 \ + --dtype float16 + ``` + +2. Build TRT-LLM engine from TRT-LLM checkpoint + + ```bash + trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/float16/decoder \ + --output_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/float16/llm/decoder \ + --paged_kv_cache disable \ + --moe_plugin disable \ + --gemm_plugin float16 \ + --bert_attention_plugin float16 \ + --gpt_attention_plugin float16 \ + --remove_input_padding enable \ + --context_fmha disable \ + --max_beam_width 1 \ + --max_batch_size 8 \ + --max_seq_len 2558 \ + --max_encoder_input_len 2048 \ + --max_input_len 1 + ``` + + The built deplot engines are located in `tmp/trt_engines/${MODEL_NAME}/1-gpu/float16`. + +3. Build TensorRT engines for visual components + + ```bash + python build_multimodal_engine.py --model_type pix2struct --model_path tmp/hf_models/${MODEL_NAME} --max_batch_size 8 --output_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/float16/vision + ``` + + The built visual engines are located in `tmp/trt_engines/${MODEL_NAME}/1-gpu/float16/vision`. + + To run the deplot pipeline with batch size > 1, change `--max_batch_size` argument to `build_multimodal_engine.py` accordingly. + +4. Assemble everything into deplot pipeline + + ```bash + python run.py \ + --max_new_tokens 100 \ + --input_text "" \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/float16 + ``` + +## Fuyu + +1. Download Huggingface weights + + ```bash + export MODEL_NAME="fuyu-8b" + git clone https://huggingface.co/adept/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` + +2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/gpt`. + The LLM portion of Fuyu uses a Persimmon model + ```bash + python ../gpt/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --dtype float16 \ + --gpt_variant persimmon + + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gemm_plugin float16 \ + --use_fused_mlp=enable \ + --max_batch_size 1 \ + --max_input_len 2048 \ + --max_seq_len 2560 \ + --max_multimodal_len 2048 + ``` + +3. Generate TensorRT engines for visual components and combine everything into final pipeline. + + ```bash + python build_multimodal_engine.py --model_type fuyu --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision + + python run.py \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu + ``` + +## Gemma3 + +**NOTE: We only support Gemma3 VLMs in Pytorch workflow.** + +Gemma3VL decoder requires a custom attention mask while processing images. During the context phase: +- Text tokens attend to other tokens in a causal fashion (standard autoregressive behavior) +- Image tokens attend to other tokens in a causal fashion AND attend to other tokens from the same image in a bidirectional manner + +**Reference:** [Gemma3 Model Documentation](https://huggingface.co/docs/transformers/en/model_doc/gemma3) + +We support this custom mask with FlashInfer attention backend. + +### Requirements + +To ensure expected behavior with Gemma3VL, the following configurations are **required**: +- **Attention Backend**: Use the FlashInfer attention backend +- **Chunked Prefill**: Must be disabled +- **KV Cache Reuse**: Must be disabled + +### Quick Start + +#### 1. Download Model Weights + +```bash +export MODEL_NAME="gemma-3-27b-it" +git clone https://huggingface.co/google/${MODEL_NAME} +``` + +#### 2. Interactive Testing + +Use the `quickstart_multimodal.py` script for quick testing: + +```bash +python3 examples/llm-api/quickstart_multimodal.py \ + --model_dir ${MODEL_NAME}/ \ + --modality image \ + --image_format pil \ + --attention_backend FLASHINFER \ + --disable_kv_cache_reuse +``` + +#### 3. Model Serving + +Serve the model using `trtllm-serve` with the required llmapi arguments mentioned in a yaml file: + +```bash +# Create the configuration file +cat > extra-llm-api-options.yaml << 'EOF' +cuda_graph_config: null +attn_backend: "FLASHINFER" +enable_chunked_prefill: false +kv_cache_config: + enable_block_reuse: false +EOF + +# Serve the model +trtllm-serve ${MODEL_NAME}/ \ + --backend pytorch \ + --tp_size 1 \ + --port 8000 \ + --max_batch_size 4 \ + --config extra-llm-api-options.yaml +``` + +### Supported Model Variants + +Currently supported Gemma3 variants: 4B, 12B, 27B + + +## InternLM-XComposer2 + +**NOTE: We only support InternLM-XComposer-VL-7b for now** + +Firstly, please install transformers with 4.45.2 +```bash + pip install -r requirements-internlm-xcomposer2.txt +``` + +1. Convert Huggingface weights to TRT-LLM checkpoint format using `examples/models/contrib/internlm/README.md`. + +2. Use `trtllm-build` command to build TRT-LLM engine for OPT. + +3. The full list of commands is as follows: + + ```bash + export MODEL_NAME=internlm-xcomposer2-vl-7b + git lfs clone https://huggingface.co/internlm/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + + python ../internlm2/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --dtype float16 \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu + + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gemm_plugin float16 \ + --lora_plugin float16 \ + --lora_dir . \ + --max_lora_rank 256 \ + --max_input_len 1536 \ + --max_batch_size 48 \ + --max_multimodal_len 58800 # 58800 = 1225(visual token/img) * 48 (max_batch_size), as each image corresponds to 1225 visual tokens in the ViT here + + python build_multimodal_engine.py \ + --model_type internlm-xcomposer2 \ + --model_path tmp/hf_models/${MODEL_NAME} \ + --output_dir trt_engines/${MODEL_NAME}/fp16/1-gpu/vision \ + --max_batch_size 48 + + python run.py \ + --max_new_tokens 200 \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir trt_engines/${MODEL_NAME}/fp16/1-gpu \ + --batch_size 1 + ``` + +## InternVL2 + +[InternVL Family](https://github.com/OpenGVLab/InternVL): Closing the Gap to Commercial Multimodal Models with Open-Source Suites —— A Pioneering Open-Source Alternative to GPT-4o. Here we show how to deploy InternVL2‑1B/InternVL2‑2B/InternVL2‑4B/InternVL2‑8B/InternVL2‑26B in TensorRT-LLM. + +Firstly, please install transformers with 4.37.2 +```bash + pip install transformers==4.37.2 +``` + +1. Download Huggingface weights + - For InternVL2-1B + ```bash + export MODEL_NAME="InternVL2-1B" + git clone https://huggingface.co/OpenGVLab/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + export LLM_MODEL_NAME="qwen" + ``` + + - For InternVL2-2B/InternVL2‑8B/InternVL2‑26B + ```bash + export MODEL_NAME="InternVL2-2B" # or InternVL2‑8B, InternVL2‑26B + git clone https://huggingface.co/OpenGVLab/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + export LLM_MODEL_NAME="internlm2" + ``` + + - For InternVL2-4B + ```bash + export MODEL_NAME="InternVL2-4B" + git clone https://huggingface.co/OpenGVLab/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + export LLM_MODEL_NAME="phi" + ``` + +2. Convert Huggingface weights into TRT-LLM checkpoints + ```bash + python ../${LLM_MODEL_NAME}/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --dtype float16 + ``` + +3. Build TRT engines + ```bash + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gemm_plugin auto \ + --max_batch_size 1 \ + --max_input_len 4096 \ + --max_seq_len 4608 \ + --max_multimodal_len 3328 + ``` + +4. Generate TensorRT engines for visual components and combine everything into final pipeline. + ```bash + python build_multimodal_engine.py --model_type internvl --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision + python run.py \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/ \ + --image_path tmp/hf_models/${MODEL_NAME}/examples/image1.jpg + ``` + +5. (Optional) FP8 and INT8 SmoothQuant quantization is supported for the InternVL2-4B variant (LLM model only). + + ```bash + # FP8 quantization + python ../../../quantization/quantize.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp8/1-gpu \ + --dtype bfloat16 \ + --qformat fp8 \ + --kv_cache_dtype fp8 + + # INT8 SmoothQuant quantization + python ../../../quantization/quantize.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/int8/1-gpu \ + --dtype bfloat16 \ + --qformat int8_sq + ``` + + Then follow the same `trtllm-build`, `build_multimodal_engine.py` and `run.py` steps as before. + + +## Kosmos-2 + +1. Download Huggingface weights + + ```bash + export MODEL_NAME="kosmos-2" + git clone https://huggingface.co/microsoft/kosmos-2-patch14-224 tmp/hf_models/${MODEL_NAME} + ``` + +2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/gpt`. + ```bash + python ../gpt/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --dtype float16 \ + --gpt_variant ${MODEL_NAME} + + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gpt_attention_plugin float16 \ + --gemm_plugin float16 \ + --max_batch_size 1 \ + --max_input_len 512 \ + --max_seq_len 1024 \ + --max_multimodal_len 64 # 1 (max_batch_size) * 64 (num_visual_features) + ``` + +3. Generate TensorRT engines for visual components and combine everything into final pipeline. + + ```bash + python build_multimodal_engine.py --model_type kosmos-2 --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision + + python run.py \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu + ``` + +## LLaVA, LLaVa-NeXT, LLaVA-OneVision and VILA + +[LLaVA](https://github.com/haotian-liu/LLaVA) and [VILA](https://github.com/Efficient-Large-Model/VILA) are both visual language models (VLM) that can be deployed in TensorRT LLM with many quantization options. [LLaVA-NeXT](https://huggingface.co/collections/llava-hf/llava-next-65f75c4afac77fd37dbbe6cf) is an extension of LLaVA. TRT-LLM currently supports [Mistral-7b](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf) and [ Nous-Hermes-2-Yi-34B](https://huggingface.co/llava-hf/llava-v1.6-34b-hf) variant of LLaVA-NeXT. [LLaVA-OneVision](https://huggingface.co/collections/llava-hf/llava-onevision-66bb1e9ce8856e210a7ed1fe) is another extension of LLaVA. + +1. Download Huggingface model weights. These models have both visual and LLM components + unlike BLIP2 example which downloads only LLM components from Huggingface. + + For LLaVA, + + ```bash + export MODEL_NAME="llava-1.5-7b-hf" # also llava-1.5-13b-hf + git clone https://huggingface.co/llava-hf/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` + For LLaVA-NeXT, + + ```bash + export MODEL_NAME="llava-v1.6-mistral-7b-hf" #for 34b variant "llava-v1.6-34b-hf" + git clone https://huggingface.co/llava-hf/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` + + For LLaVA-OneVision, + + ```bash + export MODEL_NAME="llava-onevision-qwen2-7b-ov-hf" # also llava-onevision-qwen2-0.5b-ov-hf, llava-onevision-qwen2-72b-ov-hf, etc + git clone https://huggingface.co/llava-hf/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` + + For VILA, we need a few more steps until it is added to HF model zoo + + ```bash + # install the following dependency + pip install -r requirements-vila.txt + + # clone original VILA repo + export VILA_PATH="tmp/hf_models/VILA" + git clone https://github.com/Efficient-Large-Model/VILA.git ${VILA_PATH} + + # download VILA checkpoints + export MODEL_NAME="vila1.5-3b" # NOTE: name must contain vila or VILA! it's used to identify whether we need to register the non-HF VILA codebase in HF Auto class + git clone https://huggingface.co/Efficient-Large-Model/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` + +2. Generate TRT-LLM engine for LLaMA following example in `examples/models/core/llama/README.md` and `examples/models/core/qwen/README.md` + + ```bash + python ../llama/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --dtype float16 + + # for LLaVA + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gemm_plugin float16 \ + --use_fused_mlp=enable \ + --max_batch_size 1 \ + --max_input_len 2048 \ + --max_seq_len 2560 \ + --max_multimodal_len 576 # 1 (max_batch_size) * 576 (num_visual_features) + + # for LLaVA-NeXT + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gpt_attention_plugin float16 \ + --gemm_plugin float16 \ + --use_fused_mlp=enable \ + --max_batch_size 1 \ + --max_input_len 4096 \ + --max_seq_len 5120 \ + --max_num_tokens 4096 \ + --max_multimodal_len 4096 # 1 (max_batch_size) * 4096 (max_input_len) + + # for VILA + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gemm_plugin float16 \ + --use_fused_mlp=enable \ + --max_batch_size 1 \ + --max_input_len 2048 \ + --max_seq_len 2560 \ + --max_multimodal_len 196 # 1 (max_batch_size) * 196 (num_visual_features) + + # for LLaVA-OneVision + python ../qwen/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --dtype float16 + + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gemm_plugin float16 \ + --use_fused_mlp=enable \ + --max_batch_size 1 \ + --max_input_len 7228 \ + --max_seq_len 7328 \ + --max_multimodal_len 7128 # max_batch_size * num_visual_features(depends on the image size or the specified video num frame) + ``` + +3. Build TensorRT engines for visual components + + ```bash + python build_multimodal_engine.py --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision --model_type llava # for LLaVA + + python build_multimodal_engine.py --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision --model_type llava_next --max_batch_size 5 # 1 (max_batch_size) * 5 (because LLAVA-NeXT visual encoder can have at most 5 patches) # for LLaVA-NeXT + + python build_multimodal_engine.py --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision --model_type llava_onevision --max_batch_size 32 # max_batch_size * patch for image or frame for video # for LLaVA-OneVision + + python build_multimodal_engine.py --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision --model_type vila --vila_path ${VILA_PATH} # for VILA + ``` + + ```bash + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ + --input_text "\n Which city is this?" # for LLaVA and for LLaVA-NeXT + + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ + --image_path=https://github.com/Efficient-Large-Model/VILA/raw/main/demo_images/av.png,https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png \ + --input_text "\n Please elaborate what you see in the images?","\n Which city is this?" \ + --batch_size=2 # for LLaVA + ``` + + Note that Llava can support N pairs inference batching, `--batch_size=N` should be used. There should be N images listed under `--image_path` and N text prompts listed under `--input_text`. Don't forget to set the `--max_batch_size` and `--max_multimodal_len` during engine building. + + For LLaVA-OneVision, you can use either image or video as inputs. + ```bash + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ + --input_text "What is shown in this image?" \ + --image_path image.png + + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ + --input_text "Why is this video funny?" \ + --video_path video.mp4 + --video_num_frames 8 # sample uniformly 8 frames from the video, up to 32 frames + ``` + + For VILA, you can use either local file or web url as input images. + Suppose you have a local image `av.png` downloaded from `https://github.com/Efficient-Large-Model/VILA/blob/main/demo_trt_llm/av.png` and the url of `merlion.png` + ```bash + wget -O av.png https://raw.githubusercontent.com/Efficient-Large-Model/VILA/main/demo_images/av.png + + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ + --image_path=av.png,https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png \ + --input_text="\n\n Please elaborate what you see in the images?" \ + --batch_size=1 # for VILA mode 1 + + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu \ + --image_path=av.png,https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png \ + --input_text="\n Please elaborate what you see in the images?","\n Which city is this?" \ + --batch_size=2 \ + --check_accuracy # for VILA mode 2 + ``` + + Note that VILA can support different modes in terms of batching: + - Mode 1: if you want to query N images as a whole using a prompt, `--batch_size=1` should be used (which is the default value). Example is given above. + - Mode 2: if you want to query N pairs, `--batch_size=N` should be used. There should be N images listed under `--image_path` and N text prompts listed under `--input_text`. Don't forget to set the `--max_batch_size` and `--max_multimodal_len` during engine building. + + Note: use `--run_profiling` for performance measurement, use `--check_accuracy` for accuracy check. + +4. (Optional) Different quantization methods supported in LLaMA and Qwen can be applied to LLaVA/VILA/LLaVA-OneVision as well, such as INT4/INT8 weight-only, SmoothQuant, and INT4 Activation-Aware Quantization (AWQ). Detailed instructions can be found in LLaMA [README](../llama/README.md) and Qwen [README](../qwen/README.md). + + For example, + + ```bash + # INT4 weight only + python ../llama/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --dtype float16 \ + --output_dir tmp/trt_models/${MODEL_NAME}/int4_weightonly/1-gpu \ + --use_weight_only \ + --weight_only_precision int4 + + # INT4 AWQ + python ../../../quantization/quantize.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/int4_awq/1-gpu \ + --dtype float16 \ + --qformat int4_awq \ + --calib_size 32 + ``` + + Then follow the same `trtllm-build` and `run.py` steps as before. NOTE: for `trtllm-build` command, do not use `--use_fused_mlp=enable` in these quantization modes. + +## MLLaMA + +This section shows how to build and run a LLaMA-3.2 Vision model in TensorRT-LLM. We use [Llama-3.2-11B-Vision/](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision) as an example. + +For LLaMA-3.2 text model, please refer to the [examples/models/core/llama/README.md](../llama/README.md) because it shares the model architecture of llama. + +### Support data types + * BF16 + * Tensor Parallel + * INT8 & INT4 Weight-Only + * FP8 + +### Build and run vision model + +* build engine of vision encoder model + +```bash +python examples/models/core/multimodal/build_multimodal_engine.py --model_type mllama \ + --model_path Llama-3.2-11B-Vision/ \ + --output_dir /tmp/mllama/trt_engines/vision/ +``` + +* build engine of decoder model + +```bash +python examples/models/core/mllama/convert_checkpoint.py --model_dir Llama-3.2-11B-Vision/ \ + --output_dir /tmp/mllama/trt_ckpts \ + --dtype bfloat16 + +trtllm-build --checkpoint_dir /tmp/mllama/trt_ckpts \ + --output_dir /tmp/mllama/trt_engines/llm/ \ + --max_num_tokens 4096 \ + --max_seq_len 2048 \ + --workers 1 \ + --gemm_plugin auto \ + --max_batch_size 4 \ + --max_encoder_input_len 4100 \ + --input_timing_cache model.cache +``` + +Note that for instruct Vision model, please set the `max_encoder_input_len` as `6404`. + +* Run test on multimodal/run.py with C++ runtime (LLM part only) + +```bash +python3 examples/models/core/multimodal/run.py --engine_dir /tmp/mllama/trt_engines/ \ + --hf_model_dir Llama-3.2-11B-Vision/ \ + --image_path https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg \ + --input_text "<|image|><|begin_of_text|>If I had to write a haiku for this one" \ + --max_new_tokens 50 \ + --batch_size 2 + +Use model_runner_cpp by default. To switch to model_runner, set `--session python` in the command mentioned above. + +python3 examples/models/core/multimodal/eval.py \ + --engine_dir /tmp/mllama/trt_engines/ \ + --hf_model_dir Llama-3.2-11B-Vision/ \ + --test_trtllm \ + --accuracy_threshold 65 \ + --eval_task lmms-lab/ai2d +``` + +### Run MLLaMA decoder part by FP8 + +```bash +# install modelopt 0.21.0 +pip install nvidia-modelopt[torch]~=0.21.0 + +python ./examples/quantization/quantize.py --model_dir Llama-3.2-11B-Vision/ \ + --dtype bfloat16 \ + --qformat fp8 \ + --output_dir /tmp/llama-3.2-11B-Vision/fp8/ \ + --kv_cache_dtype fp8 \ + --calib_size 512 \ + --calib_dataset scienceqa + +trtllm-build --checkpoint_dir /tmp/llama-3.2-11B-Vision/fp8/ \ + --output_dir /tmp/trt_engines/llama-3.2-11B-Vision/fp8/llm \ + --max_num_tokens 4096 \ + --max_seq_len 2048 \ + --workers 1 \ + --gemm_plugin auto \ + --max_batch_size 4 \ + --max_encoder_input_len 4100 \ + --input_timing_cache model.cache \ + --use_paged_context_fmha enable \ + --use_fp8_context_fmha enable + +# copy visiual engine directory `/tmp/mllama/trt_engines/vision/` to fp8 engine directory `/tmp/trt_engines/llama-3.2-11B-Vision/fp8/vision` + +python3 examples/models/core/multimodal/run.py --engine_dir /tmp/trt_engines/llama-3.2-11B-Vision/fp8/ \ + --hf_model_dir Llama-3.2-11B-Vision/ \ + --image_path https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg \ + --input_text "<|image|><|begin_of_text|>If I had to write a haiku for this one" \ + --max_new_tokens 50 \ + --batch_size 2 + +python3 examples/models/core/multimodal/eval.py --engine_dir /tmp/trt_engines/llama-3.2-11B-Vision/fp8/ \ + --hf_model_dir Llama-3.2-11B-Vision/ \ + --test_trtllm \ + --accuracy_threshold 65 \ + --eval_task lmms-lab/ai2d +``` + +Note that for instruct Vision model, please set the `max_encoder_input_len` as `6404`. + +## NeVA + +[NeVA](https://docs.nvidia.com/nemo-framework/user-guide/24.12/nemotoolkit/multimodal/mllm/neva.html) is a groundbreaking addition to the NeMo Multimodal ecosystem. This model seamlessly integrates large language-centric models with a vision encoder, that can be deployed in TensorRT-LLM. + +1. Generate TRT-LLM engine for NVGPT following example in `examples/models/core/gpt/README.md`. To adhere to the NVGPT conventions of the conversion script, some layer keys have to be remapped using `--nemo_rename_key`. + + ```bash + export MODEL_NAME="neva" + python ../gpt/convert_checkpoint.py \ + --nemo_ckpt_path ./${MODEL_NAME}.nemo \ + --dtype bfloat16 \ + --output_dir tmp/trt_models/${MODEL_NAME} \ + --nemo_rename_key model:model.language_model \ + attention.linear_qkv.layer_norm_bias:input_layernorm.bias \ + attention.linear_qkv.layer_norm_weight:input_layernorm.weight \ + mlp.linear_fc1.layer_norm_bias:post_attention_layernorm.bias \ + mlp.linear_fc1.layer_norm_weight:post_attention_layernorm.weight \ + linear_qkv:query_key_value \ + linear_fc1:dense_h_to_4h \ + linear_fc2:dense_4h_to_h \ + linear_proj:dense \ + decoder:encoder + + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME} \ + --output_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu/llm \ + --gpt_attention_plugin bfloat16 \ + --gemm_plugin bfloat16 \ + --max_batch_size 1 \ + --max_input_len 2048 \ + --max_seq_len 2560 \ + --max_multimodal_len 729 # 1 (max_batch_size) * 729 (num_visual_features) + ``` + +2. Build TensorRT engines for visual components + + ```bash + python build_multimodal_engine.py --model_path ./${MODEL_NAME}.nemo --model_type neva --output_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu/vision + ``` + + ```bash + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir tmp/trt_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/bf16/1-gpu \ + --input_text "Question: which city is this? Answer:" + ``` + + Note: use `--run_profiling` for performance measurement, use `--check_accuracy` for accuracy check. + +## Nougat + +1. Download Huggingface weights + + ```bash + export MODEL_NAME="nougat-base" # also nougat-small + git clone https://huggingface.co/facebook/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` + +2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/enc_dec` + + Nougat uses mBART architecture but replaces the LLM encoder with a Swin Transformer encoder. + To achieve this, we add an extra `--nougat` flag (over mBART example) to + `convert_checkpoint.py` in `examples/models/core/enc_dec` and `trtllm-build`. + + ```bash + python ../enc_dec/convert_checkpoint.py --model_type bart \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/bfloat16 \ + --tp_size 1 \ + --pp_size 1 \ + --dtype bfloat16 \ + --nougat + + trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/bfloat16/decoder \ + --output_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/bfloat16/llm/decoder \ + --paged_kv_cache disable \ + --moe_plugin disable \ + --gemm_plugin bfloat16 \ + --bert_attention_plugin bfloat16 \ + --gpt_attention_plugin bfloat16 \ + --remove_input_padding enable \ + --max_beam_width 1 \ + --max_batch_size 1 \ + --max_seq_len 101 \ + --max_input_len 1 \ + --max_encoder_input_len 588 # 1 (max_batch_size) * 588 (num_visual_features) + ``` + +3. Generate TensorRT engines for visual components and combine everything into final pipeline. + + ```bash + python build_multimodal_engine.py --model_type nougat --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/bfloat16/vision + + python run.py \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/1-gpu/bfloat16 + ``` + + Note: Nougat models usually do not need a text prompt. + + +## Phi-3-vision + +1. Download Huggingface weights + + ```bash + export MODEL_NAME="Phi-3-vision-128k-instruct" # or Phi-3.5-vision-instruct + git clone https://huggingface.co/microsoft/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` + +2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/phi`. + ```bash + python ../phi/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --dtype float16 + + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gpt_attention_plugin float16 \ + --gemm_plugin float16 \ + --max_batch_size 1 \ + --max_input_len 4096 \ + --max_seq_len 4608 \ + --max_multimodal_len 4096 + ``` + +3. Generate TensorRT engines for visual components and combine everything into final pipeline. + + ```bash + python build_multimodal_engine.py --model_type phi-3-vision --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision + + python run.py \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --kv_cache_free_gpu_memory_fraction 0.7 \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/ \ + --image_path=https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png + ``` +## Phi-4-multimodal +Navigate to the folder `TensorRT-LLM/examples/models/core/multimodal` + +1. Download Huggingface weights + + ```bash + export MODEL_NAME="Phi-4-multimodal-instruct" + export HF_DIR="tmp/hf_models/${MODEL_NAME}" + export CKPT_DIR="tmp/trt_models/${MODEL_NAME}/fp16/1-gpu" + export ENGINE_DIR="tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu" + git clone https://huggingface.co/microsoft/${MODEL_NAME} ${HF_DIR} + + ``` + +2. Convert Huggingface weights into TRT-LLM checkpoints and build TRT engines using scripts in `examples/models/core/phi`. + ```bash + python ../phi/convert_checkpoint.py \ + --model_dir ${HF_DIR} \ + --output_dir ${CKPT_DIR} \ + --dtype float16 + + trtllm-build \ + --checkpoint_dir ${CKPT_DIR} \ + --output_dir ${ENGINE_DIR} \ + --gpt_attention_plugin float16 \ + --gemm_plugin float16 \ + --max_batch_size 1 \ + --max_input_len 4096 \ + --max_seq_len 4608 \ + --max_multimodal_len 4096 + ``` + +3. Generate TensorRT engines for visual components and combine everything into final pipeline. +*Note: the encoders are not the TRT engines but are pure Pytorch ones* + + ```bash + python build_multimodal_engine.py --model_type phi-4-multimodal --model_path ${HF_DIR} --output_dir ${ENGINE_DIR} + + python run.py \ + --hf_model_dir ${HF_DIR} \ + --kv_cache_free_gpu_memory_fraction 0.7 \ + --engine_dir ${ENGINE_DIR} \ + --image_path=https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png + --audio_path=${HF_DIR}/examples/what_is_shown_in_this_image.wav + ``` +## Qwen2-VL +[Qwen2-VL Family](https://github.com/QwenLM/Qwen2-VL): is the latest version of the vision language models in the Qwen model families. Here we show how to deploy Qwen2-VL 2B and 7B in TensorRT-LLM. + +Firstly, please install transformers and qwen-vl-utils +```bash +pip install -r requirements-qwen2vl.txt +``` +### Support data types + * FP16 + * FP8 + +### Build and run vision model +* Download Huggingface weights + ```bash + export MODEL_NAME="Qwen2-VL-7B-Instruct" # or Qwen2-VL-2B-Instruct + git clone https://huggingface.co/Qwen/${MODEL_NAME} tmp/hf_models/${MODEL_NAME} + ``` +* Build engine of decoder model + + ```bash + python3 ../qwen/convert_checkpoint.py \ + --model_dir=tmp/hf_models/${MODEL_NAME} \ + --output_dir=tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --dtype float16 + + trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/llm \ + --gemm_plugin=float16 \ + --gpt_attention_plugin=float16 \ + --max_batch_size=4 \ + --max_input_len=2048 \ + --max_seq_len=3072 \ + --max_multimodal_len=1296 #(max_batch_size) * 324 (num_visual_features), this's for image_shape=[504,504] + ``` + +* Build engine of vision encoder model + ```bash + python build_multimodal_engine.py --model_type qwen2_vl --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision + ``` + +* Run test on multimodal/run.py with C++ runtime (LLM part only) + ```bash + python3 run.py \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/ + ``` +### Run Qwen2-VL decoder part by FP8 +* Build engine + ```bash + python ./examples/quantization/quantize.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --dtype float16 \ + --qformat fp8 \ + --kv_cache_dtype fp8 \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp8/1-gpu \ + --calib_size 512 + + trtllm-build --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp8/1-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp8/1-gpu/llm \ + --max_input_len=2048 \ + --max_seq_len 3072 \ + --gemm_plugin auto \ + --max_batch_size 4 \ + --max_multimodal_len=1296 + + # copy visiual engine directory `tmp/trt_engines/${MODEL_NAME}/fp16/1-gpu/vision/` to fp8 engine directory `tmp/trt_engines/${MODEL_NAME}/fp8/1-gpu/vision` + ``` +* Run test on multimodal/run.py with C++ runtime (LLM part only) + ```bash + python3 run.py \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp8/1-gpu/ + ``` ## Qwen-Image-Bench Evaluator @@ -31,3 +1183,153 @@ python examples/models/core/multimodal/qwen_image_bench_eval.py \ The script evaluates all five Qwen-Image-Bench level-1 dimensions by default: Quality, Aesthetics, Alignment, Real-world Fidelity, and Creative Generation. + +## Video NeVA + +[Video NeVA](https://github.com/NVIDIA/NeMo/blob/main/docs/source/multimodal/mllm/video_neva.rst) is a groundbreaking addition to the NeMo Multimodal ecosystem that could work with video modality. This model seamlessly integrates large language-centric models with a vision encoder, that can be deployed in TensorRT-LLM. + +1. Generate TRT-LLM engine for Nemotron model following example in `examples/models/core/nemotron/README.md`. To adhere to the NVGPT conventions of the conversion script. This will be used as our base LM for inference. + + ```bash + pip install decord # used for loading video + + python3 ../../../quantization/quantize.py \ + --nemo_ckpt_path /path/to/nemotron/model.nemo \ + --dtype bfloat16 \ + --batch_size 64 \ + --qformat full_prec \ + --output_dir nemotron-3/trt_ckpt/bf16/1-gpu + + + trtllm-build \ + --checkpoint_dir nemotron-3/trt_ckpt/bf16/1-gpu \ + --output_dir tmp/trt_engines/nemotron-3/bf16/1-gpu/llm \ + --gpt_attention_plugin bfloat16 \ + --gemm_plugin bfloat16 \ + --max_batch_size 1 \ + --max_input_len 4096 \ + --max_seq_len 4352 \ + --max_multimodal_len 3072 # 1 (max_batch_size) * (12 num_frames) * (256 image_token_len) + ``` + +2. Build TensorRT engines for visual components + + ```bash + python build_multimodal_engine.py --model_path /path/to/video/neva/projector.nemo --model_type video-neva --output_dir tmp/trt_engines/nemotron-3/visual_encoder --output_dir tmp/trt_engines/nemotron-3/bf16/1-gpu/vision + ``` + + ```bash + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir nemotron-3/trt_ckpt/bf16/1-gpu \ + --engine_dir tmp/trt_engines/nemotron-3/bf16/1-gpu \ + --input_text "Question: what is in the video? Answer:" \ + --video_path /path/to/your/local/video/file + ``` + + Note: use `--run_profiling` for performance measurement, use `--check_accuracy` for accuracy check. + +## Dataset Evaluation + +This section explains how to evaluate datasets using our provided script, including supported models and configurations. + +### Evaluation Command +To run an evaluation, use the following command: + +```bash +python ./examples/models/core/multimodal/eval.py \ + --model_type \ + --engine_dir \ + --hf_model_dir \ + --dataset_dir \ + --test_trtllm (or --test_hf, or both) \ + --accuracy_threshold \ + --eval_task \ + --max_ite 20 \ + --visual_engine_name +``` + +### Parameters +- `--model_type`: Specify the model type to evaluate. +- `--engine_dir`: Path to the model engines directory. +- `--hf_model_dir`: Path to the Hugging Face model directory. +- `--dataset_dir`: Path to the dataset directory. If not specified, will load the dataset from HF with the `--eval_task` as dataset tag. +- `--test_trtllm` or `--test_hf`: Specify which evaluation framework to use. Both can be used simultaneously. +- `--accuracy_threshold`: Set the accuracy threshold for evaluation. +- `--eval_task`: Specify the evaluation task. Supported tasks: `['lmms-lab/ai2d', 'lmms-lab/VQAv2', 'lmms-lab/MME']`. Default to `'lmms-lab/VQAv2'`. +- `--max_ite`: Maximum number of iterations, default to 20. +- `--visual_engine_name`: Name of the visual engine. + +### Supported Evaluation Tasks +The following evaluation tasks are supported: +- `lmms-lab/ai2d` +- `lmms-lab/VQAv2` +- `lmms-lab/MME` + +### Supported Model Types +The script supports the following model types: +- `blip2` +- `fuyu` +- `kosmos-2` +- `llava` +- `llava_next` +- `llava_onevision` +- `phi-3-vision` +- `qwen2_vl` +- `mllama` +- `vila` +- `cogvlm` +- `neva` +- `internvl` + +**Note:** The models `vila`, `cogvlm`, `neva`, and `internvl` do not support the `--test_hf` evaluation framework. + +## Enabling Tensor Parallelism for multi-GPU + +The LLM part of the pipeline can be run on multiple GPUs using tensor parallelism. +The visual encoder will be replicated on each GPU and operate in a data parallel fashion. + +To enable tensor parallelism, both weight conversion step (from Huggingface to FT format) +and engine building step should use additional arguments. Finally `run.py` should be prefixed +with `mpirun -n NUM_GPUS --allow-run-as-root`. + +The full set of commands to enable 2-way tensor parallelism for LLaVA is: + + ```bash + export MODEL_NAME="llava-1.5-7b-hf" + + python ../llama/convert_checkpoint.py \ + --model_dir tmp/hf_models/${MODEL_NAME} \ + --output_dir tmp/trt_models/${MODEL_NAME}/fp16/2-gpu \ + --dtype float16 --tp_size 2 + + trtllm-build \ + --checkpoint_dir tmp/trt_models/${MODEL_NAME}/fp16/2-gpu \ + --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/2-gpu/llm \ + --gemm_plugin float16 \ + --max_batch_size 1 \ + --max_input_len 2048 \ + --max_seq_len 2560 \ + --max_multimodal_len 576 + + python build_multimodal_engine.py --model_type llava --model_path tmp/hf_models/${MODEL_NAME} --output_dir tmp/trt_engines/${MODEL_NAME}/fp16/2-gpu/vision + + mpirun -n 2 --allow-run-as-root \ + python run.py \ + --max_new_tokens 30 \ + --hf_model_dir tmp/hf_models/${MODEL_NAME} \ + --engine_dir tmp/trt_engines/${MODEL_NAME}/fp16/2-gpu \ + ``` +## Enabling Embedding Table Offloading + +Embedding Table Offloading is a memory optimization technique that helps manage large embedding tables more efficiently. It offloads the embedding table to CPU memory and uses a chunked prefetching mechanism during processing. This approach is only available when operating in context chunk mode. + +To enable this feature, use the `--mm_embedding_offloading` argument: +```bash +python run.py \ + --enable_chunked_context \ + --mm_embedding_offloading true \ + --hf_model_dir ${HF_MODEL_PATH} \ + --engine_dir ${ENGINE_PATH} +``` +When not explicitly specified, this feature automatically enables if you're using a multimodal model along with context chunking enabled. diff --git a/examples/models/core/multimodal/__init__.py b/examples/models/core/multimodal/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/examples/models/core/multimodal/build_multimodal_engine.py b/examples/models/core/multimodal/build_multimodal_engine.py new file mode 100644 index 000000000000..59e8bb4ffa25 --- /dev/null +++ b/examples/models/core/multimodal/build_multimodal_engine.py @@ -0,0 +1,12 @@ +import argparse + +from tensorrt_llm.tools.multimodal_builder import (MultimodalEngineBuilder, + add_multimodal_arguments) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser = add_multimodal_arguments(parser) + args = parser.parse_args() + + builder = MultimodalEngineBuilder(args) + builder.build() diff --git a/examples/models/core/multimodal/eval.py b/examples/models/core/multimodal/eval.py new file mode 100644 index 000000000000..01804d290dee --- /dev/null +++ b/examples/models/core/multimodal/eval.py @@ -0,0 +1,317 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os + +import aiohttp +import datasets +import torch +from transformers import AutoProcessor +from utils import add_common_args + +import tensorrt_llm +import tensorrt_llm.profiler as profiler +from tensorrt_llm import logger +from tensorrt_llm.runtime import MultimodalModelRunner + +SUPPORTED_MODEL_TYPES = { + 'blip2': 'Blip2ForConditionalGeneration', + 'fuyu': 'FuyuForCausalLM', + 'kosmos-2': 'Kosmos2ForConditionalGeneration', + 'llava': 'LlavaForConditionalGeneration', + 'llava_next': 'LlavaNextForConditionalGeneration', + 'llava_onevision': 'LlavaOnevisionForConditionalGeneration', + 'phi-3-vision': 'AutoModelForCausalLM', + 'qwen2_vl': 'Qwen2VLForConditionalGeneration', # not tested for TRT-LLM yet + 'mllama': 'MllamaForConditionalGeneration', + 'vila': None, + 'cogvlm': None, # not tested for TRT-LLM yet + 'neva': None, # not tested for TRT-LLM yet + 'internvl': None, +} +EVAL_TASKS = ['lmms-lab/ai2d', 'lmms-lab/VQAv2', 'lmms-lab/MME'] + + +def parse_arguments(args=None): + parser = argparse.ArgumentParser() + parser = add_common_args(parser) + parser.add_argument('--test_trtllm', + action='store_true', + default=None, + help="Evaluate the TensorRT-LLM.") + parser.add_argument('--test_hf', + action='store_true', + default=None, + help="Evaluate the Huggingface.") + parser.add_argument('--max_ite', type=int, default=20) + parser.add_argument('--eval_task', + type=str, + choices=EVAL_TASKS, + default='lmms-lab/VQAv2') + parser.add_argument('--model_type', + type=str, + default=None, + choices=SUPPORTED_MODEL_TYPES.keys()) + parser.add_argument( + '--accuracy_threshold', + type=float, + default=None, + help= + 'used to check the accuracy of test_trtllm. Should be between 0 and 100.' + ) + parser.add_argument( + '--dataset_dir', + type=str, + default=None, + help="The local directory of the dataset for evaluation; " + "will download the dataset from huggingface hub if not specified.") + parser.add_argument( + '--dataset_cache_dir', + type=str, + default=None, + help="The local cache directory for dataset; " + "will use `~/.cache/huggingface/datasets` if not specified.") + return parser.parse_args(args=args) + + +def load_dataset(args) -> datasets.Dataset: + split_name = 'validation' if 'VQAv2' in args.eval_task else 'test' + + if args.dataset_dir is not None and os.path.exists( + os.path.join(args.dataset_dir, "dataset_info.json")): + logger.info(f"load dataset by load_from_disk from {args.dataset_dir}") + dataset = datasets.load_from_disk(args.dataset_dir) + + else: + logger.info( + f"load dataset by load_dataset from {args.dataset_dir or args.eval_task}" + ) + dataset = datasets.load_dataset( + args.dataset_dir or args.eval_task, + cache_dir=args.dataset_cache_dir, + split=split_name, + storage_options={ + 'client_kwargs': { + 'timeout': aiohttp.ClientTimeout(total=3600) + } + }, + trust_remote_code=True, + ) + return dataset + + +def load_hf_model(args): + if SUPPORTED_MODEL_TYPES[args.model_type] is None: + raise ValueError(f"Unsupported HF model_type: {args.model_type}") + profiler.start('load HF model') + model_class = getattr(__import__('transformers'), + SUPPORTED_MODEL_TYPES[args.model_type]) + hf_model = model_class.from_pretrained(args.hf_model_dir, + dtype=torch.float16, + device_map="cuda:0", + trust_remote_code=True) + profiler.stop('load HF model') + + logger.info( + f'Load HF model takes: {profiler.elapsed_time_in_sec("load HF model")} sec' + ) + return hf_model + + +def load_trtllm_model(args): + profiler.start('load TensorRT LLM model') + trtllm_model = MultimodalModelRunner(args) + profiler.stop('load TensorRT LLM model') + logger.info( + f'Load TensorRT LLM model takes: {profiler.elapsed_time_in_sec("load TensorRT LLM model")} sec' + ) + return trtllm_model + + +def prepare_prompts(task, data, model_type, processor) -> str: + prompts = None + question = data['question'] + if question[-1] != '?': + question += '?' + + if task == 'lmms-lab/ai2d': + for j, option in enumerate(data['options']): + question += f" ({j}) {option}" + + if model_type in ['blip2', 'neva']: + prompts = f"Question: {question} Answer: " + elif model_type == 'fuyu': + prompts = f"Answer the following {task} question based on the image: {question}" + elif model_type == 'kosmos-2': + prompts = f" Question: {question} Answer: " + elif model_type == 'cogvlm': + prompts = f"A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {question} ASSISTANT:" + elif model_type in ['llava', 'llava_next', 'llava_onevision']: + conversation = [ + { + "role": + "user", + "content": [ + { + "type": "image" + }, + { + "type": "text", + "text": question + }, + ], + }, + ] + prompts = processor.apply_chat_template(conversation, + add_generation_prompt=True) + elif model_type in ['vila', 'internvl']: + prompts = f"\n{question}" + elif model_type == 'phi-3-vision': + messages = [ + { + "role": "user", + "content": f"<|image_1|>\n{question}" + }, + ] + prompts = processor.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True) + elif model_type == 'qwen2_vl': + conversation = [{ + "role": + "user", + "content": [{ + "type": "image", + }, { + "type": "text", + "text": question + }] + }] + prompts = processor.apply_chat_template(conversation, + add_generation_prompt=True) + elif model_type == 'mllama': + prompts = processor.apply_chat_template(images=data['image'], + text=question + "; answer: ") + else: + raise ValueError(f"Unsupported model_type: {model_type}") + + return prompts + + +def eval(output, task, data) -> bool: + output = output.strip().lower() + if task == 'lmms-lab/VQAv2': + return any(answer['answer'] in output for answer in data['answers']) + else: + return data['answer'].lower() in output + + +os.environ["TOKENIZERS_PARALLELISM"] = "false" +args = parse_arguments() +if args.model_type not in SUPPORTED_MODEL_TYPES: + raise ValueError(f"Unsupported model_type: {args.model_type}") + +logger.set_level(args.log_level) + +runtime_rank = tensorrt_llm.mpi_rank() +dataset = load_dataset(args) +hf_model = load_hf_model(args) if args.test_hf else None +trtllm_model = load_trtllm_model(args) if args.test_trtllm else None +if SUPPORTED_MODEL_TYPES[args.model_type] is None: + hf_processor = None +else: + hf_processor = AutoProcessor.from_pretrained(args.hf_model_dir, + trust_remote_code=True) +hf_correct = trtllm_correct = 0 + +if args.model_type == 'mllama': + from tensorrt_llm.runtime.processor_wrapper import MllamaProcessorWrapper + hf_processor = MllamaProcessorWrapper(hf_processor, logger) + +torch.random.manual_seed(0) +profiler.start('evaluation') +if args.test_trtllm or args.test_hf: + for i in range(args.max_ite): + logger.debug(f"Ite: {i:3d}") + data = dataset[i] + if i > len(dataset): + break + prompts = prepare_prompts(args.eval_task, data, args.model_type, + hf_processor) + image = data['image'] + + if args.test_hf: + assert hf_model is not None, f"Unsupported HF model_type: {args.model_type}" + profiler.start('hf') + inputs = hf_processor( + images=image, + text=prompts, + return_tensors="pt", + ).to(hf_model.device, + torch.float16) # add torch.float16 for llava-onevision + input_length = inputs.input_ids.shape[-1] + hf_output = hf_model.generate(**inputs, + max_new_tokens=args.max_new_tokens) + hf_result = (hf_processor.batch_decode( + hf_output, skip_special_tokens=True)[0] if args.model_type in [ + 'blip2' + ] else hf_processor.decode(hf_output[0][input_length:], + skip_special_tokens=True)) + hf_correct += eval(hf_result, args.eval_task, data) + profiler.stop('hf') + + if args.test_trtllm: + profiler.start('tensorrt_llm') + _, output_text = trtllm_model.run( + input_text=prompts, + input_image=image, + input_audio=None, + max_new_tokens=args.max_new_tokens) + if runtime_rank == 0: + trtllm_result = output_text[0][0] + trtllm_correct += eval(trtllm_result, args.eval_task, data) + profiler.stop('tensorrt_llm') + + if runtime_rank == 0: + if args.eval_task == 'lmms-lab/VQAv2': + answer = data['answers'] + else: + answer = data['answer'] + logger.debug(f"prompts: {prompts}") + logger.debug(f"reference answer: {answer}") + if args.test_hf: + logger.debug(f"HF's answer: {hf_result}") + if args.test_trtllm: + logger.debug(f"TRT-LLM's answer: {trtllm_result}") + + if runtime_rank == 0: + logger.info(f"total iterations: {args.max_ite}") + if args.test_hf: + logger.info( + f"HF's accuracy: {100 * hf_correct / args.max_ite:4.2f}%") + if args.test_trtllm: + logger.info( + f"TRT-LLM's accuracy: {100 * trtllm_correct / args.max_ite:4.2f}%" + ) + # check if the accuracy is above the threshold + if args.accuracy_threshold is not None and args.test_trtllm: + assert trtllm_correct / args.max_ite >= args.accuracy_threshold / 100, \ + f"TRT-LLM's accuracy is below the threshold: {args.accuracy_threshold}%." +else: + logger.info("Neither enable test_trtllm nor enable test_hf") + +profiler.stop('evaluation') +logger.info( + f'Evaluation takes: {profiler.elapsed_time_in_sec("evaluation")} sec') diff --git a/examples/models/core/multimodal/requirements-eclair.txt b/examples/models/core/multimodal/requirements-eclair.txt new file mode 100644 index 000000000000..281c8ae93a91 --- /dev/null +++ b/examples/models/core/multimodal/requirements-eclair.txt @@ -0,0 +1 @@ +timm diff --git a/examples/models/core/multimodal/requirements-internlm-xcomposer2.txt b/examples/models/core/multimodal/requirements-internlm-xcomposer2.txt new file mode 100644 index 000000000000..5d27a97ebb68 --- /dev/null +++ b/examples/models/core/multimodal/requirements-internlm-xcomposer2.txt @@ -0,0 +1 @@ +transformers==4.56.0 diff --git a/examples/models/core/multimodal/requirements-llava_onevision.txt b/examples/models/core/multimodal/requirements-llava_onevision.txt new file mode 100644 index 000000000000..126cbda08e15 --- /dev/null +++ b/examples/models/core/multimodal/requirements-llava_onevision.txt @@ -0,0 +1,4 @@ +git+https://github.com/LLaVA-VL/LLaVA-NeXT.git +transformers>=4.56.0 +einops +av diff --git a/examples/models/core/multimodal/requirements-qwen2vl.txt b/examples/models/core/multimodal/requirements-qwen2vl.txt new file mode 100644 index 000000000000..50f14d1d8095 --- /dev/null +++ b/examples/models/core/multimodal/requirements-qwen2vl.txt @@ -0,0 +1,2 @@ +accelerate +qwen-vl-utils==0.0.8 # 0.0.9 has bug https://github.com/QwenLM/Qwen2-VL/pull/673, rollback until a newer version is released diff --git a/examples/models/core/multimodal/requirements-vila.txt b/examples/models/core/multimodal/requirements-vila.txt new file mode 100644 index 000000000000..00775aa0cdfa --- /dev/null +++ b/examples/models/core/multimodal/requirements-vila.txt @@ -0,0 +1,2 @@ +git+https://github.com/bfshi/scaling_on_scales.git +transformers==4.56.0 diff --git a/examples/models/core/multimodal/run.py b/examples/models/core/multimodal/run.py new file mode 100644 index 000000000000..fb13554848e7 --- /dev/null +++ b/examples/models/core/multimodal/run.py @@ -0,0 +1,128 @@ +import argparse +import os + +from utils import add_common_args, compute_str_match_rate + +import tensorrt_llm +import tensorrt_llm.profiler as profiler +from tensorrt_llm import logger +from tensorrt_llm.runtime import MultimodalModelRunner + + +def print_result(model, input_text, output_text, args): + logger.info("---------------------------------------------------------") + if model.model_type != 'nougat': + logger.info(f"\n[Q] {input_text}") + for i in range(len(output_text)): + logger.info(f"\n[A]: {output_text[i]}") + + if args.num_beams == 1: + output_ids = model.tokenizer(output_text[0][0], + add_special_tokens=False)['input_ids'] + logger.info(f"Generated {len(output_ids)} tokens") + + if args.check_accuracy: + if model.model_type != 'nougat': + if model.model_type == "vila": + for i in range(len(args.image_path.split(args.path_sep))): + if i % 2 == 0: + assert output_text[i][0].lower( + ) == "the image captures a bustling city intersection teeming with life. from the perspective of a car's dashboard camera, we see" + else: + assert output_text[i][0].lower( + ) == "the image captures the iconic merlion statue in singapore, a renowned worldwide landmark. the merlion, a mythical" + elif model.model_type == "llava": + for i in range(len(args.image_path.split(args.path_sep))): + assert output_text[i][0].lower() == 'singapore' + elif model.model_type == 'fuyu': + assert output_text[0][0].lower() == '4' + elif model.model_type == "pix2struct": + assert "characteristic | cat food, day | cat food, wet | cat treats" in output_text[ + 0][0].lower() + elif model.model_type in [ + 'blip2', 'neva', 'phi-3-vision', 'llava_next', + 'phi-4-multimodal', 'pixtral' + ]: + assert 'singapore' in output_text[0][0].lower() + elif model.model_type == 'video-neva': + assert 'robot' in output_text[0][0].lower() + elif model.model_type == 'kosmos-2': + assert 'snowman' in output_text[0][0].lower() + elif model.model_type == "mllama": + if "If I had to write a haiku for this one" in input_text: + ref_1 = ", it would be:.\\nPeter Rabbit is a rabbit.\\nHe lives in a cozy little house.\\nHe's a very good rabbit.\\" + ref_2 = "Here is a haiku for the image:\n\n" + + elif "Answer:" in input_text: + ref_1 = "2,173. A 1 2 3 4 5 6 Date Income 2005-12-17" + ref_2 = "Answer: 2,173. 1 2 3 4 5 6 Date Income 2005-12-17" + + elif "The key to life is" in input_text: + ref_1 = "to find your passion and pursue it with all your heart. For me, that passion is photography. I love capturing the beauty of the world around me" + ref_2 = "not to be found in the external world," + output = output_text[0][0] + match_rate = max(compute_str_match_rate(ref_1, output), + compute_str_match_rate(ref_2, output)) + logger.info(f"match rate: {match_rate}") + assert match_rate >= 50, \ + f"expected results: '{ref_1}' or '{ref_2}', generated results: '{output}'" + + elif model.model_type == 'llava_onevision': + if args.video_path is None: + assert 'singapore' in output_text[0][0].lower() + else: + assert 'the video is funny because the child\'s actions are' in output_text[ + 0][0].lower() + elif model.model_type == "qwen2_vl": + assert 'dog' in output_text[0][0].lower() + else: + assert output_text[0][0].lower() == 'singapore' + + if args.run_profiling: + msec_per_batch = lambda name: 1000 * profiler.elapsed_time_in_sec( + name) / args.profiling_iterations + logger.info('Latencies per batch (msec)') + logger.info('e2e generation: %.1f' % (msec_per_batch('Generate'))) + logger.info(' ' * 2 + 'Preprocessing: %.1f' % + (msec_per_batch('Preprocess'))) + logger.info(' ' * 4 + 'Vision encoder: %.1f' % + (msec_per_batch('Vision encoder'))) + if profiler.elapsed_time_in_sec('Feature transform') is not None: + logger.info(' ' * 4 + 'Feature transform: %.1f' % + (msec_per_batch('Feature transform'))) + logger.info(' ' * 2 + 'LLM generate: %.1f' % (msec_per_batch('LLM'))) + logger.info(' ' * 2 + 'Tokenizer decode: %.1f' % + (msec_per_batch('Tokenizer decode'))) + + logger.info("---------------------------------------------------------") + + +if __name__ == '__main__': + os.environ["TOKENIZERS_PARALLELISM"] = "false" + parser = argparse.ArgumentParser() + parser = add_common_args(parser) + args = parser.parse_args() + logger.set_level(args.log_level) + + model = MultimodalModelRunner(args) + visual_data = model.load_test_data(args.image_path, args.video_path) + audio_data = model.load_test_audio(args.audio_path) + + if args.run_profiling: + num_warmup_iters = 3 # Multiple iterations to load both vision and LLM engines into memory + for _ in range(num_warmup_iters): + input_text, output_text = model.run(args.input_text, visual_data, + audio_data, args.max_new_tokens) + profiler.reset() + + num_iters = args.profiling_iterations if args.run_profiling else 1 + + for _ in range(num_iters): + input_text, output_text = model.run(args.input_text, visual_data, + audio_data, args.max_new_tokens) + + runtime_rank = tensorrt_llm.mpi_rank() + if runtime_rank == 0: + print_result(model, input_text, output_text, args) + +# TODO: raise error if VILA mode 1 with C++ runtime diff --git a/examples/models/core/multimodal/utils.py b/examples/models/core/multimodal/utils.py new file mode 100644 index 000000000000..6d13fb600f40 --- /dev/null +++ b/examples/models/core/multimodal/utils.py @@ -0,0 +1,163 @@ +def add_common_args(parser): + parser.add_argument('--max_new_tokens', type=int, default=128) + parser.add_argument('--batch_size', type=int, default=1) + parser.add_argument('--log_level', type=str, default='info') + parser.add_argument('--engine_dir', + type=str, + default=None, + help='Directory containing visual and LLM TRT engines') + parser.add_argument('--visual_engine_name', + type=str, + default='model.engine', + help='Name of visual TRT engine') + parser.add_argument('--audio_engine_name', + type=str, + default='model.engine', + help='Name of audio TRT engine') + parser.add_argument('--hf_model_dir', + type=str, + default=None, + help="Directory containing tokenizer") + parser.add_argument('--input_text', + type=str, + nargs='+', + default=None, + help='Text prompt to LLM') + parser.add_argument('--num_beams', + type=int, + help="Use beam search if num_beams >1", + default=1) + parser.add_argument('--top_k', type=int, default=1) + parser.add_argument('--top_p', type=float, default=0.0) + parser.add_argument('--temperature', type=float, default=1.0) + parser.add_argument('--repetition_penalty', type=float, default=1.0) + parser.add_argument('--run_profiling', + action='store_true', + help='Profile runtime over several iterations') + parser.add_argument('--profiling_iterations', + type=int, + help="Number of iterations to run profiling", + default=20) + parser.add_argument('--check_accuracy', + action='store_true', + help='Check correctness of text output') + parser.add_argument( + '--video_path', + type=str, + default=None, + help= + 'Path to your local video file, using \'llava-onevision-accuracy\' to check the Llava-OneVision model accuracy' + ) + parser.add_argument( + '--video_num_frames', + type=int, + help= + "The number of frames sampled from the video in the Llava-OneVision model.", + default=None) + parser.add_argument("--image_path", + type=str, + nargs='+', + default=None, + help='List of input image paths, separated by symbol') + parser.add_argument("--audio_path", + type=str, + default=None, + help='input audio path') + parser.add_argument("--path_sep", + type=str, + default=",", + help='Path separator symbol') + parser.add_argument("--prompt_sep", + type=str, + default=",", + help="Prompt separator symbol") + parser.add_argument('--enable_context_fmha_fp32_acc', + action='store_true', + default=None, + help="Enable FMHA runner FP32 accumulation.") + parser.add_argument( + '--enable_chunked_context', + action='store_true', + help='Enables chunked context (only available with cpp session).', + ) + parser.add_argument( + '--mm_embedding_offloading', + type=lambda s: s.lower() == "true", + default=None, + help= + 'Enable position table offloading. When not specified, defaults to True if using a multimodal model with chunked context.' + ) + parser.add_argument( + '--session', + default='cpp_llm_only', + type=str, + choices=['python', 'cpp_llm_only', 'cpp'], + help= + 'Rumtime used to run the models. \n`cpp_llm_only`: vision engine run in python runtime, but LLM in pybind cpp runtime\n`python`: everything runs in python runtime\n`cpp`: everything runs in C++ runtime' + ) + parser.add_argument( + '--kv_cache_free_gpu_memory_fraction', + default=0.7, + type=float, + help='Specify the free gpu memory fraction.', + ) + parser.add_argument( + '--cross_kv_cache_fraction', + default=0.5, + type=float, + help= + 'Specify the kv cache fraction reserved for cross attention. Only applicable for encoder-decoder models. By default 0.5 for self and 0.5 for cross.', + ) + parser.add_argument( + '--multi_block_mode', + type=lambda s: s.lower() in + ("yes", "true", "t", "1" + ), # custom boolean function to convert input string to boolean + default=True, + help= + "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel." + ) + parser.add_argument( + '--lora_task_uids', + type=str, + default=None, + nargs="+", + help="The list of LoRA task uids; use -1 to disable the LoRA module") + parser.add_argument('--debug_mode', + default=False, + action='store_true', + help="Whether or not to turn on the debug mode") + parser.add_argument( + '--trust_remote_code', + action='store_true', + default=False, + help='Allow loading models with custom remote code from HuggingFace Hub. ' + 'Only enable this for models from trusted sources.') + return parser + + +def levenshtein_distance(s1, s2): + if len(s1) < len(s2): + return levenshtein_distance(s2, s1) + + if len(s2) == 0: + return len(s1) + + previous_row = range(len(s2) + 1) + for i, c1 in enumerate(s1): + current_row = [i + 1] + for j, c2 in enumerate(s2): + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + + return previous_row[-1] + + +def compute_str_match_rate(s1, s2): + distance = levenshtein_distance(s1, s2) + max_length = max(len(s1), len(s2)) + match_rate = (1 - distance / max_length) * 100 + return match_rate diff --git a/examples/models/core/nemotron/README_nemotron-3.md b/examples/models/core/nemotron/README_nemotron-3.md new file mode 100644 index 000000000000..0816eb995fd8 --- /dev/null +++ b/examples/models/core/nemotron/README_nemotron-3.md @@ -0,0 +1,214 @@ +# Nemotron-3 + +This document demonstrates how to build the Nemotron models using TensorRT LLM and run on a single GPU or multiple GPUs. + +- [Nemotron](#nemotron) + - [Overview](#overview) + - [Support Matrix](#support-matrix) + - [Usage](#usage) + - [Download weights from HuggingFace Transformers](#download-weights-from-huggingface-transformers) + - [Build TensorRT engine(s)](#build-tensorrt-engines) + - [FP8 Quantization](#fp8-quantization) + - [INT4 AWQ Quantization](#int4-awq-quantization) + - [Run Inference](#run-inference) + +## Overview + +The TensorRT LLM Nemotron implementation is based on the GPT model, which can be found in [`tensorrt_llm/models/gpt/model.py`](../../../../tensorrt_llm/models/gpt/model.py). The TensorRT LLM Nemotron example is located in [`examples/models/core/nemotron`](./). + +In addition, there are two shared files in the parent folder [`examples`](../../../) for inference and evaluation: + +* [`run.py`](../../../run.py) to run the inference on an input text; +* [`summarize.py`](../../../summarize.py) to summarize the articles in the [cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +## Support Matrix + * FP16/BF16 + * FP8 + * INT4 AWQ + * Tensor Parallel + * Pipeline Parallel + * Inflight Batching + * PAGED_KV_CACHE + * STRONGLY TYPED + * checkpoint type: Nemo, Huggingface (HF) + +## Nemo checkpoint - Usage + +### Download weights from HuggingFace Transformers + + +Install the dependencies and setup `git-lfs`. + +```bash +# Install dependencies +pip install -r requirements.txt + +# Setup git-lfs +git lfs install +``` + +Download one or more Nemotron models that you would like to build to TensorRT LLM engines. You can download from the [HuggingFace](https://huggingface.co) hub: + +```bash +# Download nemotron-3-8b-base-4k +git clone https://huggingface.co/nvidia/nemotron-3-8b-base-4k + +# Download nemotron-3-8b-chat-4k-sft +git clone https://huggingface.co/nvidia/nemotron-3-8b-chat-4k-sft + +# Download nemotron-3-8b-chat-4k-rlhf +git clone https://huggingface.co/nvidia/nemotron-3-8b-chat-4k-rlhf +``` + +### Build TensorRT engine(s) +The [`examples/quantization/quantize.py`](../../../quantization/quantize.py) script can quantize the Nemotron models and export to TensorRT LLM checkpoints. You may optionally skip the quantization step by specifying `--qformat full_prec` and thus export float16 or bfloat16 TensorRT LLM checkpoints. + +The `trtllm-build` command builds TensorRT LLM engines from TensorRT LLM checkpoints. The number of engine files is same to the number of GPUs used to run inference. Normally, `trtllm-build` uses one GPU by default, but if you have already more GPUs available at build time, you may enable parallel builds to make the engine building process faster by adding the `--workers` argument. + +Here are some examples: + +```bash +# single gpu, dtype bfloat16 +python3 ../../../quantization/quantize.py \ + --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ + --dtype bfloat16 \ + --batch_size 64 \ + --qformat full_prec \ + --output_dir nemotron-3-8b/trt_ckpt/bf16/1-gpu + +trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/bf16/1-gpu \ + --gpt_attention_plugin bfloat16 \ + --gemm_plugin bfloat16 \ + --output_dir nemotron-3-8b/trt_engines/bf16/1-gpu +``` + +```bash +# 2-way tensor parallelism +python3 ../../../quantization/quantize.py \ + --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ + --dtype bfloat16 \ + --batch_size 64 \ + --qformat full_prec \ + --tp_size 2 \ + --output_dir nemotron-3-8b/trt_ckpt/bf16/tp2 + +trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/bf16/tp2 \ + --gpt_attention_plugin bfloat16 \ + --gemm_plugin bfloat16 \ + --workers 2 \ + --output_dir nemotron-3-8b/trt_engines/bf16/tp2 +``` + +```bash +# 2-way tensor parallelism for both calibration and inference +mpirun -np 2 \ + python3 ../../../quantization/quantize.py \ + --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ + --dtype bfloat16 \ + --batch_size 64 \ + --qformat full_prec \ + --calib_tp_size 2 \ + --tp_size 2 \ + --output_dir nemotron-3-8b/trt_ckpt/bf16/tp2 + +trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/bf16/tp2 \ + --gpt_attention_plugin bfloat16 \ + --gemm_plugin bfloat16 \ + --workers 2 \ + --output_dir nemotron-3-8b/trt_engines/bf16/tp2 +``` + +#### FP8 Quantization + +Quantize the Nemotron models to FP8 by specifying `--qformat fp8` to `quantize.py`. + +```bash +# single gpu, fp8 quantization +python3 ../../../quantization/quantize.py \ + --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ + --dtype bfloat16 \ + --batch_size 64 \ + --qformat fp8 \ + --output_dir nemotron-3-8b/trt_ckpt/fp8/1-gpu + +trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/fp8/1-gpu \ + --gpt_attention_plugin bfloat16 \ + --output_dir nemotron-3-8b/trt_engines/fp8/1-gpu +``` + +#### INT4 AWQ Quantization + +Quantize the Nemotron models using INT4 AWQ by specifying `--qformat int4_awq` to `quantize.py`. + +```bash +# single gpu, int4 awq quantization +python3 ../../../quantization/quantize.py \ + --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ + --dtype bfloat16 \ + --batch_size 64 \ + --qformat int4_awq \ + --output_dir nemotron-3-8b/trt_ckpt/int4_awq/1-gpu + +trtllm-build --checkpoint_dir nemotron-3-8b/trt_ckpt/int4_awq/1-gpu \ + --gpt_attention_plugin bfloat16 \ + --output_dir nemotron-3-8b/trt_engines/int4_awq/1-gpu +``` + +### Run Inference + +The `summarize.py` script can run the built engines to summarize the articles from the +[cnn_dailymail](https://huggingface.co/datasets/abisee/cnn_dailymail) dataset. + +```bash +# single gpu +python3 ../../../summarize.py --test_trt_llm \ + --no_add_special_tokens \ + --engine_dir nemotron-3-8b/trt_engines/bf16/1-gpu \ + --vocab_file nemotron-3-8b/trt_ckpt/bf16/1-gpu/tokenizer.model + +# multiple gpus +mpirun -np 2 \ + python3 ../../../summarize.py --test_trt_llm \ + --no_add_special_tokens \ + --engine_dir nemotron-3-8b/trt_engines/bf16/tp2 \ + --vocab_file nemotron-3-8b/trt_ckpt/bf16/tp2/tokenizer.model +``` + +If the engines are run successfully, you will see output like: +``` +...... +[04/23/2024-09:55:54] [TRT-LLM] [I] TensorRT LLM (total latency: 14.926485538482666 sec) +[04/23/2024-09:55:54] [TRT-LLM] [I] TensorRT LLM (total output tokens: 2000) +[04/23/2024-09:55:54] [TRT-LLM] [I] TensorRT LLM (tokens per second: 133.99001357980129) +[04/23/2024-09:55:54] [TRT-LLM] [I] TensorRT LLM beam 0 result +[04/23/2024-09:55:54] [TRT-LLM] [I] rouge1 : 19.48743720965424 +[04/23/2024-09:55:54] [TRT-LLM] [I] rouge2 : 6.272381295466071 +[04/23/2024-09:55:54] [TRT-LLM] [I] rougeL : 15.011005943152721 +[04/23/2024-09:55:54] [TRT-LLM] [I] rougeLsum : 17.76145734406502 +``` + +## HF checkpoint - Usage +Support for Nemotron models was added with transformers 4.44.0 release. + +```bash +# install transformers library +pip install transformers>=4.44.0 +# Download hf minitron model +git clone https://huggingface.co/nvidia/Minitron-4B-Base + +# Convert to TensorRT LLM checkpoint +python3 ../gpt/convert_checkpoint.py --model_dir Minitron-4B-Base \ + --dtype bfloat16 \ + --output_dir minitron/trt_ckpt/bf16/1-gpu + +# Build TensorRT LLM engines +trtllm-build --checkpoint_dir minitron/trt_ckpt/bf16/1-gpu \ + --gemm_plugin auto \ + --output_dir minitron/trt_engines/bf16/1-gpu + +# Run inference +python3 ../../../run.py --engine_dir minitron/trt_engines/bf16/1-gpu \ + --tokenizer_dir Minitron-4B-Base \ + --input_text "def print_hello_world():" \ + --max_output_len 20 +``` diff --git a/examples/models/core/nemotron/requirements.txt b/examples/models/core/nemotron/requirements.txt new file mode 100644 index 000000000000..f8d1f8e1abf4 --- /dev/null +++ b/examples/models/core/nemotron/requirements.txt @@ -0,0 +1,7 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +nemo-toolkit[all]==2.0.0rc1 +megatron-core @ git+https://github.com/NVIDIA/Megatron-LM@core_r0.8.0 +datasets==3.1.0 +evaluate +rouge_score diff --git a/examples/models/core/qwen2audio/README.md b/examples/models/core/qwen2audio/README.md new file mode 100644 index 000000000000..92115345077e --- /dev/null +++ b/examples/models/core/qwen2audio/README.md @@ -0,0 +1,80 @@ +# Guide to Qwen2-Audio deployment pipeline + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +1. Download the Qwen2-Audio model. + ```bash + git lfs install + export MODEL_PATH="tmp/Qwen2-Audio-7B-Instruct" + git clone https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct $MODEL_PATH + ``` +2. Generate the TensorRT engine of audio encoder. + ```bash + export ENGINE_DIR="./trt_engines/qwen2audio/fp16" + python3 ../multimodal/build_multimodal_engine.py --model_type qwen2_audio --model_path $MODEL_PATH --max_batch_size 32 --output_dir ${ENGINE_DIR}/audio + ``` + + The TensorRT engine will be generated under `${ENGINE_DIR}/audio`. + +3. Build Qwen2 LLM TensorRT engine. +- Convert checkpoint + 1. Install packages + ```bash + pip install -r requirements.txt + ``` + 2. Convert + 2.1 FP16 checkpoint + ```bash + python3 ../qwen/convert_checkpoint.py --model_dir=$MODEL_PATH \ + --dtype=float16 \ + --output_dir=./tllm_checkpoint_1gpu_fp16 + ``` + 2.2 (Optional) INT8 Weight Only checkpoint + ```bash + python3 ../qwen/convert_checkpoint.py --model_dir=$MODEL_PATH \ + --dtype=float16 \ + --use_weight_only \ + --weight_only_precision=int8 \ + --output_dir=./tllm_checkpoint_1gpu_fp16_wo8 + ``` + +- Build TensorRT LLM engine + + NOTE: `max_prompt_embedding_table_size = query_token_num * max_batch_size`, therefore, if you change `max_batch_size`, `--max_prompt_embedding_table_size` must be reset accordingly. + ```bash + trtllm-build --checkpoint_dir=./tllm_checkpoint_1gpu_fp16 \ + --gemm_plugin=float16 --gpt_attention_plugin=float16 \ + --max_batch_size=1 --max_prompt_embedding_table_size=4096 \ + --output_dir=${ENGINE_DIR}/llm + ``` + The built Qwen engines are located in `${ENGINE_DIR}/llm`. + + You can replace the `--checkpoint_dir` with INT8 Weight Only checkpoint to build INT8 Weight Only engine as well. + For more information about Qwen, refer to the README.md in [`example/models/core/qwen`](../qwen). + +4. Assemble everything into the Qwen2-Audio pipeline. + + 4.1 Run with FP16 LLM engine + ```bash + python3 run.py \ + --tokenizer_dir=$MODEL_PATH \ + --engine_dir=${ENGINE_DIR}/llm \ + --audio_engine_path=${ENGINE_DIR}/audio/model.engine \ + --audio_url='./audio/glass-breaking-151256.mp3' + ``` + 4.2 (Optional) For multiple rounds of dialogue, you can run: + ```bash + python3 run_chat.py \ + --tokenizer_dir=$MODEL_PATH \ + --engine_dir=${ENGINE_DIR}/llm \ + --audio_engine_path=${ENGINE_DIR}/audio/model.engine \ + --max_new_tokens=256 + ``` + + Note: + - This example supports reusing the KV Cache for audio segments by assigning unique audio IDs. + - To further optimize performance, users can also cache the audio features (encoder output) to bypass the audio encoder if the original audio data remains unchanged. diff --git a/examples/models/core/qwen2audio/audio/glass-breaking-151256.mp3 b/examples/models/core/qwen2audio/audio/glass-breaking-151256.mp3 new file mode 100644 index 000000000000..150e1080f2e5 Binary files /dev/null and b/examples/models/core/qwen2audio/audio/glass-breaking-151256.mp3 differ diff --git a/examples/models/core/qwen2audio/requirements.txt b/examples/models/core/qwen2audio/requirements.txt new file mode 100644 index 000000000000..1d6d844e4b64 --- /dev/null +++ b/examples/models/core/qwen2audio/requirements.txt @@ -0,0 +1,10 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.dev0 +datasets==3.1.0 +evaluate +rouge_score +transformers>=4.45.0 +transformers-stream-generator +sentencepiece>=0.1.99 +tiktoken +einops diff --git a/examples/models/core/qwen2audio/run.py b/examples/models/core/qwen2audio/run.py new file mode 100644 index 000000000000..a0b0a68fb1c9 --- /dev/null +++ b/examples/models/core/qwen2audio/run.py @@ -0,0 +1,640 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import os +from io import BytesIO +from urllib.request import urlopen + +import librosa +import tensorrt as trt +import torch +from transformers import AutoConfig, AutoProcessor, AutoTokenizer +from utils import add_common_args + +import tensorrt_llm +import tensorrt_llm.profiler as profiler +from tensorrt_llm import logger +from tensorrt_llm.llmapi.kv_cache_type import KVCacheType +from tensorrt_llm.quantization import QuantMode +from tensorrt_llm.runtime import (PYTHON_BINDINGS, ModelConfig, ModelRunner, + SamplingConfig, Session, TensorInfo) + +if PYTHON_BINDINGS: + from tensorrt_llm.runtime import ModelRunnerCpp + + +def get_engine_name(rank): + return "rank{}.engine".format(rank) + + +def trt_dtype_to_torch(dtype): + if dtype == trt.float16: + return torch.float16 + elif dtype == trt.float32: + return torch.float32 + elif dtype == trt.int32: + return torch.int32 + else: + raise TypeError("%s is not supported" % dtype) + + +class QWenInfer(object): + + def __init__(self, + audio_engine_path, + tokenizer_dir, + engine_dir, + log_level, + output_csv, + output_npy, + num_beams, + gpu_id=0): + self.audio_engine_path = audio_engine_path + self.tokenizer_dir = tokenizer_dir + self.engine_dir = engine_dir + self.log_level = log_level + self.max_seq_len = 0 + self.runner = None + self.hf_audio_tower = None + self.tokenizer = None + self.config = None + self.sampling_config = None + self.output_csv = output_csv + self.output_npy = output_npy + self.num_beams = num_beams + self.model_config = None + self.gpu_device = torch.device("cuda", gpu_id) + + def get_model(self): + # --load the tokenizer and engines # + tokenizer = AutoTokenizer.from_pretrained( + self.tokenizer_dir, + legacy=False, + trust_remote_code=True, + ) + processor = AutoProcessor.from_pretrained(self.tokenizer_dir, + trust_remote_code=True) + config_path = os.path.join(self.engine_dir, "config.json") + with open(config_path, "r") as f: + config = json.load(f) + self.max_seq_len = config["build_config"]["max_seq_len"] + assert self.max_seq_len > 0, "max_seq_len must be positive" + + gen_config_path = os.path.join(self.tokenizer_dir, + "generation_config.json") + with open(gen_config_path, "r") as f: + gen_config = json.load(f) + top_k = gen_config["top_k"] + top_p = gen_config["top_p"] + eos_token_id = tokenizer.pad_token_id + pad_token_id = tokenizer.pad_token_id + + use_gpt_attention_plugin = config["build_config"]["plugin_config"][ + "gpt_attention_plugin"] + remove_input_padding = config["build_config"]["plugin_config"][ + "remove_input_padding"] + dtype = config["pretrained_config"]["dtype"] + tp_size = config["pretrained_config"]["mapping"]["tp_size"] + pp_size = config["pretrained_config"]["mapping"]["pp_size"] + world_size = tp_size * pp_size + assert ( + world_size == tensorrt_llm.mpi_world_size() + ), f"Engine world size ({world_size}) != Runtime world size ({tensorrt_llm.mpi_world_size()})" + num_heads = config["pretrained_config"][ + "num_attention_heads"] // world_size + max_batch_size = config["build_config"]["max_batch_size"] + hidden_size = config["pretrained_config"]["hidden_size"] // world_size + vocab_size = config["pretrained_config"]["vocab_size"] + num_layers = config["pretrained_config"]["num_hidden_layers"] + num_kv_heads = config["pretrained_config"].get("num_key_value_heads", + num_heads) + if "kv_cache_type" in config["build_config"]: + kv_cache_type = KVCacheType(config["build_config"]["kv_cache_type"]) + else: + kv_cache_type = KVCacheType.CONTINUOUS + + tokens_per_block = config["build_config"]["plugin_config"][ + "tokens_per_block"] + max_prompt_embedding_table_size = config["build_config"].get( + "max_prompt_embedding_table_size", 0) + quant_mode = QuantMode.from_quant_algo( + config["pretrained_config"]["quantization"]["quant_algo"], + config["pretrained_config"]["quantization"]["kv_cache_quant_algo"], + ) + if config["pretrained_config"].get("multi_query_mode", False): + tensorrt_llm.logger.warning( + "`multi_query_mode` config is deprecated. Please rebuild the engine." + ) + num_kv_heads = 1 + + runtime_rank = tensorrt_llm.mpi_rank() + runtime_mapping = tensorrt_llm.Mapping(world_size=world_size, + rank=runtime_rank, + tp_size=tp_size, + pp_size=pp_size) + + model_config = ModelConfig( + max_batch_size=max_batch_size, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + gpt_attention_plugin=use_gpt_attention_plugin, + kv_cache_type=kv_cache_type, + tokens_per_block=tokens_per_block, + remove_input_padding=remove_input_padding, + dtype=dtype, + quant_mode=quant_mode, + max_prompt_embedding_table_size=max_prompt_embedding_table_size, + max_beam_width=self.num_beams, + ) + sampling_config = SamplingConfig( + end_id=eos_token_id, + pad_id=pad_token_id, + num_beams=self.num_beams, + top_k=top_k, + top_p=top_p, + temperature=1.0, + ) + + engine_name = get_engine_name(runtime_rank) + serialize_path = os.path.join(self.engine_dir, engine_name) + print(f"Loading engine from {serialize_path}") + return ( + model_config, + sampling_config, + runtime_mapping, + runtime_rank, + serialize_path, + tokenizer, + processor, + eos_token_id, + pad_token_id, + ) + + def qwen_model_init(self, args): + logger.info(f"Loading audio engine from {self.audio_engine_path}") + with open(self.audio_engine_path, "rb") as f: + engine_buffer = f.read() + logger.info(f"Creating session from engine {self.audio_engine_path}") + self.session_audio = Session.from_serialized_engine(engine_buffer) + + self.config, _ = AutoConfig.from_pretrained( + self.tokenizer_dir, + return_unused_kwargs=True, + trust_remote_code=True, + ) + + ( + model_config, + sampling_config, + runtime_mapping, + runtime_rank, + serialize_path, + tokenizer, + processor, + eos_token_id, + pad_token_id, + ) = self.get_model() + runner_cls = ModelRunner if args.use_py_session else ModelRunnerCpp + runner_kwargs = dict( + engine_dir=args.engine_dir, + lora_dir=args.lora_dir, + rank=runtime_rank, + debug_mode=args.debug_mode, + lora_ckpt_source=args.lora_ckpt_source, + gpu_weights_percent=args.gpu_weights_percent, + max_output_len=args.max_new_tokens, + ) + if not args.use_py_session: + runner_kwargs.update( + is_enc_dec=False, + max_batch_size=model_config.max_batch_size, + max_input_len=self.max_seq_len - args.max_new_tokens, + max_beam_width=model_config.max_beam_width, + max_attention_window_size=args.max_attention_window_size, + sink_token_length=args.sink_token_length, + max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, + kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, + kv_cache_free_gpu_memory_fraction=args. + kv_cache_free_gpu_memory_fraction, + cross_kv_cache_fraction=None, + enable_chunked_context=args.enable_chunked_context, + multi_block_mode=args.multi_block_mode, + cuda_graph_mode=args.cuda_graph_mode, + device_ids=[args.gpu_id]) + runner_kwargs.update( + enable_context_fmha_fp32_acc=args.enable_context_fmha_fp32_acc) + self.runner = runner_cls.from_dir(**runner_kwargs) + self.tokenizer = tokenizer + self.processor = processor + self.sampling_config = sampling_config + self.model_config = model_config + + def ptuning_setup(self, prompt_table, dtype, hidden_size, tasks, input_ids): + if prompt_table is not None: + task_vocab_size = torch.tensor([prompt_table.shape[0]], + dtype=torch.int32, + device=self.gpu_device) + prompt_table = prompt_table.to( + dtype=tensorrt_llm._utils.str_dtype_to_torch(dtype), + device=self.gpu_device) + else: + prompt_table = torch.empty([1, hidden_size], device=self.gpu_device) + task_vocab_size = torch.zeros([1], device=self.gpu_device) + + if tasks is not None: + tasks = torch.tensor([int(t) for t in tasks.split(",")], + dtype=torch.int32, + device=self.gpu_device) + assert (tasks.shape[0] == input_ids.shape[0] + ), "Number of supplied tasks must match input batch size" + else: + tasks = torch.zeros([input_ids.size(0)], + dtype=torch.int32, + device=self.gpu_device) + + return [prompt_table, tasks, task_vocab_size] + + def build_user_input(self, audio=None, text=None): + assert isinstance(audio, str) or isinstance( + text, str), "audio or text must be provided as user input" + content = [] + if audio: + content.append({'type': 'audio', 'audio_url': audio}) + if text: + content.append({'type': 'text', 'text': text}) + user_input = {'role': 'user', 'content': content} + return user_input + + def get_raw_audios(self, audio_url): + audios = [] + for url in audio_url: + if os.path.isfile(url): + audio_data, _ = librosa.load( + url, sr=self.processor.feature_extractor.sampling_rate) + else: + audio_data, _ = librosa.load( + BytesIO(urlopen(url).read()), + sr=self.processor.feature_extractor.sampling_rate) + audios.append(audio_data) + return audios + + def audio_tower(self, audios, mask, stream, run_time=1): + audios = audios.to(self.gpu_device) + mask = mask.to(self.gpu_device) + audio_inputs = {"input": audios.float(), "mask": mask} + audio_output_info = self.session_audio.infer_shapes([ + TensorInfo("input", trt.DataType.FLOAT, audios.shape), + TensorInfo("mask", trt.DataType.HALF, mask.shape) + ]) + audio_outputs = { + t.name: + torch.empty(tuple(t.shape), + dtype=trt_dtype_to_torch(t.dtype), + device=self.gpu_device) + for t in audio_output_info + } + profiler.start("Audio") + for _ in range(run_time): + ok = self.session_audio.run(audio_inputs, audio_outputs, + stream.cuda_stream) + stream.synchronize() + audio_time = profiler.stop("Audio") / run_time + logger.info(f"TensorRT LLM Audio latency: {audio_time:3f} sec ") + + assert ok, "Runtime execution failed for audio session" + + audio_features = audio_outputs["output"] + + return audio_features + + def generate_for_qwen_audio( + self, + input_tokens, + args, + prompt_table=None, + extra_ids=None, + run_time=1, + ): + input_ids = torch.as_tensor(input_tokens, + device=self.gpu_device, + dtype=torch.int32) + input_lengths = torch.tensor([input_ids.size(1)], + device=self.gpu_device, + dtype=torch.int32) + max_input_length = torch.max(input_lengths).item() + max_new_tokens = min(args.max_new_tokens, + self.max_seq_len - max_input_length) + + prompt_table = prompt_table.unsqueeze(0) + profiler.start("QWen") + for _ in range(run_time): + outputs = self.runner.generate( + batch_input_ids=input_ids, + max_new_tokens=max_new_tokens, + max_attention_window_size=args.max_attention_window_size, + sink_token_length=args.sink_token_length, + end_id=self.sampling_config.end_id, + pad_id=self.sampling_config.pad_id, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + num_beams=args.num_beams, + num_return_sequences=args.num_return_sequences, + length_penalty=args.length_penalty, + early_stopping=args.early_stopping, + repetition_penalty=args.repetition_penalty, + presence_penalty=args.presence_penalty, + frequency_penalty=args.frequency_penalty, + stop_words_list=[[[151643], [151645]]], + bad_words_list=self.sampling_config.bad_words_list, + random_seed=args.random_seed, + lora_uids=args.lora_task_uids, + prompt_table=prompt_table, + prompt_tasks="0", + output_sequence_lengths=True, + no_repeat_ngram_size=args.no_repeat_ngram_size, + return_dict=True, + return_all_generated_tokens=False, + input_token_extra_ids=extra_ids) + output_ids = outputs['output_ids'] + torch.cuda.synchronize() + Qwen_time = profiler.stop("QWen") / run_time + + return output_ids, Qwen_time + + def get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor): + """ + Computes the output length of the convolutional layers and the output length of the audio encoder + """ + input_lengths = (input_lengths - 1) // 2 + 1 + output_lengths = (input_lengths - 2) // 2 + 1 + return input_lengths, output_lengths + + def qwen_infer(self, + input_text, + audios, + audio_ids, + args, + stream, + history=None, + past_audio_features=None, + run_time=1): + assert input_text, "input_text must be provided" + assert torch.cuda.is_available(), "no gpu available" + # preprocess on CPU maybe faster + device = torch.device("cpu") + if isinstance(history, list): + history.append(input_text) + full_text = self.processor.apply_chat_template( + history, add_generation_prompt=True, tokenize=False) + else: + full_text = input_text + inputs = self.processor( + text=full_text, + audios=audios, + return_tensors="pt", + padding=True, + sampling_rate=self.processor.feature_extractor.sampling_rate) + inputs = inputs.to(device) + input_ids = inputs.input_ids + + if hasattr(inputs, + 'input_features') and inputs.input_features is not None: + # audio tower + batch_size, _, max_mel_seq_len = inputs.input_features.shape + feature_attention_mask = inputs.feature_attention_mask + + audio_feat_lengths, num_audio_tokens = self.get_feat_extract_output_lengths( + feature_attention_mask.sum(-1)) + + max_seq_len = (max_mel_seq_len - 2) // 2 + 1 + # Create a sequence tensor of shape (batch_size, max_seq_len) + seq_range = (torch.arange(0, + max_seq_len, + dtype=audio_feat_lengths.dtype, + device=device).unsqueeze(0).expand( + batch_size, max_seq_len)) + lengths_expand = audio_feat_lengths.unsqueeze(1).expand( + batch_size, max_seq_len) + # Create mask + padding_mask = seq_range >= lengths_expand + + audio_attention_mask_ = padding_mask.view( + batch_size, 1, 1, max_seq_len).expand(batch_size, 1, + max_seq_len, max_seq_len) + audio_attention_mask = audio_attention_mask_.to(dtype=torch.float16, + device=device) + audio_attention_mask[audio_attention_mask_] = float("-inf") + + audio_features = self.audio_tower(inputs.input_features, + audio_attention_mask, stream, + run_time) + + # merge audio features and input ids + num_audios, max_audio_tokens, embed_dim = audio_features.shape + audio_features_mask = torch.arange( + max_audio_tokens, device=device).expand( + num_audios, + max_audio_tokens) < num_audio_tokens.unsqueeze(1) + masked_audio_features = audio_features[audio_features_mask].view( + -1, embed_dim) + batch_size, _ = input_ids.shape + + # 1. Create a mask to know where special audio tokens are + special_audio_token_mask = input_ids == self.config.audio_token_index + special_audio_token_num = special_audio_token_mask.sum().item() + if past_audio_features is not None: + assert isinstance(past_audio_features, + list), f'past_audio_features should be a list' + assert ( + special_audio_token_num == len(past_audio_features) + + num_audios + ), f'special_audio_token_num {special_audio_token_num} should be equal to len(past_audio_features) + num_audios ({len(past_audio_features)} + {num_audios})' + # split to get current audio features + cur_audio_features = torch.split(masked_audio_features, + num_audio_tokens.tolist()) + if len(past_audio_features) > 0: + # concat past and current audio features + masked_audio_features = torch.cat( + (torch.cat(past_audio_features).to( + masked_audio_features.device), + masked_audio_features)) + # get past audio tokens number + past_num_audio_tokens = torch.tensor([ + past_feat.size(0) for past_feat in past_audio_features + ]) + # concat past and current audio tokens number + num_audio_tokens = torch.cat( + (past_num_audio_tokens.to(num_audio_tokens.device), + num_audio_tokens)) + # extend past audio features, cache them in CPU memory + past_audio_features.extend( + [cur_feat.cpu() for cur_feat in cur_audio_features]) + + batch_indices, non_audio_indices = torch.where( + input_ids != self.config.audio_token_index) + + # 2. Fill the final input ids based on the mask. + batch_indices, audio_indices = torch.where( + input_ids == self.config.audio_token_index) + + vocab_size = self.config.vocab_size + fake_prompt_id = torch.arange(vocab_size, + vocab_size + num_audio_tokens.sum(), + device=device) + + input_ids[batch_indices, audio_indices] = fake_prompt_id + input_lengths = torch.tensor(input_ids.size(1), + dtype=torch.int32, + device=self.gpu_device) + dtype = self.model_config.dtype + prompt_table, tasks, task_vocab_size = self.ptuning_setup( + masked_audio_features, dtype, embed_dim, None, input_ids) + + # build extra ids + assert isinstance(audio_ids, list), "audio_ids must be a list" + assert ( + len(audio_ids) == num_audio_tokens.size(0) + ), f"audio_ids length doesn't match with num_audio_tokens ({len(audio_ids)} != {num_audio_tokens.size(0)})" + for i in audio_ids: + assert isinstance( + i, int + ) and i > 0, "audio_id should be an integer greater than 0" + extra_ids = torch.zeros_like(input_ids, + dtype=torch.int64, + device=device) + seq_extra_ids = torch.cat([ + torch.full((n, ), audio_ids[i], dtype=torch.int64) + for i, n in enumerate(num_audio_tokens) + ]).to(device) + extra_ids[batch_indices, audio_indices] = seq_extra_ids + extra_ids = extra_ids.tolist() + else: + input_ids = input_ids.to(dtype=torch.int32, device=self.gpu_device) + input_lengths = torch.tensor(input_ids.size(1), + dtype=torch.int32, + device=self.gpu_device) + dtype = self.model_config.dtype + prompt_table, tasks, task_vocab_size = self.ptuning_setup( + None, dtype, self.model_config.hidden_size, None, input_ids) + extra_ids = torch.zeros_like(input_ids, dtype=torch.int64).tolist() + + # print(f"extra_ids: {extra_ids}") + output_ids, Qwen_time = self.generate_for_qwen_audio( + input_ids, args, prompt_table, extra_ids, run_time) + + runtime_rank = tensorrt_llm.mpi_rank() + input_lengths = torch.tensor([input_ids.size(1)], + device=self.gpu_device, + dtype=torch.int32) + effective_output_token = 0 + if runtime_rank == 0: + if self.output_csv is None and self.output_npy is None: + for b in range(input_lengths.size(0)): + inputs = input_ids[b] + if self.num_beams <= 1: + outputs = output_ids[b][0, len(inputs):].tolist() + try: + effective_output_token = (effective_output_token + + outputs.index(151643)) + except: + effective_output_token = 1 + output_text = self.tokenizer.decode( + outputs, skip_special_tokens=True) + print(f'Output: "{output_text}"') + else: + for beam in range(self.num_beams): + outputs = output_ids[b][beam, len(inputs):].tolist() + output_text = self.tokenizer.decode( + outputs, skip_special_tokens=True) + print(f'Output(beam: {beam}): "{output_text}"') + logger.info(f"Input length={input_lengths[b]}") + logger.info(f"Output length={output_ids.shape}") + logger.info(f"TensorRT LLM QWen time: {Qwen_time:3f} sec ") + if isinstance(history, list): + history.append({'role': 'assistant', 'content': output_text}) + return output_text, past_audio_features + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--max_new_tokens", type=int, default=10) + parser.add_argument( + "--audio_engine_path", + type=str, + default="plan/audio_encoder/audio_encoder_fp16.plan", + ) + parser.add_argument( + "--input_text", + type=str, + default= + "<|audio_bos|><|AUDIO|><|audio_eos|>Generate the caption in English:") + parser.add_argument( + "--audio_url", + nargs="+", + type=str, + default=["./audio/glass-breaking-151256.mp3"], + ) + parser.add_argument( + "--input_tokens", + dest="input_file", + type=str, + help= + "CSV or Numpy file containing tokenized input. Alternative to text input.", + default=None, + ) + parser.add_argument( + "--output_csv", + type=str, + help="CSV file where the tokenized output is stored.", + default=None, + ) + parser.add_argument( + "--output_npy", + type=str, + help="Numpy file where the tokenized output is stored.", + default=None, + ) + parser.add_argument( + "--gpu_id", + type=int, + help= + "Specify GPU device index for running. Should be the index seen by torch, not original index", + default=0, + ) + parser = add_common_args(parser) + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_arguments() + tensorrt_llm.logger.set_level(args.log_level) + + # use cudaSetDevice before loading audio engine + torch.cuda.set_device(args.gpu_id) + qinfer = QWenInfer(args.audio_engine_path, args.tokenizer_dir, + args.engine_dir, args.log_level, args.output_csv, + args.output_npy, args.num_beams, args.gpu_id) + qinfer.qwen_model_init(args) + + audios = qinfer.get_raw_audios(args.audio_url) + gpu_device = torch.device("cuda", args.gpu_id) + stream = torch.cuda.current_stream(device=gpu_device) + qinfer.qwen_infer(args.input_text, audios, [1], args, stream, None, None, 1) diff --git a/examples/models/core/qwen2audio/run_chat.py b/examples/models/core/qwen2audio/run_chat.py new file mode 100644 index 000000000000..00d58cdc862b --- /dev/null +++ b/examples/models/core/qwen2audio/run_chat.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# isort: off +import torch +from run import QWenInfer, parse_arguments + +import tensorrt_llm +# isort: on + +if __name__ == '__main__': + args = parse_arguments() + stream = torch.cuda.current_stream() + tensorrt_llm.logger.set_level(args.log_level) + qinfer = QWenInfer( + args.audio_engine_path, + args.tokenizer_dir, + args.engine_dir, + args.log_level, + args.output_csv, + args.output_npy, + args.num_beams, + ) + qinfer.qwen_model_init(args) + + run_i = 0 + history = [] + audios = None + global_audio_id = 1 + audio_ids = [] + + while True: + input_text = None + try: + input_text = input( + "Text (type 'q' to quit, or 'audio_url:[url]' to input audio): " + ) + except: + continue + + if input_text == "clear history": + history = [] + audios = None + continue + + if input_text.lower() == 'q': + break + print('\n') + + if input_text.startswith('audio_url:'): + audio_url = input_text[len('audio_url:'):].strip() + if isinstance(audios, list): + audios.extend(qinfer.get_raw_audios([audio_url])) + else: + audios = qinfer.get_raw_audios([audio_url]) + user_input = qinfer.build_user_input(audio=audio_url) + audio_ids.append(global_audio_id) + global_audio_id += 1 + else: + user_input = qinfer.build_user_input(text=input_text) + + qinfer.qwen_infer( + user_input, + audios, + audio_ids, + args, + stream, + history, + ) diff --git a/examples/models/core/qwen2audio/utils.py b/examples/models/core/qwen2audio/utils.py new file mode 100644 index 000000000000..3252beebbf7d --- /dev/null +++ b/examples/models/core/qwen2audio/utils.py @@ -0,0 +1,130 @@ +from argparse import BooleanOptionalAction + + +def add_common_args(parser): + # sampling arguments + parser.add_argument('--num_beams', + type=int, + help="Use beam search if num_beams > 1", + default=1) + parser.add_argument('--num_return_sequences', + type=int, + help="Number of sequences to generate for each input.", + default=None) + parser.add_argument('--temperature', type=float, default=1.0) + parser.add_argument('--top_k', type=int, default=1) + parser.add_argument('--top_p', type=float, default=0.0) + parser.add_argument('--length_penalty', type=float, default=1.0) + parser.add_argument('--repetition_penalty', type=float, default=1.0) + parser.add_argument('--presence_penalty', type=float, default=0.0) + parser.add_argument('--frequency_penalty', type=float, default=0.0) + parser.add_argument('--random_seed', type=int, default=0) + parser.add_argument('--early_stopping', + type=int, + help='Use early stopping if num_beams > 1, ' + '1 for early-stopping, 0 for non-early-stopping' + 'other values for stopping by length', + default=1) + parser.add_argument('--no_repeat_ngram_size', type=int, default=None) + + # common runtime arguments + parser.add_argument('--sink_token_length', + type=int, + default=None, + help='The sink token length.') + parser.add_argument( + '--max_attention_window_size', + type=int, + default=None, + nargs="+", + help= + 'The attention window size that controls the sliding window attention kv cache behavior' + ) + parser.add_argument( + '--multi_block_mode', + type=lambda s: s.lower() in + ("yes", "true", "t", "1" + ), # custom boolean function to convert input string to boolean + default=True, + help= + "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel." + ) + parser.add_argument('--enable_context_fmha_fp32_acc', + action='store_true', + help="Enable FMHA runner FP32 accumulation.") + parser.add_argument('--cuda_graph_mode', + action='store_true', + help="Enable cuda graphs in the inference.") + parser.add_argument( + '--log_level', + type=str, + choices=['verbose', 'info', 'warning', 'error', 'internal_error'], + default='info') + parser.add_argument('--use_py_session', + default=False, + action='store_true', + help="Whether or not to use Python runtime session") + parser.add_argument('--debug_mode', + default=False, + action='store_true', + help="Whether or not to turn on the debug mode") + parser.add_argument('--lora_dir', + type=str, + default=None, + nargs="+", + help="The directory of LoRA weights") + parser.add_argument('--lora_ckpt_source', + type=str, + default="hf", + choices=["hf", "nemo"], + help="The source of lora checkpoint.") + parser.add_argument( + '--lora_task_uids', + type=str, + default=None, + nargs="+", + help="The list of LoRA task uids; use -1 to disable the LoRA module") + + # model arguments + parser.add_argument('--engine_dir', type=str, default='engine_outputs') + parser.add_argument('--hf_model_dir', '--model_dir', type=str, default=None) + parser.add_argument( + '--tokenizer_dir', + default=None, + help='tokenizer path; defaults to hf_model_dir if left unspecified') + + # memory argument + parser.add_argument( + '--gpu_weights_percent', + default=1, + type=float, + help= + 'Specify the percentage of weights that reside on GPU instead of CPU and streaming load during runtime.', + ) + parser.add_argument( + '--max_tokens_in_paged_kv_cache', + default=None, + type=int, + help= + 'Specify the maximum number of tokens in a kv cache page (only available with cpp session).', + ) + parser.add_argument( + '--kv_cache_enable_block_reuse', + default=True, + action=BooleanOptionalAction, + help= + 'Enables block reuse in kv cache (only available with cpp session).', + ) + parser.add_argument( + '--kv_cache_free_gpu_memory_fraction', + default=0.9, + type=float, + help='Specify the free gpu memory fraction.', + ) + parser.add_argument( + '--enable_chunked_context', + action='store_true', + help='Enables chunked context (only available with cpp session).', + ) + + return parser diff --git a/examples/models/core/qwenvl/README.md b/examples/models/core/qwenvl/README.md new file mode 100644 index 000000000000..27fc5d1dfce0 --- /dev/null +++ b/examples/models/core/qwenvl/README.md @@ -0,0 +1,114 @@ +# Guide to Qwen-VL deployment pipeline + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +1. Download the Qwen vision-language model (Qwen-VL). + ```bash + git lfs install + git clone https://huggingface.co/Qwen/Qwen-VL-Chat + ``` +2. Generate the Vision Transformer (ViT) ONNX model and the TensorRT engine. +- If you don't have ONNX file, run: + ```bash + python3 vit_onnx_trt.py --pretrained_model_path ./Qwen-VL-Chat + ``` + The ONNX and TensorRT engine will be generated under `./onnx/visual_encoder` and `./plan/visual_encoder` respectively. + +- If you already have an ONNX file under `./onnx/visual_encoder` and want to build a TensorRT engine with it, run: + ```bash + python3 vit_onnx_trt.py --pretrained_model_path ./Qwen-VL-Chat --only_trt + ``` + This command saves the test image tensor to `image.pt` for later pipeline inference. + +3. Build Qwen TensorRT engine. +- Convert checkpoint + 1. Install packages + ```bash + pip install -r requirements.txt + ``` + 2. Convert + ```bash + python3 ./examples/models/core/qwen/convert_checkpoint.py --model_dir=./Qwen-VL-Chat \ + --output_dir=./tllm_checkpoint_1gpu \ + --dtype float16 + ``` + +- Build TensorRT LLM engine + + NOTE: `max_prompt_embedding_table_size = query_token_num * max_batch_size`, therefore, if you change `max_batch_size`, `--max_prompt_embedding_table_size` must be reset accordingly. + ```bash + trtllm-build --checkpoint_dir=./tllm_checkpoint_1gpu \ + --gemm_plugin=float16 --gpt_attention_plugin=float16 \ + --max_input_len=2048 --max_seq_len=3072 \ + --max_batch_size=8 --max_prompt_embedding_table_size=2048 \ + --remove_input_padding=enable \ + --output_dir=./trt_engines/Qwen-VL-7B-Chat + ``` + The built Qwen engines are located in `./trt_engines/Qwen-VL-7B-Chat`. + For more information about Qwen, refer to the README.md in [`example/qwen`](../qwen). + +4. Assemble everything into the Qwen-VL pipeline. + + 4.1 Run with INT4 GPTQ weight-only quantization engine + ```bash + python3 run.py \ + --tokenizer_dir=./Qwen-VL-Chat \ + --qwen_engine_dir=./trt_engines/Qwen-VL-7B-Chat \ + --vit_engine_path=./plan/visual_encoder/visual_encoder_fp16.plan \ + --images_path='{"image": "./pics/demo.jpeg"}' + ``` + 4.2 (Optional) For multiple rounds of dialogue, you can run: + ```bash + python3 run_chat.py \ + --tokenizer_dir=./Qwen-VL-Chat \ + --qwen_engine_dir=./trt_engines/Qwen-VL-7B-Chat \ + --vit_engine_path=./plan/visual_encoder/visual_encoder_fp16.plan \ + --images_path='{"image": "./pics/demo.jpeg"}' + ``` + 4.3 (Optional) To show the bounding box result in the demo picture, install OpenCV, ZMQ, and request: + ```bash + pip install opencv-python==4.5.5.64 + pip install opencv-python-headless==4.5.5.64 + pip install zmq + pip install request + ``` + +   4.3.1 If the current program is executed on a remote machine, run the following command on a local machine: + + ```bash + python3 show_pic.py --ip=127.0.0.1 --port=8006 + ``` + +   Replace the `ip` and `port` values, where `ip` is your remote machine IP address. + +   Run the following command on the remote machine: + + ```bash + python3 run_chat.py \ + --tokenizer_dir=./Qwen-VL-Chat \ + --qwen_engine_dir=./trt_engines/Qwen-VL-7B-Chat \ + --vit_engine_path=./plan/visual_encoder/visual_encoder_fp16.plan \ + --display \ + --port=8006 + ``` + +   Replace the `port` value. + +   4.3.2 If the current program is executed on the local machine, run the following command: + + ```bash + python3 run_chat.py \ + --tokenizer_dir=./Qwen-VL-Chat \ + --qwen_engine_dir=./trt_engines/Qwen-VL-7B-Chat \ + --vit_engine_path=./plan/visual_encoder/visual_encoder_fp16.plan \ + --display \ + --local_machine + ``` + +   The question "Print the bounding box of the girl" is displayed. You should see the following image: + + ![image](./pics/1.png) diff --git a/examples/models/core/qwenvl/pics/1.png b/examples/models/core/qwenvl/pics/1.png new file mode 100644 index 000000000000..09e872ac90da Binary files /dev/null and b/examples/models/core/qwenvl/pics/1.png differ diff --git a/examples/models/core/qwenvl/pics/demo.jpeg b/examples/models/core/qwenvl/pics/demo.jpeg new file mode 100644 index 000000000000..9fdc04005062 Binary files /dev/null and b/examples/models/core/qwenvl/pics/demo.jpeg differ diff --git a/examples/models/core/qwenvl/requirements.txt b/examples/models/core/qwenvl/requirements.txt new file mode 100644 index 000000000000..9257ddb9545a --- /dev/null +++ b/examples/models/core/qwenvl/requirements.txt @@ -0,0 +1,11 @@ +-c ../../../constraints.txt +tensorrt_llm>=0.0.0.dev0 +datasets==3.1.0 +evaluate +rouge_score +transformers-stream-generator +sentencepiece>=0.1.99 +tiktoken +einops +matplotlib +torchvision diff --git a/examples/models/core/qwenvl/run.py b/examples/models/core/qwenvl/run.py new file mode 100644 index 000000000000..c996b20fd643 --- /dev/null +++ b/examples/models/core/qwenvl/run.py @@ -0,0 +1,549 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import os +from typing import List, Tuple + +import tensorrt as trt +import torch +from transformers import AutoConfig, AutoTokenizer +from vit_onnx_trt import Preprocss + +import tensorrt_llm +import tensorrt_llm.profiler as profiler +from tensorrt_llm import logger +from tensorrt_llm._deprecation import emit_engine_arch_deprecation +from tensorrt_llm.llmapi.kv_cache_type import KVCacheType +from tensorrt_llm.quantization import QuantMode +from tensorrt_llm.runtime import (ModelConfig, SamplingConfig, Session, + TensorInfo) + + +def get_engine_name(rank): + return "rank{}.engine".format(rank) + + +def trt_dtype_to_torch(dtype): + if dtype == trt.float16: + return torch.float16 + elif dtype == trt.float32: + return torch.float32 + elif dtype == trt.int32: + return torch.int32 + else: + raise TypeError("%s is not supported" % dtype) + + +class QWenInfer(object): + + def __init__( + self, + tokenizer_dir, + qwen_engine_dir, + log_level, + output_csv, + output_npy, + num_beams, + ): + self.tokenizer_dir = tokenizer_dir + self.qwen_engine_dir = qwen_engine_dir + self.log_level = log_level + self.global_max_input_len = 2048 + self.decoder = None + self.tokenizer = None + self.config = None + self.sampling_config = None + self.output_csv = output_csv + self.output_npy = output_npy + self.num_beams = num_beams + self.model_config = None + + def get_model(self): + # --load the tokenizer and engine # + tokenizer = AutoTokenizer.from_pretrained( + self.tokenizer_dir, + legacy=False, + trust_remote_code=True, + ) + config_path = os.path.join(self.qwen_engine_dir, "config.json") + with open(config_path, "r") as f: + config = json.load(f) + gen_config_path = os.path.join(self.tokenizer_dir, + "generation_config.json") + with open(gen_config_path, "r") as f: + gen_config = json.load(f) + top_k = gen_config["top_k"] + top_p = gen_config["top_p"] + chat_format = gen_config["chat_format"] + if chat_format == "raw": + eos_token_id = gen_config["eos_token_id"] + pad_token_id = gen_config["pad_token_id"] + elif chat_format == "chatml": + pad_token_id = eos_token_id = tokenizer.im_end_id + else: + raise Exception("unknown chat format ", chat_format) + + use_gpt_attention_plugin = config["build_config"]["plugin_config"][ + "gpt_attention_plugin"] + gemm_allreduce_plugin = config["build_config"]["plugin_config"][ + "gemm_allreduce_plugin"] + + remove_input_padding = config["build_config"]["plugin_config"][ + "remove_input_padding"] + dtype = config["pretrained_config"]["dtype"] + tp_size = config["pretrained_config"]["mapping"]["tp_size"] + pp_size = config["pretrained_config"]["mapping"]["pp_size"] + world_size = tp_size * pp_size + assert ( + world_size == tensorrt_llm.mpi_world_size() + ), f"Engine world size ({world_size}) != Runtime world size ({tensorrt_llm.mpi_world_size()})" + num_heads = config["pretrained_config"][ + "num_attention_heads"] // world_size + max_batch_size = config["build_config"]["max_batch_size"] + hidden_size = config["pretrained_config"]["hidden_size"] // world_size + vocab_size = config["pretrained_config"]["vocab_size"] + num_layers = config["pretrained_config"]["num_hidden_layers"] + num_kv_heads = config["pretrained_config"].get("num_key_value_heads", + num_heads) + if "kv_cache_type" in config["build_config"]: + kv_cache_type = KVCacheType(config["build_config"]["kv_cache_type"]) + else: + kv_cache_type = KVCacheType.CONTINUOUS + + tokens_per_block = config["build_config"]["plugin_config"][ + "tokens_per_block"] + max_prompt_embedding_table_size = config["build_config"].get( + "max_prompt_embedding_table_size", 0) + quant_mode = QuantMode.from_quant_algo( + config["pretrained_config"]["quantization"]["quant_algo"], + config["pretrained_config"]["quantization"]["kv_cache_quant_algo"], + ) + if config["pretrained_config"].get("multi_query_mode", False): + tensorrt_llm.logger.warning( + "`multi_query_mode` config is deprecated. Please rebuild the engine." + ) + num_kv_heads = 1 + + runtime_rank = tensorrt_llm.mpi_rank() + runtime_mapping = tensorrt_llm.Mapping(world_size=world_size, + rank=runtime_rank, + tp_size=tp_size, + pp_size=pp_size) + torch.cuda.set_device(runtime_rank % runtime_mapping.gpus_per_node) + + model_config = ModelConfig( + max_batch_size=max_batch_size, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + gpt_attention_plugin=use_gpt_attention_plugin, + gemm_allreduce_plugin=gemm_allreduce_plugin, + kv_cache_type=kv_cache_type, + tokens_per_block=tokens_per_block, + remove_input_padding=remove_input_padding, + dtype=dtype, + quant_mode=quant_mode, + max_prompt_embedding_table_size=max_prompt_embedding_table_size, + max_beam_width=self.num_beams, + ) + sampling_config = SamplingConfig( + end_id=eos_token_id, + pad_id=pad_token_id, + num_beams=self.num_beams, + top_k=top_k, + top_p=top_p, + temperature=1.0, + ) + + engine_name = get_engine_name(runtime_rank) + serialize_path = os.path.join(self.qwen_engine_dir, engine_name) + print(f"Loading engine from {serialize_path}") + return ( + model_config, + sampling_config, + runtime_mapping, + runtime_rank, + serialize_path, + tokenizer, + eos_token_id, + pad_token_id, + ) + + def qwen_model_init(self): + ( + model_config, + sampling_config, + runtime_mapping, + runtime_rank, + serialize_path, + tokenizer, + eos_token_id, + pad_token_id, + ) = self.get_model() + with open(serialize_path, "rb") as f: + engine_buffer = f.read() + self.decoder = tensorrt_llm.runtime.GenerationSession( + model_config, + engine_buffer, + runtime_mapping, + ) + self.tokenizer = tokenizer + self.sampling_config = sampling_config + self.model_config = model_config + self.config, _ = AutoConfig.from_pretrained( + self.tokenizer_dir, + return_unused_kwargs=True, + trust_remote_code=True, + ) + + def ptuning_setup(self, prompt_table, dtype, hidden_size, tasks, input_ids): + if prompt_table is not None: + task_vocab_size = torch.tensor([prompt_table.shape[1]], + dtype=torch.int32, + device="cuda") + prompt_table = prompt_table.view( + (prompt_table.shape[0] * prompt_table.shape[1], + prompt_table.shape[2])) + prompt_table = prompt_table.cuda().to( + dtype=tensorrt_llm._utils.str_dtype_to_torch(dtype)) + else: + prompt_table = torch.empty([1, hidden_size]).cuda() + task_vocab_size = torch.zeros([1]).cuda() + + if tasks is not None: + tasks = torch.tensor([int(t) for t in tasks.split(",")], + dtype=torch.int32, + device="cuda") + assert (tasks.shape[0] == input_ids.shape[0] + ), "Number of supplied tasks must match input batch size" + else: + tasks = torch.zeros([input_ids.size(0)], dtype=torch.int32).cuda() + + return [prompt_table, tasks, task_vocab_size] + + def make_context( + self, + query: str, + history: List[Tuple[str, str]] = None, + system: str = "You are a helpful assistant.", + max_window_size: int = 6144, + ): + if history is None: + history = [] + + im_start, im_end = "<|im_start|>", "<|im_end|>" + im_start_tokens = [self.tokenizer.im_start_id] # 151644 + im_end_tokens = [self.tokenizer.im_end_id] # [151645] + nl_tokens = self.tokenizer.encode("\n") + + def _tokenize_str(role, content): + return f"{role}\n{content}", self.tokenizer.encode( + role, allowed_special=set(self.tokenizer.IMAGE_ST) + ) + nl_tokens + self.tokenizer.encode( + content, allowed_special=set(self.tokenizer.IMAGE_ST)) + + system_text, system_tokens_part = _tokenize_str("system", system) + system_tokens = im_start_tokens + system_tokens_part + im_end_tokens + + raw_text = "" + context_tokens = [] + + for turn_query, turn_response in reversed(history): + query_text, query_tokens_part = _tokenize_str("user", turn_query) + query_tokens = im_start_tokens + query_tokens_part + im_end_tokens + if turn_response is not None: + response_text, response_tokens_part = _tokenize_str( + "assistant", turn_response) + response_tokens = im_start_tokens + response_tokens_part + im_end_tokens + + next_context_tokens = (nl_tokens + query_tokens + nl_tokens + + response_tokens) + prev_chat = f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}" + else: + next_context_tokens = nl_tokens + query_tokens + nl_tokens + prev_chat = f"\n{im_start}{query_text}{im_end}\n" + + current_context_size = (len(system_tokens) + + len(next_context_tokens) + + len(context_tokens)) + if current_context_size < max_window_size: + context_tokens = next_context_tokens + context_tokens + raw_text = prev_chat + raw_text + else: + break + + context_tokens = system_tokens + context_tokens + raw_text = f"{im_start}{system_text}{im_end}" + raw_text + context_tokens += (nl_tokens + im_start_tokens + + _tokenize_str("user", query)[1] + im_end_tokens + + nl_tokens + im_start_tokens + + self.tokenizer.encode("assistant") + nl_tokens) + raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n" + + return raw_text, context_tokens + + def generate_for_qwenvl( + self, + input_tokens, + max_new_tokens: int, + prompt_table=None, + tasks=None, + task_vocab_size=None, + num_beams=1, + ): + input_ids = None + input_lengths = None + input_ids = torch.as_tensor(input_tokens, + device="cuda", + dtype=torch.int32) + input_lengths = torch.tensor([input_ids.size(1)], + device="cuda", + dtype=torch.int32) + max_input_length = torch.max(input_lengths).item() + max_new_tokens = min(max_new_tokens, + self.global_max_input_len - max_input_length) + + profiler.start("QWen") + run_time = 10 + for _ in range(run_time): + self.decoder.setup( + batch_size=input_lengths.size(0), + max_context_length=max_input_length, + max_new_tokens=max_new_tokens, + beam_width=num_beams, + ) + output_ids = self.decoder.decode( + input_ids, + input_lengths, + self.sampling_config, + prompt_table, + tasks, + task_vocab_size, + ) + torch.cuda.synchronize() + profiler.stop("QWen") + Qwen_time = profiler.elapsed_time_in_sec("QWen") / run_time + + return output_ids, Qwen_time + + def qwen_infer( + self, + input_vit, + images_path, + input_text, + max_new_tokens, + num_beams=1, + history=None, + ): + if images_path is None: + content_list = [] + else: + content_list = images_path + if history is None: + history = [] + content_list.append({"text": input_text}) + query = self.tokenizer.from_list_format(content_list) + raw_text, context_tokens = self.make_context(query, history=history) + # context_tokens = self.tokenizer.encode(query) + input_ids = torch.tensor([context_tokens]).to("cuda") + bos_pos = torch.where(input_ids == self.config.visual["image_start_id"]) + eos_pos = torch.where( + input_ids == self.config.visual["image_start_id"] + 1) + assert (bos_pos[0] == eos_pos[0]).all() + img_pos = torch.stack((bos_pos[0], bos_pos[1], eos_pos[1]), dim=1) + vocab_size = self.config.vocab_size + fake_prompt_id = torch.arange( + vocab_size, + vocab_size + input_vit.shape[0] * input_vit.shape[1], + device="cuda", + ) + fake_prompt_id = fake_prompt_id.reshape(input_vit.shape[0], + input_vit.shape[1]) + for idx, (i, a, b) in enumerate(img_pos): + input_ids[i][a + 1:b] = fake_prompt_id[idx] + input_ids = input_ids.contiguous().to(torch.int32).cuda() + input_lengths = torch.tensor(input_ids.size(1), + dtype=torch.int32).cuda() + dtype = self.model_config.dtype + prompt_table, tasks, task_vocab_size = self.ptuning_setup( + input_vit, dtype, self.config.hidden_size, None, input_ids) + + output_ids, Qwen_time = self.generate_for_qwenvl( + input_ids, max_new_tokens, prompt_table, tasks, task_vocab_size, + num_beams) + + runtime_rank = tensorrt_llm.mpi_rank() + input_lengths = torch.tensor([input_ids.size(1)], + device="cuda", + dtype=torch.int32) + effective_output_token = 0 + if runtime_rank == 0: + if self.output_csv is None and self.output_npy is None: + for b in range(input_lengths.size(0)): + inputs = input_ids[b] + if content_list is not None: + print(f'Input: "{content_list}"') + print("\n") + if self.num_beams <= 1: + outputs = output_ids[b][0, len(inputs):].tolist() + try: + effective_output_token = (effective_output_token + + outputs.index(151643)) + except: + effective_output_token = 1 + output_text = self.tokenizer.decode( + outputs, skip_special_tokens=True) + print(f'Output: "{output_text}"') + print("\n") + else: + for beam in range(self.num_beams): + outputs = output_ids[b][beam, len(inputs):].tolist() + output_text = self.tokenizer.decode( + outputs, skip_special_tokens=True) + print(f'Output(beam: {beam}): "{output_text}"') + logger.info(f"Input length={input_lengths[b]}") + logger.info(f"Output length={output_ids.shape}") + logger.info(f"TensorRT LLM QWen time: {Qwen_time:3f} sec ") + history.append((query, output_text)) + return output_text + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--max_new_tokens", type=int, default=200) + parser.add_argument("--log_level", type=str, default="info") + parser.add_argument( + "--vit_engine_path", + type=str, + default="plan/visual_encoder/visual_encoder_fp16.plan", + ) + parser.add_argument( + "--qwen_engine_dir", + type=str, + default="qwen_outputs", + ) + parser.add_argument( + "--tokenizer_dir", + type=str, + default=".", + help="Directory containing the tokenizer.model.", + ) + parser.add_argument("--input_text", + type=str, + default="Describe the picture") + parser.add_argument( + "--images_path", + nargs="+", + type=json.loads, + default=[{ + "image": "./pics/demo.jpeg" + }], + ) + parser.add_argument( + "--input_tokens", + dest="input_file", + type=str, + help= + "CSV or Numpy file containing tokenized input. Alternative to text input.", + default=None, + ) + parser.add_argument( + "--output_csv", + type=str, + help="CSV file where the tokenized output is stored.", + default=None, + ) + parser.add_argument( + "--output_npy", + type=str, + help="Numpy file where the tokenized output is stored.", + default=None, + ) + parser.add_argument("--num_beams", + type=int, + help="Use beam search if num_beams >1", + default=1) + parser.add_argument("--display", default=False, action='store_true') + parser.add_argument('--port', type=str, default='8006') + parser.add_argument("--local_machine", default=False, action='store_true') + + return parser.parse_args() + + +def vit_process(image_path, vit_engine_path, stream): + img_processor = Preprocss(448) + logger.info(f"Loading engine from {vit_engine_path}") + with open(vit_engine_path, "rb") as f: + engine_buffer = f.read() + logger.info(f"Creating session from engine {vit_engine_path}") + session_vit = Session.from_serialized_engine(engine_buffer) + device = torch.device("cuda") if torch.cuda.is_available() else "cpu" + image_path_list = [] + for item in image_path: + image_path_list.append(next(iter(item.values()))) + images = img_processor.encode(image_path_list).to(device) + batch_size = images.size(0) + images = images.expand(batch_size, -1, -1, -1).contiguous() + visual_inputs = {"input": images.float()} + visual_output_info = session_vit.infer_shapes( + [TensorInfo("input", trt.DataType.FLOAT, images.shape)]) + visual_outputs = { + t.name: + torch.empty(tuple(t.shape), + dtype=trt_dtype_to_torch(t.dtype), + device="cuda") + for t in visual_output_info + } + profiler.start("ViT") + + run_time = 10 + for _ in range(run_time): + ok = session_vit.run(visual_inputs, visual_outputs, stream) + profiler.stop("ViT") + Vit_time = profiler.elapsed_time_in_sec("ViT") / run_time + logger.info(f"TensorRT LLM ViT latency: {Vit_time:3f} sec ") + + assert ok, "Runtime execution failed for vit session" + + image_embeds = visual_outputs["output"] + return image_embeds + + +if __name__ == "__main__": + emit_engine_arch_deprecation("run.py") + args = parse_arguments() + stream = torch.cuda.current_stream().cuda_stream + tensorrt_llm.logger.set_level(args.log_level) + image_embeds = vit_process(args.images_path, args.vit_engine_path, stream) + qinfer = QWenInfer( + args.tokenizer_dir, + args.qwen_engine_dir, + args.log_level, + args.output_csv, + args.output_npy, + args.num_beams, + ) + qinfer.qwen_model_init() + qinfer.qwen_infer( + image_embeds, + args.images_path, + args.input_text, + args.max_new_tokens, + args.num_beams, + history=[], + ) diff --git a/examples/models/core/qwenvl/run_chat.py b/examples/models/core/qwenvl/run_chat.py new file mode 100644 index 000000000000..1f1ba6fb6faf --- /dev/null +++ b/examples/models/core/qwenvl/run_chat.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +# isort: off +import torch +from run import QWenInfer, parse_arguments, vit_process +# isort: on + + +def make_display(port=8006): + import cv2 + import zmq + context = zmq.Context() + socket = context.socket(zmq.REP) + socket.bind(f"tcp://*:{port}") + + def func(image): + data = cv2.imencode(".jpg", image)[1].tobytes() + socket.recv() + socket.send(data) + + return func + + +def show_pic(image_path, port): + import cv2 + image = cv2.imread(image_path) + display_obj = make_display(port) + display_obj(image) + + +def show_pic_local(image_path): + import cv2 + import matplotlib.pyplot as plt + image = cv2.imread(image_path) + image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + + plt.imshow(image_rgb) + plt.pause(0.1) + + +def cooridinate_extract_show(input, history, tokenizer, local_machine, port): + pattern = r"\((\d+),(\d+)\)" + coordinates = re.findall(pattern, input) + result = "Box({},{})".format(coordinates[0][0], + coordinates[0][1]) + result += ",({},{})".format(coordinates[1][0], coordinates[1][1]) + + image = tokenizer.draw_bbox_on_latest_picture(result, history) + if image: + image.save('1.png') + if local_machine: + show_pic_local('1.png') + else: + show_pic('1.png', port) + else: + print("======No bounding boxes are detected!") + + +def exist_cooridinate(input): + pattern = r"\((\d+),(\d+)\)" + match = re.search(pattern, input) + if match: + return True + else: + return False + + +if __name__ == '__main__': + args = parse_arguments() + stream = torch.cuda.current_stream().cuda_stream + image_embeds = vit_process(args.images_path, args.vit_engine_path, stream) + qinfer = QWenInfer(args.tokenizer_dir, args.qwen_engine_dir, args.log_level, + args.output_csv, args.output_npy, args.num_beams) + qinfer.qwen_model_init() + + run_i = 0 + history = [] + if args.display: + if args.local_machine: + show_pic_local("./pics/demo.jpeg") + else: + show_pic("./pics/demo.jpeg", args.port) + + while True: + input_text = None + try: + input_text = input("Text (or 'q' to quit): ") + except: + continue + + if input_text == "clear history": + history = [] + continue + + if input_text.lower() == 'q': + break + print('\n') + + content_list = args.images_path + content_list.append({'text': input_text}) + + if run_i == 0: + query = qinfer.tokenizer.from_list_format(content_list) + else: + query = input_text + + run_i = run_i + 1 + output_text = qinfer.qwen_infer(image_embeds, None, query, + args.max_new_tokens, args.num_beams, + history) + if args.display: + if exist_cooridinate(output_text): + cooridinate_extract_show(output_text, history, qinfer.tokenizer, + args.local_machine, args.port) diff --git a/examples/models/core/qwenvl/show_pic.py b/examples/models/core/qwenvl/show_pic.py new file mode 100644 index 000000000000..f390ec860f58 --- /dev/null +++ b/examples/models/core/qwenvl/show_pic.py @@ -0,0 +1,34 @@ +import argparse + +import cv2 +import numpy as np +import zmq + +context = zmq.Context() +socket = context.socket(zmq.REQ) + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument('--ip', type=str, default='127.0.0.1') + parser.add_argument('--port', type=str, default='8006') + return parser.parse_args() + + +args = parse_arguments() +ip_addr = "tcp://" + args.ip + ":" + args.port +socket.connect(ip_addr) + +while True: + socket.send(b"a") + message = socket.recv() + if len(message) == 1 and message == b'x': + break + image = np.frombuffer(message, dtype=np.uint8) + image = cv2.imdecode(image, 1) + image = cv2.resize(image, dsize=(512, 384)) + cv2.imshow("image", image) + key = cv2.waitKey(1) & 0xFF + + if key == ord('q'): + break diff --git a/examples/models/core/qwenvl/vit_onnx_trt.py b/examples/models/core/qwenvl/vit_onnx_trt.py new file mode 100644 index 000000000000..ba21fc93ef0a --- /dev/null +++ b/examples/models/core/qwenvl/vit_onnx_trt.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +import time +from typing import List + +import requests +import tensorrt as trt +import torch +from PIL import Image +from torchvision import transforms +from torchvision.transforms import InterpolationMode +from transformers import AutoModelForCausalLM + +from tensorrt_llm._utils import release_gc, str_dtype_to_torch + + +class Preprocss: + + def __init__(self, image_size: int): + mean = (0.48145466, 0.4578275, 0.40821073) + std = (0.26862954, 0.26130258, 0.27577711) + self.image_transform = transforms.Compose([ + transforms.Resize((image_size, image_size), + interpolation=InterpolationMode.BICUBIC), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std), + ]) + + def encode(self, image_paths: List[str]): + images = [] + for image_path in image_paths: + if image_path.startswith("http://") or image_path.startswith( + "https://"): + image = Image.open(requests.get(image_path, stream=True).raw) + else: + image = Image.open(image_path) + image = image.convert("RGB") + images.append(self.image_transform(image)) + images = torch.stack(images, dim=0) + return images + + +class ONNX_TRT: + + def __init__(self, image_size): + self.image_size = image_size + + def export_onnx(self, onnx_file_path, pretrained_model_path, image_url): + print("Start converting ONNX model!") + image_pre_obj = Preprocss(self.image_size) + torch_dtype = str_dtype_to_torch("float16") + model = AutoModelForCausalLM.from_pretrained( + pretrained_model_path, + device_map="cuda", + dtype=torch_dtype, + fp16=True, + trust_remote_code=True, + ).eval() + device = torch.device("cuda") if torch.cuda.is_available() else "cpu" + image = image_pre_obj.encode(image_url).to(device) + if not os.path.exists("image.pt"): + torch.save(image, "image.pt") + + model_visual = model.transformer.visual + model_visual.eval() + del model # To save GPU memory + + torch.onnx.export( + model_visual, + image.to("cuda"), + onnx_file_path, + opset_version=17, + input_names=["input"], + output_names=["output"], + dynamic_axes={"input": { + 0: "batch" + }}, + # Required for pytorch>=2.9.0 as dynamo becomes the default and introduces bugs as it does not support opset_version=17 natively + dynamo=False) + release_gc() # Further release memory + print( + f"Export to ONNX file successfully! The ONNX file stays in {onnx_file_path}" + ) + + def generate_trt_engine(self, + onnxFile, + planFile, + minBS=1, + optBS=2, + maxBS=4): + print("Start converting TRT engine!") + logger = trt.Logger(trt.Logger.VERBOSE) + builder = trt.Builder(logger) + network = builder.create_network( + 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + profile = builder.create_optimization_profile() + config = builder.create_builder_config() + config.set_flag(trt.BuilderFlag.FP16) + parser = trt.OnnxParser(network, logger) + + with open(onnxFile, "rb") as model: + if not parser.parse(model.read(), "/".join(onnxFile.split("/"))): + print("Failed parsing %s" % onnxFile) + for error in range(parser.num_errors): + print(parser.get_error(error)) + print("Succeeded parsing %s" % onnxFile) + + nBS = -1 + nMinBS = minBS + nOptBS = optBS + nMaxBS = maxBS + inputT = network.get_input(0) + inputT.shape = [nBS, 3, self.image_size, self.image_size] + profile.set_shape( + inputT.name, + [nMinBS, 3, self.image_size, self.image_size], + [nOptBS, 3, self.image_size, self.image_size], + [nMaxBS, 3, self.image_size, self.image_size], + ) + + config.add_optimization_profile(profile) + + t0 = time.time() + engineString = builder.build_serialized_network(network, config) + t1 = time.time() + if engineString is None: + print("Failed building %s" % planFile) + else: + print("Succeeded building %s in %d s" % (planFile, t1 - t0)) + with open(planFile, "wb") as f: + f.write(engineString) + + +def parse_arguments(): + parser = argparse.ArgumentParser() + # onnx/visual_encoder + parser.add_argument("--onnxFile", + type=str, + default="visual_encoder/visual_encoder.onnx", + help="") + parser.add_argument("--pretrained_model_path", + type=str, + default="Qwen-VL-Chat", + help="") + parser.add_argument( + "--planFile", + type=str, + default="plan/visual_encoder/visual_encoder_fp16.plan", + help="", + ) + parser.add_argument( + "--only_trt", + action="store_true", + help="Run only convert the onnx to TRT engine.", + ) + parser.add_argument("--minBS", type=int, default=1) + parser.add_argument("--optBS", type=int, default=1) + parser.add_argument("--maxBS", type=int, default=4) + parser.add_argument("--image_url", nargs="+", default=["./pics/demo.jpeg"]) + args = parser.parse_args() + return args + + +if __name__ == "__main__": + args = parse_arguments() + onnx_file_dir = os.path.dirname(args.onnxFile) + if not onnx_file_dir == "" and not os.path.exists(onnx_file_dir): + os.makedirs(onnx_file_dir) + plan_file_dir = os.path.dirname(args.planFile) + if not os.path.exists(plan_file_dir): + os.makedirs(plan_file_dir) + + onnx_trt_obj = ONNX_TRT(448) # or ONNX_TRT(config.visual['image_size']) + + if args.only_trt: + onnx_trt_obj.generate_trt_engine(args.onnxFile, args.planFile, + args.minBS, args.optBS, args.maxBS) + else: + onnx_trt_obj.export_onnx(args.onnxFile, args.pretrained_model_path, + args.image_url) + onnx_trt_obj.generate_trt_engine(args.onnxFile, args.planFile, + args.minBS, args.optBS, args.maxBS) diff --git a/examples/ngram/README.md b/examples/ngram/README.md index 4b5a00ee43ac..1e4dc8792b08 100644 --- a/examples/ngram/README.md +++ b/examples/ngram/README.md @@ -1,32 +1,39 @@ # NGram Speculative Decoding -This document shows how to run a model with NGram speculative decoding -(supported as `ASSISTED_GENERATION` in transformers and vLLM, source: -[GitHub](https://github.com/apoorvumang/prompt-lookup-decoding/tree/main)) -in TensorRT LLM. +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +This document shows how to build and run a model using NGram speculative decoding (supported as `ASSISTED_GENERATION` in transformers and vLLM, source: [GitHub](https://github.com/apoorvumang/prompt-lookup-decoding/tree/main)) in TensorRT LLM on single GPU, or single node multiple GPU. ## Overview -NGram builds a pattern pool from the prompt and previously generated tokens -and proposes draft tokens by matching the tail of the current sequence -against that pool. It has 2 hyperparameters that control the process of -generation: - -- `max_draft_len`: the maximum number of tokens provided as draft tokens in - one iteration, which is usually from 4 to 10 in common usage (default - value: 4). Empirically, the larger the value is, the higher acceptance rate - but higher overhead is expected at the same time, so the right balance - based on the models and application scenarios needs to be found. -- `max_matching_ngram_size`: the maximum number of tokens extracted from the - tail of the input prompt or generated output as a pattern, which is used to - search corresponding draft tokens (default value: 2). Empirically, the - larger the value is, the more precise context can be matched from the - existed sequence, indicating higher acceptance rate, but the higher - probability of miss-match and higher overhead appear, which fall back to - normal generation (one token per iteration). +We provide two styles of workflow to run NGram (named V1 and V2 respectively) now. V1 is in TRT workflow and similar to the Draft-Target-Model workflow, running in orchestrator mode and calling `runner.generate()` multiple times to get outputs, which is more flexible for customizing but slightly more overhead. V2 is in pytorch workflow and similar to the Look-Ahead workflow, running in leader mode and calling `runner.generate()` only one time to get outputs, which provides higher performance but fixed process. -## Support Matrix +The NGram has 3 additional hyperparameters that you need to specify to control the process of generation: +- `max_draft_len`: the maximum number of tokens provided as draft tokens in one iteration, which is usually from 4 to 10 in common usage (default value: 4). Empirically, the larger the value is, the higher acceptance rate but higher overhead is expected at the same time, so the right balance based on the models and application scenarios needs to be found. +- `max_matching_ngram_size`: the maximum number of tokens extracted from the tail of the input prompt or generated output as a pattern, which is used to search corresponding draft tokens (default value: 2). Empirically, the larger the value is, the more precise context can be matched from the existed sequence, indicating higher acceptance rate, but the higher probability of miss-match and higher overhead appear, which fall back to normal generation (one token per iteration). +- `device_list`: the index list of device(s) to run the model in V1 workflow. The length of it must be the same as the TP size of the draft model engine. For instances, `device_list=[0]` means using tp_size=1 and GPU 0 for the model, `device_list=[4,5,6,7]` means using tp=4 and GPU from 4 to 7 for the model. This parameter is neddless in V2 workflow. + ++ For example, the process of getting draft tokens using `max_draft_len=2` and `max_matching_ngram_size=4` with a sentence `prefix=[..., t1, t2, t3, t4]` is like below: + +```Python +pattern = prefix[:-2] # pattern=[t3, t4] (length=2) +if pattern in pool and len(pool[pattern]) == 4: # assuming it is {(t3, t4): (t5, t6, t7, t8)} + return pool[pattern] # draft token = [t5, t6, t7, t8] +elif pattern in pool and len(pool[pattern]) == <4: # assuming it is {(t3, t4): (t9, t10, t11)} + return pool[pattern] # draft token = [t9, t10, t11] +pattern = prefix[:-1] # Try shorter pattern if no candidate of length=2 exists, pattern=[t4] (length=1) +if pattern in pool and len(pool[pattern]) == 4: # The same process as above + return pool[pattern] +elif pattern in pool and len(pool[pattern]) == <4: + return pool[pattern] +return None # No any candidate exists +``` +## Support Matrix * GPU Compute Capability >= 8.0 (Ampere or newer) * FP16 / BF16 / FP8 * Paged KV Cache @@ -34,6 +41,59 @@ generation: ## Usage +### V1 workflow + ++ We use an open-source `llama-v2-13B` models in this example. ++ `--use_paged_context_fmha=enable` must be specified since we need KVcache reuse in this approach. ++ `--speculative_decoding_mode=draft_tokens_external` must be specified. ++ `--max_draft_len` must be specified as the length maximum of the draft tokens. ++ `--ngram_config` is corresponding configuration of NGram, we can see its usage in [util.py](../util.py). + + As an example, `[10,2,[0]]` means `max_draft_len=10`, `max_matching_ngram_size=2`, and device of target model is `GPU0`. ++ `--kv_cache_enable_block_reuse` must be specified for this approach. ++ Only CPP session is supported, so `--use_py_session` must not be specified. ++ `--num_beams` can not be specified as larger than 1 since beam search is not supported in this approach yet. + +```bash +# Build engine +python3 examples/models/core/llama/convert_checkpoint.py \ + --model_dir \ + --output_dir ./ckpt-target \ + --dtype float16 + +trtllm-build \ + --checkpoint_dir ./ckpt-target \ + --output_dir ./target-engine \ + --gemm_plugin float16 \ + --use_paged_context_fmha enable \ + --speculative_decoding_mode draft_tokens_external \ + --max_draft_len 10 \ + --max_batch_size 4 \ + --max_input_len 3200 \ + --max_seq_len 4800 + +# Run decoding +python3 examples/run.py \ + --tokenizer_dir \ + --engine_dir ./target-engine \ + --ngram_config "[10,2,[0]]" \ + --max_output_len 256 \ + --kv_cache_enable_block_reuse \ + --input_text "How does Draft-Sampling work?" + +# Run summarization tasks +python examples/summarize.py \ + --test_hf \ + --test_trt_llm \ + --check_accuracy \ + --hf_model_dir \ + --engine_dir ./target-engine \ + --batch_size 1 \ + --ngram_config "[10,2,[0]]" \ + --kv_cache_enable_block_reuse +``` + +### V2 workflow + ```bash python3 examples/llm-api/quickstart_advanced.py \ --spec_decode_max_draft_len 4 \ @@ -41,8 +101,3 @@ python3 examples/llm-api/quickstart_advanced.py \ --disable_overlap_scheduler \ --disable_kv_cache_reuse ``` - -With the LLM API, configure NGram through `NGramDecodingConfig` -(`speculative_config`). See the -[speculative decoding documentation](https://nvidia.github.io/TensorRT-LLM/features/speculative-decoding.html) -for details. diff --git a/examples/ngram/run_dtm_ngram.py b/examples/ngram/run_dtm_ngram.py new file mode 100644 index 000000000000..d0cd8687ef86 --- /dev/null +++ b/examples/ngram/run_dtm_ngram.py @@ -0,0 +1,382 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast + +import numpy as np +import torch +from ordered_set import OrderedSet + +from tensorrt_llm.logger import logger +from tensorrt_llm.runtime import ModelRunnerCpp + + +class NgramPool: # Ngrams pool for Ngram + + def __init__( + self, + input_batch_size: int, + max_draft_len: int, + max_matching_ngram_size: int, + end_id: int, + max_seq_len: list[int], + is_keep_all: bool = True, + is_use_oldest: bool = True, + ): + self.input_batch_size = input_batch_size + self.max_draft_len = max_draft_len + self.max_matching_ngram_size = max_matching_ngram_size + self.end_id = end_id + self.max_seq_len = max_seq_len + self.is_keep_all = is_keep_all + self.is_use_oldest = is_use_oldest + self.pool = [{} for _ in range(input_batch_size)] + self.start_index = [0 for _ in range(input_batch_size)] + + assert self.max_draft_len > 0, f"max_draft_len must be greater than 0, but got {self.max_draft_len}" + assert self.max_matching_ngram_size > 0, f"max_matching_ngram_size must be greater than 0, but got {self.max_matching_ngram_size}" + + def print_pool(self): + """ + For debug + """ + logger.info(f"Batch size = {self.input_batch_size}") + for i, map in enumerate(self.pool): + logger.info(f"Slot {i}, size = {len(map)}") + for key, values in map.items(): + logger.info(f" {key}->{values}") + + def get_draft_tokens(self, prefix: list[torch.Tensor], + batch_slot: list[int]): + """ + Get draft tokens from a batch of requests + modified from `transformers/generation/candidate_generator.py` + """ + batch_size = len(prefix) + prefix_len = [len(prefix[bi]) for bi in range(batch_size)] + draft_tokens = [] # `logits` is useless yet + for bi in range(batch_size): + gbi = batch_slot[bi] # Global index in the input batch + chosen_ids = [self.end_id] + # Skip search if prefix is length of `max_length - 1` + if prefix_len[bi] >= self.max_seq_len[gbi] - 1: + draft_tokens.append(chosen_ids) + continue + + # Update pool + sequence = prefix[bi][self.start_index[gbi]:].tolist() + for size in range( + min(self.max_matching_ngram_size, prefix_len[bi] - 1), 0, + -1): + # Find each possible key-value combination, and use tuple for hash + for l in range(len(sequence) - size): + r = min(l + size + self.max_draft_len, len(sequence)) + key = tuple(sequence[l:l + size]) + value = tuple(sequence[l + size:r]) + if key not in self.pool[gbi] or not self.is_keep_all or \ + len(self.pool[gbi][key][0]) < self.max_draft_len: + # Update the value if + # 1. the key does not exist + # 2. we only keep the newest one value for each key (MRU) + # 3. the length of the value saved before is less than `max_draft_len` + self.pool[gbi][key] = OrderedSet((value, )) + elif value not in self.pool[gbi][key]: + # Extend the value if the key is already existed but count of values is not enough + self.pool[gbi][key].add(value) + + # Find match + for size in range( + min(self.max_matching_ngram_size, prefix_len[bi] - 1), 0, + -1): + pattern = tuple(prefix[bi][-size:].tolist()) + if pattern not in self.pool[gbi]: + continue + if self.is_use_oldest: + # Always choose the oldest match, aligned with HF + chosen_ids = self.pool[gbi][pattern][0] + else: + # Always choose the newest match + chosen_ids = self.pool[gbi][pattern][-1] + break + draft_tokens.append(chosen_ids) + self.start_index[gbi] = max( + 0, prefix_len[bi] - + (self.max_draft_len + self.max_matching_ngram_size - 1)) + + return draft_tokens, None + + +def run_dtm_ngram(batch_input_ids, + args, + runtime_rank, + end_id, + pad_id, + stop_words_list, + bad_words_list, + vocab_size, + *, + target_runner=None): + # `dtm` for Draft-Target-Model, `ngram` for NGram + is_dtm = (args.draft_target_model_config is not None) + is_ngram = (args.ngram_config is not None) + assert is_dtm ^ is_ngram, "`--draft_target_model_config` and `--ngram_config` can not be specified at the same time." + if is_dtm: + assert args.draft_engine_dir is not None, "`--draft_engine_dir` must be specified in Draft-Target-Model." + draft_len, draft_device_list, target_device_list, use_logits = ast.literal_eval( + args.draft_target_model_config) + logger.info(f"Using Draft-Target-Model speculative decoding") + logger.info(f"draft_len: {draft_len}") + logger.info(f"Device(s) for draft model: {draft_device_list}") + logger.info(f"Device(s) for target model: {target_device_list}") + logger.info(f"Use logits to accept tokens: {use_logits}") + if is_ngram: + logger.info(f"Using NGram speculative decoding V1 workflow") + max_draft_len, max_matching_ngram_size, target_device_list = ast.literal_eval( + args.ngram_config) + logger.info(f"max_draft_len: {max_draft_len}") + logger.info(f"max_matching_ngram_size: {max_matching_ngram_size}") + logger.info(f"Device(s) for the model: {target_device_list}") + use_logits = False # `logits` is useless in this approach yet + + # Variables keeping constant during decoding + input_batch_size = len(batch_input_ids) # Note as `BS` + beam_width = args.num_beams # Note as `BW` + is_compute_acceptance_ratio = logger.level == 'verbose' # Only for verbose + input_len = [len(p) for p in batch_input_ids] + max_seq_len = [i + args.max_output_len for i in input_len] + # Variables changing during decoding + n_iteration = 0 + prefix = batch_input_ids # Input for each iteration + batch_slot = list(range(input_batch_size)) # Index of requests + if is_compute_acceptance_ratio: + n_draft_token = [0 for _ in range(input_batch_size)] + n_accept_token = [0 for _ in range(input_batch_size)] + + if is_ngram: + ngram_pool = NgramPool(input_batch_size, max_draft_len, + max_matching_ngram_size, end_id, max_seq_len) + + # Repack the output like the output of function `generate` + outputs = {} + outputs["output_ids"] = torch.full( + [input_batch_size, beam_width, + max(max_seq_len)], + end_id, + dtype=torch.int32) + for bi in range(input_batch_size): + outputs["output_ids"][bi, :, :input_len[bi]] = batch_input_ids[bi] + outputs["sequence_lengths"] = torch.full([input_batch_size, beam_width], + 0, + dtype=torch.int32) + outputs["context_logits"] = None + outputs["generation_logits"] = torch.full( + [input_batch_size, beam_width, + max(max_seq_len), vocab_size], + 0, + dtype=torch.float16) + outputs['cum_log_probs'] = None + outputs['log_probs'] = None + + # Model runner + common_runner_kwargs = dict( + lora_dir=args.lora_dir, + rank=runtime_rank, + debug_mode=args.debug_mode, + lora_ckpt_source=args.lora_ckpt_source, + gpu_weights_percent=args.gpu_weights_percent, + max_output_len=args.max_output_len, + is_enc_dec=False, + max_batch_size=input_batch_size, + max_input_len=max(input_len) + args.max_output_len, + max_beam_width=beam_width, + max_attention_window_size=args.max_attention_window_size, + sink_token_length=args.sink_token_length, + max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, + kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, + kv_cache_free_gpu_memory_fraction=args. + kv_cache_free_gpu_memory_fraction, + cross_kv_cache_fraction=None, + enable_chunked_context=args.enable_chunked_context, + multi_block_mode=args.multi_block_mode, + cuda_graph_mode=args.cuda_graph_mode, + enable_context_fmha_fp32_acc=args.enable_context_fmha_fp32_acc, + is_orchestrator_mode=True, + ) + + if is_dtm: + draft_runner_kwargs = common_runner_kwargs.copy() + draft_runner_kwargs.update(engine_dir=args.draft_engine_dir, + device_ids=draft_device_list) + draft_runner = ModelRunnerCpp.from_dir(**draft_runner_kwargs) + + if target_runner is None: # Skip this constructor if we have prepared the runner before + target_runner_kwargs = common_runner_kwargs.copy() + target_runner_kwargs.update(engine_dir=args.engine_dir, + device_ids=target_device_list) + target_runner = ModelRunnerCpp.from_dir(**target_runner_kwargs) + + if is_dtm and use_logits: + assert draft_runner.gather_generation_logits and target_runner.gather_generation_logits, "`--gather_generation_logits` must be specified while building draft/target models for using logits to accept" + + common_generaion_kwargs = dict( + max_attention_window_size=args.max_attention_window_size, + sink_token_length=args.sink_token_length, + end_id=end_id, + pad_id=pad_id, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + num_beams=beam_width, + num_return_sequences=args.num_return_sequences, + length_penalty=args.length_penalty, + early_stopping=args.early_stopping, + beam_width_array=None, + repetition_penalty=args.repetition_penalty, + presence_penalty=args.presence_penalty, + frequency_penalty=args.frequency_penalty, + min_p=args.min_p, + stop_words_list=stop_words_list, + bad_words_list=bad_words_list, + random_seed=args.random_seed, + lora_uids=args.lora_task_uids, + prompt_table=args.prompt_table_path, + prompt_tasks=args.prompt_tasks, + streaming=False, + output_sequence_lengths=True, + no_repeat_ngram_size=args.no_repeat_ngram_size, + return_dict=True, + return_all_generated_tokens=args.return_all_generated_tokens, + ) + + while True: + n_iteration += 1 + # Dynamic batch_size, decreases if some requests finish + batch_size = len(prefix) + prefix_len = [len(prefix[i]) for i in range(batch_size)] + # Get draft tokens + # `d_*` means variables from draft + # `d_seq_len` includes input part, but `d_len` doesn't + if is_dtm: + draft_generation_kwargs = common_generaion_kwargs.copy() + draft_generation_kwargs.update( + batch_input_ids=prefix, + max_new_tokens=draft_len, + streaming=False, + output_sequence_lengths=True, + return_dict=True, + ) + draft = draft_runner.generate(**draft_generation_kwargs) + torch.cuda.synchronize() + + # draft["output_ids"].shape -> [BS, BW, maxSL] + # draft["sequence_lengths"].shape -> [BS, BW] + # draft["generation_logits"].shape -> [BS, BW, draft_len, vocab_size] + d_ids = [[end_id]] * batch_size + d_logits = [None] * batch_size if use_logits else None + d_seq_len = draft["sequence_lengths"][:, 0].tolist() + d_len = [d_seq_len[bi] - prefix_len[bi] for bi in range(batch_size)] + for bi in range(batch_size): + l, r = prefix_len[bi], d_seq_len[bi] + if l >= r: # No useful draft tokens + continue + d_ids[bi] = draft["output_ids"][bi, 0, l:r].tolist() + if use_logits: + d_logits[bi] = draft["generation_logits"][bi, 0, + -d_len[bi]:, :] + if is_ngram: + d_ids, d_logits = ngram_pool.get_draft_tokens(prefix, batch_slot) + d_len = [len(i) for i in d_ids] + + # Run target model + # `t_*` means variables from target model + # `t_seq_len` and `t_seq_ids` include input part, but `t_len` or `t_ids` don't + target_generation_kwargs = common_generaion_kwargs.copy() + target_generation_kwargs.update(batch_input_ids=prefix, + draft_tokens_list=d_ids, + draft_logits_list=d_logits) + if is_dtm: + max_new_tokens = draft_len + 1 + if is_ngram: + max_new_tokens = max_draft_len + 1 + target_generation_kwargs.update(max_new_tokens=max_new_tokens) + target = target_runner.generate(**target_generation_kwargs) + torch.cuda.synchronize() + + t_ids = [None] * batch_size + t_seq_ids = [None] * batch_size + t_seq_len = target["sequence_lengths"][:, 0].tolist() + t_len = [t_seq_len[bi] - prefix_len[bi] for bi in range(batch_size)] + + # Update output and tokens for next iteration + for bi in range(batch_size): + gbi = batch_slot[bi] # Global index in the input batch + l = prefix_len[bi] + r = min(t_seq_len[bi], max_seq_len[gbi]) + t_ids[bi] = target["output_ids"][bi, 0, l:r].tolist() + t_seq_ids[bi] = target["output_ids"][bi, 0, :r] + outputs["output_ids"][gbi, 0, l:r] = torch.IntTensor(t_ids[bi]) + outputs["sequence_lengths"][gbi, 0] = r + if use_logits: + outputs["generation_logits"][gbi, 0, (l - input_len[bi]):(r - input_len[bi])] = \ + target["generation_logits"][bi][0,:(r-l)].detach().cpu() + if is_compute_acceptance_ratio: + n_draft_token[gbi] += d_len[bi] + length = min(d_len[bi], t_len[bi], + max_seq_len[gbi] - prefix_len[bi]) + res = [d_ids[bi][i] == t_ids[bi][i] for i in range(length)] + n_accept_token[gbi] += \ + ((~torch.BoolTensor(res)).cumsum(axis=-1) < 1).sum() + + # Yield output if using streaming + if args.streaming and not n_iteration % args.streaming_interval: + yield outputs + + # Evaluate stop criteria and prepare inputs for next iteration + prefix_next = [] + batch_slot_next = [] + for bi in range(batch_size): + gbi = batch_slot[bi] # Global index in the input batch + # Stop due to output length + if len(t_seq_ids[bi]) >= max_seq_len[gbi]: + continue # No need to update for the stopped requests + # Stop due to the same output. Normally target should return 1 more token. + # if (d_ids is not None and np.array_equal(d_ids[bi], t_ids[bi])): + # continue + # Stop due to no change (hit early stopping) + if np.array_equal(t_seq_ids[bi].cpu().numpy(), + prefix[bi].cpu().numpy()): + continue + # Stop due to end words + if end_id in t_seq_ids[bi][prefix_len[bi]:]: + continue + # TODO: Check bad words and stop words criteria + prefix_next.append(t_seq_ids[bi]) + batch_slot_next.append(gbi) + prefix = prefix_next + batch_slot = batch_slot_next + if len(prefix) == 0: # Leave while loop if no request remained + break + + if is_compute_acceptance_ratio: + logger.debug(f"Count of iteration(s): {n_iteration}") + logger.debug(f"Acceptance ratio:") + for i, (a, d) in enumerate(zip(n_accept_token, n_draft_token)): + logger.debug(f"Request {i}: {a / d * 100 :6.2f}%") + + # Return runner in No-Streaming mode + if args.streaming: + yield outputs + else: + yield outputs, target_runner diff --git a/examples/openai_triton/README.md b/examples/openai_triton/README.md new file mode 100644 index 000000000000..b5f39d105974 --- /dev/null +++ b/examples/openai_triton/README.md @@ -0,0 +1,7 @@ +# Integration for OpenAI Triton + +The typical approach to integrate a kernel into TensorRT LLM is to create TensorRT plugins. +Specially for integrating OpenAI Triton kernels, there are two methods: + +1. Creating TensorRT plugin manually, you can refer to [manual plugin example](./manual_plugin/) for details, +2. Generate the TensorRT plugins automatically, please refer to [automatic plugin example](./plugin_autogen/) for details. diff --git a/examples/openai_triton/manual_plugin/CMakeLists.txt b/examples/openai_triton/manual_plugin/CMakeLists.txt new file mode 100644 index 000000000000..bec14231511e --- /dev/null +++ b/examples/openai_triton/manual_plugin/CMakeLists.txt @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & +# AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +cmake_minimum_required(VERSION 3.1) + +# Enable C++ +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED TRUE) + +# Define project name +set(TARGET_NAME trt_llm_custom_plugins) +project(${TARGET_NAME}) + +set(CMAKE_VERBOSE_MAKEFILE 1) + +# Compile options +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -pthread ") +set(CMAKE_C_FLAGS_DEBUG "-g -O0") +set(CMAKE_C_FLAGS_RELEASE "-O2") +set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -lstdc++") +set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) +set(CMAKE_CXX_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE}) + +set(CMAKE_BUILD_TYPE release) + +find_package(CUDA REQUIRED) +message(STATUS "CUDA library status:") +message(STATUS " config: ${CUDA_DIR}") +message(STATUS " version: ${CUDA_VERSION}") +message(STATUS " libraries: ${CUDA_LIBRARIES}") +message(STATUS " include path: ${CUDA_INCLUDE_DIRS}") + +if(NOT DEFINED TRT_INCLUDE_DIR) + set(TRT_INCLUDE_DIR "/usr/local/tensorrt/include") + if(NOT EXISTS ${TRT_INCLUDE_DIR}) + # In case of TensorRT installed from a deb package. + set(TRT_INCLUDE_DIR "/usr/include/x86_64-linux-gnu") + endif() +endif() +message(STATUS "tensorrt include path: ${TRT_INCLUDE_DIR}") +if(DEFINED TRT_LLM_INCLUDE_DIR) + message( + STATUS "openai_triton/manual_plugin example has been self-contained " + "and TRT_LLM_INCLUDE_DIR is now unnecessary to specify the path of " + "C++ runtime source files.") +endif() + +if(NOT DEFINED TRT_LIB_DIR) + set(TRT_LIB_DIR "/usr/local/tensorrt/lib") + if(NOT EXISTS ${TRT_INCLUDE_DIR}) + # In case of TensorRT installed from a deb package. + set(TRT_LIB_DIR "/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu") + endif() +endif() +find_library( + TRT_LIB_PATH nvinfer + HINTS ${TRT_LIB_DIR} + NO_DEFAULT_PATH) +find_library(TRT_LIB_PATH nvinfer REQUIRED) +message(STATUS "TRT_LIB_DIR: ${TRT_LIB_DIR}") +message(STATUS "Found nvinfer library: ${TRT_LIB_PATH}") + +if(NOT DEFINED TRT_LLM_LIB_DIR) + # Find at tensorrt_llm/libs. + execute_process( + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${PYTHONPATH}" "python" "-c" + "import tensorrt_llm; print(f'{tensorrt_llm.__path__[0]}/libs')" + OUTPUT_VARIABLE TRT_LLM_LIB_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE) + # Find /tensorrt_llm/libs. + list(APPEND TRT_LLM_LIB_DIR "../../../tensorrt_llm/libs") +endif() +find_library(TRT_LLM_LIB_PATH nvinfer_plugin_tensorrt_llm + HINTS ${TRT_LLM_LIB_DIR} NO_DEFAULT_PATH) +find_library(TRT_LLM_LIB_PATH nvinfer_plugin_tensorrt_llm REQUIRED) +message(STATUS "Found nvinfer_plugin_tensorrt_llm library: ${TRT_LLM_LIB_PATH}") + +find_library(TRT_LLM_COMMON_LIB_PATH th_common HINTS ${TRT_LLM_LIB_DIR} + NO_DEFAULT_PATH) +find_library(TRT_LLM_COMMON_LIB_PATH th_common REQUIRED) +message(STATUS "Found th_common library: ${TRT_LLM_COMMON_LIB_PATH}") + +# Declare the target library. +add_library( + ${TARGET_NAME} SHARED + tritonPlugins.cpp + TritonFlashAttentionPlugin.cpp + aot/fmha_kernel_fp16.c + aot/fmha_kernel_fp32.c + aot/fp16/fmha_kernel_d64_fp16.fbf0f274_0d1d2d3d4d5d6789.c + aot/fp32/fmha_kernel_d64_fp32.f30323ef_0d1d2d3d4d5d6789.c) + +target_link_libraries( + ${TARGET_NAME} PUBLIC cuda ${CUDA_LIBRARIES} ${TRT_LLM_LIB_PATH} + ${TRT_LLM_COMMON_LIB_PATH} ${TRT_LIB_PATH}) + +if(NOT MSVC) + set_property(TARGET ${TARGET_NAME} PROPERTY LINK_FLAGS "-Wl,--no-undefined") +endif() + +target_include_directories(${TARGET_NAME} PUBLIC /usr/local/cuda/include) +target_include_directories(${TARGET_NAME} PUBLIC ${TRT_INCLUDE_DIR}) diff --git a/examples/openai_triton/manual_plugin/README.md b/examples/openai_triton/manual_plugin/README.md new file mode 100644 index 000000000000..5c8b5d481d52 --- /dev/null +++ b/examples/openai_triton/manual_plugin/README.md @@ -0,0 +1,177 @@ +# OpenAI Triton Plugin in TensorRT-LLM + +This document describes how to build and run a custom plugin leveraging [OpenAI Triton](https://github.com/openai/triton) in TensorRT-LLM. +The workflow can be summarized as follows. + 1. Implement a kernel using Triton in Python. + 2. Compile that kernel using Triton AoT (Ahead-of-Time) compilation tool to generate C files. + 3. Implement a custom TensorRT LLM plugin to execute the compiled kernel. + 4. Build the TensorRT engine. + 5. It is ready to be executed by TensorRT. + +In this example, we show how to create a TensorRT LLM plugin to wrap a [Fused Attention]((fmha_triton.py)) kernel implemented in OpenAI Triton. +As a prerequisite, it is necessary to have the TensorRT LLM C++ runtime library. +The instructions to build that library can be found [here](../../README.md#build-from-source). + +## 1. Triton AoT Preparation + +OpenAI Triton offers an Ahead-of-Time (AoT) compilation tool to generate C files that wrap compiled GPU kernel. +To use the AoT feature, you need a Triton version posterior to the [d0c35b3](https://github.com/openai/triton/commit/d0c35b3b7d6badf0c0d56a821dddab7ace73b4de) commit +and this example has been tested on the [b43c28f](https://github.com/openai/triton/tree/b43c28fdd7a2f95b2e87180cba5d984732120d5c) commit. +```bash +git clone https://github.com/openai/triton +cd triton/python/ +git checkout d4644d6cb3ae674e1f15932cac1f28104795744f +pip install cmake && pip install . +cd - +``` + +For AoT compilation, it is necessary to provide a kernel signature and specify the values of `tl.constexpr` parameters in a comma-separated format. +Details can be found in the [compile.py](https://github.com/openai/triton/blob/main/python/triton/tools/compile.py) file in the Triton project. + +Here are examples of kernel AOT compilations for the [Fused Attention](fmha_triton.py) kernel. +```bash +# Kernel for data type=float16, BLOCK_M=128, BLOCK_DMODEL=64, BLOCK_N=128 +export TRITON_ROOT=$(pip show triton | grep Location | cut -d' ' -f2) +rm -rf aot +mkdir -p aot/fp16 +python ${TRITON_ROOT}/triton/tools/compile.py \ + fmha_triton.py \ + -n fused_attention_kernel \ + -o aot/fp16/fmha_kernel_d64_fp16 \ + --out-name fmha_d64_fp16 \ + -w 4 \ + -ns 2 \ + -s "*fp16:16, *fp32:16, *fp32:16, *fp16:16, *fp16:16, *fp16:16, fp32, i32, i32, i32, 128, 64, 128" \ + -g "(seq_len + 127) / 128, batch_size * num_heads, 1" +# Kernel for data type=float32, BLOCK_M=64, BLOCK_DMODEL=64, BLOCK_N=64 +mkdir -p aot/fp32 +python ${TRITON_ROOT}/triton/tools/compile.py \ + fmha_triton.py \ + -n fused_attention_kernel \ + -o aot/fp32/fmha_kernel_d64_fp32 \ + --out-name fmha_d64_fp32 \ + -w 4 \ + -ns 2 \ + -s "*fp32:16, *fp32:16, *fp32:16, *fp32:16, *fp32:16, *fp32:16, fp32, i32, i32, i32, 64, 64, 64" \ + -g "(seq_len + 63) / 64, batch_size * num_heads, 1" + +# Link generated headers and create dispatchers. +python ${TRITON_ROOT}/triton/tools/link.py aot/fp16/*.h -o aot/fmha_kernel_fp16 +python ${TRITON_ROOT}/triton/tools/link.py aot/fp32/*.h -o aot/fmha_kernel_fp32 +``` +The tool will generate .c and .h files to launch the GPU kernel. +Note that it is necessary to specify the kernel name using the --out-name option, it allows to define dispatcher names for the different data types. +The above invocations will generate `aot/fmha_kernel_{fp16|fp32}.{c|h}` files that contain three functions: + - the `load_fmha_d64_{fp16|fp32}` function to load the code of the GPU kernel, + - the `fmha_d64_{fp16|fp32}` function to launch the kernel, + - the `unload_fmha_d64_{fp16|fp32}` function to unload the GPU kernel. + +If GPU resources are limited, it is recommended to adjust the number of stages or warps accordingly. For example, on the V100, the aforementioned arguments might fail due to insufficient shared memory of the GPU. This can be mitigated by reducing the number of stages by one, using `-ns 1`. + + +## 2. Implement a Custom TensorRT Plugin + +This section describes how to implement a custom plugin for TensorRT LLM to execute the Triton kernel created in the previous section. +We provide an example of plugin implementation. + - TritonFlashAttentionPlugin([.cpp](TritonFlashAttentionPlugin.cpp), [.h](TritonFlashAttentionPlugin.h)): TensorRT plugin. + - [plugin.py](plugin.py): Python wrapper. + +`TritonFlashAttentionPlugin` is a TensorRT plugin that integrates a Triton kernel generated with the AoT compiler. +The `initialize` and `terminate` functions show how to initialize and terminate the TensorRT plugin. +The `enqueue` member function shows how to call the generated Triton kernel on the GPU. +Note that the name of the Triton kernel depends on the function's signature, meaning that different types or specialization leads a different kernel name. +Thus, if you change an option during AoT compilation like `-s `, you also have to update file names in CMakeLists.txt in order to match the names generated by the AoT compiler. + +To build a shared library for the custom Triton plugin, run: +```bash +mkdir -p build && cd build +cmake .. && make +cd .. +``` +As mentioned in the previous section, it is necessary to have the TensorRT LLM C++ runtime library. +If you want to specify the library paths, run: +```bash +cmake -DTRT_LIB_DIR= -DTRT_INCLUDE_DIR= -DTRT_LLM_LIB_DIR= .. +``` +If the build is successful, you should be able to find a shared library for the custom plugin at `build/libtrt_llm_custom_plugins.so`. + +A Python wrapper of the Fused Multihead Attention (FMHA) operator and the corresponding TensorRT LLM layer are implemented in [plugin.py](plugin.py). +It is similar to other TensorRT LLM operators and layers implemented in [functional.py](../../tensorrt_llm/functional.py) and [layers](../../tensorrt_llm/layers), respectively. +That FMHA operator uses the custom plugin that wraps the functions generated from the Triton kernel. + +## 3. Build and Run the TensorRT Engine + +We are now ready to build and run the TensorRT engine that uses the Triton kernel. +Here are the two commands to build and run the engine: +```bash +python build.py --num_heads 32 --head_size 64 --max_batch_size 8 --max_seq_len 512 --dtype float16 +python run.py --num_heads 32 --head_size 64 --batch_size 8 --seq_len 512 --log_level verbose --benchmark +``` + +## 4. Known Issues + +### 1. A generated dispatcher might not execute a kernel without raising an error due to a missing branch. + +The kernel dispatcher written by `link.py` has a missing branch, which can result in returning without executing a kernel. +For instance, in our example, the generated dispatcher looks like this: +```c++ +CUresult fmha_d64_fp16(CUstream stream, unsigned int gX, unsigned int gY, unsigned int gZ, CUdeviceptr Out, CUdeviceptr L, CUdeviceptr M, CUdeviceptr Q, CUdeviceptr K, CUdeviceptr V, float sm_scale, int32_t seq_len){ + if ((Out % 16 == 0) && (L % 16 == 0) && (M % 16 == 0) && (Q % 16 == 0) && (K % 16 == 0) && (V % 16 == 0)) + return fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67(stream, gX, gY, gZ, Out, L, M, Q, K, V, sm_scale, seq_len); +} +``` +It is recommended to manually update the generated functions by `link.py` to return a proper error for proper error handling. + + +### 2. The shared memory required by a generated kernel may exceed the hardware limitation. + +The AoT compiler does not verify the limitations of shared memory size during compilation time, which could potentially lead to the out-of-resource errors during runtime. +It would be helpful to verify if the requirement of the dynamic shared memory size in a generated kernel exceeds the hardware limitation. +You can find the number at the line of `cuLaunchKernel` call in the generated `.c` file. +For instance, the shared memory size is 114690 bytes in our example. +```c++ +CUresult fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67(CUstream stream, unsigned int gX, unsigned int gY, unsigned int gZ, CUdeviceptr Out, CUdeviceptr L, CUdeviceptr M, CUdeviceptr Q, CUdeviceptr K, CUdeviceptr V, float sm_scale, int32_t seq_len) { + if (fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67_func == NULL) + load_fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67(); + void *args[8] = { &Out, &L, &M, &Q, &K, &V, &sm_scale, &seq_len }; + // TODO: shared memory + if(gX * gY * gZ > 0) + return cuLaunchKernel(fmha_d64_fp16_0eb6b090_0d1d2d3d4d5d67_func, gX, gY, gZ, 4 * 32, 1, 1, 114690, stream, args, NULL); +} +``` +It may be resolved by reduing the block size. + + +### 3. AttributeError: module 'triton' has no attribute 'jit' + +This problem may arise if Triton is installed in editable mode. To resolve this issue, please install Triton using the non-editable mode. Refer https://github.com/openai/triton/issues/1693. + +### 4. Unload the same module more than once while building the engine +When the plugin is used more than once within a model, the function cuModuleUnload() will be invoked multiple times during the engine building stage. Related code is generated by Openai Triton and can be found in the folder `examples/openai_triton/manual_plugin/aot/`. One example is: + +```c++ +void unload_fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789(void) { + CUDA_CHECK(cuModuleUnload(fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789_mod)); +} +``` + +As the generated code didn't check the value of the module object, this function might unload the same module multiple times, which will cause an error as follows: + +``` +Triton Error [CUDA]: invalid resource handle\n/opt/rapids/src/cudf/cpp/build/_deps/arrow-src/cpp/src/arrow/filesystem/s3fs.cc:2904:  arrow::fs::FinalizeS3 was not called even though S3 was initialized.  This could lead to a segmentation fault at exit +``` + +The error message is ambiguous. If we use compute-sanitizer to help debug, we can get the following information: +``` +========= Program hit CUDA_ERROR_INVALID_HANDLE (error 400) due to "invalid resource handle" on CUDA API call to cuModuleUnload. +``` + +So we need to modify the above generated code as follows to avoid the above error. +```c++ +void unload_fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789(void) { + if(fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789_mod){ + CUDA_CHECK(cuModuleUnload(fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789_mod)); + } + fmha_d64_fp32_f30323ef_0d1d2d3d4d5d6789_mod=NULL; +} +``` diff --git a/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.cpp b/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.cpp new file mode 100644 index 000000000000..198d8be1ca16 --- /dev/null +++ b/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.cpp @@ -0,0 +1,386 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "TritonFlashAttentionPlugin.h" + +// Import a generated header to use generated triton kernels. +extern "C" +{ +#include "aot/fmha_kernel_fp16.h" +#include "aot/fmha_kernel_fp32.h" +} + +#include +#include +#include +#include + +using namespace nvinfer1; +using openai_triton::plugin::TritonFlashAttentionPluginCreator; +using openai_triton::plugin::TritonFlashAttentionPlugin; + +static char const* TRITON_FLASH_ATTENTION_PLUGIN_VERSION{"1"}; +static char const* TRITON_FLASH_ATTENTION_PLUGIN_NAME{"TritonFlashAttention"}; +PluginFieldCollection TritonFlashAttentionPluginCreator::mFC{}; +std::vector TritonFlashAttentionPluginCreator::mPluginAttributes; + +namespace openai_triton::plugin +{ + +// Write values into buffer +template +void writeArg(char*& buffer, T const& val) +{ + std::memcpy(buffer, &val, sizeof(T)); + buffer += sizeof(T); +} + +// Read values from buffer +template +void readArg(char const*& buffer, T& val) +{ + std::memcpy(&val, buffer, sizeof(T)); + buffer += sizeof(T); +} + +std::uintptr_t constexpr kCudaMemAlign = 128; + +int8_t* nextWorkspacePtr(int8_t* ptr, uintptr_t previousWorkspaceSize) +{ + uintptr_t addr = (uintptr_t) ptr; + addr += previousWorkspaceSize; + if (addr % kCudaMemAlign) + { + addr += kCudaMemAlign - addr % kCudaMemAlign; + } + return (int8_t*) addr; +} + +TritonFlashAttentionPlugin::TritonFlashAttentionPlugin( + int numHeads, int headSize, float softmaxScale, nvinfer1::DataType type) + : mNumHeads(numHeads) + , mHeadSize(headSize) + , mSoftmaxScale(softmaxScale) + , mType(type) +{ +} + +// Parameterized constructor +TritonFlashAttentionPlugin::TritonFlashAttentionPlugin(void const* data, size_t length) +{ + char const *d = reinterpret_cast(data), *a = d; + readArg(d, mNumHeads); + readArg(d, mHeadSize); + readArg(d, mSoftmaxScale); + readArg(d, mType); + TLLM_CHECK(d == a + length); +} + +// IPluginV2DynamicExt Methods +nvinfer1::IPluginV2DynamicExt* TritonFlashAttentionPlugin::clone() const noexcept +{ + auto* plugin = new TritonFlashAttentionPlugin(*this); + plugin->setPluginNamespace(mNamespace.c_str()); + return plugin; +} + +nvinfer1::DimsExprs TritonFlashAttentionPlugin::getOutputDimensions( + int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, nvinfer1::IExprBuilder& exprBuilder) noexcept +{ + // Output shape. + // output tensor [batchSize, seqLen, mNumHeads, head_size] + assert(outputIndex == 0); + return inputs[outputIndex]; +} + +bool TritonFlashAttentionPlugin::supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept +{ + // In this example, inputs: Q, K, V, outputs: Out + assert(nbInputs + nbOutputs == 4); + assert(0 <= pos && pos < nbInputs + nbOutputs); + + bool is_valid = false; + if (0 <= pos && pos < 3) // Q, K, V + { + is_valid = inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + } + else if (pos == nbInputs) // Out + { + is_valid = inOut[pos].type == mType && inOut[pos].format == TensorFormat::kLINEAR; + } + return is_valid; +} + +void TritonFlashAttentionPlugin::configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept +{ +} + +size_t TritonFlashAttentionPlugin::getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept +{ + // Set workspace size if needed. In this example, we need for L and m buffers. + auto const Q = inputs[0]; + int const batchSize = Q.dims.d[0]; + int const seqLen = Q.dims.d[2]; + int const numBuffers = 2; + size_t workspaces[numBuffers]; + workspaces[0] = sizeof(float) * batchSize * mNumHeads * seqLen; + workspaces[1] = sizeof(float) * batchSize * mNumHeads * seqLen; + + size_t total = 0; + for (int i = 0; i < numBuffers; i++) + { + total += workspaces[i]; + if (workspaces[i] % kCudaMemAlign) + { + total += kCudaMemAlign - (workspaces[i] % kCudaMemAlign); + } + } + return total; +} + +template +int TritonFlashAttentionPlugin::enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) +{ + assert(inputDesc[0].dims.d[1] == mNumHeads && inputDesc[0].dims.d[3] == mHeadSize); + assert(inputDesc[1].dims.d[1] == mNumHeads && inputDesc[1].dims.d[3] == mHeadSize); + assert(inputDesc[2].dims.d[1] == mNumHeads && inputDesc[2].dims.d[3] == mHeadSize); + + int batchSize = inputDesc[0].dims.d[0]; + int seqLen = inputDesc[0].dims.d[2]; + + T* Out = reinterpret_cast(outputs[0]); + + const size_t bufSize = sizeof(float) * batchSize * mNumHeads * seqLen; + float* L = reinterpret_cast(workspace); + float* M = reinterpret_cast(nextWorkspacePtr(reinterpret_cast(L), bufSize)); + + T const* Q = reinterpret_cast(inputs[0]); + T const* K = reinterpret_cast(inputs[1]); + T const* V = reinterpret_cast(inputs[2]); + + // Launch a cuda kernel generated by Triton AoT. + int res = 0; + if (std::is_same::value) + { + res = fmha_d64_fp32_default(stream, reinterpret_cast(Out), reinterpret_cast(L), + reinterpret_cast(M), reinterpret_cast(Q), reinterpret_cast(K), + reinterpret_cast(V), mSoftmaxScale, batchSize, mNumHeads, seqLen); + } + else + { + res = fmha_d64_fp16_default(stream, reinterpret_cast(Out), reinterpret_cast(L), + reinterpret_cast(M), reinterpret_cast(Q), reinterpret_cast(K), + reinterpret_cast(V), mSoftmaxScale, batchSize, mNumHeads, seqLen); + } + return res; +} + +int TritonFlashAttentionPlugin::enqueue(nvinfer1::PluginTensorDesc const* inputDesc, + nvinfer1::PluginTensorDesc const* outputDesc, void const* const* inputs, void* const* outputs, void* workspace, + cudaStream_t stream) noexcept +{ + int res = 1; + if (mType == DataType::kHALF) + { + res = enqueueImpl(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + else if (mType == DataType::kFLOAT) + { + res = enqueueImpl(inputDesc, outputDesc, inputs, outputs, workspace, stream); + } + sync_check_cuda_error(); + return res; +} + +// IPluginV2Ext Methods +nvinfer1::DataType TritonFlashAttentionPlugin::getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept +{ + assert(index == 0); + return inputTypes[0]; +} + +// IPluginV2 Methods + +char const* TritonFlashAttentionPlugin::getPluginType() const noexcept +{ + return TRITON_FLASH_ATTENTION_PLUGIN_NAME; +} + +char const* TritonFlashAttentionPlugin::getPluginVersion() const noexcept +{ + return TRITON_FLASH_ATTENTION_PLUGIN_VERSION; +} + +int TritonFlashAttentionPlugin::getNbOutputs() const noexcept +{ + return 1; +} + +int TritonFlashAttentionPlugin::initialize() noexcept +{ + // Load kernels generated by Triton AoT. + load_fmha_d64_fp32(); + load_fmha_d64_fp16(); + return 0; +} + +void TritonFlashAttentionPlugin::terminate() noexcept +{ + // Unload kernels generated by Triton AoT. + unload_fmha_d64_fp32(); + unload_fmha_d64_fp16(); +} + +size_t TritonFlashAttentionPlugin::getSerializationSize() const noexcept +{ + return sizeof(mNumHeads) + sizeof(mHeadSize) + sizeof(mSoftmaxScale) + sizeof(mType); +} + +void TritonFlashAttentionPlugin::serialize(void* buffer) const noexcept +{ + char *d = static_cast(buffer), *a = d; + writeArg(d, mNumHeads); + writeArg(d, mHeadSize); + writeArg(d, mSoftmaxScale); + writeArg(d, mType); + TLLM_CHECK(d == a + getSerializationSize()); +} + +void TritonFlashAttentionPlugin::destroy() noexcept +{ + // This gets called when the network containing plugin is destroyed + delete this; +} + +void TritonFlashAttentionPlugin::setPluginNamespace(char const* libNamespace) noexcept +{ + mNamespace = libNamespace; +} + +char const* TritonFlashAttentionPlugin::getPluginNamespace() const noexcept +{ + return mNamespace.c_str(); +} + +/////////////// + +TritonFlashAttentionPluginCreator::TritonFlashAttentionPluginCreator() +{ + // Fill PluginFieldCollection with PluginField arguments metadata + mPluginAttributes.clear(); + mPluginAttributes.emplace_back(PluginField("num_heads", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("head_size", nullptr, PluginFieldType::kINT32)); + mPluginAttributes.emplace_back(PluginField("softmax_scale", nullptr, PluginFieldType::kFLOAT32)); + mPluginAttributes.emplace_back(PluginField("type_id", nullptr, PluginFieldType::kINT32)); + mFC.nbFields = mPluginAttributes.size(); + mFC.fields = mPluginAttributes.data(); +} + +char const* TritonFlashAttentionPluginCreator::getPluginName() const noexcept +{ + return TRITON_FLASH_ATTENTION_PLUGIN_NAME; +} + +char const* TritonFlashAttentionPluginCreator::getPluginVersion() const noexcept +{ + return TRITON_FLASH_ATTENTION_PLUGIN_VERSION; +} + +PluginFieldCollection const* TritonFlashAttentionPluginCreator::getFieldNames() noexcept +{ + return &mFC; +} + +IPluginV2* TritonFlashAttentionPluginCreator::createPlugin(char const* name, PluginFieldCollection const* fc) noexcept +{ + PluginField const* fields = fc->fields; + int numHeads = 0; + int headSize = 0; + float softmaxScale = 1.0f; + nvinfer1::DataType type; + // Read configurations from each fields + for (int i = 0; i < fc->nbFields; ++i) + { + char const* attrName = fields[i].name; + if (!strcmp(attrName, "num_heads")) + { + assert(fields[i].type == PluginFieldType::kINT32); + numHeads = static_cast(*(static_cast(fields[i].data))); + } + else if (!strcmp(attrName, "head_size")) + { + assert(fields[i].type == PluginFieldType::kINT32); + headSize = static_cast(*(static_cast(fields[i].data))); + } + else if (!strcmp(attrName, "softmax_scale")) + { + assert(fields[i].type == PluginFieldType::kFLOAT32); + softmaxScale = static_cast(*(static_cast(fields[i].data))); + } + else if (!strcmp(attrName, "type_id")) + { + assert(fields[i].type == PluginFieldType::kINT32); + type = static_cast(*(static_cast(fields[i].data))); + } + } + try + { + auto* obj = new TritonFlashAttentionPlugin(numHeads, headSize, softmaxScale, type); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + std::cerr << "Caught exception: " << e.what() << std::endl; + } + return nullptr; +} + +IPluginV2* TritonFlashAttentionPluginCreator::deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept +{ + // This object will be deleted when the network is destroyed, which will + // call TritonFlashAttentionPlugin::destroy() + try + { + auto* obj = new TritonFlashAttentionPlugin(serialData, serialLength); + obj->setPluginNamespace(mNamespace.c_str()); + return obj; + } + catch (std::exception const& e) + { + std::cerr << "Caught exception: " << e.what() << std::endl; + } + return nullptr; +} + +void TritonFlashAttentionPluginCreator::setPluginNamespace(char const* libNamespace) noexcept +{ + mNamespace = libNamespace; +} + +char const* TritonFlashAttentionPluginCreator::getPluginNamespace() const noexcept +{ + return mNamespace.c_str(); +} + +} // namespace openai_triton::plugin diff --git a/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.h b/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.h new file mode 100644 index 000000000000..cd95eb48ec2a --- /dev/null +++ b/examples/openai_triton/manual_plugin/TritonFlashAttentionPlugin.h @@ -0,0 +1,113 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include +#include +#include + +#include +#include + +namespace openai_triton::plugin +{ + +class TritonFlashAttentionPlugin : public nvinfer1::IPluginV2DynamicExt +{ +public: + TritonFlashAttentionPlugin(int numHeads, int headSize, float softmaxScale, nvinfer1::DataType type); + + TritonFlashAttentionPlugin(void const* data, size_t length); + + ~TritonFlashAttentionPlugin() override = default; + + // IPluginV2DynamicExt Methods + nvinfer1::IPluginV2DynamicExt* clone() const noexcept override; + nvinfer1::DimsExprs getOutputDimensions(int outputIndex, nvinfer1::DimsExprs const* inputs, int nbInputs, + nvinfer1::IExprBuilder& exprBuilder) noexcept override; + bool supportsFormatCombination( + int pos, nvinfer1::PluginTensorDesc const* inOut, int nbInputs, int nbOutputs) noexcept override; + void configurePlugin(nvinfer1::DynamicPluginTensorDesc const* in, int nbInputs, + nvinfer1::DynamicPluginTensorDesc const* out, int nbOutputs) noexcept override; + size_t getWorkspaceSize(nvinfer1::PluginTensorDesc const* inputs, int nbInputs, + nvinfer1::PluginTensorDesc const* outputs, int nbOutputs) const noexcept override; + int enqueue(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream) noexcept override; + + template + int enqueueImpl(nvinfer1::PluginTensorDesc const* inputDesc, nvinfer1::PluginTensorDesc const* outputDesc, + void const* const* inputs, void* const* outputs, void* workspace, cudaStream_t stream); + + // IPluginV2Ext Methods + nvinfer1::DataType getOutputDataType( + int index, nvinfer1::DataType const* inputTypes, int nbInputs) const noexcept override; + + // IPluginV2 Methods + char const* getPluginType() const noexcept override; + char const* getPluginVersion() const noexcept override; + int getNbOutputs() const noexcept override; + int initialize() noexcept override; + void terminate() noexcept override; + size_t getSerializationSize() const noexcept override; + void serialize(void* buffer) const noexcept override; + void destroy() noexcept override; + void setPluginNamespace(char const* pluginNamespace) noexcept override; + char const* getPluginNamespace() const noexcept override; + +private: + const std::string mLayerName; + std::string mNamespace; + + int mNumHeads; + int mHeadSize; + float mSoftmaxScale; + nvinfer1::DataType mType; + + CUmodule mModule; + CUfunction mKernel; +}; + +class TritonFlashAttentionPluginCreator : public nvinfer1::IPluginCreator +{ +public: + TritonFlashAttentionPluginCreator(); + + char const* getPluginName() const noexcept override; + + char const* getPluginVersion() const noexcept override; + + nvinfer1::PluginFieldCollection const* getFieldNames() noexcept override; + + nvinfer1::IPluginV2* createPlugin(char const* name, nvinfer1::PluginFieldCollection const* fc) noexcept override; + + nvinfer1::IPluginV2* deserializePlugin( + char const* name, void const* serialData, size_t serialLength) noexcept override; + + void setPluginNamespace(char const* pluginNamespace) noexcept override; + + char const* getPluginNamespace() const noexcept override; + +private: + static nvinfer1::PluginFieldCollection mFC; + static std::vector mPluginAttributes; + std::string mNamespace; +}; + +} // namespace openai_triton::plugin diff --git a/examples/openai_triton/manual_plugin/build.py b/examples/openai_triton/manual_plugin/build.py new file mode 100644 index 000000000000..12b3ca883e7c --- /dev/null +++ b/examples/openai_triton/manual_plugin/build.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import math +import time +from pathlib import Path + +import tensorrt as trt +from plugin import LAYER_NAME, FmhaLayer, get_engine_name + +import tensorrt_llm +from tensorrt_llm.builder import Builder, BuilderConfig +from tensorrt_llm.logger import logger +from tensorrt_llm.network import net_guard + + +def build_engine(builder: Builder, builder_config: BuilderConfig, + engine_name: str, args: argparse.Namespace) -> trt.IHostMemory: + ''' + + @brief: Build a TensorRT engine. + @param args: The cmd line arguments. + @return: The built or refitted engine. + ''' + + # Initialize Module + softmax_scale = 1.0 / math.sqrt(args.head_size) + layer = FmhaLayer(args.num_heads, args.head_size, softmax_scale, args.dtype) + + # Module -> Network + network = builder.create_network() + network.trt_network.name = engine_name + network.plugin_config.to_legacy_setting() + with net_guard(network): + # Prepare + inputs = layer.prepare_inputs(args.max_batch_size, args.max_seq_len) + # Forward + logger.debug(f'model inputs: {inputs}') + out = layer(*inputs) + out.trt_tensor.name = 'out' + + # Network -> Engine + engine = builder.build_engine(network, builder_config) + config_path = Path(args.output_dir) / 'config.json' + builder.save_config(builder_config, str(config_path)) + return engine + + +def build(args): + tensorrt_llm.logger.set_level(args.log_level) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + builder = Builder() + cache = None + builder_config = builder.create_builder_config( + name=LAYER_NAME, + precision=args.dtype, + timing_cache=args.timing_cache if cache is None else cache, + profiling_verbosity=args.profiling_verbosity) + + engine_name = get_engine_name(args.head_size, args.dtype) + engine = build_engine(builder, builder_config, engine_name, args) + assert engine is not None + + engine_path = output_dir / engine_name + logger.info(f'Serializing engine to {str(engine_path)}...') + tik = time.time() + with engine_path.open('wb') as f: + f.write(engine) + tok = time.time() + t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) + logger.info(f'Engine serialized. Total time: {t}') + + ok = builder.save_timing_cache(builder_config, + Path(args.output_dir) / "model.cache") + assert ok, "Failed to save timing cache." + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--max_batch_size', type=int, default=4) + parser.add_argument('--max_seq_len', type=int, default=256) + parser.add_argument('--num_heads', type=int, default=8) + parser.add_argument('--head_size', type=int, default=64) + parser.add_argument('--dtype', + type=str, + default='float16', + choices=['float16', 'float32']) + parser.add_argument( + '--timing_cache', + type=str, + default='model.cache', + help='The path of to read timing cache from, will be ignored ' + 'if the file does not exist') + parser.add_argument( + '--profiling_verbosity', + type=str, + default='layer_names_only', + choices=['layer_names_only', 'detailed', 'none'], + help= + 'The profiling verbosity for the generated TRT engine. Set to detailed can inspect tactic choices and kernel parameters.' + ) + parser.add_argument('--log_level', type=str, default='info') + parser.add_argument( + '--output_dir', + type=str, + default='outputs', + help='The path to save the serialized engine files, timing cache ' + 'file and model configs') + args = parser.parse_args() + + logger.set_level(args.log_level) + logger.info('Parameters'.center(40, '=')) + for k, v in vars(args).items(): + logger.info(f' - {k.ljust(15, ".")}: {v}') + logger.info(''.center(40, '=')) + + tik = time.time() + logger.info('Build TensorRT engine.') + build(args) + tok = time.time() + t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) + logger.info(f'Total time of building TRT engine: {t}') diff --git a/examples/openai_triton/manual_plugin/fmha_triton.py b/examples/openai_triton/manual_plugin/fmha_triton.py new file mode 100644 index 000000000000..3e47dff263f0 --- /dev/null +++ b/examples/openai_triton/manual_plugin/fmha_triton.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fused attention from triton tutorial. +Modified from the original implementation + - https://github.com/openai/triton/blob/main/python/tutorials/06-fused-attention.py +=============== + +This is a Triton implementation of the Flash Attention algorithm +(see: Dao et al., https://arxiv.org/pdf/2205.14135v2.pdf; Rabe and Staats https://arxiv.org/pdf/2112.05682v2.pdf) +""" + +import torch +import triton +import triton.language as tl + + +# yapf: disable +@triton.jit +def fused_attention_kernel( + Out, L, M, # outputs + Q, K, V, + sm_scale, + batch_size, num_heads, seq_len, + BLOCK_M: tl.constexpr, BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, +): + start_m = tl.program_id(0) + off_hz = tl.program_id(1) + stride_h = BLOCK_DMODEL * seq_len + + # initialize offsets + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_DMODEL) + off_q = off_hz * stride_h + offs_m[:, None] * BLOCK_DMODEL + offs_d[None, :] + off_k = off_hz * stride_h + offs_n[None, :] * BLOCK_DMODEL + offs_d[:, None] + off_v = off_hz * stride_h + offs_n[:, None] * BLOCK_DMODEL + offs_d[None, :] + # Initialize pointers to Q, K, V + q_ptrs = Q + off_q + k_ptrs = K + off_k + v_ptrs = V + off_v + # initialize pointer to m and l + m_prev = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_prev = tl.zeros([BLOCK_M], dtype=tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32) + # load q: it will stay in SRAM throughout + q = tl.load(q_ptrs) + # loop over k, v and update accumulator + for start_n in range(0, (start_m + 1) * BLOCK_M, BLOCK_N): + # -- compute qk ---- + k = tl.load(k_ptrs) + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk *= sm_scale + qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, float("-inf")) + # compute new m + m_curr = tl.maximum(tl.max(qk, 1), m_prev) + # correct old l + l_prev *= tl.exp(m_prev - m_curr) + # attention weights + p = tl.exp(qk - m_curr[:, None]) + l_curr = tl.sum(p, 1) + l_prev + # rescale operands of matmuls + l_rcp = 1. / l_curr + p *= l_rcp[:, None] + acc *= (l_prev * l_rcp)[:, None] + # update acc + p = p.to(Q.dtype.element_ty) + v = tl.load(v_ptrs) + acc += tl.dot(p, v) + # update m_i and l_i + l_prev = l_curr + m_prev = m_curr + # update pointers + k_ptrs += BLOCK_N * BLOCK_DMODEL + v_ptrs += BLOCK_N * BLOCK_DMODEL + # rematerialize offsets to save registers + start_m = tl.program_id(0) + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + # write back l and m + l_ptrs = L + off_hz * seq_len + offs_m + m_ptrs = M + off_hz * seq_len + offs_m + tl.store(l_ptrs, l_prev) + tl.store(m_ptrs, m_prev) + # initialize pointers to output + offs_n = tl.arange(0, BLOCK_DMODEL) + off_o = off_hz * stride_h + offs_m[:, None] * BLOCK_DMODEL + offs_n[None, :] + out_ptrs = Out + off_o + tl.store(out_ptrs, acc) + + +def fused_attention(q, k, v, sm_scale, o_buf=None, l_buf=None, m_buf=None): + BLOCK = 128 if q.dtype == torch.float16 else 64 + # shape constraints + Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1] + assert Lq == Lk and Lk == Lv + assert Lk in {16, 32, 64, 128} + o = torch.empty_like(q) if o_buf is None else o_buf + grid = (triton.cdiv(q.shape[2], BLOCK), q.shape[0] * q.shape[1], 1) + shape = (q.shape[0] * q.shape[1], q.shape[2]) + L = torch.empty(shape, device=q.device, dtype=torch.float32) if l_buf is None else l_buf + m = torch.empty(shape, device=q.device, dtype=torch.float32) if m_buf is None else m_buf + + num_warps = 4 if Lk <= 64 else 8 + # Adjust num_stages for limited resource cases. + num_stages = 2 if torch.cuda.get_device_capability() >= (8, 0) else 1 + + fused_attention_kernel[grid]( + o, L, m, + q, k, v, + sm_scale, + q.shape[0], q.shape[1], q.shape[2], + # tl.constexpr + BLOCK_M=BLOCK, + BLOCK_N=BLOCK, + BLOCK_DMODEL=Lk, + num_warps=num_warps, + num_stages=num_stages, + ) + + return o +# yapf: enable diff --git a/examples/openai_triton/manual_plugin/plugin.py b/examples/openai_triton/manual_plugin/plugin.py new file mode 100644 index 000000000000..7009caaeb680 --- /dev/null +++ b/examples/openai_triton/manual_plugin/plugin.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import ctypes +from collections import OrderedDict +from pathlib import Path +from typing import List + +import numpy as np +import tensorrt as trt + +from tensorrt_llm._common import default_trtnet +from tensorrt_llm._utils import str_dtype_to_trt +from tensorrt_llm.functional import Tensor, _create_tensor +from tensorrt_llm.module import Module + +TRT_LLM_PLUGIN_NAMESPACE = 'tensorrt_llm' +LAYER_NAME = 'TritonFlashAttentionLayer' +FMHA_KERNEL_BLOCK_SIZE = 128 + + +def _load_triton_plugin_lib(): + triton_plugin_dir = Path(__file__).parent.absolute() + plugin_lib = triton_plugin_dir / 'build/libtrt_llm_custom_plugins.so' + handle = ctypes.CDLL(plugin_lib, mode=ctypes.RTLD_GLOBAL) + if handle is None: + raise ImportError('TensorRT LLM Triton Plugin is unavailable') + handle.initOpenAiTritonPlugins.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + handle.initOpenAiTritonPlugins.restype = ctypes.c_bool + assert handle.initOpenAiTritonPlugins( + None, TRT_LLM_PLUGIN_NAMESPACE.encode('utf-8')) + + +_load_triton_plugin_lib() + + +def flash_attention_op(num_heads: int, head_size: int, softmax_scale: float, + inputs: List[trt.ITensor]) -> Tensor: + # Create a plugin instance. + plugin_creator = trt.get_plugin_registry().get_plugin_creator( + 'TritonFlashAttention', '1', TRT_LLM_PLUGIN_NAMESPACE) + assert plugin_creator is not None + + pfc = trt.PluginFieldCollection([ + trt.PluginField("num_heads", np.array([num_heads], np.int32), + trt.PluginFieldType.INT32), + trt.PluginField("head_size", np.array([head_size], np.int32), + trt.PluginFieldType.INT32), + trt.PluginField("softmax_scale", np.array([softmax_scale], np.float32), + trt.PluginFieldType.FLOAT32), + trt.PluginField("type_id", np.array([int(inputs[0].dtype)], np.int32), + trt.PluginFieldType.INT32) + ]) + plugin = plugin_creator.create_plugin("flash_attention", pfc) + layer = default_trtnet().add_plugin_v2(inputs, plugin) + return _create_tensor(layer.get_output(0), layer) + + +class FmhaLayer(Module): + + def __init__(self, num_heads: int, head_size: int, softmax_scale: float, + dtype: str): + super().__init__() + self.num_heads = num_heads + self.head_size = head_size + self.softmax_scale = softmax_scale + self.dtype = str_dtype_to_trt(dtype) + + def forward(self, Q: Tensor, K: Tensor, V: Tensor): + inputs = [Q, K, V] + out = flash_attention_op(num_heads=self.num_heads, + head_size=self.head_size, + softmax_scale=self.softmax_scale, + inputs=[p.trt_tensor for p in inputs]) + out.mark_output('out', self.dtype) + return out + + def prepare_inputs(self, max_batch_size: int, max_len: int) -> List[Tensor]: + ''' + + @brief: Prepare inputs Tensors for the model, the given sizes are used to + determine the ranges of the dimensions of when using TRT dynamic shapes. + + @return: a list contains values which can be fed into the self.forward() + ''' + + bs_range = [1, (max_batch_size + 1) // 2, max_batch_size] + max_len_range = [1, (max_len + 1) // 2, max_len] + + dynamic_shape = [-1, self.num_heads, -1, self.head_size] + Q = Tensor(name='Q', + dtype=self.dtype, + shape=dynamic_shape, + dim_range=OrderedDict([ + ('batch_size', [bs_range]), + ('num_heads', [self.num_heads]), + ('seq_len', [max_len_range]), + ('head_size', [self.head_size]), + ])) + K = Tensor(name='K', + dtype=self.dtype, + shape=dynamic_shape, + dim_range=OrderedDict([ + ('batch_size', [bs_range]), + ('num_heads', [self.num_heads]), + ('seq_len', [max_len_range]), + ('head_size', [self.head_size]), + ])) + V = Tensor(name='V', + dtype=self.dtype, + shape=dynamic_shape, + dim_range=OrderedDict([ + ('batch_size', [bs_range]), + ('num_heads', [self.num_heads]), + ('seq_len', [max_len_range]), + ('head_size', [self.head_size]), + ])) + return [Q, K, V] + + +def get_engine_name(head_size, dtype): + return f'{LAYER_NAME}_{FMHA_KERNEL_BLOCK_SIZE}_d{head_size}_{dtype}.engine' diff --git a/examples/openai_triton/manual_plugin/run.py b/examples/openai_triton/manual_plugin/run.py new file mode 100644 index 000000000000..ec7cf4dd5600 --- /dev/null +++ b/examples/openai_triton/manual_plugin/run.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import math +from pathlib import Path + +import torch +from fmha_triton import fused_attention +from plugin import get_engine_name + +from tensorrt_llm import profiler +from tensorrt_llm._deprecation import emit_engine_arch_deprecation +from tensorrt_llm._utils import (str_dtype_to_torch, str_dtype_to_trt, + trt_dtype_to_torch) +from tensorrt_llm.logger import logger +from tensorrt_llm.runtime.session import Session, TensorInfo + + +def run(engine_dir, + batch_size, + seq_len, + num_heads, + head_size, + do_benchmark=False): + # Load trt engine. + engine_dir = Path(engine_dir) + config_path = engine_dir / 'config.json' + with config_path.open('r') as f: + config = json.load(f) + dtype = config['builder_config']['precision'] + serialize_path = engine_dir / get_engine_name(head_size, dtype) + + with open(serialize_path, 'rb') as f: + session = Session.from_serialized_engine(f.read()) + + # Prepare input tensors. + torch_dtype = str_dtype_to_torch(dtype) if isinstance(dtype, str) else dtype + shape = (batch_size, num_heads, seq_len, head_size) + q = torch.normal(mean=0.1, + std=0.2, + size=shape, + dtype=torch_dtype, + device='cuda') + k = torch.normal(mean=0.4, + std=0.2, + size=shape, + dtype=torch_dtype, + device='cuda') + v = torch.normal(mean=0.3, + std=0.2, + size=shape, + dtype=torch_dtype, + device='cuda') + inputs = {'Q': q, 'K': k, 'V': v} + + # Prepare output tensors. + output_info = session.infer_shapes([ + TensorInfo(name, str_dtype_to_trt(dtype), tensor.shape) + for name, tensor in inputs.items() + ]) + logger.debug(f'output info {output_info}') + outputs = { + t.name: + torch.empty(tuple(t.shape), + dtype=trt_dtype_to_torch(t.dtype), + device='cuda') + for t in output_info + } + + # Execute model inference + stream = torch.cuda.Stream() + ok = session.run(inputs=inputs, outputs=outputs, stream=stream.cuda_stream) + assert ok, 'Engine execution failed' + + # Sanity check + stream.synchronize() + sm_scale = 1.0 / math.sqrt(head_size) + ref = fused_attention(q, k, v, sm_scale) + out = outputs["out"] + logger.debug( + f'Out: vals: {out.view(1, -1)} abs_sum: {out.float().abs().sum()}') + logger.debug( + f'Ref: vals: {ref.view(1, -1)} abs_sum: {ref.float().abs().sum()}') + torch.testing.assert_close(out, ref) + + if do_benchmark: + n_repeats = 10 + + # For fair comparison, pre-allocate buffers as trt plugin does. + shape = (q.shape[0] * q.shape[1], q.shape[2]) + L = torch.empty(shape, device=q.device, dtype=torch.float32) + m = torch.empty(shape, device=q.device, dtype=torch.float32) + o = torch.empty_like(q) + + # Triton warm-up + fused_attention(q, k, v, sm_scale, l_buf=L, m_buf=m, o_buf=o) + stream.synchronize() + for _ in range(n_repeats): + profiler.start('Triton') + fused_attention(q, k, v, sm_scale, l_buf=L, m_buf=m, o_buf=o) + stream.synchronize() + profiler.stop('Triton') + + # TRT warm-up + stream.synchronize() + ok = session.run(inputs=inputs, + outputs=outputs, + stream=stream.cuda_stream) + stream.synchronize() + for _ in range(n_repeats): + profiler.start('TRT Plugin') + ok = session.run(inputs=inputs, + outputs=outputs, + stream=stream.cuda_stream) + stream.synchronize() + profiler.stop('TRT Plugin') + assert ok + profiler.summary() + + +if __name__ == '__main__': + emit_engine_arch_deprecation("run.py") + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--batch_size', type=int, default=4) + parser.add_argument('--seq_len', type=int, default=128) + parser.add_argument('--num_heads', type=int, default=8) + parser.add_argument('--head_size', type=int, default=64) + parser.add_argument('--log_level', type=str, default='info') + parser.add_argument( + '--engine_dir', + type=Path, + default='outputs', + help='The directory where serialized engine files locate.') + parser.add_argument( + '--benchmark', + action='store_true', + help='Do performance benchmark compared to triton baseline.') + args = parser.parse_args() + + logger.set_level(args.log_level) + logger.info('Parameters'.center(40, '=')) + for k, v in vars(args).items(): + logger.info(f' - {k.ljust(15, ".")}: {v}') + logger.info(''.center(40, '=')) + + assert args.engine_dir.exists(), \ + f"Engine file {str(args.engine_dir)} doesn't exists." + + logger.info('Inference using the built TensorRT engine.') + run(args.engine_dir, + args.batch_size, + args.seq_len, + args.num_heads, + args.head_size, + do_benchmark=args.benchmark) + logger.info('Done.') diff --git a/examples/openai_triton/manual_plugin/tritonPlugins.cpp b/examples/openai_triton/manual_plugin/tritonPlugins.cpp new file mode 100644 index 000000000000..27b1ece08448 --- /dev/null +++ b/examples/openai_triton/manual_plugin/tritonPlugins.cpp @@ -0,0 +1,133 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "NvInferRuntime.h" +#include "TritonFlashAttentionPlugin.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +// This singleton ensures that each plugin is only registered once for a given +// namespace and type, and attempts of duplicate registration are ignored. +class TritonPluginCreatorRegistry +{ +public: + static TritonPluginCreatorRegistry& getInstance() + { + static TritonPluginCreatorRegistry instance; + return instance; + } + + template + void addPluginCreator(void* logger, char const* libNamespace) + { + // Make accesses to the plugin creator registry thread safe + std::lock_guard lock(mRegistryLock); + + std::string errorMsg; + std::string verboseMsg; + + std::unique_ptr pluginCreator{new CreatorType{}}; + pluginCreator->setPluginNamespace(libNamespace); + + nvinfer1::ILogger* trtLogger = static_cast(logger); + std::string pluginType = std::string{pluginCreator->getPluginNamespace()} + + "::" + std::string{pluginCreator->getPluginName()} + " version " + + std::string{pluginCreator->getPluginVersion()}; + + if (mRegistryList.find(pluginType) == mRegistryList.end()) + { + bool status = getPluginRegistry()->registerCreator(*pluginCreator, libNamespace); + if (status) + { + mRegistry.push(std::move(pluginCreator)); + mRegistryList.insert(pluginType); + verboseMsg = "Registered plugin creator - " + pluginType; + } + else + { + errorMsg = "Could not register plugin creator - " + pluginType; + } + } + else + { + verboseMsg = "Plugin creator already registered - " + pluginType; + } + + if (trtLogger) + { + if (!errorMsg.empty()) + { + trtLogger->log(nvinfer1::ILogger::Severity::kERROR, errorMsg.c_str()); + } + + if (!verboseMsg.empty()) + { + trtLogger->log(nvinfer1::ILogger::Severity::kVERBOSE, verboseMsg.c_str()); + } + } + } + + ~TritonPluginCreatorRegistry() + { + std::lock_guard lock(mRegistryLock); + + // Release pluginCreators in LIFO order of registration. + while (!mRegistry.empty()) + { + mRegistry.pop(); + } + mRegistryList.clear(); + } + +private: + TritonPluginCreatorRegistry() {} + + std::mutex mRegistryLock; + std::stack> mRegistry; + std::unordered_set mRegistryList; + +public: + TritonPluginCreatorRegistry(TritonPluginCreatorRegistry const&) = delete; + void operator=(TritonPluginCreatorRegistry const&) = delete; +}; + +template +void initializeTritonPlugin(void* logger, char const* libNamespace) +{ + TritonPluginCreatorRegistry::getInstance().addPluginCreator(logger, libNamespace); +} + +} // namespace + +// New Plugin APIs + +extern "C" +{ + bool initOpenAiTritonPlugins(void* logger, char const* libNamespace) + { + initializeTritonPlugin(logger, libNamespace); + return true; + } +} // extern "C" diff --git a/examples/openai_triton/plugin_autogen/README.md b/examples/openai_triton/plugin_autogen/README.md new file mode 100644 index 000000000000..0c046330fa0b --- /dev/null +++ b/examples/openai_triton/plugin_autogen/README.md @@ -0,0 +1,97 @@ +# Integrating Triton Kernel with TensorRT Plugin Generator + +In the previous [OpenAI Triton Plugin in TensorRT-LLM](../../openai_triton/README.md) tutorial, it is demonstrated how to integrate a Triton kernel by manually writing a TensorRT plugin in C++ as well as a Python wrapper. In the latest TensorRT-LLM, we now have an end-to-end tool called PluginGen that simplifies this process. All you need to do is providing a plugin configuration. + +In this example, we will introduce the usage of the PluginGen tool and demonstrate the integration of the [Fused Attention](../openai_triton/fmha_triton.py) kernel. + + +To use the feature, you need a Triton version posterior to the [d0c35b3](https://github.com/openai/triton/commit/d0c35b3b7d6badf0c0d56a821dddab7ace73b4de) commit +and this example has been tested on the [d4644d6](https://github.com/openai/triton/tree/d4644d6cb3ae674e1f15932cac1f28104795744f) commit. + +## Introduction to the PluginGen Toolkit + +The PluginGen script can be found at `tensorrt_llm/tools/triton_integration/plugin_gen.py`. Its usage is as follows: + +```sh +usage: plugin_gen.py [-h] --workspace WORKSPACE --kernel_config KERNEL_CONFIG [--tensorrt_llm_include_path TENSORRT_LLM_INCLUDE_PATH] +``` + +There are three command-line arguments: + +1. `workspace`: This is the root directory to hold the temporary generation files. PluginGen should not alter anything outside of the workspace, +2. `kernel_config`: This is a Python file that holds a variable called `KERNELS` of type `List[KernelMetaData]`. PluginGen can process one or more kernels at a time, +3. `tensorrt_llm_include_path`: This is the path to the TensorRT LLM include directory. It is used to include the TensorRT LLM header files in the generated plugin. + +You can refer to [./kernel_config.py](./kernel_config.py) for an example of `KernelMetaData` for the Fused Attention kernel. It contains several fields: + +1. `ios` (short for "input and outputs"): This holds all the metadata of the inputs and outputs of the Triton kernel, including the data type, shape, and the name of the tensor. There are several kinds of arguments: + - `InputArg`: A common variable for this kernel. + - `OutputArg`: An output of the kernel. + - `ParamArg`: A special input that is a constant; it will be mapped to a PluginField in the generated plugin. + - `DimSizeArg`: A special input that is an expression of the input tensors' shape size; it requires an inference rule to compute the value. +2. `shape_infer_rules`: This field contains two types of rules: + a) Rules for deducing the shape of the output tensors from the input tensors. The syntax is like `input0[dim_names], input1[dim_names] -> output0[dim_names]`. + b) Rules for inferring `DimSizeArg`. The syntax is like `input0[dim_names]: some_dim_expression -> arg_name`. + +The user should provide the kernel configurations as well as the Triton kernel script, and the PluginGen toolkit will handle the following steps: + +1. Trigger the Triton AOT tool to obtain the necessary C files. +2. Generate the C++ code for a TensorRT plugin. +3. Generate the CMAKE code for compiling all the C/C++ files. +4. Perform the compilation and generate `libtriton_plugins.so`. +5. Generate a `functional.py` containing a Python wrapper for this plugin. + +After the generation, you should have `libtriton_plugins.so` and `functional.py` in the workspace. You can use them to integrate the Triton kernel by simply using the corresponding Python methods in the generated `functional.py` during the model-building stage, just like other layers located in the TensorRT LLM built-in `functional.py`. + +## End-to-End Example for FHMA Kernel Integration + +In this section, we will demonstrate the integration of the Fused Attention kernel. The steps are as follows: + +### Pre-Stage: Install Triton with a Specific Version + +In case the Triton AOT tool's update breaks compatibility, we recommend installing a specific version of Triton. The commit we tested is [d4644d6](https://github.com/openai/triton/tree/d4644d6cb3ae674e1f15932cac1f28104795744f). + +Install Triton with the following commands: + +```sh +git clone https://github.com/openai/triton +cd triton/python/ +pip install cmake && pip install . +cd - +``` + +### Step 1: Prepare the Configuration for FHMA + +To instruct the PluginGen toolkit on how to generate the plugin, please provide a Python file containing the metadata of the kernels. You can refer to [./kernel_config.py](./kernel_config.py) for an example of preparing `KernelMetaData` for the Fused Attention kernel. + +### Step 2: Run the PluginGen Tool and Generate the Plugin + +```sh +python3 {GIT_ROOT_DIR}/tensorrt_llm/tools/plugin_gen/plugin_gen.py --workspace ./tmp --kernel_config ./kernel_config.py +``` + +PluginGen will generate all the necessary files within the `./tmp` directory. The final output will be located in the `./tmp/output` directory, where you should ideally find two files: + +``` +-rw-r--r-- 1 1001 1001 2163 Sep 21 17:13 functional.py +-rwxr-xr-x 1 1001 1001 3748464 Sep 21 17:13 libtriton_plugins.so +``` + +### Post-Stage: Use the Plugin + +To use the plugin in a TensorRT LLM model, please refer to the generated `output/functional.py`. It should contain Python wrappers for all the plugins. To use the plugins, first import `functional.py` and then use the corresponding Python methods to build the model. + +For an example of using the Fused Attention plugin in a model, please refer to [build_engine.py](./build_engine.py) for building the TensorRT engine and [run_engine.py](./run_engine.py) for running the engine in the runtime. + +To run the example, you can use the following commands: + +```sh +# copy the triton script to the current directory +cp ../manual_plugin/fmha_triton.py . + +# build the TensorRT engine +python3 build_engine.py + +# run the engine +python3 run_engine.py +``` diff --git a/examples/openai_triton/plugin_autogen/build_engine.py b/examples/openai_triton/plugin_autogen/build_engine.py new file mode 100644 index 000000000000..23b829f0b7e4 --- /dev/null +++ b/examples/openai_triton/plugin_autogen/build_engine.py @@ -0,0 +1,206 @@ +import argparse +import math +# include plugins +# yapf: disable +import os +import sys +import time +from pathlib import Path +from typing import List, OrderedDict + +import tensorrt as trt + +# from plugin import LAYER_NAME, FmhaLayer, get_engine_name +import tensorrt_llm +from tensorrt_llm import Module, str_dtype_to_trt +from tensorrt_llm.builder import Builder, BuilderConfig +from tensorrt_llm.functional import Tensor +from tensorrt_llm.logger import logger +from tensorrt_llm.network import net_guard + +sys.path.append(os.environ.get('PLUGIN_GEN_WORKSPACE', './tmp')) +from functional import fused_attention_kernel # isort:skip +# yapf: enable + + +def get_engine_name(head_size: int, dtype: str) -> str: + return f'fmha_{head_size}_{dtype}.engine' + + +class FmhaLayer(Module): + + def __init__(self, num_heads: int, head_size: int, softmax_scale: float): + super().__init__() + self.num_heads = num_heads + self.head_size = head_size + self.softmax_scale = softmax_scale + self.dtype = str_dtype_to_trt('float16') + + def forward(self, Q: Tensor, K: Tensor, V: Tensor): + inputs = [Q, K, V] + Out, L, M = fused_attention_kernel(self.softmax_scale, self.num_heads, + *[p.trt_tensor for p in inputs]) + Out.mark_output('out', self.dtype) + L.mark_output('L', self.dtype) + M.mark_output('M', self.dtype) + return Out, L, M + + def prepare_inputs(self, max_batch_size: int, max_len: int) -> List[Tensor]: + ''' + + @brief: Prepare inputs Tensors for the model, the given sizes are used to + determine the ranges of the dimensions of when using TRT dynamic shapes. + + @return: a list contains values which can be fed into the self.forward() + ''' + + bs_range = [1, (max_batch_size + 1) // 2, max_batch_size] + max_len_range = [1, (max_len + 1) // 2, max_len] + + dynamic_shape = [-1, self.num_heads, -1, self.head_size] + Q = Tensor(name='Q', + dtype=trt.float16, + shape=dynamic_shape, + dim_range=OrderedDict([ + ('batch_size', [bs_range]), + ('num_heads', [self.num_heads]), + ('seq_len', [max_len_range]), + ('head_size', [self.head_size]), + ])) + K = Tensor(name='K', + dtype=trt.float16, + shape=dynamic_shape, + dim_range=OrderedDict([ + ('batch_size', [bs_range]), + ('num_heads', [self.num_heads]), + ('seq_len', [max_len_range]), + ('head_size', [self.head_size]), + ])) + V = Tensor(name='V', + dtype=trt.float16, + shape=dynamic_shape, + dim_range=OrderedDict([ + ('batch_size', [bs_range]), + ('num_heads', [self.num_heads]), + ('seq_len', [max_len_range]), + ('head_size', [self.head_size]), + ])) + return [Q, K, V] + + +def build_engine(builder: Builder, builder_config: BuilderConfig, + engine_name: str, args: argparse.Namespace) -> trt.IHostMemory: + ''' + @brief: Build a TensorRT engine. + @param args: The cmd line arguments. + @return: The built or refitted engine. + ''' + + # Initialize Module + softmax_scale = 1.0 / math.sqrt(args.head_size) + layer = FmhaLayer(args.num_heads, args.head_size, softmax_scale) + + # Module -> Network + network = builder.create_network() + network.trt_network.name = engine_name + network.plugin_config.to_legacy_setting() + with net_guard(network): + # Prepare + inputs = layer.prepare_inputs(args.max_batch_size, args.max_seq_len) + # Forward + logger.debug(f'model inputs: {inputs}') + layer(*inputs) + + print('dot:') + print(network.to_dot()) + + layer = network.get_layer_by_name(next( + network.get_layers()).name).as_layer() + print('layer', layer.plugin.plugin_type) + print('layer', layer.plugin.plugin_version) + print('layer', layer.plugin.plugin_namespace) + + # Network -> Engine + engine = builder.build_engine(network, builder_config) + config_path = Path(args.output_dir) / 'config.json' + builder.save_config(builder_config, str(config_path)) + return engine + + +def build(args): + tensorrt_llm.logger.set_level(args.log_level) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + builder = Builder() + cache = None + builder_config = builder.create_builder_config( + name='fmha_triton', + precision=args.dtype, + timing_cache=args.timing_cache if cache is None else cache, + profiling_verbosity=args.profiling_verbosity) + + engine_name = get_engine_name(args.head_size, args.dtype) + engine = build_engine(builder, builder_config, engine_name, args) + assert engine is not None + + engine_path = output_dir / engine_name + logger.info(f'Serializing engine to {str(engine_path)}...') + tik = time.time() + with engine_path.open('wb') as f: + f.write(engine) + tok = time.time() + t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) + logger.info(f'Engine serialized. Total time: {t}') + + ok = builder.save_timing_cache(builder_config, + Path(args.output_dir) / "model.cache") + assert ok, "Failed to save timing cache." + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--max_batch_size', type=int, default=4) + parser.add_argument('--max_seq_len', type=int, default=256) + parser.add_argument('--num_heads', type=int, default=8) + parser.add_argument('--head_size', type=int, default=64) + parser.add_argument('--dtype', + type=str, + default='float16', + choices=['float16', 'float32']) + parser.add_argument( + '--timing_cache', + type=str, + default='model.cache', + help='The path of to read timing cache from, will be ignored ' + 'if the file does not exist') + parser.add_argument( + '--profiling_verbosity', + type=str, + default='layer_names_only', + choices=['layer_names_only', 'detailed', 'none'], + help= + 'The profiling verbosity for the generated TRT engine. Set to detailed can inspect tactic choices and kernel parameters.' + ) + parser.add_argument('--log_level', type=str, default='info') + parser.add_argument( + '--output_dir', + type=str, + default='outputs', + help='The path to save the serialized engine files, timing cache ' + 'file and model configs') + args = parser.parse_args() + + logger.set_level(args.log_level) + logger.info('Parameters'.center(40, '=')) + for k, v in vars(args).items(): + logger.info(f' - {k.ljust(15, ".")}: {v}') + logger.info(''.center(40, '=')) + + tik = time.time() + logger.info('Build TensorRT engine.') + build(args) + tok = time.time() + t = time.strftime('%H:%M:%S', time.gmtime(tok - tik)) + logger.info(f'Total time of building TRT engine: {t}') diff --git a/examples/openai_triton/plugin_autogen/kernel_config.py b/examples/openai_triton/plugin_autogen/kernel_config.py new file mode 100644 index 000000000000..93d64c187d61 --- /dev/null +++ b/examples/openai_triton/plugin_autogen/kernel_config.py @@ -0,0 +1,55 @@ +import os + +import torch + +from tensorrt_llm.tools.plugin_gen.core import * + +openai_triton_example_root = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "manual_plugin") + + +def get_fmha_kernel_meta_data(): + block_size = 128 + num_stages = 2 if torch.cuda.get_device_capability() >= (8, 0) else 1 + + return KernelMetaData( + kernel_name='fused_attention_kernel', + ios=[ + # outputs + OutputArg('Out', Type('tensor[fp16]'), hints=['16', '16']), + OutputArg('L', Type('tensor[fp32]'), hints=['16', '16']), + OutputArg('M', Type('tensor[fp32]'), hints=['16', '16']), + # inputs + InputArg('Q', Type('tensor[fp16]'), hints=['16', '16']), + InputArg('K', Type('tensor[fp16]'), hints=['16', '16']), + InputArg('V', Type('tensor[fp16]'), hints=['16', '16']), + ParamArg('sm_scale', Type('fp32')), + DimSizeArg('batch_size'), + ParamArg('num_heads', Type('i32')), + DimSizeArg('seq_len', hints=['', '16']), + # constexprs + Constexpr(block_size), + Constexpr(64), + Constexpr(block_size), + ], + shape_infer_rules=[ + # The following rules helps to deduce the shapes of the output tensors + "Q[*] -> Out[*]", + "Q[m,n,k,*] -> L[m,n,k]", + "Q[m,n,k,*] -> M[m,n,k]", + + # The following rules helps to deduce both DimSizeArgs: batch_size and seq_len + "Q[m,n,k,*] : m -> batch_size", + "Q[m,n,k,*] : k -> seq_len", + ], + version=0, + kernel_file=f'{openai_triton_example_root}/fmha_triton.py', + num_warps=4, + num_stages=num_stages, + grid_dims=(f"(seq_len + {block_size-1}) / {block_size}", + "batch_size * num_heads", "1")) + + +KERNELS = [ + get_fmha_kernel_meta_data(), +] diff --git a/examples/openai_triton/plugin_autogen/run_engine.py b/examples/openai_triton/plugin_autogen/run_engine.py new file mode 100644 index 000000000000..9a438d1d8d87 --- /dev/null +++ b/examples/openai_triton/plugin_autogen/run_engine.py @@ -0,0 +1,170 @@ +import argparse +import json +import math +# include plugins +# yapf: disable +import sys +from pathlib import Path + +import torch +from fmha_triton import fused_attention + +from tensorrt_llm import profiler +from tensorrt_llm._utils import (str_dtype_to_torch, str_dtype_to_trt, + trt_dtype_to_torch) +from tensorrt_llm.logger import logger +from tensorrt_llm.runtime.session import Session, TensorInfo + +# from tensorrt_llm.plugin import get_engine_name + + +sys.path.append('./tmp') +from functional import fused_attention_kernel # isort:skip +# yapf: enable + + +def get_engine_name(head_size, dtype): + return f'fmha_{head_size}_{dtype}.engine' + + +def run(engine_dir, + batch_size, + seq_len, + num_heads, + head_size, + do_benchmark=False): + # Load trt engine. + engine_dir = Path(engine_dir) + config_path = engine_dir / 'config.json' + with config_path.open('r') as f: + config = json.load(f) + dtype = config['builder_config']['precision'] + serialize_path = engine_dir / get_engine_name(head_size, dtype) + + with open(serialize_path, 'rb') as f: + session = Session.from_serialized_engine(f.read()) + + # Prepare input tensors. + torch_dtype = str_dtype_to_torch(dtype) if isinstance(dtype, str) else dtype + shape = (batch_size, num_heads, seq_len, head_size) + q = torch.normal(mean=0.1, + std=0.2, + size=shape, + dtype=torch_dtype, + device='cuda') + k = torch.normal(mean=0.4, + std=0.2, + size=shape, + dtype=torch_dtype, + device='cuda') + v = torch.normal(mean=0.3, + std=0.2, + size=shape, + dtype=torch_dtype, + device='cuda') + batch_size = q.shape[0] + seq_len = q.shape[2] + + inputs = {'Q': q, 'K': k, 'V': v} + + # Prepare output tensors. + output_info = session.infer_shapes([ + TensorInfo(name, str_dtype_to_trt(dtype), tensor.shape) + for name, tensor in inputs.items() + ]) + logger.debug(f'output info {output_info}') + outputs = { + t.name: + torch.empty(tuple(t.shape), + dtype=trt_dtype_to_torch(t.dtype), + device='cuda') + for t in output_info + } + + # Execute model inference + stream = torch.cuda.current_stream() + ok = session.run(inputs=inputs, outputs=outputs, stream=stream.cuda_stream) + assert ok, 'Engine execution failed' + + # Sanity check + stream.synchronize() + sm_scale = 1.0 / math.sqrt(head_size) + ref = fused_attention(q, k, v, sm_scale) + out = outputs["out"] + logger.debug( + f'Out: vals: {out.view(1, -1)} abs_sum: {out.float().abs().sum()}') + logger.debug( + f'Ref: vals: {ref.view(1, -1)} abs_sum: {ref.float().abs().sum()}') + torch.testing.assert_close(out, ref) + + if do_benchmark: + n_repeats = 10 + + # For fair comparison, pre-allocate buffers as trt plugin does. + shape = (q.shape[0] * q.shape[1], q.shape[2]) + L = torch.empty(shape, device=q.device, dtype=torch.float32) + m = torch.empty(shape, device=q.device, dtype=torch.float32) + o = torch.empty_like(q) + + # Triton warm-up + fused_attention(q, k, v, sm_scale, l_buf=L, m_buf=m, o_buf=o) + stream.synchronize() + for _ in range(n_repeats): + profiler.start('Triton') + fused_attention(q, k, v, sm_scale, l_buf=L, m_buf=m, o_buf=o) + stream.synchronize() + profiler.stop('Triton') + + # TRT warm-up + stream.synchronize() + ok = session.run(inputs=inputs, + outputs=outputs, + stream=stream.cuda_stream) + stream.synchronize() + for _ in range(n_repeats): + profiler.start('TRT Plugin') + ok = session.run(inputs=inputs, + outputs=outputs, + stream=stream.cuda_stream) + stream.synchronize() + profiler.stop('TRT Plugin') + assert ok + profiler.summary() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--batch_size', type=int, default=4) + parser.add_argument('--seq_len', type=int, default=128) + parser.add_argument('--num_heads', type=int, default=8) + parser.add_argument('--head_size', type=int, default=64) + parser.add_argument('--log_level', type=str, default='info') + parser.add_argument( + '--engine_dir', + type=Path, + default='outputs', + help='The directory where serialized engine files locate.') + parser.add_argument( + '--benchmark', + action='store_true', + help='Do performance benchmark compared to triton baseline.') + args = parser.parse_args() + + logger.set_level(args.log_level) + logger.info('Parameters'.center(40, '=')) + for k, v in vars(args).items(): + logger.info(f' - {k.ljust(15, ".")}: {v}') + logger.info(''.center(40, '=')) + + assert args.engine_dir.exists(), \ + f"Engine file {str(args.engine_dir)} doesn't exists." + + logger.info('Inference using the built TensorRT engine.') + run(args.engine_dir, + args.batch_size, + args.seq_len, + args.num_heads, + args.head_size, + do_benchmark=args.benchmark) + logger.info('Done.') diff --git a/examples/python_plugin/README.md b/examples/python_plugin/README.md new file mode 100644 index 000000000000..8079d381109a --- /dev/null +++ b/examples/python_plugin/README.md @@ -0,0 +1,120 @@ +# TensorRT LLM Python Plugin + +TensorRT LLM provides a Python plugin interface to integrate TensorRT LLM with pure Python. + ++ `openai_triton_plugin`: plugin package ++ `build_lookup.py`: Build a TensorRT engine with TensorRT LLM Python plugin ++ `run_lookup.py`: Run the engine and compare the result with PyTorch + +## Plugin Definition + +The following code shows how to create a look-up plugin. +We only need to do a few things to define a TensorRT LLM plugin. + +1. Inherit the `PluginBase`. +2. Register the plugin class to TensorRT LLM by using `@trtllm_plugin("your_plugin_name")`. +3. Define an `__init__` function and initialize the base class. +4. Define a shape and dtype inference function. +5. Define the compute flow. + +```python +@trtllm_plugin("TritonLookUp") +class LookUpPlugin(PluginBase): + + def __init__(self, use_torch_tensor, fp32_output): + super().__init__() + self.use_torch_tensor = use_torch_tensor + self.fp32_output = fp32_output + + def shape_dtype_inference(self, inputs: Sequence[SymTensor]) -> SymTensor: + shape = inputs[1].shape + shape[0] = inputs[0].shape[0] + inputs[1].shape[0] - inputs[1].shape[0] + return SymTensor( + inputs[1].dtype if not self.fp32_output else torch.float32, shape) + + def forward(self, inputs: Sequence[TensorWrapper], + outputs: Sequence[TensorWrapper]): + assert len(inputs) == 2 + assert inputs[0].dtype in [torch.int32 or torch.int64] + assert inputs[1].dtype in [torch.float32, torch.float16, torch.bfloat16] + assert (self.fp32_output and outputs[0].dtype + == torch.float32) or outputs[0].dtype == inputs[1].dtype + + x = inputs[0] + y = inputs[1] + z = outputs[0] + if self.use_torch_tensor: + x = convert_to_torch_tensor(x) + y = convert_to_torch_tensor(y) + z = convert_to_torch_tensor(z) + MAX_BLOCK_NUM = 65536 + MAX_BLOCK_SIZE = 512 + grid = lambda meta: (min(MAX_BLOCK_NUM, x.shape[0]) * min( + MAX_BLOCK_SIZE, y.shape[1]), ) + lookup_kernel[grid](x, y, z, y.shape[0], y.shape[1], x.shape[0]) + +``` + +## Adding a TensorRT LLM Plugin to a Network + +You only need an instance of the plugin object and then call it with `tensorrt_llm.Tensor` as input arguments. + +```python +builder = tensorrt_llm.Builder() +network = builder.create_network() +with tensorrt_llm.net_guard(network): + x = Tensor(name='x', + shape=index_shape, + dtype=tensorrt_llm.str_dtype_to_trt('int32')) + y = Tensor(name='y', + shape=(vocab_size, n_embed), + dtype=torch_dtype_to_trt(dtype)) + + def lookup(x, y): + lookup_plugin = LookUpPlugin(False) + return lookup_plugin(x, y) + + output = lookup(x, y) + output.mark_output('output', torch_dtype_to_str(dtype)) +``` + +## Plugin Code Structure + +Because TensorRT LLM performs plugin registration when importing the custom TensorRT LLM plugin, there are some code structure conventions to register the plugin at runtime. + +```text +plugin_lib +├──__init__.py +├──lookup_plugin.py +└──lookup_kernel.py +``` + +The `__init__.py` file imports all the plugins in the plugin package. +With this convention, users only need to import the plugin package to register the plugins and do not need to manually import them. + +```python +# __init__.py +from .lookup_plugin import LookUpPlugin + +__all__ = ["LookUpPlugin"] +``` + +## Deserialize an Engine with TensorRT LLM Plugin + +During deserialization, TensorRT needs to find the user-defined plugin. Thus, we need to import the plugin once to register them. If the plugin follows the code structure convention, users only need to import that package to register all the custom plugins. + +```python +from tensorrt_llm.runtime.session import Session, TensorInfo + +import openai_triton_plugin # isort: skip + +if __name__ == "__main__": + + def run_engine(dtype): + output_dir = Path('tmp') / torch_dtype_to_str(dtype) + + engine_path = output_dir / "lookup.engine" + + with engine_path.open('rb') as f: + session = Session.from_serialized_engine(f.read()) +``` diff --git a/examples/python_plugin/build_lookup.py b/examples/python_plugin/build_lookup.py new file mode 100644 index 000000000000..48cccd58882e --- /dev/null +++ b/examples/python_plugin/build_lookup.py @@ -0,0 +1,61 @@ +from pathlib import Path + +import torch +from plugin_lib import LookUpPlugin + +import tensorrt_llm +from tensorrt_llm import Tensor +from tensorrt_llm._utils import torch_dtype_to_str, torch_dtype_to_trt + +if __name__ == "__main__": + + # meta data + batch_size = 10 + vocab_size = 1000 + n_embed = 1024 + + # test data + ## input index + index_shape = (batch_size, ) + index_data = torch.randint(0, vocab_size, index_shape, + dtype=torch.int32).cuda() + + def test(dtype): + builder = tensorrt_llm.Builder() + builder.strongly_typed = True + network = builder.create_network() + with tensorrt_llm.net_guard(network): + x = Tensor( + name="x", + shape=index_shape, + dtype=tensorrt_llm.str_dtype_to_trt("int32"), + ) + y = Tensor(name="y", + shape=(vocab_size, n_embed), + dtype=torch_dtype_to_trt(dtype)) + + def lookup(x, y): + lookup_plugin = LookUpPlugin(False, True) + return lookup_plugin(x, y) + + output = lookup(x, y) + + output.mark_output("output", torch_dtype_to_str(torch.float32)) + + builder_config = builder.create_builder_config("float32") + engine = builder.build_engine(network, builder_config) + assert engine is not None + + output_dir = Path("tmp") / torch_dtype_to_str(dtype) + output_dir.mkdir(parents=True, exist_ok=True) + + engine_path = output_dir / "lookup.engine" + config_path = output_dir / "config.json" + + with engine_path.open("wb") as f: + f.write(engine) + builder.save_config(builder_config, str(config_path)) + + test(torch.bfloat16) + test(torch.float16) + test(torch.float32) diff --git a/examples/python_plugin/plugin_lib/__init__.py b/examples/python_plugin/plugin_lib/__init__.py new file mode 100644 index 000000000000..f27e0ded3a1c --- /dev/null +++ b/examples/python_plugin/plugin_lib/__init__.py @@ -0,0 +1,3 @@ +from .lookup_plugin import LookUpPlugin + +__all__ = ["LookUpPlugin"] diff --git a/examples/python_plugin/plugin_lib/lookup_kernel.py b/examples/python_plugin/plugin_lib/lookup_kernel.py new file mode 100644 index 000000000000..25cf66704b96 --- /dev/null +++ b/examples/python_plugin/plugin_lib/lookup_kernel.py @@ -0,0 +1,17 @@ +import triton +import triton.language as tl + + +@triton.jit +def lookup_kernel(X, Y, Z, vocab_size, hidden_size, token_num): + pid = tl.program_id(axis=0) + while pid < token_num * hidden_size: + row_idx = pid // hidden_size + col_idx = pid % hidden_size + word_idx = tl.load(X + row_idx) + embedding = tl.load( + Y + word_idx * hidden_size + col_idx, + mask=word_idx < vocab_size, + ) + tl.store(Z + pid, embedding) + pid += tl.num_programs(0) diff --git a/examples/python_plugin/plugin_lib/lookup_plugin.py b/examples/python_plugin/plugin_lib/lookup_plugin.py new file mode 100644 index 000000000000..612ee7a4db1e --- /dev/null +++ b/examples/python_plugin/plugin_lib/lookup_plugin.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Sequence + +import torch + +from tensorrt_llm import PluginBase +from tensorrt_llm._utils import TensorWrapper, convert_to_torch_tensor +from tensorrt_llm.python_plugin import SymTensor, trtllm_plugin + +from .lookup_kernel import lookup_kernel + + +@trtllm_plugin("TritonLookUp") +class LookUpPlugin(PluginBase): + + def __init__(self, use_torch_tensor, fp32_output): + super().__init__() + self.use_torch_tensor = use_torch_tensor + self.fp32_output = fp32_output + + def shape_dtype_inference(self, inputs: Sequence[SymTensor]) -> SymTensor: + shape = inputs[1].shape + shape[0] = inputs[0].shape[0] + inputs[1].shape[0] - inputs[1].shape[0] + return SymTensor( + inputs[1].dtype if not self.fp32_output else torch.float32, shape) + + def forward(self, inputs: Sequence[TensorWrapper], + outputs: Sequence[TensorWrapper]): + assert len(inputs) == 2 + assert inputs[0].dtype in [torch.int32 or torch.int64] + assert inputs[1].dtype in [torch.float32, torch.float16, torch.bfloat16] + assert (self.fp32_output and outputs[0].dtype + == torch.float32) or outputs[0].dtype == inputs[1].dtype + + x = inputs[0] + y = inputs[1] + z = outputs[0] + if self.use_torch_tensor: + x = convert_to_torch_tensor(x) + y = convert_to_torch_tensor(y) + z = convert_to_torch_tensor(z) + MAX_BLOCK_NUM = 65536 + MAX_BLOCK_SIZE = 512 + grid = lambda meta: (min(MAX_BLOCK_NUM, x.shape[0]) * min( + MAX_BLOCK_SIZE, y.shape[1]), ) + lookup_kernel[grid](x, y, z, y.shape[0], y.shape[1], x.shape[0]) diff --git a/examples/python_plugin/run_lookup.py b/examples/python_plugin/run_lookup.py new file mode 100644 index 000000000000..055e86008c07 --- /dev/null +++ b/examples/python_plugin/run_lookup.py @@ -0,0 +1,65 @@ +from pathlib import Path + +import torch + +from tensorrt_llm import logger +from tensorrt_llm._utils import (torch_dtype_to_str, torch_dtype_to_trt, + trt_dtype_to_torch) +from tensorrt_llm.runtime.session import Session, TensorInfo + +import plugin_lib # isort: skip + +if __name__ == "__main__": + + def run_engine(dtype): + output_dir = Path('tmp') / torch_dtype_to_str(dtype) + + engine_path = output_dir / "lookup.engine" + + with engine_path.open('rb') as f: + session = Session.from_serialized_engine(f.read()) + + # meta data + batch_size = 10 + vocab_size = 1000 + n_embed = 1024 + + # test data + ## input index + index_shape = (batch_size, ) + index_data = torch.randint(0, + vocab_size, + index_shape, + dtype=torch.int32).cuda() + weight_data = torch.rand(vocab_size, n_embed, dtype=dtype).cuda() + + inputs = {"x": index_data, "y": weight_data} + + output_info = session.infer_shapes([ + TensorInfo(name, torch_dtype_to_trt(tensor.dtype), tensor.shape) + for name, tensor in inputs.items() + ]) + logger.debug(f'output info {output_info}') + outputs = { + t.name: + torch.empty(tuple(t.shape), + dtype=trt_dtype_to_torch(t.dtype), + device='cuda') + for t in output_info + } + + stream = torch.cuda.Stream() + ok = session.run(inputs=inputs, + outputs=outputs, + stream=stream.cuda_stream) + assert ok, 'Engine execution failed' + + embedding = torch.nn.Embedding.from_pretrained(weight_data) + torch_out = embedding(index_data).to(torch.float32) + trt_out = outputs['output'] + + torch.testing.assert_close(trt_out, torch_out) + + run_engine(torch.bfloat16) + run_engine(torch.float16) + run_engine(torch.float32) diff --git a/examples/quantization/README.md b/examples/quantization/README.md index 23bfa5d40c8e..b3b2e35b20ff 100644 --- a/examples/quantization/README.md +++ b/examples/quantization/README.md @@ -1,19 +1,249 @@ -# Model Quantization +# TensorRT LLM Quantization Toolkit Installation Guide -To run quantized models with TensorRT LLM: +## Introduction -- Use a pre-quantized Hugging Face checkpoint (for example the FP8/NVFP4 - checkpoints published on the [NVIDIA Hugging Face hub](https://huggingface.co/nvidia)). - Quantization settings are detected automatically when the model loads. -- To quantize your own model, use the - [NVIDIA TensorRT Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer) - Hugging Face export flow (`examples/llm_ptq` in that repository). +This document introduces: -See the [quantization feature documentation](https://nvidia.github.io/TensorRT-LLM/features/quantization.html) -for supported formats per GPU architecture. +- The steps to install the TensorRT LLM quantization toolkit. +- The Python APIs to quantize the models. -## Mixed-precision MoE checkpoints +The detailed LLM quantization recipe is distributed to the README.md of the corresponding model examples. -[`quantize_mixed_precision_moe.py`](quantize_mixed_precision_moe.py) builds a -mixed-precision MoE checkpoint from separately quantized checkpoints; see the -script's argparse help for usage. +## Installation + +The NVIDIA Model Optimizer quantization toolkit is installed automatically as a dependency of TensorRT-LLM. + +```bash +# Install the additional requirements +cd examples/quantization +pip install -r requirements.txt +``` + +## Usage + +```bash +# FP8 quantization. +python quantize.py --model_dir $MODEL_PATH --qformat fp8 --kv_cache_dtype fp8 --output_dir $OUTPUT_PATH + +# INT4_AWQ tp4 quantization. +python quantize.py --model_dir $MODEL_PATH --qformat int4_awq --awq_block_size 64 --tp_size 4 --output_dir $OUTPUT_PATH + +# INT8 SQ with INT8 kv cache. +python quantize.py --model_dir $MODEL_PATH --qformat int8_sq --kv_cache_dtype int8 --output_dir $OUTPUT_PATH + +# Auto quantization(e.g. fp8 + int4_awq + w4a8_awq) using average weights bits 5 +python quantize.py --model_dir $MODEL_PATH --autoq_format fp8,int4_awq,w4a8_awq --output_dir $OUTPUT_PATH --auto_quantize_bits 5 --tp_size 2 + +# FP8 quantization for NeMo model. +python quantize.py --nemo_ckpt_path nemotron-3-8b-base-4k/Nemotron-3-8B-Base-4k.nemo \ + --dtype bfloat16 \ + --batch_size 64 \ + --qformat fp8 \ + --output_dir nemotron-3-8b/trt_ckpt/fp8/1-gpu + +# FP8 quantization for Medusa model. +python quantize.py --model_dir $MODEL_PATH\ + --dtype float16 \ + --qformat fp8 \ + --kv_cache_dtype fp8 \ + --output_dir $OUTPUT_PATH \ + --calib_size 512 \ + --tp_size 1 \ + --medusa_model_dir /path/to/medusa_head/ \ + --num_medusa_heads 4 +``` +Checkpoint saved in `output_dir` can be directly passed to `trtllm-build`. + +### Quantization Arguments: + +- model_dir: Hugging Face model path. +- qformat: Specify the quantization algorithm applied to the checkpoint. + - nvfp4: Weights are quantized to NVFP4 block-wise with size 16. Activation global scale are calibrated. + - fp8: Weights are quantized to FP8 tensor wise. Activation ranges are calibrated tensor wise. + - fp8_pc_pt: Weights are quantized to FP8 per-channel. Activation ranges are calibrated and quantized per-token. + - int8_sq: Weights are smoothed and quantized to INT8 channel wise. Activation ranges are calibrated tensor wise. + - int4_awq: Weights are re-scaled and block-wise quantized to INT4. Block size is specified by `awq_block_size`. + - w4a8_awq: Weights are re-scaled and block-wise quantized to INT4. Block size is specified by `awq_block_size`. Activation ranges are calibrated tensor wise. + - int8_wo: Actually nothing is applied to weights. Weights are quantized to INT8 channel wise when TRTLLM building the engine. + - int4_wo: Same as int8_wo but in INT4. + - full_prec: No quantization. +- autoq_format: Specific quantization algorithms are searched in auto quantization. The algorithm must in ['fp8', 'int4_awq', 'w4a8_awq', 'int8_sq'] and you can use ',' to separate more than one quantization algorithms, such as `--autoq_format fp8,int4_awq,w4a8_awq`. Please attention that using int8_sq and fp8 together is not supported. +- auto_quantize_bits: Effective bits constraint for auto quantization. If not set, regular quantization without auto quantization search is applied. Note: it must be set within correct range otherwise it will be set by lowest value if possible. For example, the weights of LLMs have 16 bits defaultly and it results in a weight compression rate of 40% if we set `auto_quantize_bits` to 9.6 (9.6 / 16 = 0.6), which means the average bits of the weights are 9.6 but not 16. However, which format to choose is determined by solving an optimization problem, so you need to generate the according checkpoint manually if you want to customize your checkpoint formats. The format of mixed precision checkpoint is described in detail below. +- output_dir: Path to save the quantized checkpoint. +- dtype: Specify data type of model when loading from Hugging Face. +- kv_cache_dtype: Specify kv cache data type. + - int8: Use int8 kv cache. + - fp8: Use FP8 kv cache. + - None (default): Use kv cache as model dtype. +- batch_size: Batch size for calibration. Default is 1. +- calib_size: Number of samples. Default is 512. +- calib_max_seq_length: Max sequence length of calibration samples. Default is 512. +- tp_size: Checkpoint is tensor paralleled by tp_size. Default is 1. +- pp_size: Checkpoint is pipeline paralleled by pp_size. Default is 1. +- awq_block_size: AWQ algorithm specific parameter. Indicate the block size when quantizing weights. 64 and 128 are supported by TRTLLM. +- quantize_lm_head: Enable quantization of lm_head layer. This is only supported for FP8 quantization. Default is false. + +#### NeMo model specific arguments: + +- nemo_ckpt_path: NeMo checkpoint path. +- calib_tp_size: TP size for NeMo checkpoint calibration. +- calib_pp_size: PP size for NeMo checkpoint calibration. + +#### Medusa specific arguments: + +- medusa_model_dir: Model path of medusa. +- quant_medusa_head: Whether to quantize the weights of medusa heads. +- num_medusa_heads: Number of medusa heads. +- num_medusa_layers: Number of medusa layers. +- max_draft_len: Max length of draft. +- medusa_hidden_act: Activation function of medusa. + +### Building Arguments: + +There are several arguments for the building stage which relate to quantization. +- use_fp8_context_fmha: This is Hopper-only feature. Use FP8 Gemm to calculate the attention operation. + +```python +qkv scale = 1.0 +FP_O = quantize(softmax(FP8_Q * FP8_K), scale=1.0) * FP8_V +FP_O * output_scale = FP8_O +``` + +### Checkpoint Conversion Arguments (not supported by all models) + +- FP8 + - use_fp8_rowwise: Enable FP8 per-token per-channel quantization for linear layer. (FP8 from `quantize.py` is per-tensor). +- INT8 + - smoothquant: Enable INT8 quantization for linear layer. Set the α parameter (see https://arxiv.org/pdf/2211.10438.pdf) to Smoothquant the model, and output int8 weights. A good first try is 0.5. Must be in [0, 1]. + - per_channel: Using per-channel quantization for weight when `smoothquant` is enabled. + - per_token: Using per-token quantization for activation when `smoothquant` is enabled. +- Weight-Only + - use_weight_only: Weights are quantized to INT4 or INT8 channel wise. + - weight_only_precision: Indicate `int4` or `int8` when `use_weight_only` is enabled. Or `int4_gptq` when `quant_ckpt_path` is provided which means checkpoint is for GPTQ. + - quant_ckpt_path: Path of a GPTQ quantized model checkpoint in `.safetensors` format. + - group_size: Group size used in GPTQ quantization. + - per_group: Should be enabled when load from GPTQ. +- KV Cache + - int8_kv_cache: By default, we use dtype for KV cache. int8_kv_cache chooses int8 quantization for KV cache. + - fp8_kv_cache: By default, we use dtype for KV cache. fp8_kv_cache chooses fp8 quantization for KV cache. + +### Format of Mixed Precision Checkpoints + +ModelOpt can produce a mixed precision TensorRT LLM checkpoint. After producing the quantized checkpoint, you can build engine directly by `trtllm-build` command: +```bash +trtllm-build --checkpoint_dir --output_dir $OUTPUT_PATH +``` +If you have some special needs about the model weights, such as int4 for MLP and int8 for the rest, you need to generate the checkpoint and config files by yourself. + +The `trtllm-build` command consumes the same format of weights, which is presented in [TensorRT LLM checkpoint formats](https://nvidia.github.io/TensorRT-LLM/architecture/checkpoint.html), but has different quantization method for every linear. Therefore, each layer, such as layer30.mlp.fc, layer30.attention.dense, and so on, keeps the same model weights according to the quantization formats in TensorRT LLM checkpoint. What's more, the `quantization` field in `config.json` will be like this: +``` + "quantization": { + "quant_algo": "MIXED_PRECISION", + "kv_cache_quant_algo": "FP8" // The quant_algo of KV cache may change + }, +``` +There will be another file about per-layer quantization information named `quant_cfg.json` in the same directory, the format of it is like: +``` +{ + "quant_algo": "MIXED_PRECISION", + "kv_cache_quant_algo": "FP8", + "quantized_layers": { // one more filed presents per-layer's information + "transformer.layers.0.attention.qkv": { + "quant_algo": "FP8" // specific algorithm for each linear + }, + "transformer.layers.0.attention.dense": { + "quant_algo": "FP8" + }, + "transformer.layers.0.mlp.fc": { + "quant_algo": "W4A16_AWQ", + "group_size": 128, + "has_zero_point": false, + "pre_quant_scale": true + }, + "transformer.layers.0.mlp.proj": { + "quant_algo": "W8A8_SQ_PER_CHANNEL" + }, + ... + "transformer.layers.31.mlp.proj": { + "quant_algo": "FP8" + } + } +} +``` + +TensorRT LLM will automatically read `quant_cfg.json` after recogniziong the `MIXED_PRECISION` quantization method in `config.json`. All the specific algorithm keeps the same as what in `quantization` field before. If some layers are not listed, they'll be treated as no quantization. + +## APIs + +[`quantize.py`](./quantize.py) uses the quantization toolkit to calibrate the PyTorch models and export TensorRT LLM checkpoints. Each TensorRT LLM checkpoint contains a config file (in .json format) and one or several rank weight files (in .safetensors format). It will produce one another quantization config for per-layer's information when setting auto quantization. The checkpoints can be directly used by `trtllm-build` command to build TensorRT LLM engines. See this [`doc`](../../docs/source/architecture/checkpoint.md) for more details on the TensorRT LLM checkpoint format. + +> *This quantization step may take a long time to finish and requires large GPU memory. Please use a server grade GPU if a GPU out-of-memory error occurs* + +> *If the model is trained with multi-GPU with tensor parallelism, the PTQ calibration process requires the same amount of GPUs as the training time too.* + + +### PTQ (Post Training Quantization) + +PTQ can be achieved with simple calibration on a small set of training or evaluation data (typically 128-512 samples) after converting a regular PyTorch model to a quantized model. + +```python +import torch +from torch.utils.data import DataLoader +from transformers import AutoModelForCausalLM +import modelopt.torch.quantization as mtq +import modelopt.torch.utils.dataset_utils as dataset_utils + +model = AutoModelForCausalLM.from_pretrained(...) + +# Select the quantization config, for example, FP8 +config = mtq.FP8_DEFAULT_CFG + +# Prepare the calibration set and define a forward loop +calib_dataloader = DataLoader(...) +calibrate_loop = dataset_utils.create_forward_loop( + calib_dataloader, dataloader=calib_dataloader +) + +# PTQ with in-place replacement to quantized modules +with torch.no_grad(): + mtq.quantize(model, config, forward_loop=calibrate_loop) + +# or PTQ with auto quantization +with torch.no_grad(): + model, search_history = mtq.auto_quantize( + model, + data_loader=calib_dataloader, + loss_func=lambda output, batch: output.loss, + constraints={"effective_bits": auto_quantize_bits}, # The average bits of quantized weights + forward_step=lambda model, batch: model(**batch), + quantization_formats=[quant_algo1, quant_algo2,...] + [None], + num_score_steps=min( + num_calib_steps=len(calib_dataloader), + len(calib_dataloader), 128 // batch_size + ), # Limit the number of score steps to avoid long calibration time + verbose=True, + ) +``` + +### Export Quantized Model + +After the model is quantized, it can be exported to a TensorRT LLM checkpoint, which includes + +- One json file recording the model structure and metadata, and +- One or several rank weight files storing quantized model weights and scaling factors. + +The export API is + +```python +from modelopt.torch.export import export_tensorrt_llm_checkpoint + +with torch.inference_mode(): + export_tensorrt_llm_checkpoint( + model, # The quantized model. + decoder_type, # The type of the model as str, e.g gptj, llama or gptnext. + dtype, # The exported weights data type as torch.dtype. + export_dir, # The directory where the exported files will be stored. + inference_tensor_parallel=tp_size, # The tensor parallelism size for inference. + inference_pipeline_parallel=pp_size, # The pipeline parallelism size for inference. + ) +``` diff --git a/examples/quantization/quantize.py b/examples/quantization/quantize.py new file mode 100644 index 000000000000..29c2fc5ca179 --- /dev/null +++ b/examples/quantization/quantize.py @@ -0,0 +1,208 @@ +import argparse + +import torch.multiprocessing as mp + +from tensorrt_llm.quantization import (quantize_and_export, + quantize_nemo_and_export) + +if __name__ == "__main__": + mp.set_start_method("spawn", force=True) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model_dir", + help="Specify where the HuggingFace model is", + default=None) + parser.add_argument('--nemo_ckpt_path', + help="Specify where the NeMo checkpoint is", + default=None) + parser.add_argument( + '--decoder_type', + type=str, + default='gptnext', + choices=['gptnext', 'llama'], + help="Decoder type; effective for NeMo checkpoint only.") + parser.add_argument( + '--device', + help= + "The device to run calibration; effective for HuggingFace model only.", + default='cuda', + choices=['cuda', 'cpu']) + parser.add_argument( + "--device_map", + help="How to map the model on the devices", + default="auto", + choices=["auto", "sequential", "cpu", "gpu"], + ) + parser.add_argument( + '--calib_dataset', + type=str, + default='cnn_dailymail', + help= + "The huggingface dataset name or the local directory of the dataset for calibration." + ) + parser.add_argument( + '--calib_tp_size', + type=int, + default=1, + help= + "Tensor parallel size for calibration; effective for NeMo checkpoint only." + ) + parser.add_argument( + '--calib_pp_size', + type=int, + default=1, + help= + "Pipeline parallel size for calibration; effective for NeMo checkpoint only." + ) + + parser.add_argument( + '--dtype', + type=str, + default='auto', + choices=['auto', 'float16', 'bfloat16', 'float32'], + help= + "The data type for the model weights and activations of the non-quantized part, e.g., embedding and lm_head. " + "If 'auto', the data type is automatically inferred from the source model; " + "however, if the source dtype is float32, it is converted to float16.") + parser.add_argument( + "--qformat", + help="Quantization format.", + default="full_prec", + choices=[ + "nvfp4", + "fp8", + "fp8_pc_pt", + "int8_sq", + "int4_awq", + "w4a8_awq", + "int8_wo", + "int4_wo", + "full_prec", + ], + ) + parser.add_argument( + "--seed", + help="Seed the generate random numbers, the value will be used to call" + "random.seed(value) and numpy.random.seed(value)", + type=int, + default=1234) + parser.add_argument("--tokenizer_max_seq_length", + help="Max sequence length to init the tokenizers", + type=int, + default=2048) + + parser.add_argument("--batch_size", + help="Batch size for calibration.", + type=int, + default=1) + parser.add_argument("--calib_size", + help="Number of samples for calibration.", + type=int, + default=512) + parser.add_argument("--calib_max_seq_length", + help="Max sequence length for calibration", + type=int, + default=512) + parser.add_argument("--output_dir", default="exported_model") + parser.add_argument("--tp_size", type=int, default=1) + parser.add_argument("--pp_size", type=int, default=1) + parser.add_argument("--cp_size", type=int, default=1) + parser.add_argument("--awq_block_size", type=int, default=128) + parser.add_argument("--kv_cache_dtype", + help="KV Cache dtype.", + default=None, + choices=["int8", "fp8", None]) + parser.add_argument("--quantize_lm_head", + action='store_true', + default=False) + # Medusa + parser.add_argument('--num_medusa_heads', type=int, default=4) + parser.add_argument('--num_medusa_layers', type=int, default=1) + parser.add_argument('--max_draft_len', type=int, default=63) + parser.add_argument('--medusa_hidden_act', type=str, default="silu") + parser.add_argument('--medusa_model_dir', type=str, default=None) + parser.add_argument('--quant_medusa_head', + default=False, + action='store_true', + help="whether to quantize the weights of medusa heads") + + # auto quantization + parser.add_argument( + '--autoq_format', + default=None, + type=str, + help= + "Specific quantization algorithms will be searched in auto quantization." + "The algorithm must in ['fp8', 'int4_awq', 'w4a8_awq', 'int8_sq']." + "You can use ',' to separate more than one quantization algorithms(e.g. --autoq_format fp8,int4_awq,w4a8_awq)." + "Notice: fp8 and int8_sq can't be used at the same time.") + parser.add_argument( + '--auto_quantize_bits', + type=float, + default=None, + help="Effective bits constraint for auto quantization. If not set, " + "regular quantization without auto quantization search will be applied." + "You can't set it lower than the num_bits of most aggressive quantization format." + "For example, if 'int4_awq' is in autoq_format, it can't be lower than 4.0." + ) + + args = parser.parse_args() + + # auto_quantize_bits check + if args.autoq_format: + lower_bound, upper_bound = 4 if '4' in args.autoq_format else 8, 16 + if args.auto_quantize_bits is None or args.auto_quantize_bits < lower_bound or args.auto_quantize_bits > upper_bound: + print( + f"invalid auto_quantize_bits value, will be set to {lower_bound}" + ) + args.auto_quantize_bits = lower_bound + + if args.model_dir is not None: + quantize_and_export( + model_dir=args.model_dir, + device=args.device, + calib_dataset=args.calib_dataset, + dtype=args.dtype, + qformat=args.qformat + if args.auto_quantize_bits is None else args.autoq_format, + kv_cache_dtype=args.kv_cache_dtype, + calib_size=args.calib_size, + batch_size=args.batch_size, + calib_max_seq_length=args.calib_max_seq_length, + awq_block_size=args.awq_block_size, + output_dir=args.output_dir, + tp_size=args.tp_size, + pp_size=args.pp_size, + cp_size=args.cp_size, + seed=args.seed, + tokenizer_max_seq_length=args.tokenizer_max_seq_length, + num_medusa_heads=args.num_medusa_heads, + num_medusa_layers=args.num_medusa_layers, + max_draft_len=args.max_draft_len, + medusa_hidden_act=args.medusa_hidden_act, + medusa_model_dir=args.medusa_model_dir, + quant_medusa_head=args.quant_medusa_head, + auto_quantize_bits=args.auto_quantize_bits, + device_map=args.device_map, + quantize_lm_head=args.quantize_lm_head) + elif args.nemo_ckpt_path is not None: + quantize_nemo_and_export(nemo_ckpt_path=args.nemo_ckpt_path, + decoder_type=args.decoder_type, + calib_dataset=args.calib_dataset, + calib_tp_size=args.calib_tp_size, + calib_pp_size=args.calib_pp_size, + dtype=args.dtype, + qformat=args.qformat, + kv_cache_dtype=args.kv_cache_dtype, + calib_size=args.calib_size, + batch_size=args.batch_size, + calib_max_seq_length=args.calib_max_seq_length, + awq_block_size=args.awq_block_size, + output_dir=args.output_dir, + tp_size=args.tp_size, + pp_size=args.pp_size, + cp_size=args.cp_size, + seed=args.seed) + else: + raise ValueError( + "One of source checkpoint (model_dir, nemo_ckpt_path) must be specified" + ) diff --git a/examples/run.py b/examples/run.py new file mode 100755 index 000000000000..7ce36bbe9848 --- /dev/null +++ b/examples/run.py @@ -0,0 +1,710 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import ast +import csv +import os +from pathlib import Path +from typing import List, Optional + +import numpy as np +import torch +from utils import (DEFAULT_HF_MODEL_DIRS, DEFAULT_PROMPT_TEMPLATES, + add_common_args, get_beam_width_array, load_tokenizer, + prepare_enc_dec_inputs, read_model_name, + supports_inflight_batching, throttle_generator) + +import tensorrt_llm +import tensorrt_llm.profiler +from tensorrt_llm.logger import logger +from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelRunner + +if PYTHON_BINDINGS: + from tensorrt_llm.runtime import ModelRunnerCpp + +from ngram.run_dtm_ngram import run_dtm_ngram + + +def parse_arguments(args=None): + # see `add_common_args` for extended list of arguments + parser = argparse.ArgumentParser() + parser.add_argument('--max_input_length', type=int, default=923) + parser.add_argument('--max_output_len', type=int, required=True) + parser.add_argument( + '--draft_engine_dir', + type=str, + default=None, + help='Path to engine of draft model in Draft-Target-Model mode.') + parser.add_argument( + '--input_text', + type=str, + nargs='+', + default=["Born in north-east France, Soyer trained as a"]) + parser.add_argument( + '--input_file', + type=str, + help= + 'CSV or Numpy file containing tokenized input. Alternative to text input.', + default=None) + parser.add_argument('--multimodal_input_file', + type=str, + help='Path to multimodal input file.') + parser.add_argument( + '--input_token_extra_ids', + type=int, + nargs='+', + help= + 'Input token extra ids for using p-tuning and KV Cache reuse together (only available with cpp session).', + default=None) + parser.add_argument( + '--input_token_extra_ids_file', + type=str, + help= + 'CSV or Numpy file containing input token extra ids file. Alternative to text input (only available with cpp session).', + default=None) + parser.add_argument('--output_csv', + type=str, + help='CSV file where the tokenized output is stored.', + default=None) + parser.add_argument('--output_npy', + type=str, + help='Numpy file where the tokenized output is stored.', + default=None) + parser.add_argument('--output_generation_logits', + default=False, + action='store_true', + help="Enable gathering generation logits.") + parser.add_argument( + '--output_logits_npy', + type=str, + help= + 'Numpy file where the generation logits are stored. Use only when num_beams==1', + default=None) + parser.add_argument('--output_log_probs_npy', + type=str, + help='Numpy file where the log_probs are stored', + default=None) + parser.add_argument('--output_cum_log_probs_npy', + type=str, + help='Numpy file where the cum_log_probs are stored', + default=None) + parser.add_argument( + '--run_profiling', + default=False, + action='store_true', + help="Run several 10 iterations to profile the inference latencies.") + parser.add_argument( + '--fail_fast_on_attention_window_too_large', + action='store_true', + default=False, + help= + 'Exit with runtime error when attention window is too large to fit even a single sequence in the KV cache.' + ) + + parser = add_common_args(parser) + + return parser.parse_args(args=args) + + +def parse_input(tokenizer, + input_text=None, + prompt_template=None, + input_file=None, + add_special_tokens=True, + max_input_length=923, + pad_id=None, + num_prepend_vtokens=[], + model_name=None, + model_version=None): + if pad_id is None: + pad_id = tokenizer.pad_token_id + + batch_input_ids = [] + if input_file is None: + if 'whisper' in model_name.lower(): + batch_input_ids.append(tokenizer.prefix_tokens) + else: + for curr_text in input_text: + if prompt_template is not None: + curr_text = prompt_template.format(input_text=curr_text) + input_ids = tokenizer.encode( + curr_text, + add_special_tokens=add_special_tokens, + truncation=True, + max_length=max_input_length) + batch_input_ids.append(input_ids) + else: + if input_file.endswith('.csv'): + with open(input_file, 'r') as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + for line in csv_reader: + input_ids = np.array(line, dtype='int32') + batch_input_ids.append(input_ids[-max_input_length:]) + elif input_file.endswith('.npy'): + inputs = np.load(input_file) + for row in inputs: + input_ids = row[row != pad_id] + batch_input_ids.append(input_ids[-max_input_length:]) + + elif input_file.endswith('.txt'): + with open(input_file, 'r', encoding='utf-8', + errors='replace') as txt_file: + input_text = txt_file.readlines() + batch_input_ids = tokenizer( + input_text, + add_special_tokens=add_special_tokens, + truncation=True, + max_length=max_input_length)["input_ids"] + else: + print('Input file format not supported.') + raise SystemExit + + if num_prepend_vtokens: + assert len(num_prepend_vtokens) == len(batch_input_ids) + base_vocab_size = tokenizer.vocab_size + for i, length in enumerate(num_prepend_vtokens): + batch_input_ids[i] = list( + range(base_vocab_size, + base_vocab_size + length)) + batch_input_ids[i] + + if input_file is None and 'GLM' in model_name and model_version == 'glm': + for ids in batch_input_ids: + ids.append(tokenizer.sop_token_id) + + batch_input_ids = [ + torch.tensor(x, dtype=torch.int32) for x in batch_input_ids + ] + + logger.debug(f"Input token ids (batch_size = {len(batch_input_ids)}):") + for i, input_ids in enumerate(batch_input_ids): + logger.debug(f"Request {i}: {input_ids.tolist()}") + + return batch_input_ids + + +def parse_input_token_extra_ids(prompt_table_path, kv_cache_enable_block_reuse, + input_token_extra_ids, + input_token_extra_ids_file, max_input_length): + batch_extra_ids = None + if prompt_table_path and kv_cache_enable_block_reuse: + assert input_token_extra_ids or input_token_extra_ids_file, \ + "Input token extra ids must be provided when p-tuning and KV Cache reuse are both enabled" + batch_extra_ids = [] + if input_token_extra_ids_file: + if input_token_extra_ids_file.endswith('.csv'): + with open(input_token_extra_ids_file, 'r') as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + for line in csv_reader: + extra_ids = [int(num) for num in line] + batch_extra_ids.append(extra_ids[-max_input_length:]) + elif input_token_extra_ids_file.endswith('.npy'): + inputs = np.load(input_token_extra_ids_file) + for extra_ids in inputs: + batch_extra_ids.append(extra_ids[-max_input_length:]) + else: + print('Input file format not supported.') + raise SystemExit + else: + batch_extra_ids.append(input_token_extra_ids) + return batch_extra_ids + + +def print_output(tokenizer, + output_ids: torch.Tensor, + input_lengths: List[int], + sequence_lengths: torch.Tensor, + output_csv: Optional[str] = None, + output_npy: Optional[str] = None, + context_logits: Optional[torch.Tensor] = None, + generation_logits: Optional[torch.Tensor] = None, + cum_log_probs: Optional[torch.Tensor] = None, + log_probs: Optional[torch.Tensor] = None, + output_logits_npy: Optional[str] = None, + output_cum_log_probs_npy: Optional[str] = None, + output_log_probs_npy: Optional[str] = None): + num_output_sents, num_beams, _ = output_ids.size() + batch_size = len(input_lengths) + num_return_sequences = num_output_sents // batch_size + + if output_csv is None and output_npy is None and tokenizer is not None: + for i in range(batch_size * num_return_sequences): + batch_idx = i // num_return_sequences + seq_idx = i % num_return_sequences + inputs = output_ids[i][0][:input_lengths[batch_idx]].tolist() + input_text = tokenizer.decode(inputs) + if seq_idx == 0: + print(f'Input [Text {batch_idx}]: \"{input_text}\"') + + for beam in range(num_beams): + output_begin = input_lengths[batch_idx] + output_end = sequence_lengths[i][beam] + outputs = output_ids[i][beam][output_begin:output_end].tolist() + output_text = tokenizer.decode(outputs) + index_str = (f'Text {batch_idx} Seq {seq_idx} Beam {beam}' + if num_return_sequences > 1 else + f'Text {batch_idx} Beam {beam}') + print(f'Output [{index_str}]: \"{output_text}\"') + logger.debug(str(outputs)) + + output_ids = output_ids.reshape((-1, output_ids.size(2))) + + if output_csv is not None: + output_file = Path(output_csv) + output_file.parent.mkdir(exist_ok=True, parents=True) + outputs = output_ids.tolist() + with open(output_file, 'w') as csv_file: + writer = csv.writer(csv_file, delimiter=',') + writer.writerows(outputs) + + if output_npy is not None: + output_file = Path(output_npy) + output_file.parent.mkdir(exist_ok=True, parents=True) + outputs = np.array(output_ids.cpu().contiguous(), dtype='int32') + np.save(output_file, outputs) + + # Save context logits + if context_logits is not None and output_logits_npy is not None: + context_logits = torch.cat(context_logits, axis=0) + vocab_size_padded = context_logits.shape[-1] + context_logits = context_logits.reshape([1, -1, vocab_size_padded]) + + output_context_logits_npy = output_logits_npy.split( + '.npy')[0] + "_context" + output_context_logits_file = Path(output_context_logits_npy) + context_outputs = np.array( + context_logits.squeeze(0).cpu().contiguous(), + dtype='float32') # [promptLengthSum, vocabSize] + np.save(output_context_logits_file, context_outputs) + + # Save generation logits + if generation_logits is not None and output_logits_npy is not None and num_beams == 1: + output_generation_logits_npy = output_logits_npy.split( + '.npy')[0] + "_generation" + output_generation_logits_file = Path(output_generation_logits_npy) + generation_outputs = np.array(generation_logits.cpu().contiguous(), + dtype='float32') + np.save(output_generation_logits_file, generation_outputs) + + # Save cum log probs + if cum_log_probs is not None and output_cum_log_probs_npy is not None: + cum_log_probs_file = Path(output_cum_log_probs_npy) + cum_log_probs_outputs = np.array(cum_log_probs.cpu().contiguous(), + dtype='float32') + np.save(cum_log_probs_file, cum_log_probs_outputs) + + # Save cum log probs + if log_probs is not None and output_log_probs_npy is not None: + log_probs_file = Path(output_log_probs_npy) + log_probs_outputs = np.array(log_probs.cpu().contiguous(), + dtype='float32') + np.save(log_probs_file, log_probs_outputs) + + +def main(args): + runtime_rank = tensorrt_llm.mpi_rank() + logger.set_level(args.log_level) + + # different handling if encoder-decoder models + is_enc_dec = {'encoder', 'decoder'}.issubset({ + name + for name in os.listdir(args.engine_dir) + if os.path.isdir(os.path.join(args.engine_dir, name)) + }) + if is_enc_dec: + logger.warning( + "This path is an encoder-decoder model. Using different handling.") + assert not args.use_py_session, "Encoder-decoder models don't have a unified python runtime, please use its own examples/models/core/enc_dec/run.py instead." + + model_name, model_version = read_model_name( + args.engine_dir if not is_enc_dec else os.path. + join(args.engine_dir, 'encoder')) + + if args.tokenizer_dir is None and model_name in DEFAULT_HF_MODEL_DIRS: + logger.warning( + "tokenizer_dir is not specified. Try to infer from model_name, but this may be incorrect." + ) + args.tokenizer_dir = DEFAULT_HF_MODEL_DIRS[model_name] + + tokenizer, pad_id, end_id = load_tokenizer( + tokenizer_dir=args.tokenizer_dir, + vocab_file=args.vocab_file, + model_name=model_name, + model_version=model_version, + tokenizer_type=args.tokenizer_type, + ) + + if args.end_id: + end_id = args.end_id + + prompt_template = None + if args.use_prompt_template and model_name in DEFAULT_PROMPT_TEMPLATES: + prompt_template = DEFAULT_PROMPT_TEMPLATES[model_name] + + batch_input_ids = parse_input(tokenizer=tokenizer, + input_text=args.input_text, + prompt_template=prompt_template, + input_file=args.input_file, + add_special_tokens=args.add_special_tokens, + max_input_length=args.max_input_length, + pad_id=pad_id, + num_prepend_vtokens=args.num_prepend_vtokens, + model_name=model_name, + model_version=model_version) + + stop_words_list = None + if args.stop_words: + stop_words_list = tensorrt_llm.runtime.decode_words_list( + args.stop_words, tokenizer) + if model_version == 'glm4': # add default stop token ids for GLM-4 + glm4_stop_ids = [[151329], [151336], [151338]] + if stop_words_list is None: + stop_words_list = [glm4_stop_ids] * len(batch_input_ids) + else: + for req_stop_words_list in stop_words_list: + req_stop_words_list.extend(glm4_stop_ids) + + bad_words_list = None + if args.bad_words: + bad_words_list = tensorrt_llm.runtime.decode_words_list( + args.bad_words, tokenizer) + + if is_enc_dec: + encoder_input_ids, encoder_input_features, encoder_output_lengths, decoder_input_ids = prepare_enc_dec_inputs( + batch_input_ids, model_name, args.engine_dir, + args.multimodal_input_file) + + input_token_extra_ids = parse_input_token_extra_ids( + args.prompt_table_path, args.kv_cache_enable_block_reuse, + args.input_token_extra_ids, args.input_token_extra_ids_file, + args.max_input_length) + + input_lengths = [x.size(0) for x in decoder_input_ids + ] if is_enc_dec else [x.size(0) for x in batch_input_ids] + + encoder_input_lengths = [ + x.size(0) for x in (encoder_input_features or encoder_input_ids) + ] if is_enc_dec else None + + if args.beam_width_array is not None: + logger.info("Enable Variable-Beam-Width-Search (VBWS)") + assert not args.use_py_session, "`--use_py_session` is not supported in VBWS." + args.beam_width_array, args.num_beams = get_beam_width_array( + args.beam_width_array) + + if not args.use_py_session and not supports_inflight_batching( + os.path.join(args.engine_dir, "decoder") if is_enc_dec else args. + engine_dir): + logger.warning( + "The given engine does not support in-flight batching, fallback to python session" + ) + args.use_py_session = True + + if not PYTHON_BINDINGS and not args.use_py_session: + logger.warning( + "Python bindings of C++ session is unavailable, fallback to Python session." + ) + args.use_py_session = True + if args.debug_mode and not args.use_py_session: + logger.warning( + "Debug mode is not supported in C++ session for now, fallback to Python session." + ) + args.use_py_session = True + if args.return_all_generated_tokens and args.use_py_session: + raise ValueError( + "Returning all the generated tokens at each step is not supported in the Python session, use C++ session instead." + ) + if (not args.return_all_generated_tokens) and args.streaming and ( + args.num_beams > 1): + logger.warning( + "Setting return_all_generated_tokens to True since streaming AND beam search are done simultaneously. " + "Returning the full beams at each streaming step is needed because beam search + streaming can change previous outputs. " + "WARNING: using this option may increase network usage significantly (quadratically w.r.t output length)." + ) + args.return_all_generated_tokens = True + + logger.info(f"Using {'Python' if args.use_py_session else 'C++'} session") + + if args.draft_target_model_config is not None or args.ngram_config is not None: + # Speculative-Decoding of Draft-Target-Model (DTM) and NGram + # If the parameters of `runner_kwargs` and `runner.generate()` in the "else" branch change, the same change should be done for `examples/ngram/run_dtm_ngram.py` + assert args.kv_cache_enable_block_reuse, "`--kv_cache_enable_block_reuse` must be specified in speculative decoding." + assert not args.use_py_session, "`--use_py_session` is not supported in Speculative decoding." + assert not is_enc_dec, "Encoder-Decoder model is not supported in Speculative decoding." + assert args.num_beams == 1, "`--num_beams>1` is not supported in Speculative decoding." + + outputs = run_dtm_ngram(batch_input_ids, args, runtime_rank, end_id, + pad_id, stop_words_list, bad_words_list, + len(tokenizer)) + if not args.streaming: # Unpack runner from the return value in No-Streaming mode + outputs, runner = list(outputs)[0] + + else: # Normal run + runner_cls = ModelRunner if args.use_py_session else ModelRunnerCpp + runner_kwargs = dict( + engine_dir=args.engine_dir, + lora_dir=args.lora_dir, + rank=runtime_rank, + debug_mode=args.debug_mode, + lora_ckpt_source=args.lora_ckpt_source, + gpu_weights_percent=args.gpu_weights_percent, + max_output_len=args.max_output_len, + enable_context_fmha_fp32_acc=args.enable_context_fmha_fp32_acc, + fail_fast_on_attention_window_too_large=args. + fail_fast_on_attention_window_too_large, + ) + if args.medusa_choices is not None: + args.medusa_choices = ast.literal_eval(args.medusa_choices) + assert args.temperature == 1.0, "Medusa should use temperature == 1.0" + assert args.num_beams == 1, "Medusa should use num_beams == 1" + runner_kwargs.update(medusa_choices=args.medusa_choices) + if args.eagle_choices is not None or args.eagle_posterior_threshold is not None or args.eagle_use_dynamic_tree: + assert args.num_beams == 1, "Eagle should use num_beams == 1" + assert not args.use_py_session, "Eagle does not support py session" + if args.eagle_choices is not None and not args.eagle_use_dynamic_tree: + args.eagle_choices = ast.literal_eval(args.eagle_choices) + runner_kwargs.update(eagle_choices=args.eagle_choices) + if args.eagle_posterior_threshold is not None: + runner_kwargs.update( + eagle_posterior_threshold=args.eagle_posterior_threshold) + if args.eagle_use_dynamic_tree: + runner_kwargs.update( + eagle_use_dynamic_tree=args.eagle_use_dynamic_tree) + assert args.eagle_dynamic_tree_max_top_k is not None and args.eagle_dynamic_tree_max_top_k > 0 + runner_kwargs.update(eagle_dynamic_tree_max_top_k=args. + eagle_dynamic_tree_max_top_k) + if args.lookahead_config is not None: + args.lookahead_config = ast.literal_eval(args.lookahead_config) + assert len( + args.lookahead_config + ) == 3, "Lookahead needs [max_window_size, max_ngram_size, max_verification_set_size]" + runner_kwargs.update(lookahead_config=args.lookahead_config) + if not args.use_py_session: + runner_kwargs.update( + is_enc_dec=is_enc_dec, + max_batch_size=len(batch_input_ids), + max_input_len=max( + encoder_input_lengths if is_enc_dec else input_lengths), + max_beam_width=args.num_beams, + max_attention_window_size=args.max_attention_window_size, + sink_token_length=args.sink_token_length, + max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, + kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, + kv_cache_free_gpu_memory_fraction=args. + kv_cache_free_gpu_memory_fraction, + cross_kv_cache_fraction=args.cross_kv_cache_fraction + if is_enc_dec else None, + enable_chunked_context=args.enable_chunked_context, + multi_block_mode=args.multi_block_mode, + cuda_graph_mode=args.cuda_graph_mode, + gather_generation_logits=args.output_generation_logits, + use_variable_beam_width_search=(args.beam_width_array + is not None), + ) + runner = runner_cls.from_dir(**runner_kwargs) + + with torch.no_grad(): + outputs = runner.generate( + batch_input_ids=decoder_input_ids + if is_enc_dec else batch_input_ids, + encoder_input_ids=encoder_input_ids if is_enc_dec else None, + encoder_input_features=encoder_input_features + if is_enc_dec else None, + encoder_output_lengths=encoder_output_lengths + if is_enc_dec else None, + max_new_tokens=args.max_output_len, + max_attention_window_size=args.max_attention_window_size, + sink_token_length=args.sink_token_length, + end_id=end_id, + pad_id=pad_id, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + num_beams=args.num_beams, + num_return_sequences=args.num_return_sequences, + length_penalty=args.length_penalty, + early_stopping=args.early_stopping, + beam_width_array=args.beam_width_array, + repetition_penalty=args.repetition_penalty, + presence_penalty=args.presence_penalty, + frequency_penalty=args.frequency_penalty, + prompt_ignore_length=args.prompt_ignore_length, + min_p=args.min_p, + stop_words_list=stop_words_list, + bad_words_list=bad_words_list, + output_cum_log_probs=(args.output_cum_log_probs_npy != None), + output_log_probs=(args.output_log_probs_npy != None), + random_seed=args.random_seed, + lora_uids=args.lora_task_uids, + prompt_table=args.prompt_table_path, + prompt_tasks=args.prompt_tasks, + streaming=args.streaming, + output_sequence_lengths=True, + output_generation_logits=args.output_generation_logits, + no_repeat_ngram_size=args.no_repeat_ngram_size, + return_dict=True, + medusa_choices=args.medusa_choices, + eagle_choices=args.eagle_choices, + return_all_generated_tokens=args.return_all_generated_tokens, + input_token_extra_ids=input_token_extra_ids, + fail_fast_on_attention_window_too_large=args. + fail_fast_on_attention_window_too_large, + language_adapter_uids=args.language_task_uids) + torch.cuda.synchronize() + + # Receive output, print to screen or save to file + if args.streaming: + for curr_outputs in throttle_generator(outputs, + args.streaming_interval): + if runtime_rank == 0: + output_ids = curr_outputs['output_ids'] + sequence_lengths = curr_outputs['sequence_lengths'] + cum_log_probs = None + log_probs = None + if args.output_cum_log_probs_npy is not None: + cum_log_probs = curr_outputs['cum_log_probs'] + if args.output_log_probs_npy is not None: + log_probs = curr_outputs['log_probs'] + print_output( + tokenizer, + output_ids, + input_lengths, + sequence_lengths, + output_csv=args.output_csv, + output_npy=args.output_npy, + cum_log_probs=cum_log_probs, + log_probs=log_probs, + output_cum_log_probs_npy=args.output_cum_log_probs_npy, + output_log_probs_npy=args.output_log_probs_npy) + else: + if runtime_rank == 0: + output_ids = outputs['output_ids'] + sequence_lengths = outputs['sequence_lengths'] + context_logits = None + generation_logits = None + cum_log_probs = None + log_probs = None + if runner.gather_context_logits: + context_logits = outputs['context_logits'] + if runner.gather_generation_logits or args.output_generation_logits: + generation_logits = outputs['generation_logits'] + if args.output_cum_log_probs_npy is not None: + cum_log_probs = outputs['cum_log_probs'] + if args.output_log_probs_npy is not None: + log_probs = outputs['log_probs'] + print_output(tokenizer, + output_ids, + input_lengths, + sequence_lengths, + output_csv=args.output_csv, + output_npy=args.output_npy, + context_logits=context_logits, + generation_logits=generation_logits, + output_logits_npy=args.output_logits_npy, + cum_log_probs=cum_log_probs, + log_probs=log_probs, + output_cum_log_probs_npy=args.output_cum_log_probs_npy, + output_log_probs_npy=args.output_log_probs_npy) + + # Profiling + if args.run_profiling: + ite = 10 + # warmup + for _ in range(ite): + with torch.no_grad(): + outputs = runner.generate( + batch_input_ids, + max_new_tokens=args.max_output_len, + max_attention_window_size=args.max_attention_window_size, + end_id=end_id, + pad_id=pad_id, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + num_beams=args.num_beams, + length_penalty=args.length_penalty, + early_stopping=args.early_stopping, + beam_width_array=args.beam_width_array, + repetition_penalty=args.repetition_penalty, + presence_penalty=args.presence_penalty, + frequency_penalty=args.frequency_penalty, + prompt_ignore_length=args.prompt_ignore_length, + min_p=args.min_p, + stop_words_list=stop_words_list, + bad_words_list=bad_words_list, + output_cum_log_probs=(args.output_cum_log_probs_npy + is not None), + output_log_probs=(args.output_log_probs_npy is not None), + random_seed=args.random_seed, + lora_uids=args.lora_task_uids, + lookahead_config=args.lookahead_config, + prompt_table=args.prompt_table_path, + prompt_tasks=args.prompt_tasks, + streaming=args.streaming, + output_sequence_lengths=True, + return_dict=True, + return_all_generated_tokens=args. + return_all_generated_tokens, + input_token_extra_ids=input_token_extra_ids) + torch.cuda.synchronize() + + tensorrt_llm.profiler.start("tmp") + for _ in range(ite): + with torch.no_grad(): + outputs = runner.generate( + batch_input_ids, + max_new_tokens=args.max_output_len, + max_attention_window_size=args.max_attention_window_size, + end_id=end_id, + pad_id=pad_id, + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + num_beams=args.num_beams, + length_penalty=args.length_penalty, + early_stopping=args.early_stopping, + beam_width_array=args.beam_width_array, + repetition_penalty=args.repetition_penalty, + presence_penalty=args.presence_penalty, + frequency_penalty=args.frequency_penalty, + prompt_ignore_length=args.prompt_ignore_length, + stop_words_list=stop_words_list, + bad_words_list=bad_words_list, + output_cum_log_probs=(args.output_cum_log_probs_npy + != None), + output_log_probs=(args.output_log_probs_npy != None), + random_seed=args.random_seed, + lora_uids=args.lora_task_uids, + prompt_table=args.prompt_table_path, + prompt_tasks=args.prompt_tasks, + streaming=args.streaming, + output_sequence_lengths=True, + return_dict=True, + return_all_generated_tokens=args. + return_all_generated_tokens, + input_token_extra_ids=input_token_extra_ids, + fail_fast_on_attention_window_too_large=args. + fail_fast_on_attention_window_too_large) + torch.cuda.synchronize() + tensorrt_llm.profiler.stop("tmp") + + print( + f"batch_size: {len(batch_input_ids)}, avg latency of {ite} iterations: : {tensorrt_llm.profiler.elapsed_time_in_sec('tmp') / ite} sec" + ) + + +if __name__ == '__main__': + args = parse_arguments() + main(args) diff --git a/examples/sample_weight_stripping/README.md b/examples/sample_weight_stripping/README.md new file mode 100644 index 000000000000..cb3c04404902 --- /dev/null +++ b/examples/sample_weight_stripping/README.md @@ -0,0 +1,275 @@ +# Sample Weight-Stripping + +> [!WARNING] +> The `convert_checkpoint.py` / `trtllm-build` / `run.py` workflow described +> below is **legacy** and will not receive new features. New projects should use +> [`trtllm-serve`](https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html) +> or the [LLM Python API](https://nvidia.github.io/TensorRT-LLM/llm-api/index.html) instead. + +## Table Of Contents + +- [Overview](#overview) + * [Build Weights Stripped Engine](#build-weights-stripped-engine) + * [Engine Refitter](#engine-refitter) +- [Prerequisites](#prerequisites) +- [Weight-Stripping Workflow Example](#weight-stripping-workflow-example) + * [GPT-J](#gpt-j) + * [Llama-7b INT4](#llama-7b-int4) + * [Llama-7b FP16 + WoQ INT8](#llama-7b-fp16-woq-int8) + * [Llama2-70b FP8 with TP=2](#llama2-70b-fp8-with-tp2) +- [Engine Plan File Size Results](#engine-plan-file-size-results) +- [Prototype](#prototype) + * [Checkpoint Pruner](#checkpoint-pruner) + * [Pruning a TensorRT LLM Checkpoint](#pruning-a-tensorrt-llm-checkpoint) + +## Overview + +This workflow introduces a new script `trtllm-refit`. `trtllm-refit` allows you to refit the generated engine with weights from any TensorRT LLM checkpoint matching the same architecture, so long as you build the engine as refittable or stripped. + +### Build Weights Stripped Engine +TensorRT can generate refittable engines with the same performance as the non-refittable ones when TensorRT builder optimize under the assumption that the engine will be refitted with weights identical to those provide at build time. Those refittable weights can be stripped to reduce the engine plan file size, with the option to subsequently supply them via the refit interface. + +New option `--strip_plan` is introduced in `trtllm-build` + +```bash +trtllm-build --strip_plan --checkpoint_dir ${CHECKPOINT_DIR} --output_dir ${ENGINE_DIR} ... +``` + +### Engine Refitter +The refitter allows you to refit an engine with weights in a TensorRT LLM checkpoint. It does this by doing a textual match between engine and checkpoint weight names. In order for the refitter to work, the engine must be built with refitting enabled. This can be accomplished by passing `--strip_plan` to `trtllm-build`. + +After building a stripped engine via `trtllm-build`, run + +```bash +trtllm-refit --checkpoint_dir ${CHECKPOINT_DIR} --engine_dir ${ENGINE_DIR} +``` + + +## Prerequisites + +Install [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM/blob/main/README.md) either through [pip](https://github.com/NVIDIA/TensorRT-LLM/blob/main/README.md#installation) or [from the source](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/installation/build-from-source.md). + +## Weight-Stripping Workflow Example + +### GPT-J + +1. Download the weights. +```bash +# 1. Weights & config +git clone https://huggingface.co/EleutherAI/gpt-j-6b +pushd gpt-j-6b && \ + rm -f pytorch_model.bin && \ + wget https://huggingface.co/EleutherAI/gpt-j-6b/resolve/main/pytorch_model.bin && \ +popd + +# 2. Vocab and merge table +wget https://huggingface.co/EleutherAI/gpt-j-6b/resolve/main/vocab.json +wget https://huggingface.co/EleutherAI/gpt-j-6b/resolve/main/merges.txt +``` + +2. Convert the Hugging Face checkpoint into TensorRT LLM format. +Run below command lines in [`examples/models/contrib/gpt`](../gptj) directory. +```bash +# Build a float16 checkpoint using HF weights. +python convert_checkpoint.py --model_dir ./gpt-j-6b \ + --dtype float16 \ + --output_dir ./trt_ckpt/gptj_fp16_tp1/ + +# Build an int8 weight-only checkpoint using HF weights. +python convert_checkpoint.py --model_dir ./gpt-j-6b \ + --dtype float16 \ + --use_weight_only \ + --weight_only_precision int8 \ + --output_dir ./trt_ckpt/gptj_int8_tp1/ + +``` + +3. Build the weights stripped engine. +```bash +# Build with --strip_plan. Requires TRT>=10.0.0 +trtllm-build --checkpoint_dir ./trt_ckpt/gptj_fp16_tp1/ \ + --output_dir ./trt_engines/gptj_fp16_tp1/ \ + --gemm_plugin float16 \ + --max_batch_size=32 \ + --max_input_len=1919 \ + --max_seq_len=2047 \ + --strip_plan +``` + +4. Refit the engine. The refit engine lives at `${ENGINE_DIR}.refit`. +```bash +# --checkpoint_dir points to the path of the weights you want refit, in this case the original weights. +trtllm-refit --checkpoint_dir ./trt_ckpt/gptj_fp16_tp1/ --engine_dir ./trt_engines/gptj_fp16_tp1/ --output_dir ./trt_engines/gptj_fp16_tp1.refit/ +``` + +5. Verify the engine. +```bash +# Run the summarization task. +python3 ../summarize.py --engine_dir ./trt_engines/gptj_fp16_tp1.refit \ + --hf_model_dir ./gpt-j-6b \ + --batch_size 1 \ + --test_trt_llm \ + --tensorrt_llm_rouge1_threshold 14 \ + --data_type fp16 \ + --check_accuracy +``` + +### Llama-7b INT4 + +1. Download the llama-7b-hf checkpoint and saved in /llm-models/llama-models/llama-7b-hf/. + +2. Calibrate the checkpoint and convert into TensorRT LLM format. +Run below command lines in [`examples/models/core/llama`](../models/core/llama) directory. +```bash +# Calibrate INT4 using AMMO. +python ../quantization/quantize.py --model_dir /llm-models/llama-models/llama-7b-hf/ \ + --dtype float16 \ + --qformat int4_awq \ + --awq_block_size 128 \ + --output_dir ./quantized_int4-awq \ + --calib_size 32 +``` + +3. Build the weights stripped engine. +```bash +# Build with --strip_plan. Requires TRT>=10.0.0 +trtllm-build --checkpoint_dir ./quantized_int4-awq \ + --strip_plan \ + --gemm_plugin float16 \ + --output_dir trt_int4_AWQ +``` + +4. Refit the engine. +```bash +trtllm-refit --checkpoint_dir ./quantized_int4-awq \ + --engine_dir trt_int4_AWQ \ + --output_dir trt_int4_AWQ_full_from_wtless +``` + +5. Verify the engine. +```bash +python3 ../summarize.py --engine_dir trt_int4_AWQ_full_from_wtless \ + --hf_model_dir /llm-models/llama-models/llama-7b-hf/ \ + --batch_size 1 \ + --test_trt_llm \ + --check_accuracy +``` + +### Llama-7b FP16 + WoQ INT8 + +1. Download the llama-7b-hf checkpoint and saved in /llm-models/llama-models/llama-7b-hf/. + +2. Convert the checkpoint into TensorRT LLM format. +Run below command lines in [`examples/models/core/llama`](../models/core/llama) directory. +```bash +python3 convert_checkpoint.py --model_dir /llm-models/llama-models/llama-7b-hf/ \ + --output_dir ./llama-7b-hf-fp16-woq \ + --dtype float16 \ + --use_weight_only \ + --weight_only_precision int8 +``` + +3. Build the weights stripped engine. +```bash +# Build with --strip_plan. Requires TRT>=10.0.0 +trtllm-build --checkpoint_dir ./llama-7b-hf-fp16-woq \ + --output_dir ./engines/llama-7b-hf-fp16-woq-1gpu-wtless \ + --strip_plan \ + --gemm_plugin float16 +``` + +4. Refit the engine. +```bash +trtllm-refit --checkpoint_dir ./llama-7b-hf-fp16-woq \ + --engine_dir ./engines/llama-7b-hf-fp16-woq-1gpu-wtless \ + --output_dir ./engines/llama-7b-hf-fp16-woq-1gpu-wtless-to-full +``` + +5. Verify the engine. +```bash +python3 ../summarize.py --engine_dir ./engines/llama-7b-hf-fp16-woq-1gpu-wtless-to-full \ + --hf_model_dir /llm-models/llama-models/llama-7b-hf/ \ + --batch_size 1 \ + --test_trt_llm \ + --check_accuracy +``` + + +### Llama2-70b FP8 with TP=2 + +1. Download the llama-v2-70b-hf checkpoint and saved in /llm-models/llama-models-v2/llama-v2-70b-hf/. + +2. Calibrate the checkpoint and convert into TensorRT LLM format. +Run below command lines in [`examples/models/core/llama`](../models/core/llama) directory. +```bash +# Calibrate FP8 using AMMO. +python ../quantization/quantize.py --model_dir /llm-models/llama-models-v2/llama-v2-70b-hf/ \ + --dtype float16 \ + --qformat fp8 \ + --kv_cache_dtype fp8 \ + --output_dir ./llama2-70b-hf-fp8-tp2 \ + --calib_size 512 \ + --tp_size 2 +``` + +3. Build the weights stripped engine. +```bash +trtllm-build --checkpoint_dir ./llama2-70b-hf-fp8-tp2 \ + --output_dir engines/llama2-70b-hf-fp8-tp2 \ + --gemm_plugin float16 \ + --workers 2 +``` + +4. Refit the engine. +```bash +trtllm-refit --checkpoint_dir ./llama2-70b-hf-fp8-tp2 \ + --engine_dir engines/llama2-70b-hf-fp8-tp2 \ + --output_dir engines/llama2-70b-hf-fp8-tp2.refit +``` + +5. Verify the engine. +```bash +python3 ../summarize.py --engine_dir engines/llama2-70b-hf-fp8-tp2.refit \ + --hf_model_dir /llm-models/llama-models-v2/llama-v2-70b-hf/ \ + --batch_size 1 \ + --test_trt_llm \ + --check_accuracy +``` + + +## Engine Plan File Size Results + +| **Model** | **Full Engine Plan Size** | **Weight-Stripped Engine Plan Size** | +|:---------:|:----------:|:----:| +|llama-7b INT4 | 3.7GB | 5.3MB | +|llama-7b FP16 + WoQ INT8 | 6.54GB | 28.69MB | +|llama2-70b FP8 + TP=2 | 64.78GB | 60.61MB | + +## Prototype +### Checkpoint Pruner +The checkpoint pruner allows you to strip `Conv` and `Gemm` weights out of a TensorRT LLM [checkpoint](https://nvidia.github.io/TensorRT-LLM/0.21.0/architecture/checkpoint.html). Since these make up the vast majority of weights, the pruner will decrease the size of your checkpoint up to 99%. + +When building an engine with a pruned checkpoint, TensorRT LLM fills in the missing weights with random ones. These weights should later be [refit](#engine-refitter) with the original weights to preserve the intended behavior. + +Building an engine from a pruned checkpoint will also allow the engine to be [refit](#engine-refitter). + +#### Pruning a TensorRT LLM Checkpoint + +1. Install [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM/blob/main/README.md) either through [pip](https://github.com/NVIDIA/TensorRT-LLM/blob/main/README.md#installation) or [from the source](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/installation/build-from-source.md). +2. Download a model of your choice and convert it to a TensorRT LLM checkpoint ([llama instructions](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/models/core/llama/README.md#usage)). +3. (Optional) Run the `trtllm-prune` command. +```bash +# Prunes the TRT-LLM checkpoint at ${CHECKPOINT_DIR}, and stores it in the directory ${CHECKPOINT_DIR}.pruned +trtllm-prune --checkpoint_dir ${CHECKPOINT_DIR} +``` + +The pruned checkpoint lives at `${CHECKPOINT_DIR}.pruned` by default, however, this can be overridden by issuing the `--out_dir` flag. + +4. Build the stripped engine. + +```bash +# From pruned checkpoint. +trtllm-build --checkpoint_dir ${CHECKPOINT_DIR}.pruned \ + --output_dir ${ENGINE_OUT_DIR} \ + ${EXTRA_ARGS} +``` diff --git a/examples/summarize.py b/examples/summarize.py new file mode 100644 index 000000000000..1f6f8979bb7b --- /dev/null +++ b/examples/summarize.py @@ -0,0 +1,944 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import ast +import os +from pathlib import Path + +import evaluate +import numpy as np +import torch +from datasets import load_dataset +from transformers import (AutoModel, AutoModelForCausalLM, + AutoModelForSeq2SeqLM, GenerationConfig) +from utils import (DEFAULT_HF_MODEL_DIRS, add_common_args, get_beam_width_array, + load_tokenizer, read_model_name, supports_inflight_batching) + +import tensorrt_llm +import tensorrt_llm.profiler as profiler +from tensorrt_llm._utils import mpi_broadcast, str_dtype_to_torch +from tensorrt_llm.builder import EngineConfig +from tensorrt_llm.functional import RopeEmbeddingUtils, RotaryScalingType +from tensorrt_llm.layers import MropeParams +from tensorrt_llm.logger import logger +from tensorrt_llm.models.qwen.utils import make_context +from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelRunner +from tensorrt_llm.tools.ppl import ppl + +if PYTHON_BINDINGS: + from tensorrt_llm.runtime import ModelRunnerCpp + +from ngram.run_dtm_ngram import run_dtm_ngram + + +def ensemble_mrope_params(batch_input_ids, max_position_embeddings, + rotary_embedding_dim, theta): + mrope_params = MropeParams() + batch_size = len(batch_input_ids) + + _, rotary_cos_sin = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( + num_pos=max_position_embeddings, + dim=rotary_embedding_dim, + theta=1000000.0, + scale_type=RotaryScalingType.mrope, + ) + rotary_cos_sin = torch.tensor(rotary_cos_sin).to(batch_input_ids[0].device) + rotary_cos_sin = rotary_cos_sin.reshape(max_position_embeddings, + int(rotary_embedding_dim / 2), 2) + + cos_ori = rotary_cos_sin[:, :, 0] + sin_ori = rotary_cos_sin[:, :, 1] + + mrope_position_ids_padding = torch.zeros( + (batch_size, max_position_embeddings), dtype=torch.int32) + for i in range(batch_size): + seq_len = batch_input_ids[i].shape[-1] + mrope_position_ids_padding[i, :seq_len] = torch.arange( + seq_len, device=batch_input_ids[i].device) + + cos = cos_ori[mrope_position_ids_padding].unsqueeze(-1) + sin = sin_ori[mrope_position_ids_padding].unsqueeze(-1) + + mrope_params.mrope_rotary_cos_sin = torch.concatenate( + (cos, sin), axis=-1).reshape(batch_size, -1) + mrope_params.mrope_position_deltas = torch.zeros( + [batch_size, 1], device=batch_input_ids[0].device) + + return mrope_params + + +def main(args): + is_integration_test = os.getenv('INTEGRATION_TEST', '0') == '1' + if is_integration_test: + logger.info( + "Running in integration test mode - will only run one batch and skip accuracy checks" + ) + logger.info( + "Setting max_ite=1 and check_accuracy=False for integration test") + args.max_ite = 1 + args.check_accuracy = False + + runtime_rank = tensorrt_llm.mpi_rank() + logger.set_level(args.log_level) + + test_hf = args.test_hf and runtime_rank == 0 # only run hf on rank 0 + test_trt_llm = args.test_trt_llm + model_name, model_version = read_model_name( + args.engine_dir if not test_hf else args.hf_model_dir, test_hf) + if args.hf_model_dir is None: + logger.warning( + "hf_model_dir is not specified. Try to infer from model_name, but this may be incorrect." + ) + if model_name in DEFAULT_HF_MODEL_DIRS: + args.hf_model_dir = DEFAULT_HF_MODEL_DIRS[model_name] + else: + args.hf_model_dir = None + if args.tokenizer_dir is None: + args.tokenizer_dir = args.hf_model_dir + + profiler.start('load tokenizer') + tokenizer, pad_id, end_id = load_tokenizer( + tokenizer_dir=args.tokenizer_dir, + vocab_file=args.vocab_file, + model_name=model_name, + model_version=model_version, + tokenizer_type=args.tokenizer_type, + ) + profiler.stop('load tokenizer') + logger.info( + f'Load tokenizer takes: {profiler.elapsed_time_in_sec("load tokenizer")} sec' + ) + + if args.eval_task == 'code_completion': + dataset_name = "openai_humaneval" + dataset_revision = None + dataset_input_key = 'prompt' + dataset_output_key = 'canonical_solution' + dataset_split = 'test' + elif args.eval_task == 'summarize': + dataset_name = "ccdv/cnn_dailymail" + dataset_revision = "3.0.0" + dataset_input_key = 'article' + dataset_output_key = 'highlights' + dataset_split = 'test' + elif args.eval_task == 'summarize_long': + dataset_name = "tau/zero_scrolls" + dataset_revision = 'squality' + dataset_input_key = 'input' + dataset_output_key = 'output' + dataset_split = 'validation' # only this split contains reference strings + elif args.eval_task == "eval_context_ppl": + dataset_name = "SlimPajama-6B" + dataset_revision = None + dataset_input_key = 'text' + dataset_output_key = 'text' + dataset_split = 'test' + args.output_len = 1 # Only want to compute the ppl of context + args.eval_ppl = True + logger.warning( + f"Run task '{args.eval_task}', setting 'output_len' to 1, and enable 'eval_ppl'." + ) + if args.dataset_dir is not None and isinstance(args.dataset_dir, str): + args.dataset_dir = args.dataset_dir.rstrip('/') + if args.dataset_dir.endswith(dataset_name): + dataset_name = args.dataset_dir + else: + dataset_name = f"{args.dataset_dir}/{dataset_name}" + dataset = load_dataset(dataset_name, + dataset_revision, + cache_dir=args.dataset_cache_dir, + split=dataset_split, + trust_remote_code=True) + dataset = dataset.shuffle(args.random_seed) + + max_batch_size = args.batch_size + + # runtime parameters + top_k = args.top_k + top_p = args.top_p + output_len = args.output_len + test_token_num = args.max_input_length + max_attention_window_size = args.max_attention_window_size + sink_token_length = args.sink_token_length + + if args.end_id: + end_id = args.end_id + + stop_words_list = None + if args.stop_words: + stop_words_list = tensorrt_llm.runtime.decode_words_list( + args.stop_words, tokenizer) + if model_version == 'glm4': # add default stop token ids for GLM-4 + glm4_stop_ids = [[151329], [151336], [151338]] + if stop_words_list is None: + stop_words_list = [glm4_stop_ids] * args.batch_size + else: + for req_stop_words_list in stop_words_list: + req_stop_words_list.extend(glm4_stop_ids) + + bad_words_list = None + if args.bad_words: + bad_words_list = tensorrt_llm.runtime.decode_words_list( + args.bad_words, tokenizer) + + if args.beam_width_array is not None: + logger.info("Use Variable-Beam-Width-Search") + args.beam_width_array, args.num_beams = get_beam_width_array( + args.beam_width_array) + + num_beams = args.num_beams + num_return_sequences = args.num_return_sequences + num_sequences = args.num_return_sequences or num_beams + assert num_beams == 1 or num_sequences <= num_beams + + temperature = args.temperature + length_penalty = args.length_penalty + early_stopping = args.early_stopping + beam_width_array = args.beam_width_array + repetition_penalty = args.repetition_penalty + presence_penalty = args.presence_penalty + frequency_penalty = args.frequency_penalty + prompt_ignore_length = args.prompt_ignore_length + random_seed = args.random_seed + torch.manual_seed(random_seed) + + output_dir = Path(args.output_dir) if args.output_dir else None + if output_dir is not None: + output_dir.mkdir(exist_ok=True, parents=True) + if test_trt_llm: + with (output_dir / 'trtllm.out').open('w') as f: + f.write(f'Engine path: {args.engine_dir}\n') + f.write(f'Tokenizer path: {args.tokenizer_dir}\n') + if test_hf: + with (output_dir / 'hf.out').open('w') as f: + f.write(f'Model path: {args.hf_model_dir}\n') + f.write(f'Tokenizer path: {args.tokenizer_dir}\n') + + rouge_dir = args.rouge_dir if args.rouge_dir and os.path.exists( + args.rouge_dir) else "rouge" + metric_tensorrt_llm = [ + evaluate.load(rouge_dir) for _ in range(num_sequences) + ] + metric_hf = [evaluate.load(rouge_dir) for _ in range(num_sequences)] + for i in range(num_sequences): + metric_tensorrt_llm[i].seed = 0 + metric_hf[i].seed = 0 + ppls_trt_llm = [[] for _ in range(num_sequences)] + ppls_hf = [[] for _ in range(num_sequences)] + + def _prepare_inputs(batch_input_texts, + eval_task='summarize', + add_special_tokens=True, + min_input_length=0): + batch_size = len(batch_input_texts) + append_str = ' TL;DR: ' if eval_task == 'summarize' else '' + batch_input_ids = [] + for i in range(batch_size): + curr_text = batch_input_texts[i] + append_str + curr_text = curr_text.strip().replace(" n't", "n't") + + # TODO: The below lines are used to be compatible with the original code; may need fix + if 'GLM' in model_name and model_version in ('chatglm2', + 'chatglm3'): + input_ids = tokenizer.encode(curr_text, + return_tensors='pt').squeeze(0) + input_ids = input_ids[:test_token_num] + elif 'qwen' in model_name.lower() and model_version == 'qwen': + # use make_content to generate prompt + system_prompt = "You are a useful assistant, please directly output the corresponding summary according to the article entered by the user." + _, input_id_list = make_context( + tokenizer=tokenizer, + query=curr_text, + history=[], + system=system_prompt, + max_input_length=test_token_num, + ) + input_ids = torch.tensor(input_id_list) + else: + if 'qwen' in model_name.lower() and 'qwen2' in model_version: + messages = [{ + "role": + "system", + "content": + "You are a helpful assistant, please summarize the article entered by the user with one or two sentences." + }, { + "role": "user", + "content": curr_text + }] + curr_text = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True) + input_ids = tokenizer.encode( + curr_text, + return_tensors='pt', + add_special_tokens=add_special_tokens, + truncation=True, + max_length=test_token_num).squeeze(0) + + if input_ids.numel() > min_input_length: + batch_input_ids.append(input_ids) + return batch_input_ids + + def eval_trt_llm(datapoint, + eval_task='summarize', + eval_ppl=False, + add_special_tokens=True, + min_input_length=0, + runner=None): + batch_size = len(datapoint[dataset_input_key]) + batch_input_ids = _prepare_inputs(datapoint[dataset_input_key], + eval_task=eval_task, + add_special_tokens=add_special_tokens, + min_input_length=min_input_length) + # Generate mrope params for qwen model + engine_config = EngineConfig.from_json_file( + f"{args.engine_dir}/config.json") + pretrain_config = engine_config.pretrained_config + mrope_params = None + if 'qwen' in model_name.lower(): + mrope_params = ensemble_mrope_params( + batch_input_ids, + max_position_embeddings=pretrain_config.max_position_embeddings, + rotary_embedding_dim=pretrain_config.rotary_embedding_dim, + theta=pretrain_config.rotary_base, + ) + + if batch_size == 0 or len(batch_input_ids) == 0: + return [], [], [], {} + input_lengths = [x.size(0) for x in batch_input_ids] + + if args.ngram_config is not None: + # Speculative decoding of NGram + outputs = run_dtm_ngram(batch_input_ids, + args, + runtime_rank, + end_id, + pad_id, + stop_words_list, + bad_words_list, + tokenizer.vocab_size, + target_runner=runner) + if not args.streaming: # Unpack runner from the return value in No-Streaming mode + outputs, runner = list(outputs)[0] + else: # Normal run + with torch.no_grad(): + outputs = runner.generate( + batch_input_ids, + max_new_tokens=output_len, + max_attention_window_size=max_attention_window_size, + sink_token_length=sink_token_length, + end_id=end_id, + pad_id=pad_id, + temperature=temperature, + top_k=top_k, + top_p=top_p, + stop_words_list=stop_words_list, + bad_words_list=bad_words_list, + num_beams=num_beams, + num_return_sequences=num_return_sequences, + length_penalty=length_penalty, + early_stopping=early_stopping, + beam_width_array=beam_width_array, + repetition_penalty=repetition_penalty, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + prompt_ignore_length=prompt_ignore_length, + lora_uids=args.lora_task_uids, + lookahead_config=args.lookahead_config, + output_sequence_lengths=True, + output_generation_logits=eval_ppl, + return_dict=True, + random_seed=random_seed, + medusa_choices=args.medusa_choices, + eagle_choices=args.eagle_choices, + mrope_params=mrope_params) + torch.cuda.synchronize() + + # Extract a list of tensors of shape beam_width x output_ids. + if runtime_rank == 0: + output_ids = outputs['output_ids'] + output_beams_list = [ + tokenizer.batch_decode(beam_tokens[:, input_lengths[i]:], + skip_special_tokens=True) + for i, beam_tokens in enumerate(output_ids) + ] + output_ids_list = [ + beam_tokens[:, input_lengths[i]:] + for i, beam_tokens in enumerate(output_ids) + ] + + ppls = [[] for _ in range(batch_size)] + lengths_info = { + 'input_lengths': input_lengths, + 'seq_lengths': outputs["sequence_lengths"].cpu().tolist(), + } + if eval_ppl: + seq_lengths = outputs['sequence_lengths'] + context_logits = outputs['context_logits'] + # Remove the first generation logits which are same to last + # context logits. + generation_logits = outputs['generation_logits'][:, :, 1:] + for batch_idx in range(batch_size): + # [batch, beam, step] + for beam_idx in range(num_sequences): + curr_len = seq_lengths[batch_idx, beam_idx] + curr_ctx_len = input_lengths[batch_idx] + curr_gen_len = curr_len - curr_ctx_len + + curr_ids = output_ids[batch_idx, beam_idx, 1:curr_len] + curr_logits = torch.cat([ + context_logits[batch_idx], + generation_logits[batch_idx, + beam_idx, :curr_gen_len - 1] + ], + dim=0) + curr_ppl = ppl(curr_logits, curr_ids) + logger.debug(f"TensorRT LLM PPL: {curr_ppl:.3f} | " + f"Generation length: {curr_gen_len}") + ppls[batch_idx].append(curr_ppl) + return output_beams_list, output_ids_list, ppls, lengths_info + return [], [], [], {} + + def eval_hf(datapoint, + eval_task='summarize', + eval_ppl=False, + add_special_tokens=True, + min_input_length=0): + batch_size = len(datapoint[dataset_input_key]) + if batch_size > 1: + logger.warning( + f"HF does not support batch_size > 1 to verify correctness due to padding. Current batch size is {batch_size}" + ) + batch_input_ids = _prepare_inputs(datapoint[dataset_input_key], + eval_task=eval_task, + add_special_tokens=add_special_tokens, + min_input_length=min_input_length) + batch_size = len(batch_input_ids) + if batch_size == 0: + return [], [], [], [[] for _ in range(batch_size)] + input_lengths = [x.size(0) for x in batch_input_ids] + # Left padding for HF + max_length = max(input_lengths) + paddings = [ + torch.ones(max_length - l, dtype=torch.int32) * pad_id + for l in input_lengths + ] + batch_input_ids = [ + torch.cat([pad, x]) for x, pad in zip(batch_input_ids, paddings) + ] + batch_input_ids = torch.stack(batch_input_ids) + batch_input_ids = batch_input_ids.cuda() + + # specialization for HF + if early_stopping in [0, 1]: + local_early_stopping = bool(early_stopping) + else: + local_early_stopping = "never" + + with torch.no_grad(): + hf_config = {} + if num_beams == 1: + hf_config.update({ + "top_k": top_k, + "top_p": top_p, + "do_sample": True, + }) + else: + hf_config.update({ + "num_beams": num_beams, + "early_stopping": local_early_stopping, + }) + + outputs = model.generate(batch_input_ids, + max_new_tokens=output_len, + num_return_sequences=num_sequences, + temperature=temperature, + eos_token_id=end_id, + pad_token_id=pad_id, + length_penalty=length_penalty, + output_scores=True, + return_dict_in_generate=True, + **hf_config) + if eval_ppl and batch_size == 1: + # model.generate cannot return context logits? + # Will cause additional latency + context_outputs = model(batch_input_ids) + + output_ids = outputs['sequences'] + tokens_list = output_ids[:, max_length:].tolist() + output_ids = output_ids.reshape([batch_size, num_sequences, -1]) + output_lines_list = [ + tokenizer.batch_decode(output_ids[:, i, max_length:], + skip_special_tokens=True) + for i in range(num_sequences) + ] + + ppls = [[] for _ in range(batch_size)] + if eval_ppl and batch_size == 1: + # Only for batch size of 1 + seq_lens = (output_ids + != end_id).logical_and(output_ids != pad_id).sum(dim=-1) + context_logits = context_outputs['logits'] + # Remove the first generation logits which are same to last context logits + generation_logits = outputs['scores'][1:] + # When output_len is 1, generation_logits would be () and lead to error if we do torch.stack + if len(generation_logits) == 0: + generation_logits = torch.empty( + [context_logits.shape[0], 0, context_logits.shape[-1]], + device=context_logits.device) + else: + generation_logits = torch.stack(generation_logits, dim=1) + _, max_gen_len, voc_size = generation_logits.size() + generation_logits = generation_logits.view(batch_size, num_beams, + max_gen_len, voc_size) + for batch_idx in range(batch_size): + for beam_idx in range(num_sequences): + curr_len = seq_lens[batch_idx, beam_idx] + curr_ctx_len = input_lengths[batch_idx] + curr_gen_len = curr_len - curr_ctx_len + + curr_ids = output_ids[batch_idx, beam_idx, 1:curr_len] + curr_logits = torch.cat([ + context_logits[batch_idx], + generation_logits[batch_idx, + beam_idx, :curr_gen_len - 1] + ], + dim=0) + curr_ppl = ppl(curr_logits, curr_ids) + logger.debug( + f"HF PPL: {curr_ppl:.3f} | Generation length: {curr_gen_len}" + ) + ppls[batch_idx].append(curr_ppl) + + return output_lines_list, tokens_list, ppls + + if test_trt_llm: + if not supports_inflight_batching(args.engine_dir): + logger.warning( + "The given engine does not support in-flight batching, fallback to python session" + ) + args.use_py_session = True + + if not PYTHON_BINDINGS and not args.use_py_session: + logger.warning( + "Python bindings of C++ session is unavailable, fallback to Python session." + ) + args.use_py_session = True + if args.return_all_generated_tokens: + raise ValueError( + "Returning all the generated tokens at each step is not supported in summarize.py" + ) + + logger.info( + f"Using {'Python' if args.use_py_session else 'C++'} session") + + runner_cls = ModelRunner if args.use_py_session else ModelRunnerCpp + runner_kwargs = dict( + engine_dir=args.engine_dir, + rank=runtime_rank, + debug_mode=args.debug_mode, + gpu_weights_percent=args.gpu_weights_percent, + enable_context_fmha_fp32_acc=args.enable_context_fmha_fp32_acc, + ) + if not args.use_py_session: + runner_kwargs.update( + lora_dir=args.lora_dir, + lora_ckpt_source=args.lora_ckpt_source, + max_batch_size=max_batch_size, + max_input_len=test_token_num, + max_output_len=output_len, + max_beam_width=num_beams, + max_attention_window_size=max_attention_window_size, + sink_token_length=sink_token_length, + max_tokens_in_paged_kv_cache=args.max_tokens_in_paged_kv_cache, + kv_cache_enable_block_reuse=args.kv_cache_enable_block_reuse, + kv_cache_free_gpu_memory_fraction=args. + kv_cache_free_gpu_memory_fraction, + enable_chunked_context=args.enable_chunked_context, + multi_block_mode=args.multi_block_mode, + cuda_graph_mode=args.cuda_graph_mode, + gather_generation_logits=args.eval_ppl, + use_gpu_direct_storage=args.use_gpu_direct_storage, + ) + + if args.medusa_choices is not None: + args.medusa_choices = ast.literal_eval(args.medusa_choices) + assert args.temperature == 1.0, "Medusa should use temperature == 1.0" + assert args.num_beams == 1, "Medusa should use num_beams == 1" + runner_kwargs.update(medusa_choices=args.medusa_choices) + if args.eagle_choices is not None or args.eagle_posterior_threshold is not None or args.eagle_use_dynamic_tree: + assert args.num_beams == 1, "Eagle should use num_beams == 1" + if args.eagle_choices is not None and not args.eagle_use_dynamic_tree: + args.eagle_choices = ast.literal_eval(args.eagle_choices) + runner_kwargs.update(eagle_choices=args.eagle_choices) + if args.eagle_posterior_threshold is not None: + runner_kwargs.update( + eagle_posterior_threshold=args.eagle_posterior_threshold) + if args.eagle_use_dynamic_tree: + runner_kwargs.update( + eagle_use_dynamic_tree=args.eagle_use_dynamic_tree) + assert args.eagle_dynamic_tree_max_top_k is not None and args.eagle_dynamic_tree_max_top_k > 0 + runner_kwargs.update(eagle_dynamic_tree_max_top_k=args. + eagle_dynamic_tree_max_top_k) + if args.lookahead_config is not None: + args.lookahead_config = ast.literal_eval(args.lookahead_config) + assert len( + args.lookahead_config + ) == 3, "Lookahead needs [max_window_size, max_ngram_size, max_verification_set_size]" + runner_kwargs.update(lookahead_config=args.lookahead_config) + if args.ngram_config is not None: + assert args.kv_cache_enable_block_reuse, "`--kv_cache_enable_block_reuse` must be specified in speculative decoding." + assert not args.use_py_session, "`--use_py_session` is not supported in Speculative decoding." + assert args.num_beams == 1, "`--num_beams>1` is not supported in Speculative decoding." + max_draft_len, _, target_device_list = ast.literal_eval( + args.ngram_config) + args.max_output_len = output_len # Specialization for NGram + runner_kwargs.update(is_orchestrator_mode=True, + device_ids=target_device_list, + max_input_len=test_token_num + max_draft_len + + output_len) + + runner = runner_cls.from_dir(**runner_kwargs) + assert not (args.eval_ppl and not runner.gather_context_logits), \ + "PPL evaluation requires engine built with gather_context_logits enabled" + + datapoint = dataset[0:1] + output, *_ = eval_trt_llm(datapoint, + eval_task=args.eval_task, + eval_ppl=args.eval_ppl, + add_special_tokens=args.add_special_tokens, + min_input_length=args.min_input_length, + runner=runner) + if runtime_rank == 0 and args.eval_task != "eval_context_ppl": + logger.info( + "---------------------------------------------------------") + logger.info("TensorRT LLM Generated: ") + logger.info(f" Input: {datapoint[dataset_input_key]}") + logger.info(f"\n Reference: {datapoint[dataset_output_key]}") + logger.info(f"\n Output: {output}") + logger.info( + "---------------------------------------------------------") + + ite_count = 0 + data_point_idx = 0 + total_output_token_count_trt_llm = 0 # only valid for runtime_rank == 0 + while (data_point_idx < len(dataset)) and (ite_count < args.max_ite): + if runtime_rank == 0: + logger.debug( + f"run data_point {data_point_idx} ~ {data_point_idx + max_batch_size}" + ) + datapoint = dataset[data_point_idx:(data_point_idx + + max_batch_size)] + + profiler.start('tensorrt_llm') + output_tensorrt_llm, output_ids_trt_llm, curr_ppls_trt_llm, lengths_info = eval_trt_llm( + datapoint, + eval_task=args.eval_task, + eval_ppl=args.eval_ppl, + add_special_tokens=args.add_special_tokens, + min_input_length=args.min_input_length, + runner=runner) + profiler.stop('tensorrt_llm') + + empty_batch = runtime_rank == 0 and len(output_tensorrt_llm) == 0 + empty_batch = mpi_broadcast(empty_batch, 0) + if empty_batch: + # No valid samples in the current batch, skip this iteration + data_point_idx += max_batch_size + continue + + if runtime_rank == 0: + input_lengths = lengths_info['input_lengths'] + seq_lengths = lengths_info['seq_lengths'] + output_token_count_trt_llm = sum( + beam_len - input_lengths[batch_idx] + for batch_idx, beam_lens in enumerate(seq_lengths) + for beam_len in beam_lens) + total_output_token_count_trt_llm += output_token_count_trt_llm + for batch_idx, output_beams in enumerate(output_tensorrt_llm): + reference = datapoint[dataset_output_key][batch_idx] + for beam_idx, output_beam in enumerate(output_beams): + metric_tensorrt_llm[beam_idx].add_batch( + predictions=[output_beam], references=[reference]) + if args.eval_ppl: + ppls_trt_llm[beam_idx].append( + curr_ppls_trt_llm[batch_idx][beam_idx]) + if output_dir is not None: + for i in range(len(output_tensorrt_llm[0])): + for beam_idx in range(num_sequences): + with (output_dir / 'trtllm.out').open('a') as f: + f.write( + f'[{data_point_idx + i}] [Beam {beam_idx}] {output_tensorrt_llm[beam_idx][i]}\n' + ) + + logger.debug('-' * 100) + logger.debug(f"Input: {datapoint[dataset_input_key]}") + logger.debug(f'TensorRT LLM Output: {output_tensorrt_llm}') + logger.debug(f"Reference: {datapoint[dataset_output_key]}") + + data_point_idx += max_batch_size + ite_count += 1 + del runner + + if test_hf and runtime_rank == 0: + profiler.start('load HF model') + dtype_alias_mapping = { + 'fp32': 'float32', + 'fp16': 'float16', + 'bf16': 'bfloat16' + } + args.hf_data_type = dtype_alias_mapping.get(args.hf_data_type, + args.hf_data_type) + if 'GLM' in model_name and model_version == 'glm': + auto_model_cls = AutoModelForSeq2SeqLM + elif 'GLM' in model_name and model_version == 'chatglm': + auto_model_cls = AutoModel + else: + auto_model_cls = AutoModelForCausalLM + # TODO: args.hf_device_map_auto is not being correctly set + # remove in future version + if model_name == 'DeepseekV2ForCausalLM': + args.hf_device_map_auto = True + model = auto_model_cls.from_pretrained( + args.hf_model_dir, + trust_remote_code=True, + dtype=str_dtype_to_torch(args.hf_data_type), + device_map='auto' if args.hf_device_map_auto else None) + try: + model.to_bettertransformer() + except Exception as e: + logger.warning( + f'Fail to call model.to_bettertransformer(), exception:\n{str(e)}' + ) + if not args.hf_device_map_auto: + model.cuda() + if model_name == 'qwen': + model.generation_config = GenerationConfig.from_pretrained( + args.hf_model_dir, trust_remote_code=True) + profiler.stop('load HF model') + logger.info( + f'Load HF model takes: {profiler.elapsed_time_in_sec("load HF model")} sec' + ) + + datapoint = dataset[0:1] + output, *_ = eval_hf(datapoint, + eval_task=args.eval_task, + eval_ppl=args.eval_ppl, + add_special_tokens=args.add_special_tokens, + min_input_length=args.min_input_length) + if runtime_rank == 0 and args.eval_task != "eval_context_ppl": + logger.info( + "---------------------------------------------------------") + logger.info("HF Generated: ") + logger.info(f" Input: {datapoint[dataset_input_key]}") + logger.info(f"\n Reference: {datapoint[dataset_output_key]}") + logger.info(f"\n Output: {output}") + logger.info( + "---------------------------------------------------------") + + ite_count = 0 + data_point_idx = 0 + total_output_token_count_hf = 0 # only valid for runtime_rank == 0 + while (data_point_idx < len(dataset)) and (ite_count < args.max_ite): + if runtime_rank == 0: + logger.debug( + f"run data_point {data_point_idx} ~ {data_point_idx + max_batch_size}" + ) + datapoint = dataset[data_point_idx:(data_point_idx + + max_batch_size)] + + profiler.start('hf') + output_hf, token_list, curr_ppls_hf = eval_hf( + datapoint, + eval_task=args.eval_task, + eval_ppl=args.eval_ppl, + add_special_tokens=args.add_special_tokens, + min_input_length=args.min_input_length) + profiler.stop('hf') + + # HF model runs on rank 0 only + empty_batch = len(output_hf) == 0 + if empty_batch: + # No valid samples in the current batch, skip this iteration + data_point_idx += max_batch_size + continue + + if runtime_rank == 0: + seq_lengths = [len(tokens) for tokens in token_list] + total_output_token_count_hf += sum(seq_lengths) + for beam_idx in range(num_sequences): + for batch_idx in range(len(output_hf[beam_idx])): + metric_hf[beam_idx].add_batch( + predictions=[output_hf[beam_idx][batch_idx]], + references=[ + datapoint[dataset_output_key][batch_idx] + ]) + if args.eval_ppl and args.batch_size == 1: + ppls_hf[beam_idx].append( + curr_ppls_hf[batch_idx][beam_idx]) + if output_dir is not None: + for i in range(len(output_hf[0])): + for beam_idx in range(num_sequences): + with (output_dir / 'hf.out').open('a') as f: + f.write( + f'[{data_point_idx + i}] [Beam {beam_idx}] {output_hf[beam_idx][i]}\n' + ) + + logger.debug('-' * 100) + logger.debug(f"Input: {datapoint[dataset_input_key]}") + logger.debug(f'HF Output: {output_hf}') + logger.debug(f"Reference: {datapoint[dataset_output_key]}") + + data_point_idx += max_batch_size + ite_count += 1 + del model + + if runtime_rank == 0 and args.max_ite > 0: + if test_trt_llm: + np.random.seed(0) # rouge score use sampling to compute the score + logger.info( + f'TensorRT LLM (total latency: {profiler.elapsed_time_in_sec("tensorrt_llm")} sec)' + ) + + logger.info( + f'TensorRT LLM (total output tokens: {total_output_token_count_trt_llm})' + ) + logger.info( + f'TensorRT LLM (tokens per second: {total_output_token_count_trt_llm / profiler.elapsed_time_in_sec("tensorrt_llm")})' + ) + for beam_idx in range(num_sequences): + logger.info(f"TensorRT LLM beam {beam_idx} result") + if args.eval_task != "eval_context_ppl": + if args.estimate_accuracy_std_dev: + computed_metrics_tensorrt_llm = metric_tensorrt_llm[ + beam_idx].compute(use_aggregator=False) + computed_std_dev_tensorrt_llm = { + key: np.std(scores) + for key, scores in + computed_metrics_tensorrt_llm.items() + } + computed_metrics_tensorrt_llm = { + key: np.mean(scores) + for key, scores in + computed_metrics_tensorrt_llm.items() + } + for key in computed_metrics_tensorrt_llm.keys(): + logger.info( + f" {key}: {computed_metrics_tensorrt_llm[key]*100} ({computed_std_dev_tensorrt_llm[key]*100})" + ) + else: + computed_metrics_tensorrt_llm = metric_tensorrt_llm[ + beam_idx].compute() + for key in computed_metrics_tensorrt_llm.keys(): + logger.info( + f" {key}: {computed_metrics_tensorrt_llm[key]*100}" + ) + if args.check_accuracy and beam_idx == 0: + rouge1 = computed_metrics_tensorrt_llm['rouge1'] * 100 + assert rouge1 > args.tensorrt_llm_rouge1_threshold, f"[FAILED] rouge1 ({rouge1}) is smaller than threshold ({args.tensorrt_llm_rouge1_threshold})." + if args.eval_ppl: + logger.info( + f" Per-token perplexity: {np.mean(ppls_trt_llm[beam_idx])}" + ) + if args.check_accuracy and beam_idx == 0: + avg_ppl = np.mean(ppls_trt_llm[beam_idx]) + assert avg_ppl < args.tensorrt_llm_ppl_threshold, f"[FAILED] average PPL ({avg_ppl}) is larger than threshold ({args.tensorrt_llm_ppl_threshold})." + if test_hf: + np.random.seed(0) # rouge score use sampling to compute the score + logger.info( + f'Hugging Face (total latency: {profiler.elapsed_time_in_sec("hf")} sec)' + ) + logger.info( + f'Hugging Face (total output tokens: {total_output_token_count_hf})' + ) + logger.info( + f'Hugging Face (tokens per second: {total_output_token_count_hf / profiler.elapsed_time_in_sec("hf")})' + ) + + for beam_idx in range(num_sequences): + logger.info(f"HF beam {beam_idx} result") + computed_metrics_hf = metric_hf[beam_idx].compute() + if args.eval_task != "eval_context_ppl": + for key in computed_metrics_hf.keys(): + logger.info(f' {key}: {computed_metrics_hf[key]*100}') + if args.eval_ppl and args.batch_size == 1: + logger.info( + f" Per-token perplexity: {np.mean(ppls_hf[beam_idx])}") + + +if __name__ == '__main__': + # see `add_common_args` for extended list of arguments + parser = argparse.ArgumentParser() + parser.add_argument('--test_hf', action='store_true') + parser.add_argument('--test_trt_llm', action='store_true') + parser.add_argument('--eval_task', + type=str, + default='summarize', + choices=[ + 'summarize', 'summarize_long', 'code_completion', + 'eval_context_ppl' + ]) + parser.add_argument('--check_accuracy', action='store_true') + parser.add_argument('--estimate_accuracy_std_dev', action='store_true') + parser.add_argument('--tensorrt_llm_rouge1_threshold', + type=float, + default=15.0) + parser.add_argument('--eval_ppl', action='store_true') + parser.add_argument('--tensorrt_llm_ppl_threshold', + type=float, + default=15.0) + parser.add_argument( + '--dataset_dir', + type=str, + default=None, + help="The local directory of the dataset for evaluation; " + "will download the dataset from huggingface hub if not specified.") + parser.add_argument( + '--dataset_cache_dir', + type=str, + default=None, + help="The local cache directory for dataset; " + "will use `~/.cache/huggingface/datasets` if not specified.") + parser.add_argument('--batch_size', type=int, default=1) + parser.add_argument('--max_ite', type=int, default=20) + parser.add_argument('--output_len', type=int, default=100) + parser.add_argument('--max_input_length', type=int, default=923) + parser.add_argument( + '--min_input_length', + type=int, + default=0, + help='skip the sentences which are shorter than min_input_length.') + parser.add_argument( + '--output_dir', + type=str, + default=None, + help="Directory where to save output sentences. 'trtllm.out' for " + "TensorRT LLM outputs, and 'hf.out' for HF outputs. If None, do not " + "save outputs.") + parser.add_argument( + '--rouge_dir', + default=None, + type=str, + help= + "evaluate.load('rouge') will attempt to pull rouge package from HF. Use cached rouge can avoid network outage of host or HF." + ) + parser.add_argument("--use_gpu_direct_storage", + default=False, + action="store_true", + help="Use GPUDirect Storage (GDS) to load the engine") + parser = add_common_args(parser) + args = parser.parse_args() + + main(args) diff --git a/examples/utils.py b/examples/utils.py new file mode 100644 index 000000000000..c75cd255850f --- /dev/null +++ b/examples/utils.py @@ -0,0 +1,658 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import ast +import json +import os +import subprocess +import sys +from argparse import BooleanOptionalAction +from functools import partial +from pathlib import Path +from typing import List, Optional + +import torch +from transformers import AutoTokenizer, LlamaTokenizer + +from tensorrt_llm._utils import supports_inflight_batching # noqa +from tensorrt_llm._utils import (mpi_barrier, mpi_rank, mpi_world_size, + str_dtype_to_torch) +from tensorrt_llm.builder import get_engine_version + + +class SentencePieceTokenizer: + """Minimal SentencePiece-backed tokenizer with a transformers-like API. + + transformers v5 replaced the pure-Python SentencePiece backend of + ``T5Tokenizer`` / ``LlamaTokenizer`` with the Rust ``tokenizers`` backend, + so instantiating them with a raw SentencePiece ``.model`` vocab file no + longer reads the underlying vocabulary. This wrapper preserves the old + behavior for NEMO-style checkpoints (e.g. gpt-next) by delegating to + ``sentencepiece.SentencePieceProcessor`` directly. + """ + + def __init__(self, + vocab_file: str, + padding_side: str = 'left', + truncation_side: str = 'left'): + import sentencepiece as spm + sp = spm.SentencePieceProcessor() + sp.Load(vocab_file) + self.sp_model = sp + self.padding_side = padding_side + self.truncation_side = truncation_side + self.vocab_size = sp.GetPieceSize() + + def _opt(i: int) -> int | None: + return i if i >= 0 else None + + self.pad_token_id = _opt(sp.pad_id()) + self.eos_token_id = _opt(sp.eos_id()) + self.bos_token_id = _opt(sp.bos_id()) + self.unk_token_id = _opt(sp.unk_id()) + + def encode(self, + text: str, + return_tensors: Optional[str] = None, + add_special_tokens: bool = True, + truncation: bool = False, + max_length: Optional[int] = None, + **kwargs): + ids = self.sp_model.EncodeAsIds(text) + if add_special_tokens and self.bos_token_id is not None: + ids = [self.bos_token_id] + ids + if truncation and max_length is not None and len(ids) > max_length: + ids = ids[ + -max_length:] if self.truncation_side == 'left' else ids[: + max_length] + if return_tensors == 'pt': + return torch.tensor([ids], dtype=torch.long) + return ids + + def decode(self, ids, skip_special_tokens: bool = False, **kwargs) -> str: + if isinstance(ids, torch.Tensor): + ids = ids.tolist() + if skip_special_tokens: + special = {self.pad_token_id, self.eos_token_id, self.bos_token_id} + ids = [t for t in ids if t not in special] + return self.sp_model.DecodeIds(list(ids)) + + def batch_decode(self, + sequences, + skip_special_tokens: bool = False, + **kwargs) -> List[str]: + if isinstance(sequences, torch.Tensor): + sequences = sequences.tolist() + return [ + self.decode(seq, skip_special_tokens=skip_special_tokens) + for seq in sequences + ] + + +DEFAULT_HF_MODEL_DIRS = { + 'BaichuanForCausalLM': 'baichuan-inc/Baichuan-13B-Chat', + 'BaiChuanForCausalLM': 'baichuan-inc/Baichuan-13B-Chat', + 'BloomForCausalLM': 'bigscience/bloom-560m', + 'GLMModel': 'THUDM/glm-10b', + 'ChatGLMModel': 'THUDM/chatglm3-6b', + 'ChatGLMForCausalLM': 'THUDM/chatglm3-6b', + 'RWForCausalLM': 'tiiuae/falcon-rw-1b', + 'FalconForCausalLM': 'tiiuae/falcon-rw-1b', + 'GPT2LMHeadModel': 'gpt2', + 'GPT2LMHeadCustomModel': 'gpt2', + 'Starcoder2ForCausalLM': 'bigcode/starcoder2-3b', + 'GPTForCausalLM': 'gpt2', + 'GPTJForCausalLM': 'EleutherAI/gpt-j-6b', + 'GPTNeoXForCausalLM': 'EleutherAI/gpt-neox-20b', + 'InternLMForCausalLM': 'internlm/internlm-chat-7b', + 'InternLM2ForCausalLM': 'internlm/internlm2-chat-7b', + 'LlamaForCausalLM': 'meta-llama/Llama-2-7b-hf', + 'MPTForCausalLM': 'mosaicml/mpt-7b', + 'PhiForCausalLM': 'microsoft/phi-2', + 'OPTForCausalLM': 'facebook/opt-350m', + 'QWenLMHeadModel': 'Qwen/Qwen-7B', + 'QWenForCausalLM': 'Qwen/Qwen-7B', + 'Qwen2ForCausalLM': 'Qwen/Qwen1.5-7B', + 'Qwen2MoeForCausalLM': 'Qwen/Qwen1.5-MoE-A2.7B', + 'RecurrentGemmaForCausalLM': 'google/recurrentgemma-2b', +} + +INTERNLM_META_INSTRUCTION = """You are an AI assistant whose name is InternLM (书生·浦语). +- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless. +- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文. +""" + +QWEN_PROMPT_TEMPLATE = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{input_text}<|im_end|>\n<|im_start|>assistant\n" + +DEFAULT_PROMPT_TEMPLATES = { + 'InternLMForCausalLM': "<|User|>:{input_text}\n<|Bot|>:", + 'InternLM2ForCausalLM': "<|im_start|>system\n" + INTERNLM_META_INSTRUCTION + + "<|im_end|>\n<|im_start|>user\n{input_text}<|im_end|>\n<|im_start|>assistant\n", + 'QWenLMHeadModel': QWEN_PROMPT_TEMPLATE, + 'QWenForCausalLM': QWEN_PROMPT_TEMPLATE, + 'Qwen2ForCausalLM': QWEN_PROMPT_TEMPLATE, + 'Qwen2MoeForCausalLM': QWEN_PROMPT_TEMPLATE, +} + + +def read_decoder_start_token_id(engine_dir): + with open(Path(engine_dir) / "config.json", 'r') as f: + config = json.load(f) + return config['pretrained_config']['decoder_start_token_id'] + + +def read_is_enc_dec(engine_dir: str, is_hf: bool = False): + if is_hf: + with open(Path(engine_dir) / "config.json", 'r') as f: + config = json.load(f) + is_enc_dec = config.get('is_encoder_decoder', False) + else: + is_enc_dec = {'encoder', 'decoder'}.issubset({ + name + for name in os.listdir(engine_dir) + if os.path.isdir(os.path.join(engine_dir, name)) + }) + return is_enc_dec + + +def read_model_name(engine_dir: str, is_hf: bool = False): + with open(Path(engine_dir) / "config.json", 'r') as f: + config = json.load(f) + + if is_hf: + model_arch = config['architectures'][0] + model_version = config.get('model_type', None) + return model_arch, model_version + + engine_version = get_engine_version(engine_dir) + if engine_version is None: + return config['builder_config']['name'], None + + model_arch = config['pretrained_config']['architecture'] + model_version = None + if 'GLM' in model_arch: + model_version = config['pretrained_config']['chatglm_version'] + if 'qwen' in model_arch.lower(): + model_version = config['pretrained_config']['qwen_type'] + return model_arch, model_version + + +def throttle_generator(generator, stream_interval): + for i, out in enumerate(generator): + if not i % stream_interval: + yield out + + if i % stream_interval: + yield out + + +# Load tokenizer impl, it will be called in external wrapper to avoid loading tokenizer bug under MPI env. +def _load_tokenizer(tokenizer_dir: Optional[str] = None, + vocab_file: Optional[str] = None, + model_name: str = 'GPTForCausalLM', + model_version: Optional[str] = None, + tokenizer_type: Optional[str] = None): + if vocab_file is None: + if 'whisper' in model_name.lower(): + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_dir or 'openai/whisper-large-v3', + language='english', + task='transcribe', + predict_timestamps=False, + ) + elif tokenizer_type == 'language_adapter': + tokenizer = None + else: + use_fast = True + if tokenizer_type is not None and tokenizer_type == "llama": + use_fast = False + # Should set both padding_side and truncation_side to be 'left' + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_dir, + legacy=False, + padding_side='left', + truncation_side='left', + trust_remote_code=True, + tokenizer_type=tokenizer_type, + use_fast=use_fast) + elif model_name == 'GemmaForCausalLM' or model_name == 'RecurrentGemmaForCausalLM': + from transformers import GemmaTokenizer + + # Initialize tokenizer from vocab file. + tokenizer = GemmaTokenizer(vocab_file=vocab_file, + padding_side='left', + truncation_side='left', + legacy=False) + elif model_name == 'Grok1ModelForCausalLM': + tokenizer = LlamaTokenizer(vocab_file=vocab_file, + padding_side='left', + truncation_side='left', + legacy=False, + use_fast=False) + else: + # For gpt-next, directly load from the SentencePiece ``tokenizer.model`` + # file. transformers v5 removed the pure-Python SentencePiece backend, + # so ``T5Tokenizer(vocab_file=...)`` no longer reads the vocabulary and + # reports vocab_size=104 with all tokens decoding to . + tokenizer = SentencePieceTokenizer(vocab_file=vocab_file, + padding_side='left', + truncation_side='left') + if 'qwen' in model_name.lower() and model_version == 'qwen': + with open(Path(tokenizer_dir) / "generation_config.json") as f: + gen_config = json.load(f) + pad_id = gen_config['pad_token_id'] + end_id = gen_config['eos_token_id'] + elif 'GLM' in model_name and model_version == 'glm': + pad_id = tokenizer.pad_token_id + end_id = tokenizer.eop_token_id + elif tokenizer_type == 'language_adapter': + pad_id = 0 + end_id = 2 + else: + if tokenizer.pad_token_id is None: + tokenizer.pad_token_id = tokenizer.eos_token_id + pad_id = tokenizer.pad_token_id + end_id = tokenizer.eos_token_id + + return tokenizer, pad_id, end_id + + +def load_tokenizer(tokenizer_dir: Optional[str] = None, + vocab_file: Optional[str] = None, + model_name: str = 'GPTForCausalLM', + model_version: Optional[str] = None, + tokenizer_type: Optional[str] = None): + func = partial(_load_tokenizer, tokenizer_dir, vocab_file, model_name, + model_version, tokenizer_type) + if mpi_world_size() > 1: + # Under MPI env, load tokenizer will result in multiple processes to download the same file to the same folder. + # This will result some random bug. Force loading on rank0 to warmup the tokenizer to avoid this issue. + if mpi_rank() == 0: + func() + mpi_barrier() + return func() + + +def prepare_enc_dec_inputs(batch_input_ids: List[torch.Tensor], model_name: str, + engine_dir: str, + multimodal_input_file: Optional[str]): + encoder_input_features = None + encoder_input_ids = None + if 'whisper' in model_name.lower(): + # cannot directly import whisper due to name collision + sys.path.append(f"{os.path.dirname(__file__)}/models/core/whisper") + from whisper_utils import log_mel_spectrogram + + config_path = os.path.join(engine_dir, 'encoder', 'config.json') + with open(config_path, 'r') as f: + config = json.load(f) + n_mels = config['pretrained_config']['n_mels'] + dtype = config['pretrained_config']['dtype'] + + # download mel filters file + subprocess.run([ + "wget", "-nc", f"--directory-prefix={engine_dir}", + "https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz" + ], + check=True) + + mel, total_duration = log_mel_spectrogram(multimodal_input_file, + n_mels, + return_duration=True, + mel_filters_dir=engine_dir) + mel = mel.type(str_dtype_to_torch(dtype)) # [featureDim, seqLen] + decoder_input_ids = batch_input_ids + encoder_input_features = [torch.einsum('DL->LD', mel)] + encoder_output_lengths = [encoder_input_features[0].shape[0] // 2] + else: + encoder_input_ids = batch_input_ids + decoder_start_token_id = read_decoder_start_token_id( + os.path.join(engine_dir, "decoder")) + decoder_input_ids = [ + torch.tensor([decoder_start_token_id], dtype=torch.int32) + for _ in batch_input_ids + ] + encoder_output_lengths = None + return encoder_input_ids, encoder_input_features, encoder_output_lengths, decoder_input_ids + + +def get_beam_width_array(bwa: str = None): + bwa = ast.literal_eval(bwa) # Short for "beam_width_array" + if isinstance(bwa, str): + bwa = ast.literal_eval(bwa) # parse again for string + + def parse_one_bwa(row): + assert isinstance(row, list), f"Beam width array must be a list." + assert len( + row + ) <= 8, "Length of beam width array must not be greater than 8 now." + assert all([isinstance(beam, int) for beam in row + ]), "Numbers in beam width array must be integer." + bwa_tensor = torch.zeros([8], dtype=torch.int32) + for j in range(len(row)): + bwa_tensor[j] = row[j] + bwa_tensor[len(row):] = row[-1] + return bwa_tensor, max(row) + + if isinstance(bwa, list): # Only one BWA + bwa_tensor, max_beam_width = parse_one_bwa(bwa) + elif isinstance(bwa, tuple): # BWA for respective requests + bwa_tensor_list = [] + max_beam_width = 0 + for row in bwa: + bwa_tensor, beam_width = parse_one_bwa(row) + bwa_tensor_list.append(bwa_tensor) + max_beam_width = max(max_beam_width, beam_width) + bwa_tensor = torch.stack(bwa_tensor_list, dim=0) + else: + raise ValueError(f"Invalid beam width array: {bwa}") + + return bwa_tensor.tolist(), max_beam_width + + +def add_common_args(parser): + # sampling arguments + parser.add_argument('--num_beams', + type=int, + help="Use beam search if num_beams > 1", + default=1) + parser.add_argument('--num_return_sequences', + type=int, + help="Number of sequences to generate for each input.", + default=None) + parser.add_argument('--temperature', type=float, default=1.0) + parser.add_argument('--top_k', type=int, default=1) + parser.add_argument('--top_p', type=float, default=0.0) + parser.add_argument('--length_penalty', type=float, default=1.0) + parser.add_argument('--repetition_penalty', type=float, default=1.0) + parser.add_argument('--presence_penalty', type=float, default=0.0) + parser.add_argument('--frequency_penalty', type=float, default=0.0) + parser.add_argument('--prompt_ignore_length', type=int, default=0) + parser.add_argument('--min_p', type=float, default=0.0) + parser.add_argument('--beam_search_diversity_rate', type=float, default=0.0) + parser.add_argument('--random_seed', type=int, default=0) + parser.add_argument('--early_stopping', + type=int, + help='Use early stopping if num_beams > 1, ' + '1 for early-stopping, 0 for non-early-stopping' + 'other values for stopping by length', + default=1) + parser.add_argument( + '--beam_width_array', + type=str, + default=None, + help= + 'Beam width array for each step. E.g.: --beam_width_array="[2,4,6,8]"', + ) + parser.add_argument( + '--end_id', + default=None, + type=int, + help="Override tokenizer end_id to stop on given end_id token.") + parser.add_argument( + '--stop_words', + default=None, + type=str, + nargs="+", + action='append', + help= + 'Set stop words for a batch. Successive invocations of --stop_words set stop words for other batches.' + ' E.g.: --stop_words " London" " chef" --stop_words "eventually became" "was not"', + ) + parser.add_argument( + '--bad_words', + default=None, + type=str, + nargs="+", + action='append', + help= + 'Set bad words for a batch. Successive invocations of --bad_words set bad words for other batches.' + ' E.g.: --bad_words " London" " chef" --bad_words "eventually became" "was not"', + ) + parser.add_argument('--no_repeat_ngram_size', type=int, default=None) + + # common runtime arguments + parser.add_argument('--sink_token_length', + type=int, + default=None, + help='The sink token length.') + parser.add_argument( + '--max_attention_window_size', + type=int, + default=None, + nargs="+", + help= + 'The attention window size that controls the sliding window attention kv cache behavior' + ) + parser.add_argument( + '--multi_block_mode', + type=lambda s: s.lower() in + ("yes", "true", "t", "1" + ), # custom boolean function to convert input string to boolean + default=True, + help= + "Distribute the work across multiple CUDA thread-blocks on the GPU for masked MHA kernel." + ) + parser.add_argument('--enable_context_fmha_fp32_acc', + action='store_true', + help="Enable FMHA runner FP32 accumulation.") + parser.add_argument('--cuda_graph_mode', + action='store_true', + help="Enable cuda graphs in the inference.") + parser.add_argument( + '--log_level', + type=str, + choices=['verbose', 'info', 'warning', 'error', 'internal_error'], + default='info') + parser.add_argument( + '--no_prompt_template', + dest='use_prompt_template', + default=True, + action='store_false', + help= + "Whether or not to use default prompt template to wrap the input text.") + parser.add_argument('--use_py_session', + default=False, + action='store_true', + help="Whether or not to use Python runtime session") + parser.add_argument('--debug_mode', + default=False, + action='store_true', + help="Whether or not to turn on the debug mode") + parser.add_argument('--streaming', default=False, action='store_true') + parser.add_argument('--streaming_interval', + type=int, + help="How often to return tokens when streaming.", + default=5) + parser.add_argument( + '--prompt_table_path', + type=str, + help="Path to .npy file, exported by nemo_prompt_convert.py") + parser.add_argument( + '--prompt_tasks', + help="Comma-separated list of tasks for prompt tuning, e.g., 0,3,1,0") + parser.add_argument('--lora_dir', + type=str, + default=None, + nargs="+", + help="The directory of LoRA weights") + parser.add_argument('--lora_ckpt_source', + type=str, + default="hf", + choices=["hf", "nemo"], + help="The source of lora checkpoint.") + parser.add_argument( + '--lora_task_uids', + type=str, + default=None, + nargs="+", + help="The list of LoRA task uids; use -1 to disable the LoRA module") + parser.add_argument( + '--num_prepend_vtokens', + nargs="+", + type=int, + help="Number of (default) virtual tokens to prepend to each sentence." + " For example, '--num_prepend_vtokens=10' will prepend the tokens" + " [vocab_size, vocab_size + 1, ..., vocab_size + 9] to the sentence.") + parser.add_argument( + '--draft_target_model_config', + type=str, + default=None, + help= + "Configuration of Draft-Target-Model decoding, see `examples/draft_target_model/README.md` for more information." + " E.g.: [4, [0], [1], False] for [draft_len, draft_model_device_list, target_model_device_list, use_logits]." + ) + parser.add_argument( + '--ngram_config', + type=str, + default=None, + help= + "Configuration of NGram decoding, see `examples/ngram/README.md` for more information." + " E.g.: [10,2,[0]] for [max_draft_len, max_matching_ngram_size, device_list].", + ) + parser.add_argument( + '--medusa_choices', + type=str, + default=None, + help="Configuration of Medusa decoding." + " E.g.: [[0, 0, 0, 0], [0, 1, 0], [1, 0], [1, 1]] for 9 medusa tokens." + ) + parser.add_argument( + '--eagle_choices', + type=str, + default=None, + help="Configuration of Eagle-1 decoding." + " E.g.: [[0, 0, 0, 0], [0, 1, 0], [1, 0], [1, 1]] for 9 draft tokens." + ) + parser.add_argument( + '--eagle_posterior_threshold', + type=float, + default=None, + help="Minimum token probability threshold for typical acceptance. " + "Enables typical acceptance in Eagle. " + "Corresponds to epsilon in https://arxiv.org/pdf/2401.10774.") + parser.add_argument('--eagle_use_dynamic_tree', + action='store_true', + help="Whether to use Ealge-2") + parser.add_argument( + '--eagle_dynamic_tree_max_top_k', + default=None, + type=int, + help= + "The maximum number of draft tokens to expand for each node in Eagle-2." + ) + parser.add_argument( + '--lookahead_config', + type=str, + default=None, + help="Configuration of executor and request lookahead decoding." + " E.g.: [5, 6, 7] for [max_window_size, max_ngram_size, max_verification_set_size]." + ) + # model arguments + parser.add_argument('--engine_dir', type=str, default='engine_outputs') + parser.add_argument( + '--tokenizer_type', + help= + 'Specify that argument when providing a .model file as the tokenizer_dir. ' + 'It allows AutoTokenizer to instantiate the correct tokenizer type.') + parser.add_argument('--vocab_file', + help="Used for sentencepiece tokenizers") + parser.add_argument('--no_add_special_tokens', + dest='add_special_tokens', + default=True, + action='store_false', + help="Whether or not to add special tokens") + parser.add_argument('--hf_model_dir', '--model_dir', type=str, default=None) + parser.add_argument( + '--tokenizer_dir', + default=None, + help='tokenizer path; defaults to hf_model_dir if left unspecified') + + # memory argument + parser.add_argument( + '--gpu_weights_percent', + default=1, + type=float, + help= + 'Specify the percentage of weights that reside on GPU instead of CPU and streaming load during runtime.', + ) + parser.add_argument( + '--max_tokens_in_paged_kv_cache', + default=None, + type=int, + help= + 'Specify the maximum number of tokens in a kv cache page (only available with cpp session).', + ) + parser.add_argument( + '--kv_cache_enable_block_reuse', + default=True, + action=BooleanOptionalAction, + help= + 'Enables block reuse in kv cache (only available with cpp session).', + ) + parser.add_argument( + '--kv_cache_free_gpu_memory_fraction', + default=0.9, + type=float, + help='Specify the free gpu memory fraction.', + ) + parser.add_argument( + '--cross_kv_cache_fraction', + default=0.5, + type=float, + help= + 'Specify the kv cache fraction reserved for cross attention. Only applicable for encoder-decoder models. By default 0.5 for self and 0.5 for cross.', + ) + parser.add_argument( + '--enable_chunked_context', + action='store_true', + help='Enables chunked context (only available with cpp session).', + ) + + # hf model argument (if use hf model) + parser.add_argument( + '--hf_data_type', + '--data_type', + type=str, + choices=['fp32', 'fp16', 'bf16', 'float32', 'float16', 'bfloat16'], + default='fp16', + help="The data type for hf model.") + parser.add_argument( + '--hf_device_map_auto', + action='store_true', + help="Use device map 'auto' to load a pretrained HF model. This may " + "help to test a large model that cannot fit into a singlue GPU.") + + parser.add_argument( + "--return_all_generated_tokens", + default=False, + action="store_true", + help="This option changes the token output only for streaming. " + "If not specified, return only generated tokens at each step. " + "If specified, return the full beams/outputs at each step. " + "It is automatically enabled for num_beams>1 (only available with cpp session). " + "WARNING: using this option may increase network usage significantly (quadratically w.r.t output length)." + ) + + parser.add_argument( + '--language_task_uids', + type=int, + nargs='+', + default=None, + help= + "language task id indicating which adapter to use in language adapter. Please include 1 locale per input text" + ) + parser.add_argument('--backend', type=str, default=None) + + return parser diff --git a/jenkins/Build.groovy b/jenkins/Build.groovy index b1d541c04b13..a5d3966bb5c8 100644 --- a/jenkins/Build.groovy +++ b/jenkins/Build.groovy @@ -33,7 +33,7 @@ AARCH64_TRIPLE = "aarch64-linux-gnu" LLM_DOCKER_IMAGE = env.dockerImage // Always use x86_64 image for agent -AGENT_IMAGE = env.dockerImage.replace("aarch64", "x86_64").replace("sbsa", "x86_64") +AGENT_IMAGE = env.dockerImage.replace("aarch64", "x86_64") POD_TIMEOUT_SECONDS_BUILD = env.podTimeoutSeconds ? env.podTimeoutSeconds : "43200" @@ -384,7 +384,7 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) sh "ccache -sv" sh "rm -rf **/*.xml *.tar.gz" - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) if (env.alternativeTRT) { sh "cd ${LLM_ROOT} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" } @@ -421,12 +421,24 @@ def runLLMBuild(pipeline, buildFlags, tarName, is_linux_x86_64) def buildJobs = buildFlags[BUILD_JOBS_FOR_CONFIG] ?: BUILD_JOBS withCredentials([usernamePassword(credentialsId: "urm-artifactory-creds", usernameVariable: 'CONAN_LOGIN_USERNAME', passwordVariable: 'CONAN_PASSWORD')]) { - sh "cd ${LLM_ROOT} && python3 scripts/build_wheel.py --use_ccache -G Ninja -j ${buildJobs} -a '${buildFlags[WHEEL_ARCHS]}' ${buildFlags[WHEEL_EXTRA_ARGS]}" + sh "cd ${LLM_ROOT} && python3 scripts/build_wheel.py --use_ccache -G Ninja -j ${buildJobs} -a '${buildFlags[WHEEL_ARCHS]}' ${buildFlags[WHEEL_EXTRA_ARGS]} --benchmarks" } + if (is_linux_x86_64) { + sh "cd ${LLM_ROOT} && python3 scripts/build_cpp_examples.py" + } + // Step 3: packaging wheels into tarfile sh "cp ${LLM_ROOT}/build/tensorrt_llm-*.whl TensorRT-LLM/" - // Step 4: packaging attribution files into tarfile when they exist + // Step 4: packaging benchmark and required cpp dependencies into tarfile + sh "mkdir -p TensorRT-LLM/benchmarks/cpp" + sh "cp ${LLM_ROOT}/cpp/build/benchmarks/bertBenchmark TensorRT-LLM/benchmarks/cpp" + sh "cp ${LLM_ROOT}/cpp/build/benchmarks/gptManagerBenchmark TensorRT-LLM/benchmarks/cpp" + sh "cp ${LLM_ROOT}/cpp/build/benchmarks/disaggServerBenchmark TensorRT-LLM/benchmarks/cpp" + sh "cp ${LLM_ROOT}/cpp/build/tensorrt_llm/libtensorrt_llm.so TensorRT-LLM/benchmarks/cpp" + sh "cp ${LLM_ROOT}/cpp/build/tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so TensorRT-LLM/benchmarks/cpp" + + // Step 5: packaging attribution files into tarfile when they exist sh "mkdir -p TensorRT-LLM/attribution" sh "cp ${LLM_ROOT}/cpp/build/attribution/missing_files.json TensorRT-LLM/attribution/ || true" sh "cp ${LLM_ROOT}/cpp/build/attribution/import_payload.json TensorRT-LLM/attribution/ || true" @@ -459,7 +471,7 @@ def buildWheelInContainer(pipeline, libraries=[], triple=X86_64_TRIPLE, clean=fa sh "cat ${CCACHE_DIR}/ccache.conf" // Step 1: cloning tekit source code - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) if (env.alternativeTRT) { trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver) sh "cd ${LLM_ROOT} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" @@ -559,7 +571,7 @@ def launchStages(pipeline, cpu_arch, enableFailFast, globalVars) stage(key) { stage("[${key}] Run") { echoNodeAndGpuInfo(pipeline, key) - buildWheelInContainer(pipeline, [], X86_64_TRIPLE, false, false, "cp312", "-a '90-real' -b Debug --micro_benchmarks") + buildWheelInContainer(pipeline, [], X86_64_TRIPLE, false, false, "cp312", "-a '90-real' -b Debug --benchmarks --micro_benchmarks") } } }) diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index d22c9dc23c69..c41b999b8920 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -300,7 +300,7 @@ def buildImage(config, imageKeyToTag) stage (config.stageName) { // Step 1: Clone TRT-LLM source codes // If using a forked repo, svc_tensorrt needs to have the access to the forked repo. - trtllm_utils.checkoutSource(LLM_REPO, LLM_COMMIT_OR_BRANCH, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, LLM_COMMIT_OR_BRANCH, LLM_ROOT, false, true) } // Step 2: Build the images @@ -453,26 +453,15 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { ] def release_action = params.action - def stageNames = [ - internalReleaseX86: "Build Internal release (x86_64 trtllm)", - internalReleaseSBSA: "Build Internal release (SBSA trtllm)", - ciImageX86: "Build CI Image (x86_64 tritondevel)", - ciImageSBSA: "Build CI Image (SBSA tritondevel)", - ciImageRockyPy310: "Build CI Image (RockyLinux8 Python310)", - ciImageRockyPy312: "Build CI Image (RockyLinux8 Python312)", - ciImageSBSAUbuntu: "Build CI Image (SBSA Ubuntu24.04 Python312)", - ngcReleaseX86: "Build NGC devel And release (x86_64)", - ngcReleaseSBSA: "Build NGC devel And release (SBSA)", - ] def buildConfigs = [ - (stageNames.internalReleaseX86): [ + "Build Internal release (x86_64 trtllm)": [ target: "trtllm", action: release_action, customTag: LLM_BRANCH_TAG + "-x86_64", build_wheel: true, dockerfileStage: "release", ], - (stageNames.internalReleaseSBSA): [ + "Build Internal release (SBSA trtllm)": [ target: "trtllm", action: release_action, customTag: LLM_BRANCH_TAG + "-sbsa", @@ -480,27 +469,27 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { arch: "arm64", dockerfileStage: "release", ], - (stageNames.ciImageX86): [:], - (stageNames.ciImageSBSA): [ + "Build CI Image (x86_64 tritondevel)": [:], + "Build CI Image (SBSA tritondevel)": [ arch: "arm64", ], - (stageNames.ciImageRockyPy310): [ + "Build CI Image (RockyLinux8 Python310)": [ target: "rockylinux8", args: "PYTHON_VERSION=3.10.12", postTag: "-py310", ], - (stageNames.ciImageRockyPy312): [ + "Build CI Image (RockyLinux8 Python312)": [ target: "rockylinux8", args: "PYTHON_VERSION=3.12.3", postTag: "-py312", ], - (stageNames.ciImageSBSAUbuntu): [ + "Build CI Image (SBSA Ubuntu24.04 Python312)": [ arch: "arm64", target: "ubuntu24", args: "PYTHON_VERSION=3.12.3", postTag: "-py312", ], - (stageNames.ngcReleaseX86): [ + "Build NGC devel And release (x86_64)": [ target: "ngc-release", action: release_action, args: "DOCKER_BUILD_OPTS='--load --platform linux/amd64'", @@ -511,7 +500,7 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { ], dockerfileStage: "release", ], - (stageNames.ngcReleaseSBSA): [ + "Build NGC devel And release (SBSA)": [ target: "ngc-release", action: release_action, args: "DOCKER_BUILD_OPTS='--load --platform linux/arm64'", @@ -524,19 +513,6 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { dockerfileStage: "release", ], ] - def enabledStages = [] - if (params.buildInternalRelease) { - enabledStages += [stageNames.internalReleaseX86, stageNames.internalReleaseSBSA] - } - if (params.buildCiImage) { - enabledStages += [stageNames.ciImageX86, stageNames.ciImageSBSA, stageNames.ciImageRockyPy310, stageNames.ciImageRockyPy312, stageNames.ciImageSBSAUbuntu] - } - if (params.buildNgcRelease) { - enabledStages += [stageNames.ngcReleaseX86, stageNames.ngcReleaseSBSA] - } - buildConfigs = buildConfigs.findAll { key, config -> key in enabledStages } - echo "Running stages: ${buildConfigs.keySet()}" - // Override all fields in build config with default values buildConfigs.each { key, config -> defaultBuildConfig.each { defaultKey, defaultValue -> @@ -605,21 +581,6 @@ pipeline { choices: ["build", "push"], description: "Docker image generation action. build: only perform image build step; push: build docker image and push it to artifacts" ) - booleanParam( - name: "buildInternalRelease", - defaultValue: true, - description: "Build internal release images (x86_64 and SBSA trtllm)" - ) - booleanParam( - name: "buildCiImage", - defaultValue: true, - description: "Build CI images (tritondevel and OS variant images)" - ) - booleanParam( - name: "buildNgcRelease", - defaultValue: true, - description: "Build NGC devel and release images (x86_64 and SBSA)" - ) } options { // Check the valid options at: https://www.jenkins.io/doc/book/pipeline/syntax/ diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 3f276522ee54..255d9f6e442d 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1,19 +1,3 @@ -/* - * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - @Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ import java.lang.InterruptedException @@ -140,12 +124,7 @@ def DETAILED_LOG = "detailed_log" @Field def CBTS_RESULT = "cbts_result" @Field -def CBTS_COVERAGE = "cbts_coverage" -@Field def DISABLE_CBTS = "disable_cbts" -// Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1. -@Field -def ENABLE_CBTS_COVERAGE = true def testFilter = [ (REUSE_TEST): gitlabParamsFromBot.get(REUSE_TEST, null), @@ -165,7 +144,6 @@ def testFilter = [ (AUTO_TRIGGER_TAG_LIST): [], (DETAILED_LOG): gitlabParamsFromBot.get(DETAILED_LOG, false), (CBTS_RESULT): null, - (CBTS_COVERAGE): false, (DISABLE_CBTS): gitlabParamsFromBot.get((DISABLE_CBTS), false), ] @@ -186,7 +164,7 @@ def globalVars = [ (CACHED_CHANGED_FILE_LIST): null, (ACTION_INFO): gitlabParamsFromBot.get('action_info', null), (IMAGE_KEY_TO_TAG): [:], - (TARGET_BRANCH): gitlabParamsFromBot.get('target_branch', 'main'), + (TARGET_BRANCH): gitlabParamsFromBot.get('target_branch', null), ] // If not running all test stages in the L0 pre-merge, we will not update the GitLab status at the end. @@ -325,10 +303,10 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) // NB: getContainerURIs reads files in ${LLM_ROOT}/jenkins/ if (env.gitlabMergeRequestLastCommit) { env.gitlabCommit = env.gitlabMergeRequestLastCommit - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) } else { branch = env.gitlabBranch ? env.gitlabBranch : "main" - trtllm_utils.checkoutSource(LLM_REPO, branch, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, branch, LLM_ROOT, false, true) checkoutCommit = sh (script: "cd ${LLM_ROOT} && git rev-parse HEAD",returnStdout: true).trim() env.gitlabCommit = checkoutCommit } @@ -338,9 +316,6 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) - // Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag. - testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/) - pipeline.echo("CBTS coverage eligible: ${testFilter[(CBTS_COVERAGE)]}") getContainerURIs().each { k, v -> globalVars[k] = v } @@ -451,7 +426,7 @@ def launchReleaseCheck(pipeline, globalVars) sh "pip3 config set global.break-system-packages true" sh "git config --global --add safe.directory \"*\"" // Step 1: Clone TRT-LLM source codes - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) sh "cd ${LLM_ROOT} && git config --unset-all core.hooksPath" // Step 2: Run guardwords scan @@ -774,9 +749,6 @@ def getCbtsResult(pipeline, testFilter, globalVars) // pyyaml is needed by main.py's blocks.py to parse test-db YAMLs. sh "apt-get update -qq && apt-get install -y -qq python3-yaml" - // Shadow audit: download the latest merged touch DB and log its health + HEAD coverage gap (diagnostic only). - _cbtsCoverageAudit(pipeline) - // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py --list-needed-diffs", @@ -845,33 +817,6 @@ def getCbtsResult(pipeline, testFilter, globalVars) } } -// Download the latest merged touch DB and run coverage_audit.py on it; best-effort, never changes the CBTS decision. -def _cbtsCoverageAudit(pipeline) -{ - try { - def covDir = "${LLM_ROOT}/cbts_cov" - def url = sh( - script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py --print-url || true", - returnStdout: true, - ).trim() - if (!url) { - pipeline.echo("CBTS audit: no coverage DB artifact found — skipping") - return - } - sh "mkdir -p ${covDir}" - // wget the tarball (retrying) and extract the sqlite. - trtllm_utils.llmExecStepWithRetry(pipeline, script: - "wget -nv '${url}' -O ${covDir}/cbts_pystart_report.tar.gz && " + - "tar xzf ${covDir}/cbts_pystart_report.tar.gz -C ${covDir}") - sh "python3 ${LLM_ROOT}/jenkins/scripts/cbts/tools/coverage_audit.py " + - "--db ${covDir}/cbts_touchmap.sqlite" - } catch (InterruptedException e) { - throw e - } catch (Exception e) { - pipeline.echo("CBTS audit: skipped (non-fatal): ${e.message}") - } -} - // Post one CBTS decision record to OpenSearch (best-effort; never blocks CI). // decisionJson null for deferred; reason used only then. Context/creds via env. def _cbtsReportDecision(pipeline, globalVars, String status, String reason, String decisionJson) @@ -984,6 +929,7 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "cpp/include/tensorrt_llm/runtime/worldConfig.h", "cpp/tensorrt_llm/batch_manager/", "cpp/tensorrt_llm/executor/", + "cpp/tensorrt_llm/executor_worker/", "cpp/tensorrt_llm/kernels/communicationKernels/", "cpp/tensorrt_llm/kernels/customAllReduceKernels.cu", "cpp/tensorrt_llm/kernels/customAllReduceKernels.h", @@ -998,6 +944,13 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "cpp/tensorrt_llm/kernels/userbuffers/", "cpp/tensorrt_llm/kernels/xqaDispatcher.cpp", "cpp/tensorrt_llm/kernels/xqaDispatcher.h", + "cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.cpp", + "cpp/tensorrt_llm/plugins/cpSplitPlugin/cpSplitPlugin.h", + "cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.cpp", + "cpp/tensorrt_llm/plugins/gptAttentionCommon/gptAttentionCommon.h", + "cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.cpp", + "cpp/tensorrt_llm/plugins/gptAttentionPlugin/gptAttentionPlugin.h", + "cpp/tensorrt_llm/plugins/ncclPlugin/", "cpp/tensorrt_llm/nanobind/", "cpp/tensorrt_llm/runtime/ipcUtils.cpp", "cpp/tensorrt_llm/runtime/ncclCommunicator.cpp", @@ -1007,6 +960,8 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "cpp/tensorrt_llm/thop/allgatherOp.cpp", "cpp/tensorrt_llm/thop/allreduceOp.cpp", "cpp/tensorrt_llm/thop/reducescatterOp.cpp", + "cpp/tests/e2e_tests/batch_manager/", + "cpp/tests/e2e_tests/executor/", "cpp/tests/unit_tests/multi_gpu/", "jenkins/L0_Test.groovy", "tensorrt_llm/_ipc_utils.py", @@ -1067,18 +1022,22 @@ def getMultiGpuFileChanged(pipeline, testFilter, globalVars) "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node2_gpu8.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node4_gpu16.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node4_gpu16.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml", + "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16.yml", "tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb300.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32.yml", + "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node4_gpu16.yml", "tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_node2_gpu8.yml", @@ -1213,7 +1172,16 @@ def collectTestResults(pipeline, testFilter, globalVars) echo "Result File Number: ${resultFileNumber}, Downloaded: ${resultFileDownloadedNumber}" sh "find . -name results-\\*.tar.gz -type f -exec tar -zxvf {} \\; || true" - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) + if (testFilter[(IS_POST_MERGE)]) { + try { + sh "python3 llm/scripts/generate_duration.py --duration-file=new_test_duration.json" + trtllm_utils.uploadArtifacts("new_test_duration.json", "${UPLOAD_PATH}/test-results/") + } catch (Exception e) { + // No need to fail the stage if the duration file generation fails + echo "An error occurred while generating or uploading the duration file: ${e.toString()}" + } + } junit(testResults: '**/results*.xml', allowEmptyResults : true) @@ -1273,27 +1241,34 @@ def collectTestResults(pipeline, testFilter, globalVars) try { stage("Test Coverage") { sh "ls" + def CUR_PATH = sh(returnStdout: true, script: 'pwd').replaceAll("\\s","") + sh "echo ${CUR_PATH}" sh "rm -rf cov && mkdir -p cov" - // Collect the per-process PY_START data files from every stage's results tarball. - sh "find . -type f -name '.cbtscov.*.sqlite' -exec mv -t cov/ {} + || true" - sh "cd cov && ls -la" - def fileCount = sh(returnStdout: true, script: 'find cov -name ".cbtscov.*.sqlite" | wc -l').replaceAll("\\s","").toInteger() + sh "find . -type f -wholename '*/.coverage.*' -exec mv {} cov/ \\; || true" + sh "cd cov && find . -type f" + def fileCount = sh(returnStdout: true, script: 'find cov -type f | wc -l').replaceAll("\\s","").toInteger() if (fileCount == 0) { - echo "CBTS coverage skipped: no PY_START data files." + echo "Test coverage is skipped because there is no test data file." return } - // Merge into the indexed touch DB, a per-file HTML report, and the coverage rate. + trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install coverage") + sh "coverage --version" + + sh "cp llm/examples/openai_triton/manual_plugin/fmha_triton.py llm/examples/openai_triton/plugin_autogen/" + def coverageConfigFile = "cov/.coveragerc" sh """ - python3 llm/jenkins/scripts/cbts/coverage_utils/pystart_report.py \ - --glob 'cov/.cbtscov.*.sqlite' \ - --out-sqlite cov/cbts_touchmap.sqlite \ - --out-dir cov/cbts_report \ - --source-root llm/tensorrt_llm + echo '[paths]' > ${coverageConfigFile} + echo 'source1=\n ${CUR_PATH}/llm/examples/\n */TensorRT-LLM/src/examples/' >> ${coverageConfigFile} + echo 'source2=\n ${CUR_PATH}/llm/tensorrt_llm/\n */tensorrt_llm/' >> ${coverageConfigFile} + cat ${coverageConfigFile} """ - // Upload compressed only: the guardword scanner byte-matches raw files but not archives. - sh "cd cov && tar czf cbts_pystart_report.tar.gz cbts_touchmap.sqlite cbts_report" - trtllm_utils.uploadArtifacts("cov/cbts_pystart_report.tar.gz", "${UPLOAD_PATH}/cbts-coverage/") - echo "CBTS coverage (touch DB + report): https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/cbts-coverage/cbts_pystart_report.tar.gz" + + sh "cd cov && coverage combine" + sh "cd cov && find . -type f" + sh "cd cov && coverage report -i" // -i: ignore errors. Ignore the error that the source code file cannot be found. + sh "cd cov && coverage html -d test_coverage_html -i" + trtllm_utils.uploadArtifacts("cov/test_coverage_html/*", "${UPLOAD_PATH}/test-results/coverage-report/") + echo "Test coverage report: https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/test-results/coverage-report/index.html" } // Test coverage } catch (InterruptedException e) @@ -1505,10 +1480,6 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Skipping x86_64 tests (GenPostMergeBuilds mode: builds only)" return } - if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { - echo "Skipping x86_64 tests (PLC container scanning)" - return - } testStageName = "[Test-x86_64-Single-GPU] Remote Run" def singleGpuTestFailed = false @@ -1622,11 +1593,6 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) return } - if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { - echo "Skipping SBSA tests (PLC container scanning)" - return - } - testStageName = "[Test-SBSA-Single-GPU] Remote Run" def singleGpuTestFailed = false stage(testStageName) { @@ -1773,88 +1739,6 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Build-Docker-Images job is set explicitly. Both x86_64-Linux and SBSA-Linux sub-pipelines will be disabled." } - def plcContainerScanningJob = [ - "PLC Container Scanning": { - script { - stage("[Build-Release-Docker-Images] Remote Run") { - try { - def branch = env.gitlabBranch ? env.gitlabBranch : "main" - if (globalVars[GITHUB_PR_API_URL]) { - branch = "github-pr-" + globalVars[GITHUB_PR_API_URL].split('/').last() - } - - // Force the image tag suffix to be this L0_MergeRequest BUILD_NUMBER - // instead of the BuildDockerImages helper job's own counter. - def shortCommit = env.gitlabCommit ? env.gitlabCommit.substring(0, 7) : "undefined" - def branchTag = branch.replaceAll('/', '_') - def defaultTag = "${shortCommit}-${branchTag}-${env.BUILD_NUMBER}" - - def additionalParameters = [ - 'branch': branch, - 'action': "push", - 'triggerType': "post-merge", - 'runSanityCheck': false, - 'defaultTag': defaultTag, - 'buildInternalRelease': false, - 'buildCiImage': false, - 'artifactPath': ARTIFACT_PATH, - 'nspect_id': "", - 'uploadPath': UPLOAD_PATH - ] - launchJob(pipeline, "/LLM/helpers/BuildDockerImages", false, enableFailFast, globalVars, "x86_64", additionalParameters) - } catch (InterruptedException e) { - throw e - } catch (Exception e) { - if (BUILD_CHECK_CHOICE == STAGE_CHOICE_IGNORE) { - catchError( - buildResult: 'SUCCESS', - stageResult: 'FAILURE') { - error "Build-Docker-Images job failed but ignored due to Jenkins configuration" - } - } else { - throw e - } - } - } - stage("[NGC-Container-Compliance-Check] Run") { - echo "Triggering OSS Compliance (PLC) container scan for ref: " - try { - def params = [ - string(name: 'postMergePipelineName', value: env.JOB_NAME), - string(name: 'postMergeBuildNumber', value: env.BUILD_NUMBER), - string(name: 'scanMode', value: 'pre_merge'), - string(name: 'runSourceCodeScanning', value: 'false'), - string(name: 'runContainerScanning', value: 'true'), - string(name: 'runSonarQube', value: 'false'), - ] - def logger = new Logger(pipeline) - def handle = build( - job: "/LLM/helpers/PLCScanningSetup", - parameters: params, - propagate: false - ) - if (handle.result != "SUCCESS") { - catchError(buildResult: currentBuild.result ?: 'SUCCESS', stageResult: 'UNSTABLE') { - error "Risks detected on NGC Containers" - } - } - } catch (InterruptedException e) { - throw e - } catch (Exception e) { - catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { - error "OSS Compliance Check failed: ${e.getMessage()}" - } - } - } - } - } - ] - if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { - stages += plcContainerScanningJob - testFilter[(TEST_STAGE_LIST)]?.remove("NGC-Container-Scanning") - echo "Will run job to build ngc containers and running in-pipeline scanning for them" - } - parallelJobs = stages.collectEntries{key, value -> [key, { script { stage(key) { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index ee9654166f9c..047dd63dd2b9 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -59,7 +59,6 @@ linuxPkgName = ( env.targetArch == AARCH64_TRIPLE ? "tensorrt-llm-sbsa-release-s // available tags can be found in: https://urm.nvidia.com/artifactory/sw-tensorrt-docker/tensorrt-llm/ // [base_image_name]-[arch]-[os](-[python_version])-[trt_version]-[torch_install_type]-[stage]-[date]-[mr_id] LLM_DOCKER_IMAGE = env.dockerImage -X86_64_DOCKER_IMAGE = LLM_DOCKER_IMAGE.replace("aarch64", "x86_64").replace("sbsa", "x86_64") LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE = env.wheelDockerImagePy310 LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = env.wheelDockerImagePy312 LLM_WHEEL_DOCKER_IMAGE = env.wheelDockerImage @@ -152,17 +151,6 @@ SLURM_INFRA_RETRY_MAX = 1 // to avoid nesting with the inner SLURM retry. K8S_INFRA_RETRY_MAX = 1 -// Per-stage override of the above: set `infraRetryMax` in a stage's opts map (the -// 3rd element of its parallel-jobs config tuple, alongside singleAttempt) to cap -// or disable stage-level infra retries for resource-scarce hardware pools -- -// `infraRetryMax: 0` disables retries entirely (1 attempt). It may only reduce the -// budget: values above the scope global are clamped down to it (resolveInfraRetryMax), -// so it can never increase retries past these caps. It applies to whichever -// stage-level retry the stage uses: the SLURM retry (runLLMTestlistOnSlurm) for -// dispatcher pods, or the K8s pod retry (runKubernetesPodWithInfraRetry) for regular -// test pods. It does NOT touch the dispatcher-pod launch-retry (relaunching a cheap -// Blossom pod doesn't tax the scarce hardware). Null/absent = use the globals above. - // Fallback discriminator for SLURM timeouts. // If we can't reach the SLURM node for an authoritative reason, // we apply a heuristic: if the job needed more than this of its budget @@ -186,29 +174,6 @@ ENABLE_NGC_RELEASE_IMAGE_TEST = params.enableNgcReleaseImageTest ?: false COMMON_SSH_OPTIONS = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o TCPKeepAlive=no -o ServerAliveInterval=30 -o ServerAliveCountMax=20" -// Per-stage CBTS coverage exclusions applied on top of the upstream eligibility decision. -CBTS_EXCLUDE_STAGES = [] as Set - -def isCbtsStage(String stageName) { - // Pipeline-level eligibility (post-merge gate + kill switch) is decided in L0_MergeRequest.groovy and propagated via testFilter. - if (!(testFilter[(CBTS_COVERAGE)] ?: false)) { - return false - } - // Perf stages skip coverage to avoid skewing measurements (perfMode == stageName.contains("Perf")). - if (stageName.contains("Perf")) { - return false - } - // Skip stages with no product Python coverage: TensorRT (legacy), CPP (gtest), AutoDeploy (leaving L0). - if (stageName.contains("TensorRT") || stageName.contains("CPP") || stageName.contains("AutoDeploy")) { - return false - } - // Phase 1: single-GPU only; multi-GPU / multi-node stages carry the "_GPUs" / "_Nodes" token. - if (stageName.contains("_GPUs") || stageName.contains("_Nodes")) { - return false - } - return !CBTS_EXCLUDE_STAGES.contains(stageName) -} - def scpFromRemoteCmd(Map remote, String remotePath, String localPath) { String portOpt = remote.port ? "-P ${remote.port} " : "" if (remote.privateKeyPath) { @@ -217,82 +182,11 @@ def scpFromRemoteCmd(Map remote, String remotePath, String localPath) { return "sshpass -p '${remote.passwd}' scp ${portOpt}-r -p ${COMMON_SSH_OPTIONS} ${remote.user}@${remote.host}:${remotePath} ${localPath}" } -// Print the last `lines` lines of the remote Slurm job log file to the console when the Slurm job fails. -// If the log file does not exist, print a message to the console. -def echoRemoteLogTail(def pipeline, Map remote, String remotePath, int lines = 200) { - pipeline.echo("===== Last ${lines} lines of ${remotePath} on ${remote.host} =====") - try { - def tailOut = Utils.exec( - pipeline, - script: Utils.sshUserCmd(remote, - "\"bash -c 'if [ -f \\\"${remotePath}\\\" ]; then tail -n ${lines} -- \\\"${remotePath}\\\"; " + - "else echo \\\"[log not found: ${remotePath}]\\\"; fi'\""), - returnStdout: true, - numRetries: 1, - )?.trim() - pipeline.echo(tailOut ?: "") - } catch (Exception tailEx) { - pipeline.echo("Ignorable warning: could not tail ${remotePath} on ${remote.host}: ${tailEx.message}") - } -} - -// Scrape the SLURM job output log for a device / driver / interconnect fault -// signature and return the matched signature itself, or "" for no match. -// -// Device faults (CUDA/NVLink/ECC/driver) print into job-output.log but never -// reach the stage exception chain -- the tracker squashes a failed job to -// `exit 1` -- so classify() otherwise sees only a generic failure and cannot -// steer the retry off the bad node. This is a GATE only: the returned signature -// is folded into a fresh exception so FailureClassifier.PATTERN_CATALOG (the -// authoritative list) makes the real retry/severity decision. A signature the -// catalog does not recognize simply falls through to a normal rethrow. -// App-induced CUDA errors (illegal memory access, unspecified launch failure, -// OOM) are deliberately excluded -- the OpenSearch stage data shows those are -// overwhelmingly code regressions, not node faults, and must not trigger a -// node-avoiding retry. -// -// grep -o returns only the matched signature (not the whole line), so a long -// log line cannot truncate the signature out of the result before it reaches -// classify(). Each alternative must therefore be catalog-exact: it must match -// (via `.` wildcards for shell-hostile chars) the full catalog substring, so -// grep -o emits text that still contains the catalog pattern. -def scrapeSlurmLogForDeviceFault(def pipeline, Map remote, String remoteLogPath) { - def deviceFaultRegex = "cudaErrorMapBufferObjectFailed|mapping of buffer object failed|" + - "uncorrectable NVLink error|cudaErrorNvlinkUncorrectable|CUDA_ERROR_SYSTEM_NOT_READY|" + - "uncorrectable ECC error|CUDA_ERROR_ECC_UNCORRECTABLE|has fallen off the bus|GPU is lost|" + - "Unable to determine the device handle for GPU|RmInitAdapter failed|Failed to initialize NVML|" + - "could... communicate with the NVIDIA driver|CUDA_ERROR_DEVICE_UNAVAILABLE|" + - "no CUDA-capable device is detected|CUDA_ERROR_UNKNOWN: 999|CUDA unknown error|" + - "CUDA-capable device.s. is/are busy or unavailable" - try { - // Wrap the body in `bash -c` so it is shell-agnostic: cluster login shells - // are often csh/tcsh, which can't parse this bash test/pipe/redirection - // syntax. The login shell only has to run `bash -c ''`. - return Utils.exec( - pipeline, - script: Utils.sshUserCmd(remote, - "\"bash -c 'if [ -f \\\"${remoteLogPath}\\\" ]; then grep -aioE \\\"${deviceFaultRegex}\\\" \\\"${remoteLogPath}\\\" 2>/dev/null | tail -n 1 | cut -c1-500; fi'\""), - returnStdout: true, - numRetries: 1, - )?.trim() - } catch (InterruptedException e) { - throw e - } catch (Exception scrapeEx) { - pipeline.echo("Ignorable warning: could not scrape ${remoteLogPath} for device faults on ${remote.host}: ${scrapeEx.message}") - return "" - } -} - // `postTag` uniquifies the uploaded tar filename, the Artifactory guard key and // the locally-staged result XMLs when the same stageName is uploaded more than // once in a build (e.g. SLURM infra-failure retries). First attempt passes "". def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String nodeName, String stageName, Boolean stageIsInterrupted, String postTag="") { - CloudManager.withSlurmSshCredentialRemotes(pipeline, clusterName, cluster) { remotes -> - // Pin one reachable frontend for the whole collect: every download targets - // the same node workspace (/home/svc_tensorrt/bloom/scripts/${nodeName}), - // so the find + scps must all hit the login node that holds those files. - // No whole-closure failover here -- uploadArtifacts/junit must not re-run. - def remote = CloudManager.selectReachableSlurmRemote(pipeline, remotes) + CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> def hasTimeoutTest = false def downloadResultSucceed = false def downloadPerfResultSucceed = false @@ -335,41 +229,6 @@ def uploadResults(def pipeline, SlurmCluster cluster, String clusterName, String downloadPerfResultSucceed = Utils.exec(pipeline, script: scpFromRemoteCmd(remote, scpSources, "${stageName}/"), returnStatus: true, numRetries: 3) == 0 } - // Pull this stage's per-process .cbtscov files as one archive into ${stageName}/cbts/; bounded and non-fatal. - if (isCbtsStage(stageName)) { - def remoteWs = "/home/svc_tensorrt/bloom/scripts/${nodeName}" - def cbtsLocalDir = "${stageName}/cbts" - def cbtsArchive = "cbts_coverage_${stageName}.tar.gz" - sh "mkdir -p ${cbtsLocalDir}" - try { - timeout(time: 5, unit: 'MINUTES') { - // Pack on the login node; drop a partial archive when nothing matches so the scp fails cleanly. - Utils.exec( - pipeline, - script: Utils.sshUserCmd( - remote, - "\"cd '${remoteWs}' && tar czf '${cbtsArchive}' .cbtscov.${stageName}* 2>/dev/null || rm -f '${cbtsArchive}'\"" - ), - returnStatus: true, - numRetries: 3, - ) - def gotArchive = Utils.exec( - pipeline, - script: scpFromRemoteCmd(remote, "${remoteWs}/${cbtsArchive}", "${cbtsLocalDir}/"), - returnStatus: true, - numRetries: 3, - ) == 0 - if (gotArchive) { - sh "tar xzf ${cbtsLocalDir}/${cbtsArchive} -C ${cbtsLocalDir}/ && rm -f ${cbtsLocalDir}/${cbtsArchive}" - } else { - echo "CBTS: no coverage archive retrieved for ${stageName} (no coverage data or transfer skipped)." - } - } - } catch (Exception e) { - echo "CBTS: coverage pull for ${stageName} skipped (${e.message}); continuing." - } - } - echo "hasTimeoutTest: ${hasTimeoutTest}, downloadResultSucceed: ${downloadResultSucceed}, downloadPerfResultSucceed: ${downloadPerfResultSucceed}" if (hasTimeoutTest || downloadResultSucceed || downloadPerfResultSucceed) { // On retry attempts, rename freshly-downloaded result XMLs so that @@ -431,6 +290,7 @@ def runIsolatedTests(preprocessedLists, testCmdLine, llmSrc, stageName) { isolateTestCmdLine += ["--test-prefix=${stageName}"] isolateTestCmdLine += ["--csv=${WORKSPACE}/${stageName}/report_isolated_${i}.csv"] isolateTestCmdLine += ["--periodic-junit-xmlpath ${WORKSPACE}/${stageName}/results_isolated_${i}.xml"] + isolateTestCmdLine += ["--cov-append"] // Append coverage data to avoid overwriting previous data try { sh """ @@ -663,7 +523,7 @@ def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, du } def cleanUpSlurmResources(def pipeline, SlurmCluster cluster, String clusterName, String jobUID){ - CloudManager.withSlurmFrontendFailover(pipeline, clusterName, cluster) { remote -> + CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> def jobWorkspace = "/home/svc_tensorrt/bloom/scripts/${jobUID}" Utils.exec(pipeline, script: "echo Sleeping to allow Slurm job completion; sleep 30") @@ -706,7 +566,7 @@ def cleanUpSlurmResources(def pipeline, SlurmCluster cluster, String clusterName pipeline, script: Utils.sshUserCmd( remote, - Utils.bashWrappedRemoteCmd(cleanupCommands) + "\"${cleanupCommands}\"" ) ) @@ -722,7 +582,7 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName, Utils.exec(pipeline, script: "echo Sleeping to allow node destruction; sleep 30") - CloudManager.withSlurmFrontendFailover(pipeline, clusterName, cluster) { remote -> + CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> Utils.exec(pipeline, script: "echo Slurm job ID: ${slurmJobID}") Utils.exec( @@ -747,7 +607,7 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName, pipeline, script: Utils.sshUserCmd( remote, - Utils.bashWrappedRemoteCmd(cleanupCommands) + "\"${cleanupCommands}\"" ) ) @@ -768,7 +628,7 @@ def querySlurmJobState(def pipeline, SlurmCluster cluster, String clusterName, S } String state = null try { - CloudManager.withSlurmFrontendFailover(pipeline, clusterName, cluster) { remote -> + CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> // -X: allocation row only (skip .batch/.extern steps). -Pn: // parsable, no header. First line's first token is the job state; // SLURM renders cancellations as "CANCELLED by ", so we keep @@ -812,7 +672,7 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, try { // Run ssh command to start node in desired cluster via SLURM - CloudManager.withSlurmFrontendFailover(pipeline, partition.clusterName, cluster) { remote -> + CloudManager.withSlurmSshCredentials(pipeline, partition.clusterName, cluster) { remote -> stage('Request Node Via Slurm') { println("Selected Cluster: ${cluster.name}") @@ -881,24 +741,19 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, def jobRunningStartMs = null stage('Check If Node Is Online') { - CloudManager.withSlurmSshCredentialRemotes(pipeline, partition.clusterName, cluster) { remotes -> + CloudManager.withSlurmSshCredentials(pipeline, partition.clusterName, cluster) { remote -> // Check the SLURM job once; if it is no longer active, raise a typed // InfraFailure(SLURM) so the retry layer routes it via instanceof (scope=SLURM). def checkSlurmJobActive = { try { - CloudManager.withSlurmFrontendFailover(pipeline, remotes) { statusRemote -> - SlurmConfig.checkJobStatus(pipeline, cluster, slurmJobID, statusRemote) - } + SlurmConfig.checkJobStatus(pipeline, cluster, slurmJobID, remote) } catch (InterruptedException e) { throw e } catch (Exception e) { if (e.message?.contains("is no longer active")) { - def slurmLogPath = "/home/svc_tensorrt/slurm-logs/slurm-${slurmJobID}-${nodeName}.out" - echoRemoteLogTail(pipeline, remote, slurmLogPath) throw new InfraFailure( - "${e.message}. Check SLURM logs at ${slurmLogPath} on ${cluster.host}", - e, InfraFailure.TRANSIENT, InfraFailure.SLURM, "" - ) + "${e.message}. Check SLURM logs at /home/svc_tensorrt/slurm-logs/slurm-${slurmJobID}-${nodeName}.out on ${cluster.host}", + e, InfraFailure.TRANSIENT, InfraFailure.SLURM, "") } // Otherwise, log the error but continue (SSH might be temporarily unavailable) pipeline.echo("Warning: Could not check SLURM job status: ${e.message}") @@ -911,8 +766,8 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, // which overflowed the per-stage step cap). Release the held job every 10 // iterations (~30 min). 300 iterations * 3 min = 15h budget. // Exit codes: 0 = job RUNNING, 3 = job no longer active, 4 = timed out. - def sacctStateCmd = CloudManager.sshUserCmdWithSlurmFrontendFailover(remotes, "\"sacct -j ${slurmJobID} --format=State -Pn --allocations\"") - def releaseCmd = CloudManager.sshUserCmdWithSlurmFrontendFailover(remotes, "\"scontrol release ${slurmJobID} || true\"") + def sacctStateCmd = Utils.sshUserCmd(remote, "\"sacct -j ${slurmJobID} --format=State -Pn --allocations\"") + def releaseCmd = Utils.sshUserCmd(remote, "\"scontrol release ${slurmJobID} || true\"") def waitRc = pipeline.sh(returnStatus: true, script: """ set +e counter=0 @@ -1033,7 +888,7 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, def setupLogPath = "/home/svc_tensorrt/slurm-logs/slurm-${slurmJobID}-${nodeName}.out" def enrootLog = Utils.exec( pipeline, - script: CloudManager.sshUserCmdWithSlurmFrontendFailover(remotes, Utils.bashWrappedRemoteCmd("grep '\\[ENROOT\\]' ${setupLogPath} 2>/dev/null || true")), + script: Utils.sshUserCmd(remote, "\"grep '\\[ENROOT\\]' ${setupLogPath} 2>/dev/null || true\""), returnStdout: true, numRetries: 3 ).trim() @@ -1168,6 +1023,7 @@ def getPytestBaseCommandLine( String waivesFilePath, Boolean perfMode, String outputPath, + String trtllmWheelPath, String coverageConfigFile, String pytestUtil = "", List extraArgs = [], @@ -1176,7 +1032,6 @@ def getPytestBaseCommandLine( ) { def extraInternalEnv = "" def pytestTestTimeout = "3600" - def cbtsMode = isCbtsStage(stageName) // TRT uses half of the host logic cores for engine building which is bad for multi-GPU machines. extraInternalEnv = "__LUNOWUD=\"-thread_pool_size=${TESTER_CORES}\"" @@ -1186,13 +1041,6 @@ def getPytestBaseCommandLine( extraInternalEnv += " NCCL_DEBUG=INFO" // Pass stage name to perf sanity tests for OpenSearch tracking extraInternalEnv += " stageName=${stageName}" - // CBTS stages put cbts_plugin on PYTHONPATH (via ${VAR:-} for set -u safety) plus the marker/config env vars sitecustomize.py reads in subprocesses. - if (cbtsMode) { - def cbtsScriptDir = "${llmSrc}/jenkins/scripts/cbts/coverage_utils" - extraInternalEnv += " PYTHONPATH=${cbtsScriptDir}:\${PYTHONPATH:-}" - extraInternalEnv += " CBTS_COVERAGE_CONFIG=${coverageConfigFile}" - extraInternalEnv += " CBTS_MARKER_FILE=${outputPath}/cbts_current_test.txt" - } // Container port allocation environment variables for avoiding port conflicts def portEnvVars = "" @@ -1226,8 +1074,11 @@ def getPytestBaseCommandLine( "--output-dir=${outputPath}/", "--csv=${outputPath}/report.csv", "-o junit_logging=${jUnitLogging}", - // Coverage capture: only CBTS (post-merge) stages instrument, via cbts_plugin / sitecustomize. - cbtsMode ? "-p cbts_plugin" : "", + "--cov=${llmSrc}/examples/", + "--cov=${llmSrc}/tensorrt_llm/", + "--cov=${trtllmWheelPath}/tensorrt_llm/", + "--cov-report=", + "--cov-config=${coverageConfigFile}", "--periodic-junit", "--periodic-junit-xmlpath ${outputPath}/results.xml", "--periodic-batch-size=1", @@ -1247,9 +1098,6 @@ def getPytestBaseCommandLine( } def unittestMarkExpr = (stageName.startsWith("CPU-")) ? "cpu_only and not disabled" : "not cpu_only" testCmdLine += ["--unittest-markexpr='${unittestMarkExpr}'"] - if (ENABLE_UPLOAD_TEST_RESULTS) { - testCmdLine += ["-o console_output_style=progress-even-when-capture-no"] - } if (extraArgs) { testCmdLine += extraArgs } @@ -1321,7 +1169,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), ]) { - CloudManager.withSlurmFrontendFailover(pipeline, partition.clusterName, cluster) { remote -> + CloudManager.withSlurmSshCredentials(pipeline, partition.clusterName, cluster) { remote -> def tarName = BUILD_CONFIGS[config][TARNAME] def llmTarfile = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/${tarName}" def llmPath = sh (script: "realpath .", returnStdout: true).trim() @@ -1392,20 +1240,11 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def makoOptsJson = transformMakoArgsToJson(["Mako options:"] + makoArgs) String clusterNameForDurations = useClusterDurations ? partition.clusterName.replaceAll('[^a-zA-Z0-9]', '_') : null def testListPathLocal = renderTestDB(pipeline, testList, llmSrcLocal, stageName, makoOptsJson, clusterNameForDurations) - // Copy the test list atomically. A retry that reuses a still-active job - // re-copies over ${testListPathNode} while that job may be reading it via - // --test-list; scp truncates-then-streams, so a concurrent read could see a - // partial list and silently run a subset. Stage to a temp path and mv into - // place (same-dir rename is atomic) so a reader sees the whole old or new file. Utils.copyFileToRemoteHost( pipeline, remote, testListPathLocal, - "${testListPathNode}.tmp" - ) - Utils.exec( - pipeline, - script: Utils.sshUserCmd(remote, "\"mv -f ${testListPathNode}.tmp ${testListPathNode}\"") + testListPathNode ) // Download and Merge waives.txt @@ -1436,20 +1275,16 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG ) } - // Generate .coveragerc: CBTS stages render coveragerc.template; all other stages get an empty rcfile (no coverage). - if (isCbtsStage(stageName)) { - // @TRTLLM_WHEEL_PATH@ stays a placeholder; slurm_run.sh substitutes it on the worker. - sh """ - cp ${llmSrcLocal}/jenkins/scripts/cbts/coverage_utils/coveragerc.template ./.coveragerc - sed -i \\ - -e 's|@JOB_WORKSPACE@|${jobWorkspace}|g' \\ - -e 's|@STAGE_NAME@|${stageName}|g' \\ - ./.coveragerc - cat ./.coveragerc - """ - } else { - sh "touch ./.coveragerc" - } + // generate .coveragerc in workspace and add file path to pytest command + sh """ + touch ./.coveragerc + echo '[run]' > ./.coveragerc + echo 'branch = True' >> ./.coveragerc + echo 'data_file = ${jobWorkspace}/.coverage.${stageName}' >> ./.coveragerc + echo '[paths]' >> ./.coveragerc + echo 'source =\n ${llmSrcNode}/tensorrt_llm/\n ---wheel_path---/tensorrt_llm//tensorrt_llm/' >> ./.coveragerc + cat ./.coveragerc + """ Utils.copyFileToRemoteHost( pipeline, @@ -1484,10 +1319,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "--s3-upload-path=${uploadPath}/${stageName}", ] if (ENABLE_S3_ECHO_STDOUT) { - extraArgs += [ - "--s3-echo-stdout", - "--s3-capture-mode=timestamped", - ] + extraArgs += ["--s3-echo-stdout"] } } def pytestCommand = getPytestBaseCommandLine( @@ -1496,6 +1328,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG waivesListPathNode, perfMode, jobWorkspace, + "__PLACEHOLDER_TRTLLM_WHL_PATH__", "$jobWorkspace/.coveragerc", pytestUtil, extraArgs, @@ -1752,26 +1585,13 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG set -xEeuo pipefail trap 'rc=\$?; echo "Error in file \${BASH_SOURCE[0]} on line \$LINENO: \$BASH_COMMAND (exit \$rc)"; exit \$rc' ERR - # Reuse an already-active job after an ambiguous frontend disconnect. + # Clean up previous job intermediate files so that retry can work if [ -f "${jobWorkspace}/slurm_job_id.txt" ]; then previous_job_id=\$(cat "${jobWorkspace}/slurm_job_id.txt") echo "Found previous Slurm job ID: \${previous_job_id}" - previous_state=\$(sacct -j "\${previous_job_id}" --format=State -Pn --allocations 2>/dev/null | head -1 | cut -d'|' -f1 | awk '{print \$1}' || true) - if [ -z "\${previous_state}" ]; then - previous_state=\$(scontrol show job "\${previous_job_id}" 2>/dev/null | tr ' ' '\\n' | sed -n 's/^JobState=//p' | head -1 || true) - fi - case "\${previous_state}" in - RUNNING|PENDING|CONFIGURING|COMPLETING|REQUEUED|RESIZING|SUSPENDED|SIGNALING|STOPPED) - echo "Reusing active Slurm job \${previous_job_id} in state \${previous_state}" - exit 0 - ;; - *) - echo "Previous Slurm job \${previous_job_id} is not active (state='\${previous_state:-UNKNOWN}'). Cleaning it up before resubmission." - scancel "\${previous_job_id}" || true - # Wait for 120 seconds to ensure the previous job is canceled - sleep 120 - ;; - esac + scancel "\${previous_job_id}" || true + # Wait for 120 seconds to ensure the previous job is canceled + sleep 120 fi # Clean up workspace: remove all files/dirs not in the keep list @@ -1800,15 +1620,14 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG } stage("[${stageName}] Run Pytest") { - // Submit the Slurm job. Submit/metadata/track all run on the one - // frontend the enclosing withSlurmFrontendFailover pinned, so they - // share the job workspace (slurm_job_id.txt, scripts) on that login - // node; a frontend disconnect fails the whole closure over to a fresh - // frontend as a unit (the submit script reuses an active job). + // Submit the Slurm job Utils.exec( pipeline, timeout: false, - script: Utils.sshUserCmd(remote, scriptSubmitPathNode), + script: Utils.sshUserCmd( + remote, + scriptSubmitPathNode + ), numRetries: 3 ) @@ -1817,7 +1636,10 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG if (!slurmJobId) { slurmJobId = Utils.exec( pipeline, - script: Utils.sshUserCmd(remote, "\"cat ${jobWorkspace}/slurm_job_id.txt\""), + script: Utils.sshUserCmd( + remote, + "\"cat ${jobWorkspace}/slurm_job_id.txt\"" + ), returnStdout: true, numRetries: 3 ).trim() @@ -1908,12 +1730,15 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG true ) - // Track the Slurm job on the same pinned frontend. + // Track the Slurm job try { Utils.exec( pipeline, timeout: false, - script: Utils.sshUserCmd(remote, scriptTrackPathNode), + script: Utils.sshUserCmd( + remote, + scriptTrackPathNode + ), numRetries: 3 ) } catch (InterruptedException e) { @@ -1936,30 +1761,13 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "Original failure: ${e.message}", e) } - // A terminal FAILED state may be a node/device fault whose - // signature (CUDA/NVLink/ECC/driver) printed only into the SLURM - // job output log, never into this exception chain. Scrape the log - // and, on a hit, surface the matched line into a fresh exception - // so the authoritative catalog (FailureClassifier.classify at the - // runLLMTestlistWithSbatch caller) can match it and steer the retry - // off the bad node. A miss falls through to the plain rethrow. - if (slurmState == "FAILED") { - def deviceHit = scrapeSlurmLogForDeviceFault(pipeline, remote, slurmJobLogPath) - if (deviceHit) { - echo "[INFRA-RETRY] ${stageName}: device-fault signature in SLURM job ${slurmJobId} log; " + - "surfacing to classifier: ${deviceHit}" - throw new Exception( - "Device/interconnect fault on SLURM node during job ${slurmJobId} for ${stageName}: " + - "${deviceHit} | original: ${e.message}") - } - } echo "[INFRA-RETRY] ${stageName}: SLURM job ${slurmJobId} terminal state=${slurmState ?: 'unknown'}; " + "deferring to failure classifier." throw e } } echo "Finished test stage execution." - } // end CloudManager.withSlurmFrontendFailover + } // end CloudManager.withSlurmSshCredentials } // end withCredentials } catch (InterruptedException e) { stageIsInterrupted = true @@ -2013,15 +1821,10 @@ def cbtsResizeSplits(configs) { return resized } -def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, runWithSbatch=false, skipInstallWheel=false, cpver="cp312", String outerAttemptTag="", boolean useClusterDurations=false, Integer infraRetryMax=null) +def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, runWithSbatch=false, skipInstallWheel=false, cpver="cp312", String outerAttemptTag="", boolean useClusterDurations=false) { echo "Run Slurm job with native sbatch: $runWithSbatch" - // Per-stage override of the SLURM infra-retry budget (from opts.infraRetryMax, - // threaded via the dispatcher's retryContext). Lets resource-scarce pools cap - // or disable stage-level retries (0 = no retry). Null falls back to the global. - int slurmInfraRetryMax = resolveInfraRetryMax(InfraFailure.SLURM, infraRetryMax) - def attempt = 0 // Avoided SLURM nodes keyed by cluster. The platform can resolve to a // different cluster on each attempt (auto: platforms pick one at random), and @@ -2038,7 +1841,7 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p ] try { if (attempt > 1) { - echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${slurmInfraRetryMax + 1}" + echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${SLURM_INFRA_RETRY_MAX + 1}" if (!avoidedSlurmNodeListsByCluster.isEmpty()) { echo "[INFRA-RETRY] ${stageName}: avoiding prior SLURM node list(s): " + avoidedSlurmNodeListsByCluster.collect { c, ns -> "${c}: ${ns.join(' ')}" }.join('; ') @@ -2085,7 +1888,7 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p rememberAvoidedSlurmNodeLists(avoidedSlurmNodeListsByCluster, attemptPlacementContext.lastSlurmClusterName, attemptPlacementContext.lastSlurmNodeList, stageName) - def effectiveMax = (c.severity == InfraFailure.PERSISTENT) ? Math.min(1, slurmInfraRetryMax) : slurmInfraRetryMax + def effectiveMax = (c.severity == InfraFailure.PERSISTENT) ? 1 : SLURM_INFRA_RETRY_MAX if (attempt > effectiveMax) { echo "[INFRA-RETRY] ${stageName}: Infrastructure failure (${c.detectedPattern}) " + @@ -2172,9 +1975,6 @@ def DEBUG_MODE = "debug" def DETAILED_LOG = "detailed_log" @Field def CBTS_RESULT = "cbts_result" -// Pipeline-level CBTS coverage eligibility, decided in L0_MergeRequest.groovy. -@Field -def CBTS_COVERAGE = "cbts_coverage" // Suffix for CBTS-narrowed stages so their results aren't reused by non-CBTS runs. // A suffix (not prefix) keeps the GPU type as the first '-' token for positional parsers. @Field @@ -2198,7 +1998,6 @@ def testFilter = [ (AUTO_TRIGGER_TAG_LIST): [], (DETAILED_LOG): false, (CBTS_RESULT): null, - (CBTS_COVERAGE): false, ] @Field @@ -2268,27 +2067,12 @@ long retrySafetyMarginMs(String scope) return scope == InfraFailure.SLURM ? 20L * 60L * 1000L : 15L * 60L * 1000L } -// Resolve a per-stage infra-retry override (opts.infraRetryMax) against its -// scope's global budget. The override may only CAP or DISABLE retries, never -// exceed the global (which bounds CI time / worst-case attempts), so it is -// clamped to [0, scopeDefault]; e.g. with the default of 1, infraRetryMax=2 -// still yields 1, and infraRetryMax=0 disables. Null falls back to the default. -int resolveInfraRetryMax(String scope, Integer override) +int retryMaxForFailure(String scope, InfraFailure failure) { - int scopeDefault = (scope == InfraFailure.SLURM) ? SLURM_INFRA_RETRY_MAX : K8S_INFRA_RETRY_MAX - if (override == null) { - return scopeDefault + if (failure.severity == InfraFailure.PERSISTENT) { + return 1 } - return Math.max(0, Math.min(override, scopeDefault)) -} - -int retryMaxForFailure(String scope, InfraFailure failure, Integer infraRetryMax=null) -{ - // Per-stage override caps the scope default (see resolveInfraRetryMax); a - // PERSISTENT failure is still capped to at most one retry, so infraRetryMax=0 - // disables retries for every severity. - int base = resolveInfraRetryMax(scope, infraRetryMax) - return (failure.severity == InfraFailure.PERSISTENT) ? Math.min(1, base) : base + return scope == InfraFailure.SLURM ? SLURM_INFRA_RETRY_MAX : K8S_INFRA_RETRY_MAX } boolean hasBudgetForInfraRetry(def pipeline, String stageName, String scope, InfraFailure failure, int attempt, int effectiveMax, long backoffMs, boolean logDecision) @@ -2320,7 +2104,7 @@ boolean retryContextAllowsRetry(def pipeline, Map retryContext, Throwable error, } Long parsedAttempt = trtllm_utils.parseCiBudgetLong(retryContext.attempt) int attempt = parsedAttempt != null ? parsedAttempt as int : 1 - int effectiveMax = retryMaxForFailure(scope, classified, retryContext.infraRetryMax as Integer) + int effectiveMax = retryMaxForFailure(scope, classified) Long parsedBackoffMs = trtllm_utils.parseCiBudgetLong(retryContext.backoffMs) long backoffMs = parsedBackoffMs != null ? parsedBackoffMs : 60L * 1000L return hasBudgetForInfraRetry(pipeline, retryContext.stageName ?: "Unknown", scope, classified, attempt, effectiveMax, backoffMs, logDecision) @@ -2371,7 +2155,7 @@ def readSlurmWorkspaceFile(def pipeline, Map remote, String path, String stageNa pipeline, script: Utils.sshUserCmd( remote, - Utils.bashWrappedRemoteCmd("cat ${path} 2>/dev/null || true") + "\"cat ${path} 2>/dev/null || true\"" ), returnStdout: true, numRetries: numRetries @@ -2380,13 +2164,6 @@ def readSlurmWorkspaceFile(def pipeline, Map remote, String path, String stageNa } catch (InterruptedException e) { throw e } catch (Exception e) { - // A dead frontend must propagate so the enclosing withSlurmFrontendFailover - // fails over to another remote; swallowing it as "" would strand the stage on - // the unreachable frontend. Any other read failure (missing file, transient) - // is non-fatal -- the metadata is best-effort, so return "" and carry on. - if (CloudManager.isSlurmFrontendConnectionFailure(e)) { - throw e - } echo "[INFRA-RETRY] ${stageName}: unable to read SLURM metadata file ${path}: ${e.toString()}" return "" } @@ -2441,7 +2218,7 @@ def captureSlurmJobNodeList(def pipeline, SlurmCluster cluster, String clusterNa def capturedJobID = slurmJobID def nodeList = null try { - CloudManager.withSlurmFrontendFailover(pipeline, clusterName, cluster) { remote -> + CloudManager.withSlurmSshCredentials(pipeline, clusterName, cluster) { remote -> def metadata = captureSlurmWorkspaceMetadata(pipeline, remote, jobWorkspace, placementContext, stageName) capturedJobID = capturedJobID ?: metadata.slurmJobId nodeList = metadata.nodeList @@ -2953,7 +2730,7 @@ def runLLMDocBuild(pipeline, config) sh "pwd && ls -alh" sh "env | sort" // allow to checkout from forked repo, svc_tensorrt needs to have access to the repo, otherwise clone will fail - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) sh "mkdir TensorRT-LLM" sh "cp -r ${LLM_ROOT}/ TensorRT-LLM/src/" trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"") @@ -3161,6 +2938,10 @@ def getMakoArgsFromStageName(stageName, parseSysinfo=false) { // If stageName contains "-PyTorch-", add "backend=pytorch" to makoArgs // At this point, only tests with backend=pytorch or unspecified backend will be run makoArgs += ["backend=pytorch"] + } else if (stageName.contains("-TensorRT-")) { + // If stageName contains "-TensorRT-", add "backend=tensorrt" to makoArgs + // At this point, only tests with backend=tensorrt or unspecified backend will be run + makoArgs += ["backend=tensorrt"] } else if (stageName.contains("-CPP-")) { // If stageName contains "-CPP-", add "backend=cpp" to makoArgs // At this point, only tests with backend=cpp or unspecified backend will be run @@ -3185,7 +2966,7 @@ def getMakoArgsFromStageName(stageName, parseSysinfo=false) { // At this point, only tests with backend=verl or unspecified backend will be run makoArgs += ["backend=verl"] } else { - // If stageName does not contain "-PyTorch-", "-CPP-", "-Triton-", "-FMHA-", "-AutoDeploy-", or "-Verl-", do not add any backend + // If stageName does not contain "-PyTorch-", "-TensorRT-", "-CPP-", "-Triton-", "-FMHA-", "-AutoDeploy-", or "-Verl-", do not add any backend // At this point, all tests will be run // For cases where backend is not specified in makoArgs, we will match all types of backends and tests without specified backend } @@ -3985,19 +3766,14 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO sh "echo ${TRTLLM_WHL_PATH}" def coverageConfigFile = "${llmSrc}/${stageName}/.coveragerc" sh "mkdir -p ${llmSrc}/${stageName} && touch ${coverageConfigFile}" - // CBTS stages render coveragerc.template; all other stages leave the rcfile empty (no coverage). Keep in sync with the SLURM branch. - if (isCbtsStage(stageName)) { - // K8s runner knows TRTLLM_WHL_PATH here, so all placeholders are substituted at controller time (no worker-side sed). - sh """ - cp ${llmSrc}/jenkins/scripts/cbts/coverage_utils/coveragerc.template ${coverageConfigFile} - sed -i \\ - -e 's|@TRTLLM_WHEEL_PATH@|${TRTLLM_WHL_PATH}|g' \\ - -e 's|@JOB_WORKSPACE@|${WORKSPACE}/${stageName}|g' \\ - -e 's|@STAGE_NAME@|${stageName}|g' \\ - ${coverageConfigFile} - cat ${coverageConfigFile} - """ - } + sh """ + echo '[run]' > ${coverageConfigFile} + echo 'branch = True' >> ${coverageConfigFile} + echo 'data_file = ${WORKSPACE}/${stageName}/.coverage.${stageName}' >> ${coverageConfigFile} + echo '[paths]' >> ${coverageConfigFile} + echo 'source =\n ${llmSrc}/tensorrt_llm/\n ${TRTLLM_WHL_PATH}/tensorrt_llm/' >> ${coverageConfigFile} + cat ${coverageConfigFile} + """ echoNodeAndGpuInfo(pipeline, stageName) // Allocate a unique port section for this container to avoid port conflicts @@ -4016,10 +3792,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO "--s3-upload-path=${uploadPath}/${stageName}", ] if (ENABLE_S3_ECHO_STDOUT) { - extraArgs += [ - "--s3-echo-stdout", - "--s3-capture-mode=timestamped", - ] + extraArgs += ["--s3-echo-stdout"] } } def pytestCommand = getPytestBaseCommandLine( @@ -4028,6 +3801,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO "${llmSrc}/tests/integration/test_lists/waives.txt", perfMode, "${WORKSPACE}/${stageName}", + TRTLLM_WHL_PATH, coverageConfigFile, "", // pytestUtil extraArgs, // extraArgs @@ -4113,15 +3887,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO error "No tests were executed for stage ${stageName}, please check the test list and test-db rendering result." } } - - // CBTS coverage liveness signal: log this stage's touch counts (no artifacts); never fails the stage (|| true). - if (isCbtsStage(stageName)) { - sh """ - cd ${WORKSPACE}/${stageName} && \ - python3 ${llmSrc}/jenkins/scripts/cbts/coverage_utils/pystart_report.py \ - --glob '.cbtscov.${stageName}*' || true - """ - } } // Generate comprehensive rerun report if any reruns occurred @@ -4376,7 +4141,7 @@ def runLLMBuild( sh "env | sort" sh "ccache -sv" - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, "tensorrt_llm", true, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, "tensorrt_llm", false, true) if (env.alternativeTRT) { sh "cd tensorrt_llm/ && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt" } @@ -4519,12 +4284,9 @@ def runPackageSanityCheck(pipeline, wheel_path, reinstall_dependencies=false, cp trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${pkgUrl}") sh "tar -zvxf ${linuxPkgName}" - // TODO: The steps below drove the removed TensorRT engine flow (trtllm-build / examples/run.py). - // When re-enabling this sanity check, use PyTorch backend test samples instead - // (e.g. examples/llm-api/quickstart_example.py). - // trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core/gpt && python3 ../../../generate_checkpoint_config.py --architecture GPTForCausalLM --dtype float16'") - // trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core//gpt && trtllm-build --model_config config.json --log_level verbose'") - // trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core/gpt && python3 ../../../run.py --max_output_len 4 --end_id -1'") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core/gpt && python3 ../../../generate_checkpoint_config.py --architecture GPTForCausalLM --dtype float16'") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core//gpt && trtllm-build --model_config config.json --log_level verbose'") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "bash -c 'cd tensorrt_llm/examples/models/core/gpt && python3 ../../../run.py --max_output_len 4 --end_id -1'") } def checkStageNameSet(stageNames, jobKeys, paramName) { @@ -4644,12 +4406,6 @@ def runInKubernetes(pipeline, podSpec, containerName) def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerName, String stageName, Closure runner) { boolean singleAttempt = opts.singleAttempt ?: false - // Per-stage override of the K8s pod-level infra-retry budget (opts.infraRetryMax, - // 0 = no retry) so resource-scarce pools can cap or disable stage retries. Applies - // to the outer test-pod retry loop below; the singleAttempt launch-retry keeps the - // global (relaunching a dispatcher pod doesn't tax the scarce test hardware). SLURM - // dispatchers carry the same opt through to their inner SLURM retry via retryContext. - int k8sInfraRetryMax = resolveInfraRetryMax(InfraFailure.K8S, opts.infraRetryMax as Integer) // DEBUG_MODE preserves the existing 2-hour-input human-inspection workflow // inside runLLMTestlistOnPlatform's finallyRunner: a single attempt only. @@ -4727,7 +4483,7 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN Map attemptPlacementContext = [:] try { if (attempt > 1) { - echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${k8sInfraRetryMax + 1}" + echo "[INFRA-RETRY] ${stageName}: Starting attempt ${attempt} of ${K8S_INFRA_RETRY_MAX + 1}" if (!avoidedKubernetesHostNodes.isEmpty()) { echo "[INFRA-RETRY] ${stageName}: avoiding prior Kubernetes host node(s): ${avoidedKubernetesHostNodes.join(', ')}" } @@ -4748,14 +4504,13 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN // cacheErrorAndUploadResult does not suppress synthetic stage-fail // XML / junit() on what would otherwise look (to it) like just // another intermediate attempt. - def effectiveMaxThisAttempt = (lastSeverity == InfraFailure.PERSISTENT) ? Math.min(1, k8sInfraRetryMax) : k8sInfraRetryMax + def effectiveMaxThisAttempt = (lastSeverity == InfraFailure.PERSISTENT) ? 1 : K8S_INFRA_RETRY_MAX boolean isFinalAttempt = (attempt > effectiveMaxThisAttempt) def retryContext = [ scope: InfraFailure.K8S, stageName: stageName, attempt: attempt, backoffMs: 60L * 1000L, - infraRetryMax: opts.infraRetryMax, excludedKubernetesHostNodes: avoidedKubernetesHostNodes.collect(), ] def attemptPodSpec = trtllm_utils.withKubernetesHostNodeExclusion(podSpec, avoidedKubernetesHostNodes) @@ -4781,7 +4536,7 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN rememberAvoidedKubernetesHostNodes(avoidedKubernetesHostNodes, attemptPlacementContext.lastHostNode, stageName) - def effectiveMax = (c.severity == InfraFailure.PERSISTENT) ? Math.min(1, k8sInfraRetryMax) : k8sInfraRetryMax + def effectiveMax = (c.severity == InfraFailure.PERSISTENT) ? 1 : K8S_INFRA_RETRY_MAX if (attempt > effectiveMax) { echo "[INFRA-RETRY] ${stageName}: Infrastructure failure (${c.detectedPattern}) " + @@ -4837,11 +4592,12 @@ def launchTestJobs(pipeline, testFilter) x86TestConfigs = [ "CPU-Generic-x86-1": ["cpu", "l0_cpu_x86", 1, 1], "DGX_H100-4_GPUs-CPP-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], - "A10-PyTorch-1": ["a10", "l0_a10", 1, 3], - "A10-PyTorch-2": ["a10", "l0_a10", 2, 3], - "A10-PyTorch-3": ["a10", "l0_a10", 3, 3], + "A10-PyTorch-1": ["a10", "l0_a10", 1, 2], + "A10-PyTorch-2": ["a10", "l0_a10", 2, 2], + "A10-TensorRT-1": ["a10", "l0_a10", 1, 1], "A30-PyTorch-1": ["a30", "l0_a30", 1, 2], "A30-PyTorch-2": ["a30", "l0_a30", 2, 2], + "A10-CPP-1": ["a10", "l0_a10", 1, 1], "A30-CPP-1": ["a30", "l0_a30", 1, 1], "A30-AutoDeploy-1": ["a30", "l0_a30", 1, 1], "A100X-PyTorch-1": ["a100x", "l0_a100", 1, 1], @@ -4850,29 +4606,56 @@ def launchTestJobs(pipeline, testFilter) "H100_PCIe-PyTorch-Ray-1": ["h100-cr", "l0_h100", 1, 1], "H100_PCIe-AutoDeploy-1": ["h100-cr", "l0_h100", 1, 1], "H100_PCIe-CPP-1": ["h100-cr", "l0_h100", 1, 1], + "H100_PCIe-TensorRT-1": ["h100-cr", "l0_h100", 1, 1], "RTX5090-PyTorch-1": ["rtx-5090", "l0_gb202", 1, 1], - "RTX5080-PyTorch-1": ["rtx-5080", "l0_gb203", 1, 2], - "RTX5080-PyTorch-2": ["rtx-5080", "l0_gb203", 2, 2], + "RTX5080-TensorRT-1": ["rtx-5080", "l0_gb203", 1, 2], + "RTX5080-TensorRT-2": ["rtx-5080", "l0_gb203", 2, 2], // Currently post-merge test stages only run tests with "stage: post_merge" mako // in the test-db. This behavior may change in the future. - "A10-PyTorch-Post-Merge-1": ["a10", "l0_a10", 1, 4], - "A10-PyTorch-Post-Merge-2": ["a10", "l0_a10", 2, 4], - "A10-PyTorch-Post-Merge-3": ["a10", "l0_a10", 3, 4], - "A10-PyTorch-Post-Merge-4": ["a10", "l0_a10", 4, 4], + "A10-PyTorch-Post-Merge-1": ["a10", "l0_a10", 1, 1], + "A10-TensorRT-Post-Merge-1": ["a10", "l0_a10", 1, 3], + "A10-TensorRT-Post-Merge-2": ["a10", "l0_a10", 2, 3], + "A10-TensorRT-Post-Merge-3": ["a10", "l0_a10", 3, 3], "A10-FMHA-Post-Merge-1": ["a10", "l0_a10", 1, 1], + // "A30-TensorRT-Post-Merge-1": ["a30", "l0_a30", 1, 6], + // "A30-TensorRT-Post-Merge-2": ["a30", "l0_a30", 2, 6], + // "A30-TensorRT-Post-Merge-3": ["a30", "l0_a30", 3, 6], + // "A30-TensorRT-Post-Merge-4": ["a30", "l0_a30", 4, 6], + // "A30-TensorRT-Post-Merge-5": ["a30", "l0_a30", 5, 6], + // "A30-TensorRT-Post-Merge-6": ["a30", "l0_a30", 6, 6], "A30-CPP-Post-Merge-1": ["a30", "l0_a30", 1, 2], "A30-CPP-Post-Merge-2": ["a30", "l0_a30", 2, 2], // "A30-Triton-Post-Merge-1": ["a30", "l0_a30", 1, 2], // "A30-Triton-Post-Merge-2": ["a30", "l0_a30", 2, 2], - "A100X-PyTorch-Post-Merge-1": ["a100x", "l0_a100", 1, 1], - "L40S-PyTorch-Post-Merge-1": ["l40s", "l0_l40s", 1, 1], + // "A100X-TensorRT-Post-Merge-1": ["a100x", "l0_a100", 1, 6], + // "A100X-TensorRT-Post-Merge-2": ["a100x", "l0_a100", 2, 6], + // "A100X-TensorRT-Post-Merge-3": ["a100x", "l0_a100", 3, 6], + // "A100X-TensorRT-Post-Merge-4": ["a100x", "l0_a100", 4, 6], + // "A100X-TensorRT-Post-Merge-5": ["a100x", "l0_a100", 5, 6], + // "A100X-TensorRT-Post-Merge-6": ["a100x", "l0_a100", 6, 6], + // "L40S-TensorRT-Post-Merge-1": ["l40s", "l0_l40s", 1, 5], + // "L40S-TensorRT-Post-Merge-2": ["l40s", "l0_l40s", 2, 5], + // "L40S-TensorRT-Post-Merge-3": ["l40s", "l0_l40s", 3, 5], + // "L40S-TensorRT-Post-Merge-4": ["l40s", "l0_l40s", 4, 5], + // "L40S-TensorRT-Post-Merge-5": ["l40s", "l0_l40s", 5, 5], "L40S-FMHA-Post-Merge-1": ["l40s", "l0_l40s", 1, 1], "H100_PCIe-AutoDeploy-Post-Merge-1": ["h100-cr", "l0_h100", 1, 1], + "H100_PCIe-CPP-Post-Merge-1": ["h100-cr", "l0_h100", 1, 1], + // "H100_PCIe-TensorRT-Post-Merge-1": ["h100-cr", "l0_h100", 1, 5], + // "H100_PCIe-TensorRT-Post-Merge-2": ["h100-cr", "l0_h100", 2, 5], + // "H100_PCIe-TensorRT-Post-Merge-3": ["h100-cr", "l0_h100", 3, 5], + // "H100_PCIe-TensorRT-Post-Merge-4": ["h100-cr", "l0_h100", 4, 5], + // "H100_PCIe-TensorRT-Post-Merge-5": ["h100-cr", "l0_h100", 5, 5], "H100_PCIe-FMHA-Post-Merge-1": ["h100-cr", "l0_h100", 1, 1], + // "B200_PCIe-TensorRT-Post-Merge-1": ["b100-ts2", "l0_b200", 1, 2], + // "B200_PCIe-TensorRT-Post-Merge-2": ["b100-ts2", "l0_b200", 2, 2], "H100_PCIe-PyTorch-Perf-1": ["h100-cr", "l0_perf", 1, 1], "DGX_H200-8_GPUs-PyTorch-Post-Merge-1": ["dgx-h200-x8", "l0_dgx_h200", 1, 1, 8], "DGX_H200-4_GPUs-PyTorch-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 1, 4], "DGX_H200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["dgx-h200-x8", "l0_dgx_h200_perf_sanity", 1, 1, 8], + // "DGX_H200-4_GPUs-TensorRT-Post-Merge-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 3, 4], + // "DGX_H200-4_GPUs-TensorRT-Post-Merge-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 3, 4], + // "DGX_H200-4_GPUs-TensorRT-Post-Merge-3": ["dgx-h200-x4", "l0_dgx_h200", 3, 3, 4], // Disable RTXPro6000 stages due to nodes will be offline temporarily. // [TODO] Split tests between RTXPro6000 and RTXPro6000D and move reasonable mount of tests to pre-merge. // "RTXPro6000-PyTorch-Post-Merge-1": ["rtx-pro-6000", "l0_rtx_pro_6000", 1, 1], @@ -4958,35 +4741,27 @@ def launchTestJobs(pipeline, testFilter) // VisualGen PerfSanity post-merge test "DGX_B200-8_GPUs-PyTorch-VisualGen-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_visual_gen_perf_sanity", 1, 1, 8, 1, true], // PerfSanity post-merge tests - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 1, 4, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 2, 4, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 3, 4, 8, 1, true], - "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 4, 4, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 1, 6, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 2, 6, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 3, 6, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 4, 6, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 5, 6, 8, 1, true], + "DGX_B200-8_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:dgx-b200-flex", "l0_b200_multi_gpus_perf_sanity", 6, 6, 8, 1, true], ] - // B200 PerfSanity pre-merge disaggregated (functional-only: perf regressions do not fail CI) - // 2 Nodes - x86SlurmTestConfigs += buildStageConfigs( - "DGX_B200-16_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE1-GPU8", - "auto:dgx-b200-flex", - "l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8", - 1, - 16, - 2 - ) // B200 PerfSanity post-merge disaggregated // 2 Nodes x86SlurmTestConfigs += buildStageConfigs( "DGX_B200-16_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU8-Post-Merge", "auto:dgx-b200-flex", "l0_b200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu8", - 2, + 6, 16, 2 ) x86SlurmTestConfigs = cbtsResizeSplits(x86SlurmTestConfigs) fullSet += x86SlurmTestConfigs.keySet() - parallelSlurmJobs = x86SlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(X86_64_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> + parallelSlurmJobs = x86SlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> // attemptTag comes from runKubernetesPodWithInfraRetry for the outer // dispatcher pod (when retry is enabled — see opts below) and is // threaded into runLLMTestlistOnSlurm so a future re-enable of outer @@ -5000,7 +4775,7 @@ def launchTestJobs(pipeline, testFilter) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, false, retryContext?.infraRetryMax) + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag) }, [singleAttempt: true]]]} // SLURM dispatcher pods run their own inner retry loop // (runLLMTestlistOnSlurm with SLURM_INFRA_RETRY_MAX). Disabling the outer @@ -5015,7 +4790,7 @@ def launchTestJobs(pipeline, testFilter) // SBSA machines from the Blossom machine pool SBSATestConfigs = [ "CPU-Generic-arm-1": ["cpu", "l0_cpu_arm", 1, 1], - "GH200-PyTorch-Post-Merge-1": ["gh200", "l0_gh200", 1, 1], + "GH200-TensorRT-Post-Merge-1": ["gh200", "l0_gh200", 1, 1], // DGX Spark is also named as GB10 Grace Blackwell Superchip. "GB10-PyTorch-1": ["gb10x", "l0_gb10", 1, 1], ] @@ -5041,10 +4816,15 @@ def launchTestJobs(pipeline, testFilter) "GB200-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 1, 2, 4], "GB200-4_GPUs-PyTorch-PerfSanity-2": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 2, 2, 4], // PerfSanity post-merge tests - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 1, 4, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 2, 4, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 3, 4, 4], - "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 4, 4, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 1, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 2, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 3, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 4, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 5, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 6, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-7": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 7, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-8": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 8, 9, 4], + "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-9": ["auto:gb200-x4-split", "l0_gb200_multi_gpus_perf_sanity", 9, 9, 4], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 3, 4], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 3, 4], "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 3, 4], @@ -5071,23 +4851,13 @@ def launchTestJobs(pipeline, testFilter) 8, 2 ) - // PerfSanity pre-merge disaggregated (functional-only: perf regressions do not fail CI) - // 2 Nodes - multiNodesSBSAConfigs += buildStageConfigs( - "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4", - "auto:gb200-flex", - "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", - 1, - 8, - 2 - ) // PerfSanity post-merge disaggregated // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU2-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu2", - 1, + 4, 8, 2 ) @@ -5095,6 +4865,14 @@ def launchTestJobs(pipeline, testFilter) "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU1-GEN1-NODE1-GPU4-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu1_gen1_node1_gpu4", + 7, + 8, + 2 + ) + multiNodesSBSAConfigs += buildStageConfigs( + "GB200-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4-Post-Merge", + "auto:gb200-flex", + "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4", 5, 8, 2 @@ -5112,7 +4890,7 @@ def launchTestJobs(pipeline, testFilter) "GB200-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 3, + 8, 12, 3 ) @@ -5135,6 +4913,14 @@ def launchTestJobs(pipeline, testFilter) 5 ) // 6 Nodes + multiNodesSBSAConfigs += buildStageConfigs( + "GB200-24_GPUs-6_Nodes-PyTorch-Disagg-PerfSanity-CTX2-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge", + "auto:gb200-flex", + "l0_gb200_multi_nodes_perf_sanity_ctx2_node1_gpu4_gen1_node4_gpu16", + 2, + 24, + 6 + ) multiNodesSBSAConfigs += buildStageConfigs( "GB200-24_GPUs-6_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE2-GPU8-GEN1-NODE4-GPU16-Post-Merge", "auto:gb200-flex", @@ -5148,36 +4934,45 @@ def launchTestJobs(pipeline, testFilter) "GB200-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge", "auto:gb200-flex", "l0_gb200_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32", - 8, + 12, 36, 9 ) + // 10 Nodes + multiNodesSBSAConfigs += buildStageConfigs( + "GB200-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE2-GPU8-GEN1-NODE8-GPU32-Post-Merge", + "auto:gb200-flex", + "l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32", + 1, + 40, + 10 + ) // GB300 PerfSanity post-merge aggregated // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB300-8_GPUs-2_Nodes-PyTorch-PerfSanity-Node2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_node2_gpu8", - 2, + 3, 8, 2 ) // GB300 PerfSanity post-merge disaggregated - // 3 Nodes (pre-merge, functional-only) + // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( - "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8", + "GB300-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4-Post-Merge", "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 1, - 12, - 3 + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4", + 3, + 8, + 2 ) // 3 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node2_gpu8", - 2, + 5, 12, 3 ) @@ -5195,26 +4990,26 @@ def launchTestJobs(pipeline, testFilter) "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node8_gpu32", - 1, + 2, 36, 9 ) // GB300 GLM-5 disaggregated (ctx DEP2) - // 3 Nodes (pre-merge, functional-only) + // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( - "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-FUNCTIONAL-ONLY-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8", + "GB300-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE1-GPU4-Post-Merge", "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", + "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node1_gpu4", 1, - 12, - 3 + 8, + 2 ) // 3 Nodes multiNodesSBSAConfigs += buildStageConfigs( "GB300-12_GPUs-3_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node2_gpu8", - 2, + 5, 12, 3 ) @@ -5223,46 +5018,10 @@ def launchTestJobs(pipeline, testFilter) "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU2-GEN1-NODE8-GPU32-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu2_gen1_node8_gpu32", - 1, - 36, - 9 - ) - // 9 Nodes: ctx1 (1 node, 4 GPUs) + gen4 (2 nodes, 8 GPUs each) = 36 GPUs - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-36_GPUs-9_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN4-NODE2-GPU8-Post-Merge", - "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen4_node2_gpu8", 2, 36, 9 ) - // 10 Nodes: ctx6 (1 node, 4 GPUs each) + gen1 (4 nodes, 16 GPUs) = 40 GPUs - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge", - "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16", - 2, - 40, - 10 - ) - // 11 Nodes: ctx3 (1 node, 4 GPUs each) + gen1 (8 nodes, 32 GPUs) = 44 GPUs - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-44_GPUs-11_Nodes-PyTorch-Disagg-PerfSanity-CTX3-NODE1-GPU4-GEN1-NODE8-GPU32-Post-Merge", - "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx3_node1_gpu4_gen1_node8_gpu32", - 2, - 44, - 11 - ) - // 14 Nodes: ctx12 (1 node, 4 GPUs each) + gen1 (2 nodes, 8 GPUs) = 56 GPUs - multiNodesSBSAConfigs += buildStageConfigs( - "GB300-56_GPUs-14_Nodes-PyTorch-Disagg-PerfSanity-CTX12-NODE1-GPU4-GEN1-NODE2-GPU8-Post-Merge", - "auto:gb300-flex", - "l0_gb300_multi_nodes_perf_sanity_ctx12_node1_gpu4_gen1_node2_gpu8", - 2, - 56, - 14 - ) multiNodesSBSAConfigs = cbtsResizeSplits(multiNodesSBSAConfigs) fullSet += multiNodesSBSAConfigs.keySet() @@ -5275,7 +5034,7 @@ def launchTestJobs(pipeline, testFilter) // singleAttempt:true disables the outer K8s pod retry; see the x86 // SLURM closure above for the full rationale (cap nested retry budget // so consistently-timing-out tests don't burn ~36h on retry cascades). - parallelSlurmJobs = SBSASlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(X86_64_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> + parallelSlurmJobs = SBSASlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> // attemptTag is threaded into runLLMTestlistOnSlurm as the outer // dispatcher pod's tag so the inner SLURM retry's postTag can't // collide with a previous dispatcher pod's upload. See the x86 @@ -5287,13 +5046,13 @@ def launchTestJobs(pipeline, testFilter) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false, retryContext?.infraRetryMax) + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 1, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false) }, [singleAttempt: true]]]} parallelJobs += parallelSlurmJobs // Add SBSA multi node Slurm jobs // singleAttempt:true disables the outer K8s pod retry; see above. - parallelMultiNodesSBSAJobs = multiNodesSBSAConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(X86_64_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> + parallelMultiNodesSBSAJobs = multiNodesSBSAConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null -> def config = LINUX_AARCH64_CONFIG if (key.contains("single-device")) { config = SINGLE_DEVICE_CONFIG @@ -5301,7 +5060,7 @@ def launchTestJobs(pipeline, testFilter) if (key.contains("llvm")) { config = LLVM_CONFIG } - runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 2, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false, retryContext?.infraRetryMax) + runLLMTestlistOnSlurm(pipeline, values[0], values[1], config, key.contains("-Perf-"), key, values[2], values[3], values[4] ?: 1, values[5] ?: 2, values[6] ?: false, false, "cp312", attemptTag, values[7] ?: false) }, [singleAttempt: true]]]} parallelJobs += parallelMultiNodesSBSAJobs @@ -5473,7 +5232,7 @@ def launchTestJobs(pipeline, testFilter) trtllm_utils.llmExecStepWithRetry(pipeline, script: 'rm -rf $(python3 -c "import site; print(site.getsitepackages()[0])")/nvidia_cutlass_dsl*') } trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y python3-pip git rsync curl wget") - trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 config set global.break-system-packages true") trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install requests") trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 uninstall -y tensorrt") @@ -5615,6 +5374,7 @@ def launchTestJobs(pipeline, testFilter) def backendMode = testFilter[(TEST_BACKEND)].collect { it.toLowerCase() } def changeMap = [ "pytorch": "-PyTorch-", + "tensorrt": "-TensorRT-", "cpp": "-CPP-", "triton": "-Triton-", "fmha": "-FMHA-", @@ -5641,9 +5401,9 @@ def launchTestJobs(pipeline, testFilter) } else { echo "ONLY_ONE_GROUP_CHANGED mode is true. The group is: ${testFilter[(ONLY_ONE_GROUP_CHANGED)]}." def excludedBackends = new HashMap() - excludedBackends["PyTorch"] = ["-CPP-", "-FMHA-"] // Only pytorch file change also need to run triton tests - excludedBackends["Triton"] = ["-PyTorch-", "-CPP-", "-FMHA-"] - excludedBackends["FMHA"] = ["-PyTorch-", "-CPP-", "-Triton-"] + excludedBackends["PyTorch"] = ["-CPP-", "-TensorRT-", "-FMHA-"] // Only pytorch file change also need to run triton tests + excludedBackends["Triton"] = ["-PyTorch-", "-CPP-", "-TensorRT-", "-FMHA-"] + excludedBackends["FMHA"] = ["-PyTorch-", "-CPP-", "-TensorRT-", "-Triton-"] def group = testFilter[(ONLY_ONE_GROUP_CHANGED)] if (excludedBackends.containsKey(group)) { parallelJobsFiltered = parallelJobsFiltered.findAll { key, value -> @@ -5679,8 +5439,8 @@ def launchTestJobs(pipeline, testFilter) // CBTS Layer 2: replace `parallelJobsFiltered` with affected stages plus // PackageSanityCheck (kept iff sanity_required) and PerfSanity (kept iff - // perfsanity_required). Pure -Perf- stages run only when CBTS selects them - // (present in affected_stages). Post-Merge stages are never force-kept; + // perfsanity_required). Pure -Perf- stages always excluded (own trigger + // model, full-list benchmarks). Post-Merge stages are never force-kept; // they only run when explicitly listed in affected_stages. def cbts = testFilter[(CBTS_RESULT)] if (cbts != null) { @@ -5690,6 +5450,7 @@ def launchTestJobs(pipeline, testFilter) def needsSanity = cbts.sanity_required def needsPerfSanity = cbts.perfsanity_required parallelJobsFiltered = parallelJobs.findAll { key, _ -> + if (key =~ /-Perf-/) return false if (key =~ /Post-Merge/) return affectedSet.contains(key) return affectedSet.contains(key) || (needsSanity && key =~ /PackageSanityCheck/) || @@ -5727,15 +5488,7 @@ def launchTestJobs(pipeline, testFilter) // for SLURM dispatcher pods to disable nested pod retry). def opts = (values.size() >= 3 && values[2] instanceof Map) ? values[2] : [:] runKubernetesPodWithInfraRetry(opts, pipeline, values[0], "trt-llm", key, { attemptTag, isFinalAttempt, retryContext = null -> - // Carry a per-stage infra-retry override (opts.infraRetryMax) to the - // inner runner via retryContext -- the SLURM dispatcher's runner reads - // it and passes it to runLLMTestlistOnSlurm. Preserve the null default - // when no override is set so non-SLURM runners are unaffected. - def innerRetryContext = retryContext - if (opts.infraRetryMax != null) { - innerRetryContext = (retryContext ?: [:]) + [infraRetryMax: opts.infraRetryMax] - } - values[1](attemptTag, isFinalAttempt, innerRetryContext) + values[1](attemptTag, isFinalAttempt, retryContext) }) } else { values() diff --git a/jenkins/TensorRT_LLM_PLC.groovy b/jenkins/TensorRT_LLM_PLC.groovy index 651e431ea96f..dc3c1735e2d1 100644 --- a/jenkins/TensorRT_LLM_PLC.groovy +++ b/jenkins/TensorRT_LLM_PLC.groovy @@ -140,7 +140,7 @@ def checkoutSource () def LLM_REPO = getLLMRepo() sh "git config --global --add safe.directory ${env.WORKSPACE}" def ref = params.ref - trtllm_utils.checkoutSource(LLM_REPO, ref, env.WORKSPACE, true, true) + trtllm_utils.checkoutSource(LLM_REPO, ref, env.WORKSPACE, false, true) } def getPulseToken(serviceId, scopes) { diff --git a/jenkins/UpdateTestDurations.groovy b/jenkins/UpdateTestDurations.groovy deleted file mode 100644 index 78938d084e76..000000000000 --- a/jenkins/UpdateTestDurations.groovy +++ /dev/null @@ -1,236 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _ - -LLM_ROOT = "llm" - -UBUNTU_24_04_IMAGE = "urm.nvidia.com/docker/ubuntu:24.04" -DURATION_FILE_PATH = "tests/integration/defs/.test_durations" -// Target repository the updated duration file is committed straight back into. -TARGET_REPO = "NVIDIA/TensorRT-LLM" -LLM_REPO = "https://github.com/${TARGET_REPO}.git" - -def createKubernetesPodConfig(image, arch = "amd64") -{ - def archSuffix = arch == "arm64" ? "arm" : "amd" - def jnlpImage = "urm.nvidia.com/sw-ipp-blossom-sre-docker-local/lambda/custom_jnlp_images_${archSuffix}_linux:jdk17" - - def podConfig = [ - cloud: "kubernetes-cpu", - namespace: "sw-tensorrt", - yaml: """ - apiVersion: v1 - kind: Pod - spec: - nodeSelector: - nvidia.com/node_type: builder - kubernetes.io/os: linux - containers: - - name: trt-llm - image: ${image} - command: ['cat'] - volumeMounts: - - name: sw-tensorrt-pvc - mountPath: "/mnt/sw-tensorrt-pvc" - readOnly: false - tty: true - resources: - requests: - cpu: 2 - memory: 5Gi - ephemeral-storage: 25Gi - limits: - cpu: 2 - memory: 5Gi - ephemeral-storage: 25Gi - imagePullPolicy: Always - - name: jnlp - image: ${jnlpImage} - args: ['\$(JENKINS_SECRET)', '\$(JENKINS_NAME)'] - resources: - requests: - cpu: '2' - memory: 5Gi - ephemeral-storage: 25Gi - limits: - cpu: '2' - memory: 5Gi - ephemeral-storage: 25Gi - qosClass: Guaranteed - volumes: - - name: sw-tensorrt-pvc - persistentVolumeClaim: - claimName: sw-tensorrt-pvc - """.stripIndent(), - ] - - return podConfig -} - -pipeline { - agent { - kubernetes createKubernetesPodConfig(UBUNTU_24_04_IMAGE) - } - options { - timestamps() - timeout(time: 1, unit: 'HOURS') - } - triggers { - cron('H 2 * * 1') - } - parameters { - string( - name: 'DAYS', - defaultValue: '7', - description: 'Number of days to look back in OpenSearch for test durations (e.g. 3, 7, 14). ') - string( - name: 'SOURCE_REPO', - defaultValue: 'NVIDIA/TensorRT-LLM', - description: 'GitHub repo to checkout scripts from (e.g. EmmaQiaoCh/TensorRT-LLM for testing).') - string( - name: 'TARGET_BRANCH', - defaultValue: 'main', - description: 'Branch of the target repo to commit the updated duration file to.') - booleanParam( - name: 'DRY_RUN', - defaultValue: false, - description: 'When true, generate the duration file but skip the commit/push.') - } - environment { - OPEN_SEARCH_DB_BASE_URL = credentials('open_search_db_base_url') - } - stages { - stage('Setup') { - steps { - container('trt-llm') { - sh """ - apt-get update -qq && \ - apt-get install -y -qq git python3-pip curl && \ - pip3 install --quiet --break-system-packages requests pyyaml - """ - } - } - } // stage Setup - - stage('Checkout') { - steps { - container('trt-llm') { - script { - def sourceRepo = "https://github.com/${params.SOURCE_REPO}.git" - trtllm_utils.checkoutSource(sourceRepo, params.TARGET_BRANCH, LLM_ROOT, false, false) - } - } - } - } // stage Checkout - - stage('Generate Duration File') { - steps { - container('trt-llm') { - sh """ - cd ${LLM_ROOT} - python3 jenkins/scripts/generate_duration.py \ - --days ${params.DAYS} \ - --duration-file new_test_durations.json - echo "Generated file size: \$(wc -l < new_test_durations.json) lines" - echo "Sample output (first 5 lines):" - head -5 new_test_durations.json - - """ - - // Always archive the freshly generated file so the user can download - // it and upload manually if the job later refuses to auto-commit. - archiveArtifacts( - artifacts: "${LLM_ROOT}/new_test_durations.json", - fingerprint: true) - - // Sanity gate: if the new file diverges too much from the one in use, - // fail the job instead of committing a possibly-broken duration file. - script { - def countItems = { path -> - sh(script: "python3 -c \"import json; print(len(json.load(open('${path}'))))\"", - returnStdout: true).trim() as Integer - } - def oldCount = countItems("${LLM_ROOT}/${DURATION_FILE_PATH}") - def newCount = countItems("${LLM_ROOT}/new_test_durations.json") - echo "Duration-file item counts -> old: ${oldCount}, new: ${newCount}" - if (oldCount == 0) { - error("Existing duration file is empty or missing; aborting.") - } - def diffPct = Math.abs(newCount - oldCount) * 100.0 / oldCount - echo "Item-count difference: ${String.format('%.1f', diffPct)}%" - if (diffPct >= 50.0) { - error("Item-count difference ${String.format('%.1f', diffPct)}% " + - ">= 50%; refusing to auto-commit. Download the archived " + - "new_test_durations.json, review, and upload manually.") - } - } - } - } - } // stage Generate Duration File - - stage('Commit and Push') { - when { - expression { !params.DRY_RUN } - } - steps { - container('trt-llm') { - script { - // Overwrite the checked-in duration file with the freshly generated one. - sh """ - cd ${LLM_ROOT} - git config --global --add safe.directory \$(pwd) - git config user.email "90828364+tensorrt-cicd@users.noreply.github.com" - git config user.name "TensorRT LLM" - cp new_test_durations.json ${DURATION_FILE_PATH} - """ - - def changeCount = sh( - script: "cd ${LLM_ROOT} && git status --porcelain ${DURATION_FILE_PATH} | wc -l", - returnStdout: true).trim() - echo "Changed duration-file count: ${changeCount}" - if (changeCount == "0") { - echo "No update to the duration file; nothing to commit." - return - } - - sh """ - cd ${LLM_ROOT} - git add ${DURATION_FILE_PATH} - git commit -s -m "[None][infra] Auto-update test durations from OpenSearch (last ${params.DAYS} days)" - """ - - withCredentials([usernamePassword( - credentialsId: 'github-cred-trtllm-ci', - usernameVariable: 'NOT_IN_USE', - passwordVariable: 'GITHUB_API_TOKEN')]) { - def authedUrl = LLM_REPO.replaceFirst( - 'https://', "https://svc_tensorrt:${GITHUB_API_TOKEN}@") - // Rebase onto the latest target branch before pushing to avoid - // clobbering commits landed since checkout. - sh """ - cd ${LLM_ROOT} - git remote set-url origin ${authedUrl} - git fetch origin ${params.TARGET_BRANCH} - git rebase origin/${params.TARGET_BRANCH} - git push origin HEAD:${params.TARGET_BRANCH} - """ - } - } - } - } - } // stage Commit and Push - } // stages -} // pipeline diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index 8dc020d3f65e..504fd8f234a8 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm -LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-skip-tritondevel-202607211045-16608 -LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-skip-tritondevel-202607211045-16608 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-skip-tritondevel-202607211045-16608 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-skip-tritondevel-202607211045-16608 -LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-skip-tritondevel-202607211045-16608 +LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607151440-16194 +LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607151440-16194 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-trt10.16.1.11-skip-tritondevel-202607151440-16194 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-trt10.16.1.11-skip-tritondevel-202607151440-16194 +LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.16.1.11-skip-tritondevel-202607151440-16194 diff --git a/jenkins/runPerfSanityTriage.groovy b/jenkins/runPerfSanityTriage.groovy index bc31ad53e318..86591da57cc7 100644 --- a/jenkins/runPerfSanityTriage.groovy +++ b/jenkins/runPerfSanityTriage.groovy @@ -89,7 +89,7 @@ pipeline { container("trt-llm") { script { sh "pwd && ls -alh" - trtllm_utils.checkoutSource(LLM_REPO, params.BRANCH, LLM_ROOT, true, false) + trtllm_utils.checkoutSource(LLM_REPO, params.BRANCH, LLM_ROOT, false, false) def commandsBase64 = params.COMMANDS.bytes.encodeBase64().toString() sh """ cd ${LLM_ROOT}/jenkins/scripts/perf && python3 perf_sanity_triage.py \ diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 5a4ce6fd84fc..b6ce6dab3620 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -11,7 +11,7 @@ CBTS narrows test cases only; Build always runs. | Layer | Where | Action | |---|---|---| -| **2. Stage** | `L0_Test.groovy::launchTestJobs` | Set `parallelJobsFiltered` to affected stages plus PackageSanityCheck (kept iff `sanity_required`) and PerfSanity (kept iff `perfsanity_required`). Pure `-Perf-` stages run only when present in `affected_stages` (not force-kept). Empty affectedSet + nothing force-kept → no-op. | +| **2. Stage** | `L0_Test.groovy::launchTestJobs` | Set `parallelJobsFiltered` to affected stages plus PackageSanityCheck (kept iff `sanity_required`) and PerfSanity (kept iff `perfsanity_required`). Pure `-Perf-` stages always excluded. Empty affectedSet + nothing force-kept → no-op. | | **2.5. Split-resize** | `L0_Test.groovy::launchTestJobs` (`cbtsResizeSplits`) | Keep only shards `1..k` per narrowed stage, where `k` (duration-sized to ~2h/shard) is `affected_stage_split_counts`. | | **3. Within-stage tests** | `L0_Test.groovy::renderTestDB` | Point trt-test-db at the narrowed `cbts_test_db/`. Each affected block's `tests:` is restricted to entries in the filter prefix subtree; unaffected blocks are dropped. | diff --git a/jenkins/scripts/cbts/blocks.py b/jenkins/scripts/cbts/blocks.py index 9ce689073682..8eb3ea400ab5 100644 --- a/jenkins/scripts/cbts/blocks.py +++ b/jenkins/scripts/cbts/blocks.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -437,6 +437,7 @@ def _classify_map_var(var_name: str) -> Optional[str]: # jenkins/L0_Test.groovy (line ~2079). IMPORTANT: keep this list in sync. _BACKEND_PATTERNS = [ ("-PyTorch-", "pytorch"), + ("-TensorRT-", "tensorrt"), ("-CPP-", "cpp"), ("-Triton-", "triton"), ("-FMHA-", "fmha"), @@ -507,9 +508,6 @@ def parse_stages_from_groovy( current_arch: Optional[str] = None for line in groovy_path.read_text().splitlines(): - # Skip commented-out lines: disabled stage configs must not be parsed. - if line.lstrip().startswith("//"): - continue open_match = _MAP_OPEN_RE.search(line.rstrip()) if open_match: # Reset on every map opening — including unfamiliar ones — so a diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py deleted file mode 100644 index 74bb0159fffb..000000000000 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Resolve and fetch the latest merged CBTS touch DB from Artifactory. - -The tarball is uploaded per post-merge run to -`//cbts-coverage/cbts_pystart_report.tar.gz` (sqlite at -the tar root plus `cbts_report/`). - -`latest_tarball_url()` reads the newest build number from the Jenkins REST API, -then walks builds down, probing Artifactory with a 1-byte ranged GET until it -finds one whose tarball exists. - -Two entry points for the Groovy wiring: - * `--print-url` — resolve and print the tarball URL only (no download). - * `--dest DIR` — download + extract, printing the local sqlite path. -""" - -from __future__ import annotations - -import argparse -import json -import shutil -import sys -import tarfile -import urllib.error -import urllib.request -from pathlib import Path -from typing import Optional - -# Merged-artifact base for the main-branch L0_PostMerge job. -ARTIFACT_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/L0_PostMerge" -TARBALL_NAME = "cbts_pystart_report.tar.gz" -SQLITE_NAME = "cbts_touchmap.sqlite" - -_URM = "https://urm.nvidia.com/artifactory" -_JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" -# Max builds to walk back when recent builds have no tarball. -_MAX_PROBE = 10 -# Per-request timeout in seconds. -_TIMEOUT = 15 - - -def _get(url: str) -> tuple[Optional[int], Optional[bytes]]: - try: - with urllib.request.urlopen(url, timeout=_TIMEOUT) as resp: - return resp.status, resp.read() - except urllib.error.HTTPError as e: - return e.code, None - except OSError as e: - print(f"[artifact] error fetching {url}: {e}", file=sys.stderr) - return None, None - - -def _exists(url: str) -> bool: - """True if the artifact exists — a 1-byte ranged GET; 200/206 means present.""" - req = urllib.request.Request(url, headers={"Range": "bytes=0-0"}) - try: - with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp: - return resp.status in (200, 206) - except urllib.error.HTTPError: - return False - except OSError as e: - print(f"[artifact] error probing {url}: {e}", file=sys.stderr) - return False - - -def latest_build_number(jenkins_base: str = _JENKINS_BASE) -> Optional[int]: - """Newest build number via the Jenkins REST API (lastBuild, then lastCompletedBuild).""" - for kind in ("lastBuild", "lastCompletedBuild"): - status, data = _get(f"{jenkins_base}/{kind}/api/json") - if status == 200 and data: - try: - return int(json.loads(data)["number"]) - except (json.JSONDecodeError, KeyError, ValueError): - pass - return None - - -def tarball_url(build: int, artifact_base: str = ARTIFACT_BASE) -> str: - return f"{_URM}/{artifact_base}/{build}/cbts-coverage/{TARBALL_NAME}" - - -def latest_tarball_url( - artifact_base: str = ARTIFACT_BASE, - jenkins_base: str = _JENKINS_BASE, - max_probe: int = _MAX_PROBE, -) -> Optional[str]: - """URL of the newest build whose coverage tarball actually exists, or None.""" - build = latest_build_number(jenkins_base) - if build is None: - print("[artifact] could not resolve latest build number", file=sys.stderr) - return None - floor = max(0, build - max_probe) - while build > floor: - url = tarball_url(build, artifact_base) - if _exists(url): - return url - print(f"[artifact] build {build} has no tarball, trying {build - 1}", file=sys.stderr) - build -= 1 - print(f"[artifact] no tarball in the last {max_probe} builds", file=sys.stderr) - return None - - -def extract_touch_db(tarball: Path | str, dest_dir: Path | str) -> Optional[Path]: - """Extract `cbts_touchmap.sqlite` from a downloaded tarball; return its path.""" - dest_dir = Path(dest_dir) - dest_dir.mkdir(parents=True, exist_ok=True) - with tarfile.open(tarball) as tf: - member = next((m for m in tf.getmembers() if m.name.endswith(SQLITE_NAME)), None) - if member is None: - return None - member.name = SQLITE_NAME - tf.extract(member, dest_dir) - return dest_dir / SQLITE_NAME - - -def fetch_latest_touch_db(dest_dir: Path | str, url: Optional[str] = None) -> Optional[Path]: - """Download + extract the latest post-merge touch DB; return local sqlite Path or None. - - `url` pins an explicit tarball (skips latest-build resolution); any failure returns None. - """ - dest_dir = Path(dest_dir) - dest_dir.mkdir(parents=True, exist_ok=True) - url = url or latest_tarball_url() - if url is None: - return None - tarball = dest_dir / TARBALL_NAME - try: - with urllib.request.urlopen(url, timeout=_TIMEOUT) as resp, open(tarball, "wb") as f: - shutil.copyfileobj(resp, f) - return extract_touch_db(tarball, dest_dir) - except OSError as e: - print(f"[artifact] download/extract failed {url}: {e}", file=sys.stderr) - return None - - -def main(argv: Optional[list[str]] = None) -> int: - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - ap.add_argument("--dest", help="download + extract into DIR; prints the local sqlite path") - ap.add_argument( - "--print-url", action="store_true", help="resolve and print the tarball URL only" - ) - ap.add_argument( - "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" - ) - args = ap.parse_args(argv) - - url = tarball_url(args.build) if args.build is not None else None - - if args.print_url: - url = url or latest_tarball_url() - if url is None: - return 1 - print(url) - return 0 - - if args.dest: - path = fetch_latest_touch_db(args.dest, url=url) - if path is None: - return 1 - print(path) - return 0 - - ap.error("one of --print-url or --dest is required") - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/jenkins/scripts/cbts/coverage_selection/touch_db.py b/jenkins/scripts/cbts/coverage_selection/touch_db.py deleted file mode 100644 index d6ff12abc686..000000000000 --- a/jenkins/scripts/cbts/coverage_selection/touch_db.py +++ /dev/null @@ -1,232 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Read-only accessor for the merged CBTS touch DB (`cbts_touchmap.sqlite`). - -Implements the TOUCH_DB_CONTRACT.md queries. Two real-data details: - - the `test` column is `/` (stage-prefixed); - - unit tests are recorded wrapped as - `test_unittests.py::test_unittests_v2[]`, while test-db YAML lists - the bare `` entry. -""" - -from __future__ import annotations - -import re -import sqlite3 -from pathlib import Path -from typing import Optional - -from blocks import normalize_test_id - -# Canonicalize an absolute co_filename to the DB `file` form (`tensorrt_llm/...`). -_CANON_RE = re.compile(r"(tensorrt_llm/.*)$") - -# A DB test value is `/`; unit tests wrap the inner entry. -_UNITTEST_WRAP_RE = re.compile(r"::test_unittests_v2\[(?P.+)\]$") - -# Completeness-heuristic constants consumed by `untrusted_tests()`. -_WORKER_SENTINEL = "tensorrt_llm/_torch/pyexecutor/py_executor.py" -_LAUNCH_MARKERS: tuple[tuple[str, str], ...] = ( - ("tensorrt_llm/llmapi/llm.py", "generate"), - ("tensorrt_llm/executor/executor.py", "GenerationExecutor.generate"), -) -_SERVING_PATH_MARKERS: tuple[str, ...] = ("disaggregated/",) -_MIN_FUNCS = 30 - - -def canon(path: str) -> str: - """Canonicalize a path to the DB's `file` form (`tensorrt_llm/...`).""" - m = _CANON_RE.search(path) - return m.group(1) if m else path - - -def split_stage(test: str) -> tuple[str, str]: - """Split a DB `test` value `/` into `(stage, nodeid)`; `("", test)` if no `/`.""" - stage, sep, nodeid = test.partition("/") - return (stage, nodeid) if sep else ("", test) - - -def unwrap_unittest(nodeid: str) -> Optional[str]: - """Return the inner `unittest/...` entry of a wrapped unittest nodeid, else None. - - `test_unittests.py::test_unittests_v2[unittest/x.py -m "part0"]` - -> `unittest/x.py -m "part0"` - """ - m = _UNITTEST_WRAP_RE.search(nodeid) - return m.group("inner") if m else None - - -def db_key(entry: str) -> Optional[str]: - """Map a test-db YAML `tests:` entry to the DB nodeid form, or None if not 1:1. - - Unit tests wrap as `test_unittests.py::test_unittests_v2[]`; a `-k` - keyword entry expands to many nodeids (no single DB key) -> None. - """ - e = normalize_test_id(entry) - if e.startswith("unittest/"): - return f"test_unittests.py::test_unittests_v2[{e}]" - if " -k " in e: - return None - return e - - -class TouchDB: - """Read-only view over a merged `cbts_touchmap.sqlite`.""" - - def __init__(self, conn: sqlite3.Connection) -> None: - self._conn = conn - - @classmethod - def open(cls, sqlite_path: Path | str) -> "TouchDB": - """Open the DB read-only (`mode=ro`) and verify the `touch` schema.""" - uri = f"file:{Path(sqlite_path).resolve()}?mode=ro" - conn = sqlite3.connect(uri, uri=True) - cols = {row[1] for row in conn.execute("PRAGMA table_info(touch)")} - if not {"test", "file", "qualname"} <= cols: - conn.close() - raise ValueError(f"unexpected touch schema, columns={sorted(cols)}") - return cls(conn) - - def close(self) -> None: - self._conn.close() - - def __enter__(self) -> "TouchDB": - return self - - def __exit__(self, *_exc) -> None: - self.close() - - # -- meta (every key optional; read with a default) -- - - def meta(self, key: str, default: Optional[str] = None) -> Optional[str]: - try: - row = self._conn.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone() - except sqlite3.OperationalError: - return default - return row[0] if row is not None else default - - def schema_version(self) -> Optional[str]: - return self.meta("schema_version") - - def collection_commit(self) -> Optional[str]: - """Commit the DB was collected at, or None if not recorded.""" - return self.meta("commit") or self.meta("collection_commit") - - # -- reverse lookup (the core of selection); always `test != ''` -- - - def tests_touching_file(self, file: str) -> set[str]: - """Stage-prefixed tests that entered any function in `file` (file-level).""" - return { - row[0] - for row in self._conn.execute( - "SELECT DISTINCT test FROM touch WHERE file=? AND test!=''", (file,) - ) - } - - def tests_touching_func(self, file: str, qualname: str) -> set[str]: - """Stage-prefixed tests that entered `qualname` in `file` (function-level).""" - return { - row[0] - for row in self._conn.execute( - "SELECT DISTINCT test FROM touch WHERE file=? AND qualname=? AND test!=''", - (file, qualname), - ) - } - - def file_has_touch_rows(self, file: str) -> bool: - """True iff any instrumented test entered a function in `file`.""" - row = self._conn.execute( - "SELECT 1 FROM touch WHERE file=? AND test!='' LIMIT 1", (file,) - ).fetchone() - return row is not None - - # -- universe / per-stage -- - - def known_tests(self) -> set[str]: - """Every stage-prefixed test with coverage data.""" - return { - row[0] for row in self._conn.execute("SELECT DISTINCT test FROM touch WHERE test!=''") - } - - def per_test_footprint(self) -> dict[str, int]: - """`{test -> functions entered}` over all stage-prefixed tests.""" - return { - row[0]: row[1] - for row in self._conn.execute( - "SELECT test, COUNT(*) FROM touch WHERE test!='' GROUP BY test" - ) - } - - def instrumented_stages(self) -> set[str]: - """Stage names the DB has data for — the stages coverage may narrow.""" - return {stage for stage, _ in map(split_stage, self.known_tests()) if stage} - - def known_by_stage(self) -> dict[str, set[str]]: - """`{stage -> {bare nodeid, ...}}` over all known tests.""" - out: dict[str, set[str]] = {} - for test in self.known_tests(): - stage, nodeid = split_stage(test) - if stage: - out.setdefault(stage, set()).add(nodeid) - return out - - # -- forward lookup (debug / explain-why) -- - - def files_touched_by(self, test: str) -> list[tuple[str, str]]: - """`(file, qualname)` rows for a stage-prefixed `test`.""" - return [ - (row[0], row[1]) - for row in self._conn.execute("SELECT file, qualname FROM touch WHERE test=?", (test,)) - ] - - # -- coverage-completeness heuristic -- - - def untrusted_tests( - self, - worker_file: str, - launch_markers: tuple[tuple[str, str], ...], - serving_path_markers: tuple[str, ...], - min_funcs: int, - ) -> set[str]: - """Stage-prefixed tests whose per-test capture looks incomplete (must always run). - - Flags a test that drove execution/serving but is missing `worker_file` — - matched by a `launch_markers` `(file, qualname_substring)` call or a - `serving_path_markers` nodeid substring — or that entered fewer than - `min_funcs` functions total. - """ - drove_execution: set[str] = set() - for file, qual_substr in launch_markers: - drove_execution |= { - row[0] - for row in self._conn.execute( - "SELECT DISTINCT test FROM touch WHERE file=? AND qualname LIKE ? AND test!=''", - (file, f"%{qual_substr}%"), - ) - } - if serving_path_markers: - drove_execution |= { - test - for test in self.known_tests() - if any(marker in test for marker in serving_path_markers) - } - missing_worker = drove_execution - self.tests_touching_file(worker_file) - tiny = { - row[0] - for row in self._conn.execute( - "SELECT test FROM touch WHERE test!='' GROUP BY test HAVING COUNT(*) < ?", - (min_funcs,), - ) - } - return missing_worker | tiny diff --git a/jenkins/scripts/cbts/coverage_utils/README.md b/jenkins/scripts/cbts/coverage_utils/README.md deleted file mode 100644 index 07edb9676e8b..000000000000 --- a/jenkins/scripts/cbts/coverage_utils/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# CBTS Layer C — Coverage Utils - -CI tooling that captures per-test **function/class-level** coverage (which product functions each -test entered), including subprocesses, on single-GPU L0 stages. CI infrastructure only — nothing -ships in the product wheel, and every file is a no-op unless `CBTS_COVERAGE_CONFIG` is set in the -environment. - -Capture uses `sys.monitoring` `PY_START` (Python 3.12+): each function a test enters fires once, -then that code object is disabled until the next test — so overhead scales with functions entered, -not lines executed (far cheaper than line tracing). - -## Files - -| File | Role | -|---|---| -| `cbts_pystart.py` | The tracker: a `sys.monitoring` (tool id 4) `PY_START` tool that records, per test context, the set of product `(file, qualname)` entered. Writes one `.cbtscov...sqlite` per process; these leave the node only inside compressed tarballs, so the publish-artifacts guardword/secret scanner (which byte-matches product paths in raw files but does not recurse into archives) never sees the paths stored inside. | -| `sitecustomize.py` | Starts the tracker in each Python process under `CBTS_COVERAGE_CONFIG` (except dependency build/install tools — `pip`, `setup.py`, `cmake`, … — which opt out themselves and their spawned subtree). Reads `source` + `data_file` from the rcfile. Long-lived non-pytest processes (e.g. `trtllm-serve`) poll a marker file to switch context; `mpi4py.futures` pool workers use the inherited `CBTS_TEST_ID` context plus the atexit save. | -| `cbts_plugin.py` | Pytest plugin (`-p cbts_plugin`): per test, writes the marker file, sets `CBTS_TEST_ID`, and switches the tracker context via `sitecustomize.switch_test_context`; also patches `mpi_session._start_mpi_pool` so workers inherit the coverage env. | -| `pystart_report.py` | Merges all `.cbtscov.*.sqlite` (union per test; legacy `.json`/`.json.gz` also accepted) and emits any of: `--out-sqlite` (indexed `touch(test, file, qualname)` DB — the selector artifact), `--out-dir` (per-file split HTML report: index + one page per file), `--out-json` (full `test -> [file::qualname]` map). With `--source-root` also computes the file/function coverage rate. Prints a one-line touch-count summary. | -| `coveragerc.template` | Template for the runtime rcfile; only `[run] source` + `data_file` are used. | -| `make_coveragerc.sh` | Substitutes `@...@` placeholders in the template; writes `$JOB_WORKSPACE/.coveragerc`. | - -## When it runs - -`jenkins/L0_MergeRequest.groovy` decides pipeline-level eligibility and propagates it to the runner via `testFilter[cbts_coverage]`: - -- `ENABLE_CBTS_COVERAGE` (global kill-switch) AND the post-merge gate — coverage runs only on the official post-merge pipeline - -`isCbtsStage()` in `jenkins/L0_Test.groovy` then gates each stage on that propagated flag plus: - -- not a perf stage, and not a TensorRT / CPP / AutoDeploy stage -- single-GPU only — stages named with `-_GPUs` or `-_Nodes` (multi-GPU / multi-node) are disabled in phase 1 and enabled incrementally later -- `CBTS_EXCLUDE_STAGES` (per-stage skip) - -Non-CBTS stages get an empty `.coveragerc` and run uninstrumented. - -## Granularity - -- **Integration tests**: the outer pytest carries `-p cbts_plugin`, so each test-db entry (one pytest item) is its own context. -- **Unit tests** (`test_unittests_v2[entry]`): the inner pytest carries no plugin, so the whole batch runs under the one inherited `CBTS_TEST_ID` context = the test-db entry. This matches CBTS's selection granularity (entry level). -- `co_qualname` gives `Class.method`, so results roll up to function → class → file. Comprehension / generator / lambda frames are skipped. - -## Output - -- Per-process `.cbtscov...pid.X.sqlite` files ride back in the standard `results-.tar.gz` under `cbts/`. Riding inside a compressed tarball keeps their plaintext product paths away from the artifact guardword/secret scanner, which byte-matches raw files but does not recurse into archives. -- `L0_MergeRequest.groovy`'s Test Coverage stage merges all stages' files via `pystart_report.py` and uploads one tarball to `${UPLOAD_PATH}/cbts-coverage/`: - - `cbts_pystart_report.tar.gz` — contains `cbts_touchmap.sqlite` (indexed touch DB / selector artifact, with a `meta` table holding the coverage rate) and `cbts_report/` (the split HTML report; open `cbts_report/index.html` after extracting). Bundled compressed so the touch DB's plaintext paths never reach the guardword scanner. - -## Query the touch DB - -Which tests to run for a change, from `cbts_touchmap.sqlite`: - -```python -import sqlite3 -c = sqlite3.connect("cbts_touchmap.sqlite") -# file-level (phase 1): any test that entered a function in the changed file -c.execute("SELECT DISTINCT test FROM touch WHERE file = ?", - ("tensorrt_llm/_torch/pyexecutor/py_executor.py",)).fetchall() -# function-level (phase 2): tests that entered a specific function/method -c.execute("SELECT DISTINCT test FROM touch WHERE file = ? AND qualname = ?", - ("tensorrt_llm/_torch/pyexecutor/py_executor.py", "PyExecutor.forward")).fetchall() -# coverage rate -dict(c.execute("SELECT key, value FROM meta")) -``` - -## Smoke test - -```bash -COV_DIR=jenkins/scripts/cbts/coverage_utils -export TRTLLM_WHEEL_PATH=/usr/local/lib/python3.12/dist-packages -export JOB_WORKSPACE=/tmp/cbts_smoke STAGE_NAME=smoke -"${COV_DIR}/make_coveragerc.sh" -export PYTHONPATH="${COV_DIR}:${PYTHONPATH:-}" -export CBTS_COVERAGE_CONFIG="${JOB_WORKSPACE}/.coveragerc" -export CBTS_MARKER_FILE="${JOB_WORKSPACE}/cbts_current_test.txt" -cd tests/integration/defs -pytest -p cbts_plugin -vs "accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4" -cd "${JOB_WORKSPACE}" && python3 "${OLDPWD}/${COV_DIR}/pystart_report.py" --glob '.cbtscov.smoke*.sqlite' -``` diff --git a/jenkins/scripts/cbts/coverage_utils/cbts_plugin.py b/jenkins/scripts/cbts/coverage_utils/cbts_plugin.py deleted file mode 100644 index 56a24646f730..000000000000 --- a/jenkins/scripts/cbts/coverage_utils/cbts_plugin.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Pytest plugin for CBTS Layer C per-test coverage attribution.""" - -import inspect -import os -import sys - -MARKER_FILE = os.environ.get("CBTS_MARKER_FILE", "/tmp/cbts/current_test.txt") - -_ENV_WHITELIST_PREFIXES = ("TRTLLM", "TLLM", "COVERAGE_", "CBTS_", "PYTHON") - -_PATCHED_MARKER = "_cbts_patched_start_mpi_pool" - - -def install_mpi_pool_patch(*, raise_on_refactor=True): - """Widen ``MpiPoolSession._start_mpi_pool``'s env whitelist; idempotent.""" - try: - from mpi4py.futures import MPIPoolExecutor # noqa: F401 - - import tensorrt_llm.llmapi.mpi_session as _ms - except ImportError: - return False - - method = _ms.MpiPoolSession._start_mpi_pool - if getattr(method, _PATCHED_MARKER, False): - return False - - src = inspect.getsource(method) - if "TRTLLM" not in src or "MPIPoolExecutor" not in src: - msg = ( - "CBTS: tensorrt_llm.llmapi.mpi_session.MpiPoolSession." - "_start_mpi_pool has been refactored upstream; the " - "monkeypatch in cbts_plugin.py needs to be updated. See " - "jenkins/scripts/cbts/coverage_utils/README.md" - ) - if raise_on_refactor: - raise RuntimeError(msg) - return False - - def _patched_start_mpi_pool(self): - """Widened env whitelist so COVERAGE_* and PYTHON* reach workers.""" - import sys as _sys - - from mpi4py.futures import MPIPoolExecutor as _MPE - - assert not self.mpi_pool, "MPI session already started" - env = {k: v for k, v in os.environ.items() if k.startswith(_ENV_WHITELIST_PREFIXES)} - self.mpi_pool = _MPE( - max_workers=self.n_workers, - path=_sys.path, - env=env, - ) - - setattr(_patched_start_mpi_pool, _PATCHED_MARKER, True) - _ms.MpiPoolSession._start_mpi_pool = _patched_start_mpi_pool - return True - - -def _switch_test_context(nodeid): - """Switch the per-test tracking context via the sitecustomize bootstrap, if active.""" - try: - import sitecustomize - - switch = getattr(sitecustomize, "switch_test_context", None) - except ImportError: - switch = None - if switch is not None: - switch(nodeid) - - -# Bind pytest only when already loaded, so importing this module for install_mpi_pool_patch stays cheap. -if "pytest" in sys.modules: - import pytest - - def pytest_configure(config): # noqa: D401 - pytest hook - """Apply ``mpi_session`` monkeypatch with a compatibility guard.""" - del config - install_mpi_pool_patch(raise_on_refactor=True) - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_protocol(item, nextitem): # noqa: D401 - pytest hook - """Per-test marker write + switch the tracking context to the current test.""" - del nextitem - nodeid = item.nodeid - - marker_dir = os.path.dirname(MARKER_FILE) - if marker_dir: - os.makedirs(marker_dir, exist_ok=True) - with open(MARKER_FILE, "w") as f: - f.write(nodeid) - f.flush() - - # Propagate nodeid via env so subprocesses pick it up in sitecustomize.py. - os.environ["CBTS_TEST_ID"] = nodeid - - _switch_test_context(nodeid) - - yield diff --git a/jenkins/scripts/cbts/coverage_utils/cbts_pystart.py b/jenkins/scripts/cbts/coverage_utils/cbts_pystart.py deleted file mode 100644 index c1c8dde9aeb5..000000000000 --- a/jenkins/scripts/cbts/coverage_utils/cbts_pystart.py +++ /dev/null @@ -1,138 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Function/class-level per-test coverage tracker via sys.monitoring PY_START (Python 3.12+).""" - -import os -import secrets -import socket -import sqlite3 -import sys - -_MON = getattr(sys, "monitoring", None) -_DEFAULT_TOOL_ID = int(os.environ.get("CBTS_PYSTART_TOOL_ID", "4")) - - -class PyStartTracker: - """Per-process tracker; one SQLite data file per process, merged (unioned) downstream.""" - - def __init__(self, source_roots, data_dir, stage="stage", tool_id=_DEFAULT_TOOL_ID): - self.source_roots = tuple(os.path.abspath(p).rstrip("/") + "/" for p in source_roots if p) - self.data_dir = data_dir - self.stage = stage - self.tool_id = tool_id - self._ctx = os.environ.get("CBTS_TEST_ID", "") or "" - self._data = {} # context -> set((filename, qualname)) - self._file_ok = {} # co_filename -> bool (cached source-membership) - self._active = False - self._new_suffix() - - @property - def available(self): - return _MON is not None and bool(self.source_roots) - - def _new_suffix(self): - self._suffix = f"{socket.gethostname()}.pid{os.getpid()}.X{secrets.token_urlsafe(6)}" - - def _in_source(self, filename): - if not filename or filename[0] == "<": - return False - return os.path.abspath(filename).startswith(self.source_roots) - - # Skip synthetic comprehension / genexpr / lambda frames; keep real functions, methods, module bodies. - _SKIP_QUALNAMES = frozenset(("", "", "", "", "")) - - def _on_py_start(self, code, offset): - try: - fn = code.co_filename - ok = self._file_ok.get(fn) - if ok is None: - ok = self._file_ok[fn] = self._in_source(fn) - if ok: - qual = code.co_qualname - if "" not in qual and qual not in self._SKIP_QUALNAMES: - self._data.setdefault(self._ctx, set()).add((fn, qual)) - except Exception: - # A tracker fault must never propagate into monitored host code. - pass - # Disable this code object's PY_START (for this tool) until the next test's restart_events(). - return _MON.DISABLE - - def start(self): - if not self.available: - return False - try: - _MON.use_tool_id(self.tool_id, "cbts-pystart") - except ValueError: - return False - try: - _MON.register_callback(self.tool_id, _MON.events.PY_START, self._on_py_start) - _MON.set_events(self.tool_id, _MON.events.PY_START) - except Exception: - try: - _MON.free_tool_id(self.tool_id) - except Exception: - pass - return False - self._active = True - try: - os.register_at_fork(after_in_child=self._after_fork_child) - except (AttributeError, ValueError): - pass - return True - - def _after_fork_child(self): - # The child writes its own data file and rediscovers what it runs. - self._new_suffix() - self._data = {} - if self._active: - try: - _MON.restart_events() - except Exception: - pass - - def switch_test_context(self, nodeid): - self._ctx = nodeid or "" - if self._active: - _MON.restart_events() - - def save(self): - # Write a per-process SQLite the downstream merge reads directly; uploaded compressed only. - snap = self._data.copy() # atomic shallow copy; each set snapshotted below - if not snap: - return None - os.makedirs(self.data_dir, exist_ok=True) - path = os.path.join(self.data_dir, f".cbtscov.{self.stage}.{self._suffix}.sqlite") - tmp = path + ".tmp" - if os.path.exists(tmp): - os.remove(tmp) - con = sqlite3.connect(tmp) - try: - con.execute("CREATE TABLE touch (test TEXT, file TEXT, qualname TEXT)") - rows = ((ctx, f, q) for ctx, fs in snap.items() for (f, q) in fs.copy()) - con.executemany("INSERT INTO touch VALUES (?, ?, ?)", rows) - con.commit() - finally: - con.close() - os.replace(tmp, path) - return path - - def stop(self): - if not self._active: - return - self._active = False - try: - _MON.set_events(self.tool_id, 0) - _MON.free_tool_id(self.tool_id) - except Exception: - pass diff --git a/jenkins/scripts/cbts/coverage_utils/coveragerc.template b/jenkins/scripts/cbts/coverage_utils/coveragerc.template deleted file mode 100644 index 3ae3e32e326e..000000000000 --- a/jenkins/scripts/cbts/coverage_utils/coveragerc.template +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# CBTS runtime config template; only [run] source and data_file are consumed by sitecustomize.py. - -[run] -source = - @TRTLLM_WHEEL_PATH@/tensorrt_llm/ -data_file = @JOB_WORKSPACE@/.coverage.@STAGE_NAME@ diff --git a/jenkins/scripts/cbts/coverage_utils/make_coveragerc.sh b/jenkins/scripts/cbts/coverage_utils/make_coveragerc.sh deleted file mode 100755 index 814f334b1539..000000000000 --- a/jenkins/scripts/cbts/coverage_utils/make_coveragerc.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Materialize $JOB_WORKSPACE/.coveragerc from coveragerc.template by substituting env placeholders. - -set -euo pipefail - -: "${TRTLLM_WHEEL_PATH:?TRTLLM_WHEEL_PATH is required}" -: "${JOB_WORKSPACE:?JOB_WORKSPACE is required}" -: "${STAGE_NAME:?STAGE_NAME is required}" - -TEMPLATE_DIR="$(cd "$(dirname "$0")" && pwd)" -TEMPLATE_FILE="${TEMPLATE_DIR}/coveragerc.template" -OUTPUT_FILE="${JOB_WORKSPACE}/.coveragerc" - -if [[ ! -f "${TEMPLATE_FILE}" ]]; then - echo "error: template not found at ${TEMPLATE_FILE}" >&2 - exit 1 -fi - -mkdir -p "${JOB_WORKSPACE}" - -sed \ - -e "s|@TRTLLM_WHEEL_PATH@|${TRTLLM_WHEEL_PATH}|g" \ - -e "s|@JOB_WORKSPACE@|${JOB_WORKSPACE}|g" \ - -e "s|@STAGE_NAME@|${STAGE_NAME}|g" \ - "${TEMPLATE_FILE}" > "${OUTPUT_FILE}" - -echo "${OUTPUT_FILE}" diff --git a/jenkins/scripts/cbts/coverage_utils/pystart_report.py b/jenkins/scripts/cbts/coverage_utils/pystart_report.py deleted file mode 100644 index d429adcf953f..000000000000 --- a/jenkins/scripts/cbts/coverage_utils/pystart_report.py +++ /dev/null @@ -1,322 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Merge per-process CBTS PY_START data files (streamed, deduped) into the touch DB / HTML report / JSON map.""" - -import argparse -import ast -import glob -import gzip -import hashlib -import json -import os -import re -import sqlite3 -import tempfile -from collections import defaultdict -from html import escape - - -def canon(path): - """Collapse an absolute install/src path to the ``tensorrt_llm/...`` product-relative form.""" - m = re.search(r"(tensorrt_llm/.*)$", path) - return m.group(1) if m else path - - -def _iter_rows(fp): - """Yield (test, file, qualname) rows from one per-process data file (SQLite or legacy JSON).""" - if fp.endswith(".sqlite"): - con = sqlite3.connect(f"file:{fp}?mode=ro", uri=True) - try: - yield from con.execute("SELECT test, file, qualname FROM touch") - finally: - con.close() - else: - opener = gzip.open if fp.endswith(".gz") else open - with opener(fp, "rt") as f: - data = json.load(f) - for ctx, pairs in data.items(): - for file, qual in pairs: - yield ctx, file, qual - - -def _safe_rows(fp): - """Yield canon'd (test, file, qualname) rows from one input file; stop early on a corrupt/unreadable input.""" - try: - for test, file, qual in _iter_rows(fp): - yield test, canon(file), qual - except (OSError, ValueError, sqlite3.Error): - return - - -def merge_to_sqlite(pattern, out_path): - """Stream every per-process file into a deduped touch DB. Returns (connection, n_data_files).""" - files = sorted(glob.glob(pattern)) - if os.path.exists(out_path): - os.remove(out_path) - con = sqlite3.connect(out_path) - con.execute("PRAGMA journal_mode=OFF") - con.execute("PRAGMA synchronous=OFF") - con.execute( - "CREATE TABLE touch (test TEXT, file TEXT, qualname TEXT, UNIQUE(test, file, qualname))" - ) - con.execute("CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)") - for fp in files: - batch = [] - # Input errors are recovered in _safe_rows; destination-write errors abort the merge. - for row in _safe_rows(fp): - batch.append(row) - if len(batch) >= 20000: - con.executemany("INSERT OR IGNORE INTO touch VALUES (?, ?, ?)", batch) - batch = [] - if batch: - con.executemany("INSERT OR IGNORE INTO touch VALUES (?, ?, ?)", batch) - con.execute("CREATE INDEX ix_file ON touch(file)") - con.execute("CREATE INDEX ix_func ON touch(file, qualname)") - con.execute("CREATE INDEX ix_test ON touch(test)") - con.commit() - return con, len(files) - - -def _collect_defs(node, prefix, rel, funcs): - """Record functions/methods reachable through class and control-flow nesting (matches PY_START co_qualname).""" - for child in ast.iter_child_nodes(node): - if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): - funcs.add((rel, prefix + child.name)) - elif isinstance(child, ast.ClassDef): - _collect_defs(child, f"{prefix}{child.name}.", rel, funcs) - else: - _collect_defs(child, prefix, rel, funcs) - - -def enumerate_defs(source_root): - """Return (coverable_files, funcs) for the coverage-rate denominator, matching what the tracker records.""" - funcs = set() - for dirpath, _dirs, names in os.walk(source_root): - for nm in names: - if not nm.endswith(".py"): - continue - full = os.path.join(dirpath, nm) - rel = canon(os.path.abspath(full)) - try: - tree = ast.parse(open(full, encoding="utf-8").read()) - except (OSError, SyntaxError): - continue - _collect_defs(tree, "", rel, funcs) - return {f for f, _q in funcs}, funcs - - -_CSS = ( - "" -) - - -def _page_name(file): - # Readable slug plus a path digest so distinct files (a-b.py vs a_b.py) never share a page. - slug = re.sub(r"[^A-Za-z0-9]", "_", file) - return f"{slug}-{hashlib.sha1(file.encode('utf-8')).hexdigest()[:10]}.html" - - -def write_html_tree(out_dir, con, rate_line, n_tests, n_data_files): - """Split HTML report, queried per file from the merged DB (bounded memory per page).""" - files_dir = os.path.join(out_dir, "files") - os.makedirs(files_dir, exist_ok=True) - files = [ - r[0] for r in con.execute("SELECT DISTINCT file FROM touch WHERE test != '' ORDER BY file") - ] - - for file in files: - # one file's (qualname -> tests) held at a time - funcs = defaultdict(list) - for qual, test in con.execute( - "SELECT qualname, test FROM touch WHERE file = ? AND test != '' ORDER BY qualname, test", - (file,), - ): - funcs[qual].append(test) - h = [f"{escape(file)}{_CSS}"] - h.append( - f"

    ← index

    {escape(file)}

    " - ) - h.append(f"

    {len(funcs)} functions touched

    ") - for qual in sorted(funcs): - tests = funcs[qual] - h.append( - f"
    {escape(qual)} " - f"← {len(tests)} tests" - ) - for t in tests: - h.append(f"
    {escape(t)}
    ") - h.append("
    ") - with open(os.path.join(files_dir, _page_name(file)), "w") as fh: - fh.write("\n".join(h)) - - # index: files grouped by directory, with per-file counts (SQL aggregate) - counts = { - f: (nf, nt) - for f, nf, nt in con.execute( - "SELECT file, COUNT(DISTINCT qualname), COUNT(DISTINCT test) " - "FROM touch WHERE test != '' GROUP BY file" - ) - } - by_dir = defaultdict(list) - for f in files: - by_dir[os.path.dirname(f)].append(f) - h = [f"CBTS coverage{_CSS}"] - h.append("

    CBTS function-level coverage (PY_START)

    ") - h.append( - f"

    {n_tests} tests · {len(files)} product files touched · {n_data_files} data files merged.

    " - ) - if rate_line: - h.append(f"

    {escape(rate_line)}

    ") - for d in sorted(by_dir): - flist = by_dir[d] - h.append( - f"
    {escape(d or '.')} ({len(flist)} files)" - ) - h.append("") - for f in flist: - nf, nt = counts.get(f, (0, 0)) - h.append( - f"" - f"" - ) - h.append("
    filefuncstests
    {escape(os.path.basename(f))}{nf}{nt}
    ") - with open(os.path.join(out_dir, "index.html"), "w") as fh: - fh.write("\n".join(h)) - - -def write_json(con, out_path): - """Stream the per-test touch map to JSON without holding it all in memory.""" - with open(out_path, "w") as f: - f.write("{") - first = True - cur_test = None - touched = [] - rows = con.execute( - "SELECT test, file, qualname FROM touch WHERE test != '' ORDER BY test, file, qualname" - ) - for test, file, qual in rows: - if test != cur_test: - if cur_test is not None: - f.write( - ("," if not first else "") - + json.dumps(cur_test) - + ":" - + json.dumps(sorted(touched)) - ) - first = False - cur_test = test - touched = [] - touched.append(f"{file}::{qual}") - if cur_test is not None: - f.write( - ("," if not first else "") - + json.dumps(cur_test) - + ":" - + json.dumps(sorted(touched)) - ) - f.write("}") - - -def main(): - ap = argparse.ArgumentParser(description="Merge CBTS PY_START data into touch artifacts.") - ap.add_argument("--glob", default=".cbtscov.*.sqlite", help="glob for per-process data files") - ap.add_argument( - "--out-sqlite", default=None, help="indexed touch(test,file,qualname) DB for the selector" - ) - ap.add_argument( - "--out-dir", - default=None, - help="directory for the split HTML report (index + per-file pages)", - ) - ap.add_argument( - "--out-json", default=None, help="optional full per-test -> [file::qualname] map" - ) - ap.add_argument( - "--source-root", - default=None, - help="product tree (…/tensorrt_llm); when given, compute file/function coverage rate", - ) - a = ap.parse_args() - - # Merge into a SQLite DB (bounded memory): --out-sqlite directly, else a unique per-invocation temp file. - keep = a.out_sqlite is not None - if keep: - db_path = a.out_sqlite - else: - fd, db_path = tempfile.mkstemp(prefix="cbts_merge_", suffix=".sqlite") - os.close(fd) - con, n_data_files = merge_to_sqlite(a.glob, db_path) - - n_tests = con.execute("SELECT COUNT(DISTINCT test) FROM touch WHERE test != ''").fetchone()[0] - n_files = con.execute("SELECT COUNT(DISTINCT file) FROM touch WHERE test != ''").fetchone()[0] - n_funcs = con.execute( - "SELECT COUNT(*) FROM (SELECT DISTINCT file, qualname FROM touch WHERE test != '')" - ).fetchone()[0] - print( - f"CBTS PY_START: merged {n_data_files} data files -> " - f"{n_tests} tests, {n_files} product files, {n_funcs} functions touched" - ) - - meta = {"tests": str(n_tests), "files": str(n_files), "functions": str(n_funcs)} - rate_line = "" - if a.source_root: - coverable_files, all_funcs = enumerate_defs(a.source_root) - touched_files = { - r[0] for r in con.execute("SELECT DISTINCT file FROM touch WHERE test != ''") - } & coverable_files - touched_funcs = { - t for t in con.execute("SELECT DISTINCT file, qualname FROM touch WHERE test != ''") - } & all_funcs - fr = 100.0 * len(touched_files) / len(coverable_files) if coverable_files else 0.0 - qr = 100.0 * len(touched_funcs) / len(all_funcs) if all_funcs else 0.0 - rate_line = ( - f"coverage rate: files {len(touched_files)}/{len(coverable_files)} " - f"({fr:.1f}%), functions {len(touched_funcs)}/{len(all_funcs)} ({qr:.1f}%)" - ) - print(rate_line) - meta.update( - { - "file_rate_pct": f"{fr:.2f}", - "func_rate_pct": f"{qr:.2f}", - "total_files": str(len(coverable_files)), - "total_functions": str(len(all_funcs)), - } - ) - - con.executemany("INSERT OR REPLACE INTO meta VALUES (?, ?)", list(meta.items())) - con.commit() - if keep: - print(f"wrote {a.out_sqlite}") - if a.out_json: - write_json(con, a.out_json) - print(f"wrote {a.out_json}") - if a.out_dir: - write_html_tree(a.out_dir, con, rate_line, n_tests, n_data_files) - print(f"wrote {a.out_dir}/index.html") - - con.close() - if not keep: - try: - os.remove(db_path) - except OSError: - pass - - -if __name__ == "__main__": - main() diff --git a/jenkins/scripts/cbts/coverage_utils/sitecustomize.py b/jenkins/scripts/cbts/coverage_utils/sitecustomize.py deleted file mode 100644 index 93b807cf9323..000000000000 --- a/jenkins/scripts/cbts/coverage_utils/sitecustomize.py +++ /dev/null @@ -1,186 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Per-process function-level coverage bootstrap for CBTS Layer C, active only when CBTS_COVERAGE_CONFIG is set.""" - -import os -import sys - - -def _parent_is_pytest(): - """Return True if our parent process is also running pytest.""" - try: - with open(f"/proc/{os.getppid()}/cmdline", "rb") as f: - parent_cmdline = f.read().split(b"\x00") - except OSError: - return False - return any(b"pytest" in part for part in parent_cmdline) - - -def _is_dependency_build_process(): - """Return True for pip / setuptools / native build-tool processes, which opt the subtree out.""" - argv = getattr(sys, "orig_argv", sys.argv) or [""] - # Scan each token's basename: the tool may be in argv[0] (bare) or argv[1] (shebang / setup.py). - tools = {"pip", "pip3", "cmake", "ninja", "ninja-build", "meson"} - for a in argv: - base = os.path.basename(a or "").lower() - if base in tools or base == "setup.py" or (a or "").endswith("setup.py"): - return True - joined = " ".join(argv) - return any(n in joined for n in ("-m pip", "-m build", "_in_process", "pyproject_hooks")) - - -# Drop the gate var so build tooling and everything it spawns opts out of instrumentation. -if os.getenv("CBTS_COVERAGE_CONFIG") and _is_dependency_build_process(): - os.environ.pop("CBTS_COVERAGE_CONFIG", None) - - -if os.getenv("CBTS_COVERAGE_CONFIG"): - import atexit - import configparser - import threading - - sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - from cbts_pystart import PyStartTracker - - _CONFIG = os.getenv("CBTS_COVERAGE_CONFIG") - - # Read [run] source (product roots) and data_file (dir + stage name) from the rendered rcfile. - def _read_config(path): - cp = configparser.ConfigParser() - try: - cp.read(path) - except configparser.Error: - return [], ".", "stage" - src = [ln.strip() for ln in cp.get("run", "source", fallback="").splitlines() if ln.strip()] - data_file = cp.get("run", "data_file", fallback="") - data_dir = os.path.dirname(data_file) or "." - base = os.path.basename(data_file) - stage = base.split(".coverage.", 1)[1] if ".coverage." in base else "stage" - return src, data_dir, stage - - _src, _data_dir, _stage = _read_config(_CONFIG) - - _PERIODIC_SAVE_SECONDS = 5 - _stop_event = threading.Event() - - _tracker = PyStartTracker(_src, _data_dir, _stage) - _tracker.start() - - def switch_test_context(nodeid): - """Switch the active test context; each test's entered functions are recorded separately.""" - if _stop_event.is_set(): - return - _tracker.switch_test_context(nodeid or "") - - def _save_active(): - try: - _tracker.save() - except Exception as e: - print(f"[cbts] periodic save failed in pid {os.getpid()}: {e!r}", file=sys.stderr) - - def _final_save(): - _stop_event.set() - try: - _tracker.save() - _tracker.stop() - except Exception as e: - print(f"[cbts] final save failed in pid {os.getpid()}: {e!r}", file=sys.stderr) - - atexit.register(_final_save) - - # sys.orig_argv preserves the launching cmdline; sys.argv has not yet gained "pytest" when sitecustomize runs. - _orig_argv = getattr(sys, "orig_argv", sys.argv) - _is_pytest_main = any("pytest" in a for a in _orig_argv[:4]) - _is_nested_pytest = _parent_is_pytest() and _is_pytest_main - # mpi4py.futures pool workers serve one test via the inherited CBTS_TEST_ID and the atexit save; no daemons. - _is_mpi_pool_worker = any("mpi4py.futures" in a for a in _orig_argv) - _skip_daemons = _is_pytest_main or _is_mpi_pool_worker - - # Subprocesses inherit the current nodeid via CBTS_TEST_ID; the outer pytest re-switches per test via the plugin. - _initial_nodeid = os.environ.get("CBTS_TEST_ID", "").strip() - if _initial_nodeid: - switch_test_context(_initial_nodeid) - - if _is_nested_pytest: - # Inner pytest: apply the mpi_session env-whitelist patch synchronously instead of via the watcher thread. - try: - from cbts_plugin import install_mpi_pool_patch - - install_mpi_pool_patch(raise_on_refactor=False) - except Exception as _exc: - print( - f"[cbts] nested-pytest mpi patch skipped in pid {os.getpid()}: {_exc!r}", - file=sys.stderr, - ) - - if not _skip_daemons: - - def _watch_mpi_session(): - # Wait until the host has imported the target modules so this daemon triggers no racing import. - while not _stop_event.is_set(): - mod = sys.modules.get("tensorrt_llm.llmapi.mpi_session") - if ( - mod is not None - and hasattr(mod, "MpiPoolSession") - and "mpi4py.futures" in sys.modules - ): - try: - from cbts_plugin import install_mpi_pool_patch - - install_mpi_pool_patch(raise_on_refactor=False) - except Exception as exc: - print( - f"[cbts] mpi_session patch in pid {os.getpid()} failed: {exc!r}", - file=sys.stderr, - ) - return - _stop_event.wait(0.1) - - threading.Thread( - target=_watch_mpi_session, - daemon=True, - name="cbts-mpi-patcher", - ).start() - - def _periodic_save(): - while not _stop_event.wait(_PERIODIC_SAVE_SECONDS): - _save_active() - - threading.Thread( - target=_periodic_save, - daemon=True, - name="cbts-periodic-save", - ).start() - - _MARKER_FILE = os.environ.get("CBTS_MARKER_FILE", "/tmp/cbts/current_test.txt") - - def _poll_marker(): - last_seen = _initial_nodeid - while not _stop_event.is_set(): - try: - with open(_MARKER_FILE) as f: - nodeid = f.read().strip() - if nodeid and nodeid != last_seen: - # Long-lived non-pytest processes (e.g. trtllm-serve) switch context on marker change. - switch_test_context(nodeid) - last_seen = nodeid - except (FileNotFoundError, OSError): - pass - _stop_event.wait(0.1) - - threading.Thread( - target=_poll_marker, - daemon=True, - name="cbts-context-poller", - ).start() diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 9b5fd8890b3d..ce4060de64ac 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/_helpers.py b/jenkins/scripts/cbts/rules/_helpers.py index e84207c7c850..ad5aedde86d1 100644 --- a/jenkins/scripts/cbts/rules/_helpers.py +++ b/jenkins/scripts/cbts/rules/_helpers.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/auto_deploy_rule.py b/jenkins/scripts/cbts/rules/auto_deploy_rule.py index 0f44ac22bbb9..48bf826f11d8 100644 --- a/jenkins/scripts/cbts/rules/auto_deploy_rule.py +++ b/jenkins/scripts/cbts/rules/auto_deploy_rule.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/base.py b/jenkins/scripts/cbts/rules/base.py index f4f71d0335e2..018a05438584 100644 --- a/jenkins/scripts/cbts/rules/base.py +++ b/jenkins/scripts/cbts/rules/base.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/out_of_scope_rule.py b/jenkins/scripts/cbts/rules/out_of_scope_rule.py index 95b15a3e2347..7017abc65ad1 100644 --- a/jenkins/scripts/cbts/rules/out_of_scope_rule.py +++ b/jenkins/scripts/cbts/rules/out_of_scope_rule.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/spec_dec_rule.py b/jenkins/scripts/cbts/rules/spec_dec_rule.py index 15e1db7a86f6..cc92134e8d69 100644 --- a/jenkins/scripts/cbts/rules/spec_dec_rule.py +++ b/jenkins/scripts/cbts/rules/spec_dec_rule.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/test_list_rule.py b/jenkins/scripts/cbts/rules/test_list_rule.py index 118eeefcaa67..34634d637378 100644 --- a/jenkins/scripts/cbts/rules/test_list_rule.py +++ b/jenkins/scripts/cbts/rules/test_list_rule.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/tests_def_rule.py b/jenkins/scripts/cbts/rules/tests_def_rule.py index 9f85039f5e01..36658a383580 100644 --- a/jenkins/scripts/cbts/rules/tests_def_rule.py +++ b/jenkins/scripts/cbts/rules/tests_def_rule.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/visual_gen_rule.py b/jenkins/scripts/cbts/rules/visual_gen_rule.py index 45cf4e48c476..fefd9ded65e5 100644 --- a/jenkins/scripts/cbts/rules/visual_gen_rule.py +++ b/jenkins/scripts/cbts/rules/visual_gen_rule.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/rules/waives_rule.py b/jenkins/scripts/cbts/rules/waives_rule.py index 184193c53600..f1608b875733 100644 --- a/jenkins/scripts/cbts/rules/waives_rule.py +++ b/jenkins/scripts/cbts/rules/waives_rule.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/tools/coverage_audit.py b/jenkins/scripts/cbts/tools/coverage_audit.py deleted file mode 100644 index b9e2309b6152..000000000000 --- a/jenkins/scripts/cbts/tools/coverage_audit.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -r"""Audit a CBTS touch DB (`cbts_touchmap.sqlite`) — format, scale, and coverage completeness. - -Reports the format (stage prefix, schema_version, collection commit), scale, -per-stage known counts, the per-test footprint distribution, and the tests -whose capture looks incomplete (the same heuristic the selector uses). - -Example:: - - python3 jenkins/scripts/cbts/tools/coverage_audit.py \\ - --db cbts_touchmap.sqlite --list-untrusted -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -THIS = Path(__file__).resolve() -CBTS = THIS.parent.parent -sys.path.insert(0, str(CBTS)) -sys.path.insert(0, str(CBTS / "coverage_selection")) - -from blocks import YAMLIndex, block_matches_stage, parse_stages_from_groovy # noqa: E402 -from touch_db import ( # noqa: E402 - _LAUNCH_MARKERS, - _MIN_FUNCS, - _SERVING_PATH_MARKERS, - _WORKER_SENTINEL, - TouchDB, - db_key, - split_stage, -) - -_DEFAULT_TEST_DB = CBTS.parents[2] / "tests/integration/test_lists/test-db" -_DEFAULT_GROOVY = CBTS.parents[2] / "jenkins/L0_Test.groovy" - - -def _fmt_pct(n: int, d: int) -> str: - return f"{100.0 * n / d:.0f}%" if d else "n/a" - - -def main(argv: list[str] | None = None) -> int: - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - ap.add_argument("--db", required=True, help="path to cbts_touchmap.sqlite") - ap.add_argument("--list-untrusted", action="store_true", help="print every untrusted test") - ap.add_argument( - "--min-funcs", type=int, default=_MIN_FUNCS, help=f"near-empty floor (default {_MIN_FUNCS})" - ) - ap.add_argument( - "--test-db", - default=str(_DEFAULT_TEST_DB), - help="test-db dir to diff against the DB (HEAD coverage gap); '' to skip", - ) - ap.add_argument( - "--groovy", default=str(_DEFAULT_GROOVY), help="Groovy file to parse stage defs from" - ) - ap.add_argument( - "--list-not-in-db", action="store_true", help="print every gap case (default: first 15)" - ) - args = ap.parse_args(argv) - - db = TouchDB.open(args.db) - known = db.known_tests() - stages = db.known_by_stage() - footprint = db.per_test_footprint() - - print(f"=== CBTS coverage DB audit: {args.db} ===\n") - - # -- Format -- - print("## Format") - if stages: - print(f" test field: stage-prefixed ({len(stages)} instrumented stage(s) derivable)") - elif known: - print( - " test field: BARE nodeid !! WARNING: no stage prefix -> per-stage narrowing impossible" - ) - sv = db.schema_version() - commit = db.collection_commit() - print(f" schema_version: {sv or 'MISSING (selector cannot hard-fail on format drift)'}") - print( - f" collection commit: {commit or 'MISSING (no staleness gating; zero-touch lever stays off)'}" - ) - - # -- Scale -- - print("\n## Scale") - print( - f" known tests: {len(known)} | meta: tests={db.meta('tests')} files={db.meta('files')} " - f"functions={db.meta('functions')}" - ) - fr, qr = db.meta("file_rate_pct"), db.meta("func_rate_pct") - if fr or qr: - print(f" coverage rate: files {fr}% functions {qr}%") - - # -- Per-stage -- - print("\n## Instrumented stages") - for stage in sorted(stages): - print(f" {stage}: {len(stages[stage])} known") - - # -- Completeness -- - untrusted = db.untrusted_tests( - _WORKER_SENTINEL, _LAUNCH_MARKERS, _SERVING_PATH_MARKERS, args.min_funcs - ) - - def reason(test: str) -> str: - if any(m in test for m in _SERVING_PATH_MARKERS): - return "disagg-path (servers uninstrumented)" - if footprint[test] < args.min_funcs: - return f"near-empty (<{args.min_funcs} funcs)" - return "worker-lost (drove inference, no py_executor)" - - print("\n## Coverage completeness") - if footprint: - print( - f" per-test footprint (functions entered): min={min(footprint.values())} " - f"max={max(footprint.values())} (few funcs => likely lost subprocess capture)" - ) - else: - print(" per-test footprint: none (no test != '' rows — no usable per-test coverage)") - trusted_fp = [footprint[t] for t in known if t not in untrusted] - untrusted_fp = [footprint[t] for t in untrusted] - if trusted_fp and untrusted_fp: - print( - f" footprint gap: untrusted max={max(untrusted_fp)} | trusted min={min(trusted_fp)}" - ) - print( - f" UNTRUSTED (incomplete capture): {len(untrusted)}/{len(known)} ({_fmt_pct(len(untrusted), len(known))})" - ) - by_reason: dict[str, int] = {} - by_stage: dict[str, int] = {} - for t in untrusted: - by_reason[reason(t)] = by_reason.get(reason(t), 0) + 1 - by_stage[split_stage(t)[0]] = by_stage.get(split_stage(t)[0], 0) + 1 - for r, n in sorted(by_reason.items(), key=lambda kv: -kv[1]): - print(f" - {r}: {n}") - print(f" by stage: {dict(sorted(by_stage.items()))}") - print( - f"\n TRUSTED skippable universe: {len(known) - len(untrusted)}/{len(known)} " - f"(only these may ever be skipped)" - ) - - if args.list_untrusted: - print("\n## Untrusted tests") - for t in sorted(untrusted): - print(f" [{footprint[t]:>5} funcs] {t}\n -> {reason(t)}") - - # -- HEAD coverage gap: cases on an instrumented stage with no DB row -- - if args.test_db and Path(args.test_db).is_dir() and Path(args.groovy).is_file(): - yaml_index = YAMLIndex.load(Path(args.test_db)) - all_stages = parse_stages_from_groovy(Path(args.groovy), include_post_merge=True) - bare_known = {split_stage(t)[1] for t in known} - per_stage: dict[str, set[str]] = {} - for name in sorted(set(all_stages) & set(stages)): - stage = all_stages[name] - missing = { - entry - for block in yaml_index.blocks - if block.yaml_stem == stage.yaml_stem and block_matches_stage(block, stage) - for entry in block.tests - if (k := db_key(entry)) is not None and k not in bare_known - } - if missing: - per_stage[name] = missing - gap = sorted(set().union(*per_stage.values())) if per_stage else [] - print("\n## HEAD coverage gap on instrumented stages") - print( - f" {len(gap)} unique case(s) render on an instrumented single-GPU stage but have " - f"NO DB row -> always must-run (new/renamed or never captured)" - ) - for name in sorted(per_stage, key=lambda n: -len(per_stage[n])): - print(f" {name}: {len(per_stage[name])} not-in-DB") - preview = gap if args.list_not_in_db else gap[:15] - if preview: - print(" cases:") - for t in preview: - print(f" - {t}") - if not args.list_not_in_db and len(gap) > 15: - print(f" ... (+{len(gap) - 15}; use --list-not-in-db)") - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/jenkins/scripts/cbts/tools/dryrun.py b/jenkins/scripts/cbts/tools/dryrun.py index 2293e35ebc2b..0b5ab785d865 100644 --- a/jenkins/scripts/cbts/tools/dryrun.py +++ b/jenkins/scripts/cbts/tools/dryrun.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index dc5127e57343..6b9e1ffb02cf 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/jenkins/scripts/generate_duration.py b/jenkins/scripts/generate_duration.py deleted file mode 100644 index 74f9e1687c7b..000000000000 --- a/jenkins/scripts/generate_duration.py +++ /dev/null @@ -1,237 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import glob -import json -import os -import re -import time - -OPENSEARCH_INDEX = "df-swdl-trtllm-infra-ci-prod-test_info-*" - -# Test-list YAML entries may carry trailing turtle directives that are NOT part -# of the test identity and never appear in an OpenSearch s_turtle_name: -# - "TIMEOUT (90)" (also the rare no-space form "TIMEOUT(60)") -# - "ISOLATION" -# They can be chained, e.g. "... TIMEOUT (90) ISOLATION". pytest-native flags -# such as -k / -m DO change what runs and MUST be preserved. No node-id -# parameter list contains a space, so anchoring to the end is safe. -_TURTLE_DIRECTIVE_RE = re.compile(r"(?:\s+(?:TIMEOUT\s*\(\d+\)|ISOLATION))+\s*$") - -# Default location of the turtle test-db lists, relative to the repo root. -# This file lives at /jenkins/scripts/, so go up three levels. -_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -DEFAULT_TEST_LIST_DIR = os.path.join(_REPO_ROOT, "tests", "integration", "test_lists", "test-db") - - -def normalize_test_spec(name): - """Strip trailing turtle directives (TIMEOUT/ISOLATION), keep -k/-m flags.""" - if not name: - return name - return _TURTLE_DIRECTIVE_RE.sub("", name).strip() - - -def load_test_list_specs(test_list_dir): - """Collect the set of normalized test specs declared in the test-db YAMLs.""" - import yaml - - specs = set() - yml_files = sorted(glob.glob(os.path.join(test_list_dir, "*.yml"))) - for path in yml_files: - with open(path) as f: - try: - data = yaml.safe_load(f) or {} - except yaml.YAMLError as e: - print(f" Warning: failed to parse {path}: {e}") - continue - for value in data.values(): - if not isinstance(value, list): - continue - for block in value: - if not isinstance(block, dict): - continue - for test in block.get("tests", []) or []: - if isinstance(test, str): - specs.add(normalize_test_spec(test)) - return specs, yml_files - - -def query_opensearch_durations(days): - import sys - - sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - import requests - from open_search_db import DEFAULT_RETRY_COUNT, OPEN_SEARCH_DB_BASE_URL, QUERY_TIMEOUT_SECONDS - - since_ms = int((time.time() - days * 86400) * 1000) - search_url = f"{OPEN_SEARCH_DB_BASE_URL}/opensearch/{OPENSEARCH_INDEX}/_search" - headers = {"Content-Type": "application/json", "Accept-Charset": "UTF-8"} - - test_durations = {} - after_key = None - page = 0 - - while True: - composite_agg = { - "size": 1000, - "sources": [{"test_name": {"terms": {"field": "s_turtle_name"}}}], - } - if after_key is not None: - composite_agg["after"] = after_key - - query = { - "size": 0, - "query": { - "bool": { - "must": [ - {"term": {"s_status": "PASSED"}}, - {"range": {"ts_created": {"gte": since_ms}}}, - ], - "must_not": [{"term": {"s_turtle_name": "Stage Failed"}}], - } - }, - "aggs": { - "by_test": { - "composite": composite_agg, - "aggs": {"avg_duration_ms": {"avg": {"field": "l_e2e_time_ms"}}}, - } - }, - } - - query_str = json.dumps(query) - res = None - for attempt in range(DEFAULT_RETRY_COUNT): - res = requests.get( - search_url, data=query_str, headers=headers, timeout=QUERY_TIMEOUT_SECONDS - ) - if res.status_code in (200, 201, 202): - break - print( - f" Warning: OpenSearch returned {res.status_code}, attempt {attempt + 1}/{DEFAULT_RETRY_COUNT}" - ) - else: - raise RuntimeError( - f"OpenSearch query failed after {DEFAULT_RETRY_COUNT} attempts: " - f"{res.status_code} {res.text[:200]}" - ) - - data = res.json() - buckets = data["aggregations"]["by_test"]["buckets"] - page += 1 - print(f" Page {page}: got {len(buckets)} buckets") - - for bucket in buckets: - test_name = bucket["key"]["test_name"] - avg_ms = bucket["avg_duration_ms"]["value"] - if avg_ms is not None: - test_durations[test_name] = avg_ms / 1000.0 - - after_key = data["aggregations"]["by_test"].get("after_key") - if not after_key or len(buckets) == 0: - break - - return test_durations - - -def main(): - # Parse command-line arguments - parser = argparse.ArgumentParser(description="Generate test duration file.") - parser.add_argument( - "--duration-file", - type=str, - default="new_test_duration.json", - help="Path to the output duration file (default: new_test_duration.json)", - ) - parser.add_argument( - "--cluster", - type=str, - default=None, - help="Cluster name (e.g. 'aws_dfw'). When set, writes " - "tests/integration/defs/.test_durations_ relative to the " - "repo root instead of --duration-file.", - ) - parser.add_argument( - "--days", - type=int, - default=7, - help="Number of days to look back in OpenSearch (default: 7).", - ) - parser.add_argument( - "--test-list-dir", - type=str, - default=DEFAULT_TEST_LIST_DIR, - help="Directory of turtle test-db YAML lists used to filter OpenSearch " - "results. Only turtle names present in these lists are written " - f"(default: {DEFAULT_TEST_LIST_DIR}).", - ) - parser.add_argument( - "--no-filter", - action="store_true", - default=False, - help="Skip filtering OpenSearch results against the test-db lists.", - ) - args = parser.parse_args() - - # Resolve output path - if args.cluster: - NEW_TEST_DURATION = os.path.join( - _REPO_ROOT, "tests", "integration", "defs", f".test_durations_{args.cluster}" - ) - else: - NEW_TEST_DURATION = args.duration_file - - print(f"Querying OpenSearch for last {args.days} day(s)...") - test_durations = query_opensearch_durations(args.days) - raw_count = len(test_durations) - - # Filter against the turtle test-db lists: an aggregated turtle name may be - # a stale entry or a subtest that is no longer scheduled. Keep only names - # that still appear in the checked-in lists. - dropped = 0 - if args.no_filter: - print("Filtering disabled (--no-filter); writing all turtle names.") - specs = None - else: - specs, yml_files = load_test_list_specs(args.test_list_dir) - print( - f"Loaded {len(specs)} test specs from {len(yml_files)} list(s) in {args.test_list_dir}" - ) - if not specs: - print(" Warning: no test specs loaded; writing all turtle names unfiltered.") - else: - filtered = { - name: dur - for name, dur in test_durations.items() - if normalize_test_spec(name) in specs - } - dropped = raw_count - len(filtered) - test_durations = filtered - - with open(NEW_TEST_DURATION, "w") as file: - json.dump(test_durations, file, indent=3) - file.write("\n") - - print("\nSummary:") - print(f" OpenSearch index : {OPENSEARCH_INDEX}") - print(f" Days looked back : {args.days}") - print(f" Turtle names from query: {raw_count}") - print(f" Dropped (not in lists) : {dropped}") - print(f" Unique tests in output : {len(test_durations)}") - print(f" Output written to : {NEW_TEST_DURATION}") - - -if __name__ == "__main__": - main() diff --git a/jenkins/scripts/perf/local/slurm_install.sh b/jenkins/scripts/perf/local/slurm_install.sh index 2524c7c8ac79..91bb7d664e62 100755 --- a/jenkins/scripts/perf/local/slurm_install.sh +++ b/jenkins/scripts/perf/local/slurm_install.sh @@ -21,7 +21,7 @@ slurm_build_wheel() { fi echo "Building wheel on node ${SLURM_NODEID:-0}, task ${SLURM_LOCALID:-0}" - retry_command bash -c "cd $llmSrcNode && rm -rf .venv-3.12 && python3 ./scripts/build_wheel.py --use_ccache --cuda_architectures '100-real' --clean -c" + retry_command bash -c "cd $llmSrcNode && rm -rf .venv-3.12 && python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --benchmarks --use_ccache --cuda_architectures '100-real' --clean -c" cd $jobWorkspace echo "(Writing build wheel lock) Lock file: $build_lock_file" diff --git a/jenkins/scripts/perf/local/submit.py b/jenkins/scripts/perf/local/submit.py index c12798aa847e..f71a825936c3 100755 --- a/jenkins/scripts/perf/local/submit.py +++ b/jenkins/scripts/perf/local/submit.py @@ -250,43 +250,25 @@ def get_hardware_config(config, runtime_mode, benchmark_mode, test_name=None): } -def _join_env(*parts): - """Space-join non-empty env-var strings (drops falsy entries).""" - return " ".join(p for p in parts if p) - - def get_env_config(config, runtime_mode, benchmark_mode=None, server_name=None): """Get worker / server / benchmark env vars from the yaml. Aggregated yaml stores env vars per server config under `server_configs[i].server_env_var`. Disaggregated yaml stores them at the - top-level `environment.{worker,server,benchmark}_env_var`, plus optional - `environment.{ctx,gen}_worker_env_var` for role-specific extras (appended - to the shared `worker_env_var`). + top-level `environment.{worker,server,benchmark}_env_var`. ctx_only is a hybrid: the launch path is aggregated, but the yaml is the disagg one, so the agg launch's "server_env_var" comes from - `environment.worker_env_var` (merged with ctx-side extras when present). + `environment.worker_env_var`. - Returns: {worker_env_var (shared, back-compat), - ctx_worker_env_var, gen_worker_env_var, - server_env_var, benchmark_env_var}. + Returns: {worker_env_var, server_env_var, benchmark_env_var}. """ env = config.get("environment", {}) or {} - common = env.get("worker_env_var", "") or "" - ctx_extra = env.get("ctx_worker_env_var", "") or "" - gen_extra = env.get("gen_worker_env_var", "") or "" - ctx_env = _join_env(common, ctx_extra) - gen_env = _join_env(common, gen_extra) if runtime_mode == "aggregated": if benchmark_mode == "ctx_only": return { - "worker_env_var": common, - "ctx_worker_env_var": ctx_env, - "gen_worker_env_var": gen_env, - # ctx_only launches through the aggregated single-pytest path; - # the ctx-merged env is what actually runs. - "server_env_var": ctx_env, + "worker_env_var": env.get("worker_env_var", "") or "", + "server_env_var": env.get("worker_env_var", "") or "", "benchmark_env_var": env.get("benchmark_env_var", "") or "", } agg_server_env_var = "" @@ -296,15 +278,11 @@ def get_env_config(config, runtime_mode, benchmark_mode=None, server_name=None): break return { "worker_env_var": "", - "ctx_worker_env_var": "", - "gen_worker_env_var": "", "server_env_var": agg_server_env_var, "benchmark_env_var": "", } return { - "worker_env_var": common, - "ctx_worker_env_var": ctx_env, - "gen_worker_env_var": gen_env, + "worker_env_var": env.get("worker_env_var", "") or "", "server_env_var": env.get("server_env_var", "") or "", "benchmark_env_var": env.get("benchmark_env_var", "") or "", } @@ -808,22 +786,19 @@ def main(): server_env_vars = "" benchmark_env_var = "" if runtime_mode == "disaggregated": - # Build worker env vars (split into ctx and gen for role-specific - # settings). get_env_config already merged the shared worker_env_var - # with any per-role ctx_worker_env_var / gen_worker_env_var from yaml. - ctx_worker_env_var = env_config.get("ctx_worker_env_var", "") - gen_worker_env_var = env_config.get("gen_worker_env_var", "") + # Build worker env vars (split into ctx and gen for role-specific settings) + common_worker_env_var = env_config.get("worker_env_var", "") ctx_worker_env_vars = ( f"TLLM_PROFILE_START_STOP='{ctx_tllm_profile_start_stop}' " f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " f"HF_HOME=/tmp/hf_home " - f"{ctx_worker_env_var}" + f"{common_worker_env_var}" ) gen_worker_env_vars = ( f"TLLM_PROFILE_START_STOP='{gen_tllm_profile_start_stop}' " f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " f"HF_HOME=/tmp/hf_home " - f"{gen_worker_env_var}" + f"{common_worker_env_var}" ) server_env_vars = env_config.get("server_env_var", "") benchmark_env_var = env_config.get("benchmark_env_var", "") diff --git a/jenkins/scripts/perf/submit.py b/jenkins/scripts/perf/submit.py index d68e776f036e..a621d4ca1f8c 100755 --- a/jenkins/scripts/perf/submit.py +++ b/jenkins/scripts/perf/submit.py @@ -219,43 +219,25 @@ def get_hardware_config(config, runtime_mode, benchmark_mode, server_name): } -def _join_env(*parts): - """Space-join non-empty env-var strings (drops falsy entries).""" - return " ".join(p for p in parts if p) - - def get_env_config(config, runtime_mode, benchmark_mode, server_name): """Get worker / server / benchmark env vars from the yaml. Aggregated yaml stores env vars per server config under `server_configs[i].server_env_var`. Disaggregated yaml stores them at the - top-level `environment.{worker,server,benchmark}_env_var`, plus optional - `environment.{ctx,gen}_worker_env_var` for role-specific extras (appended - to the shared `worker_env_var`). + top-level `environment.{worker,server,benchmark}_env_var`. ctx_only is a hybrid: the launch path is aggregated, but the yaml is the disagg one, so the agg launch's "server_env_var" comes from - `environment.worker_env_var` (merged with ctx-side extras when present). + `environment.worker_env_var`. - Returns: {worker_env_var (shared, back-compat), - ctx_worker_env_var, gen_worker_env_var, - server_env_var, benchmark_env_var}. + Returns: {worker_env_var, server_env_var, benchmark_env_var}. """ env = config.get("environment", {}) or {} - common = env.get("worker_env_var", "") or "" - ctx_extra = env.get("ctx_worker_env_var", "") or "" - gen_extra = env.get("gen_worker_env_var", "") or "" - ctx_env = _join_env(common, ctx_extra) - gen_env = _join_env(common, gen_extra) if runtime_mode == "aggregated": if benchmark_mode == "ctx_only": return { - "worker_env_var": common, - "ctx_worker_env_var": ctx_env, - "gen_worker_env_var": gen_env, - # ctx_only launches through the aggregated single-pytest path; - # the ctx-merged env is what actually runs. - "server_env_var": ctx_env, + "worker_env_var": env.get("worker_env_var", "") or "", + "server_env_var": env.get("worker_env_var", "") or "", "benchmark_env_var": env.get("benchmark_env_var", "") or "", } agg_server_env_var = "" @@ -265,15 +247,11 @@ def get_env_config(config, runtime_mode, benchmark_mode, server_name): break return { "worker_env_var": "", - "ctx_worker_env_var": "", - "gen_worker_env_var": "", "server_env_var": agg_server_env_var, "benchmark_env_var": "", } return { - "worker_env_var": common, - "ctx_worker_env_var": ctx_env, - "gen_worker_env_var": gen_env, + "worker_env_var": env.get("worker_env_var", "") or "", "server_env_var": env.get("server_env_var", "") or "", "benchmark_env_var": env.get("benchmark_env_var", "") or "", } @@ -535,13 +513,13 @@ def main(): srun_args_lines.append("--container-env=pytestCommand") else: # Disaggregated (e2e or gen_only). - base_prefix = ( - "FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${SLURM_LOCALID} HF_HOME=/tmp/hf_home" + base_worker_env_vars = ( + f"FLASHINFER_JIT_DIR=/tmp/flashinfer_jit_cache_\\${{SLURM_LOCALID}} " + f"HF_HOME=/tmp/hf_home " + f"{env_config['worker_env_var']}" ) - # ctx / gen env vars: shared worker_env_var + optional per-role extras - # from the yaml. get_env_config already merged them. - ctx_worker_env_vars = f"{base_prefix} {env_config['ctx_worker_env_var']}".rstrip() - gen_worker_env_vars = f"{base_prefix} {env_config['gen_worker_env_var']}".rstrip() + ctx_worker_env_vars = base_worker_env_vars + gen_worker_env_vars = base_worker_env_vars server_env_vars = env_config["server_env_var"] # gen_only_no_context comes from yaml's benchmark.mode, not the test diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index e7d5f1a56d2c..1b9391d20fca 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -7,6 +7,28 @@ trap 'rc=$?; echo "Error in file ${BASH_SOURCE[0]} on line $LINENO: $BASH_COMMAN cd $resourcePathNode llmSrcNode=$resourcePathNode/TensorRT-LLM/src +set_value_in_command() { + # Parameters + local key="$1" + local value="$2" + local command="$3" + + # Transform the key + local placeholder="__PLACEHOLDER_${key}__" + + # Check if placeholder exists + if [[ "$command" != *"$placeholder"* ]]; then + echo "Error: placeholder '$placeholder' not found in the command" >&2 + return 1 + fi + + # Replace all occurrences + local result="${command//${placeholder}/${value}}" + + # Return the result + echo "$result" +} + # Only the first process will set the git config if [ $SLURM_PROCID -eq 0 ]; then # Update HOME/.gitconfig @@ -32,14 +54,18 @@ llmapiLaunchScript="$llmSrcNode/tensorrt_llm/llmapi/trtllm-llmapi-launch" chmod +x $llmapiLaunchScript cd $llmSrcNode/tests/integration/defs -# Wheel path for the CBTS .coveragerc @TRTLLM_WHEEL_PATH@ substitution below. +# get trtllm wheel path and add to pytest command trtllmWhlPath=$(pip3 show tensorrt_llm | grep Location | cut -d ' ' -f 2) trtllmWhlPath=$(echo "$trtllmWhlPath" | sed 's/[[:space:]]+/_/g') echo "TRTLLM WHEEL PATH: $trtllmWhlPath" +# In disaggregated mode, we only set coverage config file in benchmark pytest. +if [[ -z "${DISAGG_SERVING_TYPE:-}" || "${DISAGG_SERVING_TYPE}" == "BENCHMARK" ]]; then + pytestCommand=$(set_value_in_command "TRTLLM_WHL_PATH" "$trtllmWhlPath" "$pytestCommand") +fi # Only the first process will save the coverage config file if [ $SLURM_PROCID -eq 0 ]; then - sed -i "s|@TRTLLM_WHEEL_PATH@|$trtllmWhlPath|g" "$coverageConfigFile" + sed -i "s|---wheel_path---|$trtllmWhlPath|g" "$coverageConfigFile" else # Sleep 30 seconds to wait for the coverage config file to be saved sleep 30 diff --git a/legacy-files.txt b/legacy-files.txt index 021a4fb301a8..d6c8b23308aa 100644 --- a/legacy-files.txt +++ b/legacy-files.txt @@ -1,6 +1,14 @@ .devcontainer/make_env.py .github/scripts/label_community_user.py .github/scripts/pr_checklist_check.py +benchmarks/cpp/__init__.py +benchmarks/cpp/prepare_dataset.py +benchmarks/cpp/utils/__init__.py +benchmarks/cpp/utils/convert_nemo_dataset.py +benchmarks/cpp/utils/generate_rand_loras.py +benchmarks/cpp/utils/prepare_real_data.py +benchmarks/cpp/utils/prepare_synthetic_data.py +benchmarks/cpp/utils/utils.py cpp/conanfile.py cpp/kernels/fmha_v2/conftest.py cpp/kernels/fmha_v2/fmha_test.py @@ -25,17 +33,58 @@ cpp/micro_benchmarks/gen-moe-benchmark-file.py cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py +cpp/tests/resources/scripts/build_chatglm_engines.py +cpp/tests/resources/scripts/build_eagle_engines.py +cpp/tests/resources/scripts/build_enc_dec_engines.py +cpp/tests/resources/scripts/build_engines_utils.py +cpp/tests/resources/scripts/build_gpt_engines.py +cpp/tests/resources/scripts/build_gptj_engines.py +cpp/tests/resources/scripts/build_llama_engines.py +cpp/tests/resources/scripts/build_mamba_engines.py +cpp/tests/resources/scripts/build_medusa_engines.py +cpp/tests/resources/scripts/build_recurrentgemma_engines.py +cpp/tests/resources/scripts/build_redrafter_engines.py +cpp/tests/resources/scripts/generate_expected_chatglm_output.py +cpp/tests/resources/scripts/generate_expected_eagle_output.py +cpp/tests/resources/scripts/generate_expected_enc_dec_output.py +cpp/tests/resources/scripts/generate_expected_gpt_output.py +cpp/tests/resources/scripts/generate_expected_gptj_output.py +cpp/tests/resources/scripts/generate_expected_llama_output.py +cpp/tests/resources/scripts/generate_expected_mamba_output.py +cpp/tests/resources/scripts/generate_expected_medusa_output.py +cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py +cpp/tests/resources/scripts/generate_expected_redrafter_output.py +cpp/tests/resources/scripts/generate_hf_gpt_output.py cpp/tests/resources/scripts/generate_test_lora_weights.py +cpp/tests/resources/scripts/io_converter.py docs/source/conf.py docs/source/helper.py examples/apps/chat.py examples/apps/fastapi_server.py +examples/bindings/executor/example_advanced.py +examples/bindings/executor/example_basic.py +examples/bindings/executor/example_debug.py +examples/bindings/executor/example_logits_processor.py examples/disaggregated/clients/disagg_client.py examples/disaggregated/slurm/benchmark/submit.py +examples/dora/normalize_weights.py +examples/eagle/convert_checkpoint.py +examples/eval_long_context.py +examples/generate_checkpoint_config.py +examples/generate_xgrammar_tokenizer_info.py +examples/hf_lora_convert.py examples/infinitebench/args.py examples/infinitebench/compute_scores.py examples/infinitebench/construct_synthetic_dataset.py examples/infinitebench/eval_utils.py +examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py +examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py +examples/llm-api/_tensorrt_engine/llm_inference_customize.py +examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py +examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py +examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py +examples/llm-api/_tensorrt_engine/llm_quantization.py +examples/llm-api/_tensorrt_engine/quickstart_example.py examples/llm-api/llm_guided_decoding.py examples/llm-api/llm_inference.py examples/llm-api/llm_inference_async.py @@ -55,17 +104,122 @@ examples/llm-api/quickstart_advanced.py examples/llm-api/quickstart_example.py examples/llm-api/quickstart_multimodal.py examples/llm-api/star_attention.py +examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py examples/longbench/eval_longbench_v1.py +examples/medusa/convert_checkpoint.py +examples/mmlu.py +examples/models/contrib/baichuan/convert_checkpoint.py +examples/models/contrib/bloom/convert_checkpoint.py +examples/models/contrib/chatglm-6b/tokenization_chatglm.py +examples/models/contrib/chatglm2-6b/tokenization_chatglm.py +examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py +examples/models/contrib/cogvlm/convert_checkpoint.py +examples/models/contrib/dbrx/convert_checkpoint.py +examples/models/contrib/deepseek_v1/__init__.py +examples/models/contrib/deepseek_v1/convert_checkpoint.py +examples/models/contrib/deepseek_v2/convert_checkpoint.py +examples/models/contrib/dit/convert_checkpoint.py +examples/models/contrib/dit/diffusion.py +examples/models/contrib/dit/sample.py +examples/models/contrib/dit/utils_modelopt.py +examples/models/contrib/dit/vae_decoder_trt.py +examples/models/contrib/falcon/convert_checkpoint.py +examples/models/contrib/gptj/convert_checkpoint.py +examples/models/contrib/gptneox/convert_checkpoint.py +examples/models/contrib/grok/convert_checkpoint.py +examples/models/contrib/mmdit/convert_checkpoint.py +examples/models/contrib/mmdit/sample.py +examples/models/contrib/mpt/convert_checkpoint.py +examples/models/contrib/opt/convert_checkpoint.py +examples/models/contrib/sdxl/build_sdxl_unet.py +examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py +examples/models/contrib/sdxl/run_sdxl.py +examples/models/contrib/stdit/aspect.py +examples/models/contrib/stdit/convert_checkpoint.py +examples/models/contrib/stdit/pipeline_tllm.py +examples/models/contrib/stdit/sample.py +examples/models/contrib/stdit/scheduler.py +examples/models/contrib/stdit/text_encoder.py +examples/models/contrib/stdit/utils.py +examples/models/contrib/stdit/vae.py +examples/models/contrib/stdit/video_transforms.py +examples/models/core/bert/__init__.py +examples/models/core/bert/convert_checkpoint.py +examples/models/core/bert/run.py +examples/models/core/bert/utils.py +examples/models/core/commandr/convert_checkpoint.py +examples/models/core/enc_dec/__init__.py +examples/models/core/enc_dec/convert_checkpoint.py +examples/models/core/enc_dec/helper.py +examples/models/core/enc_dec/run.py +examples/models/core/gemma/convert_checkpoint.py +examples/models/core/glm-4-9b/convert_checkpoint.py +examples/models/core/glm-4-9b/tokenization_chatglm.py +examples/models/core/gpt/convert_checkpoint.py +examples/models/core/gpt/merge_ptuning_tables.py +examples/models/core/gpt/nemo_lora_convert.py +examples/models/core/gpt/nemo_prompt_convert.py +examples/models/core/gpt/run_hf.py examples/models/core/gpt_oss/openai_chat_client_function_calling.py +examples/models/core/internlm2/convert_checkpoint.py examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py +examples/models/core/llama/convert_checkpoint.py +examples/models/core/llama/summarize_long.py +examples/models/core/mamba/convert_checkpoint.py +examples/models/core/mllama/convert_checkpoint.py +examples/models/core/multimodal/__init__.py +examples/models/core/multimodal/build_multimodal_engine.py +examples/models/core/multimodal/eval.py +examples/models/core/multimodal/run.py +examples/models/core/multimodal/utils.py +examples/models/core/nemotron_nas/calibration_utils.py +examples/models/core/nemotron_nas/convert_checkpoint.py +examples/models/core/phi/convert_checkpoint.py +examples/models/core/qwen/convert_checkpoint.py +examples/models/core/qwen2audio/run.py +examples/models/core/qwen2audio/run_chat.py +examples/models/core/qwen2audio/utils.py +examples/models/core/qwenvl/run.py +examples/models/core/qwenvl/run_chat.py +examples/models/core/qwenvl/show_pic.py +examples/models/core/qwenvl/vit_onnx_trt.py +examples/models/core/recurrentgemma/convert_checkpoint.py +examples/models/core/vit/convert_checkpoint.py +examples/models/core/whisper/convert_checkpoint.py +examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py +examples/models/core/whisper/run.py +examples/models/core/whisper/tokenizer.py +examples/models/core/whisper/whisper_utils.py +examples/ngram/run_dtm_ngram.py +examples/openai_triton/manual_plugin/build.py +examples/openai_triton/manual_plugin/fmha_triton.py +examples/openai_triton/manual_plugin/plugin.py +examples/openai_triton/manual_plugin/run.py +examples/openai_triton/plugin_autogen/build_engine.py +examples/openai_triton/plugin_autogen/kernel_config.py +examples/openai_triton/plugin_autogen/run_engine.py +examples/python_plugin/build_lookup.py +examples/python_plugin/plugin_lib/__init__.py +examples/python_plugin/plugin_lib/lookup_kernel.py +examples/python_plugin/plugin_lib/lookup_plugin.py +examples/python_plugin/run_lookup.py +examples/quantization/quantize.py examples/quantization/quantize_mixed_precision_moe.py examples/ray_orchestrator/llm_inference_async_ray.py examples/ray_orchestrator/llm_inference_distributed_ray.py +examples/redrafter/convert_checkpoint.py +examples/run.py examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py examples/scaffolding/contrib/DeepConf/run_generation.py examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py examples/scaffolding/contrib/TreeInference/run_mcts_example.py examples/scaffolding/contrib/TreeInference/run_tot_example.py +examples/scaffolding/contrib/mcp/e2b/e2bserver.py +examples/scaffolding/contrib/mcp/e2b/main.py +examples/scaffolding/contrib/mcp/mcptest.py +examples/scaffolding/contrib/mcp/weather/weather.py +examples/scaffolding/contrib/mcp/websearch/main.py +examples/scaffolding/contrib/mcp/websearch/websearch.py examples/scaffolding/run_basic_generation.py examples/scaffolding/run_best_of_n_with_reward.py examples/scaffolding/run_majority_vote_aime24.py @@ -75,6 +229,8 @@ examples/serve/openai_chat_client_for_multimodal.py examples/serve/openai_completion_client.py examples/serve/openai_completion_client_for_lora.py examples/serve/openai_completion_client_json_schema.py +examples/summarize.py +examples/utils.py examples/wide_ep/ep_load_balancer/generate_eplb_config.py examples/wide_ep/ep_load_balancer/report_load_statistics.py examples/wide_ep/ep_load_balancer/utils.py @@ -82,10 +238,12 @@ examples/wide_ep/slurm_scripts/process_gen_iterlog.py jenkins/scripts/mergeWaiveList.py jenkins/scripts/open_search_db.py jenkins/scripts/test_rerun.py +scripts/build_cpp_examples.py scripts/build_wheel.py scripts/check_test_list.py scripts/dco_check.py scripts/format_test_list.py +scripts/generate_duration.py scripts/generate_lock_file.py scripts/get_wheel_from_package.py scripts/git_replace.py @@ -96,6 +254,7 @@ scripts/test_to_stage_mapping.py setup.py tensorrt_llm/__init__.py tensorrt_llm/_ray_utils.py +tensorrt_llm/_tensorrt_engine/__init__.py tensorrt_llm/_torch/__init__.py tensorrt_llm/_torch/attention_backend/__init__.py tensorrt_llm/_torch/attention_backend/flashinfer.py @@ -299,6 +458,7 @@ tensorrt_llm/_torch/pyexecutor/grammar_matcher.py tensorrt_llm/_torch/pyexecutor/guided_decoder.py tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py tensorrt_llm/_torch/pyexecutor/handle_logits.py +tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -336,6 +496,11 @@ tensorrt_llm/bench/benchmark/utils/__init__.py tensorrt_llm/bench/benchmark/utils/asynchronous.py tensorrt_llm/bench/benchmark/utils/general.py tensorrt_llm/bench/benchmark/utils/processes.py +tensorrt_llm/bench/build/__init__.py +tensorrt_llm/bench/build/build.py +tensorrt_llm/bench/build/dataclasses.py +tensorrt_llm/bench/build/tuning.py +tensorrt_llm/bench/build/utils.py tensorrt_llm/bench/dataclasses/__init__.py tensorrt_llm/bench/dataclasses/configuration.py tensorrt_llm/bench/dataclasses/engine.py @@ -345,9 +510,13 @@ tensorrt_llm/bench/dataclasses/reporting.py tensorrt_llm/bench/dataclasses/statistics.py tensorrt_llm/bench/utils/__init__.py tensorrt_llm/bench/utils/data.py +tensorrt_llm/builder.py tensorrt_llm/commands/__init__.py tensorrt_llm/commands/bench.py +tensorrt_llm/commands/build.py tensorrt_llm/commands/eval.py +tensorrt_llm/commands/prune.py +tensorrt_llm/commands/refit.py tensorrt_llm/commands/serve.py tensorrt_llm/evaluate/__init__.py tensorrt_llm/evaluate/cnn_dailymail.py @@ -383,7 +552,23 @@ tensorrt_llm/inputs/evs.py tensorrt_llm/inputs/multimodal.py tensorrt_llm/inputs/registry.py tensorrt_llm/inputs/utils.py +tensorrt_llm/layers/__init__.py +tensorrt_llm/layers/activation.py +tensorrt_llm/layers/attention.py +tensorrt_llm/layers/cast.py +tensorrt_llm/layers/conv.py +tensorrt_llm/layers/embedding.py +tensorrt_llm/layers/language_adapter.py +tensorrt_llm/layers/linear.py +tensorrt_llm/layers/lora.py +tensorrt_llm/layers/mlp.py +tensorrt_llm/layers/moe.py +tensorrt_llm/layers/normalization.py +tensorrt_llm/layers/pooling.py +tensorrt_llm/layers/recurrent.py +tensorrt_llm/layers/ssm.py tensorrt_llm/llmapi/__init__.py +tensorrt_llm/llmapi/build_cache.py tensorrt_llm/llmapi/disagg_utils.py tensorrt_llm/llmapi/kv_cache_type.py tensorrt_llm/llmapi/llm.py @@ -406,17 +591,179 @@ tensorrt_llm/metrics/collector.py tensorrt_llm/metrics/enums.py tensorrt_llm/models/__init__.py tensorrt_llm/models/automodel.py +tensorrt_llm/models/baichuan/__init__.py +tensorrt_llm/models/baichuan/config.py +tensorrt_llm/models/baichuan/convert.py +tensorrt_llm/models/baichuan/model.py +tensorrt_llm/models/bert/__init__.py +tensorrt_llm/models/bert/config.py +tensorrt_llm/models/bert/convert.py +tensorrt_llm/models/bert/model.py +tensorrt_llm/models/bloom/__init__.py +tensorrt_llm/models/bloom/model.py +tensorrt_llm/models/chatglm/__init__.py +tensorrt_llm/models/chatglm/config.py +tensorrt_llm/models/chatglm/convert.py +tensorrt_llm/models/chatglm/model.py +tensorrt_llm/models/clip/__init__.py +tensorrt_llm/models/clip/model.py +tensorrt_llm/models/cogvlm/__init__.py +tensorrt_llm/models/cogvlm/config.py +tensorrt_llm/models/cogvlm/convert.py +tensorrt_llm/models/cogvlm/model.py +tensorrt_llm/models/commandr/__init__.py +tensorrt_llm/models/commandr/config.py +tensorrt_llm/models/commandr/model.py tensorrt_llm/models/convert_utils.py +tensorrt_llm/models/dbrx/__init__.py +tensorrt_llm/models/dbrx/config.py +tensorrt_llm/models/dbrx/model.py +tensorrt_llm/models/deepseek_v1/__init__.py +tensorrt_llm/models/deepseek_v1/config.py +tensorrt_llm/models/deepseek_v1/convert.py +tensorrt_llm/models/deepseek_v1/model.py +tensorrt_llm/models/deepseek_v2/__init__.py +tensorrt_llm/models/deepseek_v2/config.py +tensorrt_llm/models/deepseek_v2/convert.py +tensorrt_llm/models/deepseek_v2/model.py +tensorrt_llm/models/dit/__init__.py +tensorrt_llm/models/dit/model.py +tensorrt_llm/models/eagle/__init__.py +tensorrt_llm/models/eagle/config.py +tensorrt_llm/models/eagle/model.py +tensorrt_llm/models/enc_dec/__init__.py +tensorrt_llm/models/enc_dec/model.py +tensorrt_llm/models/falcon/__init__.py +tensorrt_llm/models/falcon/config.py +tensorrt_llm/models/falcon/convert.py +tensorrt_llm/models/falcon/model.py +tensorrt_llm/models/gemma/__init__.py +tensorrt_llm/models/gemma/config.py +tensorrt_llm/models/gemma/convert.py +tensorrt_llm/models/gemma/model.py +tensorrt_llm/models/gemma/smoothquant.py +tensorrt_llm/models/gemma/utils/__init__.py +tensorrt_llm/models/gemma/utils/layers.py +tensorrt_llm/models/gemma/utils/modules.py +tensorrt_llm/models/gemma/utils/params.py +tensorrt_llm/models/gemma/utils/positional_embeddings.py +tensorrt_llm/models/gemma/utils/sampler.py +tensorrt_llm/models/gemma/utils/transformer.py +tensorrt_llm/models/gemma/weight.py +tensorrt_llm/models/generation_mixin.py +tensorrt_llm/models/gpt/__init__.py +tensorrt_llm/models/gpt/config.py +tensorrt_llm/models/gpt/convert.py +tensorrt_llm/models/gpt/model.py +tensorrt_llm/models/gptj/__init__.py +tensorrt_llm/models/gptj/config.py +tensorrt_llm/models/gptj/convert.py +tensorrt_llm/models/gptj/model.py +tensorrt_llm/models/gptneox/__init__.py +tensorrt_llm/models/gptneox/model.py +tensorrt_llm/models/grok/__init__.py +tensorrt_llm/models/grok/convert.py +tensorrt_llm/models/grok/model.py +tensorrt_llm/models/grok/weight.py +tensorrt_llm/models/llama/__init__.py +tensorrt_llm/models/llama/config.py +tensorrt_llm/models/llama/convert.py +tensorrt_llm/models/llama/model.py +tensorrt_llm/models/mamba/__init__.py +tensorrt_llm/models/mamba/config.py +tensorrt_llm/models/mamba/convert.py +tensorrt_llm/models/mamba/model.py +tensorrt_llm/models/medusa/__init__.py +tensorrt_llm/models/medusa/config.py +tensorrt_llm/models/medusa/model.py +tensorrt_llm/models/medusa/weight.py +tensorrt_llm/models/mllama/__init__.py +tensorrt_llm/models/mllama/config.py +tensorrt_llm/models/mllama/model.py +tensorrt_llm/models/mmdit_sd3/__init__.py +tensorrt_llm/models/mmdit_sd3/config.py +tensorrt_llm/models/mmdit_sd3/model.py +tensorrt_llm/models/model_weights_loader.py tensorrt_llm/models/modeling_utils.py +tensorrt_llm/models/mpt/__init__.py +tensorrt_llm/models/mpt/model.py +tensorrt_llm/models/multimodal_encoders/__init__.py +tensorrt_llm/models/multimodal_encoders/config.py +tensorrt_llm/models/multimodal_encoders/model.py +tensorrt_llm/models/nemotron_nas/__init__.py +tensorrt_llm/models/nemotron_nas/config.py +tensorrt_llm/models/nemotron_nas/convert.py +tensorrt_llm/models/nemotron_nas/layer_config.py +tensorrt_llm/models/nemotron_nas/model.py +tensorrt_llm/models/opt/__init__.py +tensorrt_llm/models/opt/model.py +tensorrt_llm/models/phi/__init__.py +tensorrt_llm/models/phi/config.py +tensorrt_llm/models/phi/convert.py +tensorrt_llm/models/phi/model.py +tensorrt_llm/models/phi3/__init__.py +tensorrt_llm/models/phi3/config.py +tensorrt_llm/models/phi3/convert.py +tensorrt_llm/models/phi3/model.py +tensorrt_llm/models/phi3/split_weights.py +tensorrt_llm/models/qwen/__init__.py +tensorrt_llm/models/qwen/config.py +tensorrt_llm/models/qwen/convert.py +tensorrt_llm/models/qwen/model.py +tensorrt_llm/models/qwen/utils.py +tensorrt_llm/models/recurrentgemma/__init__.py +tensorrt_llm/models/recurrentgemma/model.py +tensorrt_llm/models/redrafter/__init__.py +tensorrt_llm/models/redrafter/drafter.py +tensorrt_llm/models/redrafter/model.py +tensorrt_llm/models/redrafter/redrafter_helper.py +tensorrt_llm/models/stdit/__init__.py +tensorrt_llm/models/stdit/config.py +tensorrt_llm/models/stdit/model.py +tensorrt_llm/models/unet/__init__.py +tensorrt_llm/models/unet/attention.py +tensorrt_llm/models/unet/embeddings.py +tensorrt_llm/models/unet/pp/__init__.py +tensorrt_llm/models/unet/pp/attention.py +tensorrt_llm/models/unet/pp/conv2d.py +tensorrt_llm/models/unet/pp/groupnorm.py +tensorrt_llm/models/unet/pp/unet_pp.py +tensorrt_llm/models/unet/resnet.py +tensorrt_llm/models/unet/unet_2d_blocks.py +tensorrt_llm/models/unet/unet_2d_condition.py +tensorrt_llm/models/unet/weights.py +tensorrt_llm/network.py +tensorrt_llm/parameter.py +tensorrt_llm/plugin/__init__.py +tensorrt_llm/plugin/plugin.py tensorrt_llm/quantization/__init__.py tensorrt_llm/quantization/functional.py +tensorrt_llm/quantization/image_processing.py +tensorrt_llm/quantization/layers.py tensorrt_llm/quantization/mode.py +tensorrt_llm/quantization/quantize.py +tensorrt_llm/quantization/quantize_by_modelopt.py tensorrt_llm/quantization/utils/__init__.py tensorrt_llm/quantization/utils/fp4_utils.py tensorrt_llm/quantization/utils/fp8_utils.py tensorrt_llm/ray_stub.py tensorrt_llm/runtime/__init__.py +tensorrt_llm/runtime/enc_dec_model_runner.py +tensorrt_llm/runtime/generation.py +tensorrt_llm/runtime/kv_cache_manager.py +tensorrt_llm/runtime/medusa_utils.py tensorrt_llm/runtime/memory_pools/__init__.py +tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py +tensorrt_llm/runtime/memory_pools/pool.py +tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py +tensorrt_llm/runtime/model_runner.py +tensorrt_llm/runtime/model_runner_cpp.py +tensorrt_llm/runtime/multimodal_model_runner.py +tensorrt_llm/runtime/processor_wrapper/__init__.py +tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py +tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py +tensorrt_llm/runtime/redrafter_utils.py +tensorrt_llm/runtime/session.py tensorrt_llm/scaffolding/__init__.py tensorrt_llm/scaffolding/benchmark.py tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py @@ -471,7 +818,12 @@ tensorrt_llm/serve/tool_parser/utils.py tensorrt_llm/tokenizer/tokenizer.py tensorrt_llm/tools/__init__.py tensorrt_llm/tools/importlib_utils.py +tensorrt_llm/tools/multimodal_builder.py +tensorrt_llm/tools/onnx_utils.py tensorrt_llm/tools/plugin_gen/__init__.py +tensorrt_llm/tools/plugin_gen/core.py +tensorrt_llm/tools/plugin_gen/plugin_gen.py +tensorrt_llm/tools/plugin_gen/shape_infer.py tensorrt_llm/tools/ppl.py tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py tensorrt_llm/version.py @@ -480,7 +832,9 @@ tests/integration/defs/accuracy/__init__.py tests/integration/defs/accuracy/accuracy_core.py tests/integration/defs/accuracy/scripts/collect_evaluated_accuracies.py tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py +tests/integration/defs/accuracy/test_cli_flow.py tests/integration/defs/accuracy/test_disaggregated_serving.py +tests/integration/defs/accuracy/test_llm_api.py tests/integration/defs/accuracy/test_llm_api_autodeploy.py tests/integration/defs/accuracy/test_llm_api_pytorch.py tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py @@ -489,29 +843,63 @@ tests/integration/defs/common.py tests/integration/defs/conftest.py tests/integration/defs/cpp/conftest.py tests/integration/defs/cpp/cpp_common.py +tests/integration/defs/cpp/test_e2e.py tests/integration/defs/cpp/test_multi_gpu.py tests/integration/defs/cpp/test_unit_tests.py +tests/integration/defs/deterministic/mixtral_deterministic.py +tests/integration/defs/deterministic/test_mixtral_deterministic.py tests/integration/defs/disaggregated/test_auto_scaling.py tests/integration/defs/disaggregated/test_disaggregated.py tests/integration/defs/disaggregated/test_disaggregated_etcd.py tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py tests/integration/defs/disaggregated/test_workers.py +tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py tests/integration/defs/examples/run_llm_quickstart_atexit.py tests/integration/defs/examples/serve/test_serve.py tests/integration/defs/examples/serve/test_serve_negative.py tests/integration/defs/examples/test_ad_guided_decoding.py +tests/integration/defs/examples/test_bert.py +tests/integration/defs/examples/test_bindings.py +tests/integration/defs/examples/test_chatglm.py +tests/integration/defs/examples/test_commandr.py +tests/integration/defs/examples/test_draft_target_model.py +tests/integration/defs/examples/test_eagle.py +tests/integration/defs/examples/test_enc_dec.py +tests/integration/defs/examples/test_exaone.py +tests/integration/defs/examples/test_gemma.py tests/integration/defs/examples/test_gpt.py +tests/integration/defs/examples/test_gptj.py +tests/integration/defs/examples/test_granite.py +tests/integration/defs/examples/test_internlm.py +tests/integration/defs/examples/test_llama.py tests/integration/defs/examples/test_llm_api_with_mpi.py +tests/integration/defs/examples/test_mamba.py +tests/integration/defs/examples/test_medusa.py +tests/integration/defs/examples/test_mistral.py +tests/integration/defs/examples/test_mixtral.py +tests/integration/defs/examples/test_multimodal.py +tests/integration/defs/examples/test_nemotron.py +tests/integration/defs/examples/test_nemotron_nas.py +tests/integration/defs/examples/test_ngram.py +tests/integration/defs/examples/test_openai.py tests/integration/defs/examples/test_phi.py +tests/integration/defs/examples/test_qwen.py +tests/integration/defs/examples/test_qwen2audio.py +tests/integration/defs/examples/test_qwenvl.py tests/integration/defs/examples/test_ray.py +tests/integration/defs/examples/test_recurrentgemma.py +tests/integration/defs/examples/test_redrafter.py +tests/integration/defs/examples/test_whisper.py tests/integration/defs/llmapi/__init__.py tests/integration/defs/llmapi/_run_llmapi_llm.py tests/integration/defs/llmapi/test_llm_api_connector.py tests/integration/defs/llmapi/test_llm_api_qa.py +tests/integration/defs/llmapi/test_llm_e2e.py tests/integration/defs/llmapi/test_llm_examples.py tests/integration/defs/local_venv.py tests/integration/defs/perf/__init__.py tests/integration/defs/perf/allowed_configs.py +tests/integration/defs/perf/build.py tests/integration/defs/perf/create_perf_comparison_report.py tests/integration/defs/perf/data.py tests/integration/defs/perf/data_export.py @@ -532,21 +920,38 @@ tests/integration/defs/test_e2e.py tests/integration/defs/test_fmha.py tests/integration/defs/test_list_parser.py tests/integration/defs/test_list_validation.py +tests/integration/defs/test_mlpf_results.py tests/integration/defs/test_sanity.py tests/integration/defs/test_unittests.py tests/integration/defs/triton_server/__init__.py +tests/integration/defs/triton_server/build_engines.py tests/integration/defs/triton_server/common.py tests/integration/defs/triton_server/conftest.py +tests/integration/defs/triton_server/local_venv.py +tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py +tests/integration/defs/triton_server/runner_interface.py tests/integration/defs/triton_server/test_list_parser.py +tests/integration/defs/triton_server/test_triton.py +tests/integration/defs/triton_server/test_triton_llm.py +tests/integration/defs/triton_server/test_triton_memleak.py +tests/integration/defs/triton_server/test_triton_multi_node.py +tests/integration/defs/triton_server/test_triton_rcca.py tests/integration/defs/triton_server/trt_test_alternative.py tests/integration/defs/trt_test_alternative.py tests/integration/defs/utils/__init__.py tests/integration/defs/utils/periodic_junit.py tests/integration/defs/utils/timeout_manager.py tests/microbenchmarks/all_reduce.py +tests/microbenchmarks/build_time_benchmark.py +tests/microbenchmarks/build_time_dashboard.py tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py tests/scripts/allreduce_perf/allreduce_perf_viz.py tests/scripts/iteration_log_parser.py +tests/scripts/perf-sanity/parse_benchmark_results.py +tests/scripts/perf-sanity/run_benchmark_serve.py +tests/unittest/_torch/attention/sparse/test_dsa_indexer.py +tests/unittest/_torch/attention/sparse/test_flash_mla.py +tests/unittest/_torch/attention/sparse/test_rocketkv.py tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py tests/unittest/_torch/attention/test_attention.py tests/unittest/_torch/attention/test_attention_mla.py @@ -567,6 +972,7 @@ tests/unittest/_torch/misc/test_share_tensor.py tests/unittest/_torch/misc/test_virtual_memory.py tests/unittest/_torch/modeling/test_modeling_bert.py tests/unittest/_torch/modeling/test_modeling_clip.py +tests/unittest/_torch/modeling/test_modeling_exaone4.py tests/unittest/_torch/modeling/test_modeling_gemma3.py tests/unittest/_torch/modeling/test_modeling_gpt_oss.py tests/unittest/_torch/modeling/test_modeling_llama.py @@ -590,6 +996,8 @@ tests/unittest/_torch/modules/test_moe_load_balancer.py tests/unittest/_torch/modules/test_moe_routing.py tests/unittest/_torch/modules/test_rotary_embedding.py tests/unittest/_torch/modules/test_triton_linear.py +tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py +tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py tests/unittest/_torch/multi_gpu/test_allreduce.py tests/unittest/_torch/multi_gpu/test_alltoall.py tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py @@ -616,10 +1024,24 @@ tests/unittest/_torch/ray_orchestrator/single_gpu/test_cache_transceiver_comm.py tests/unittest/_torch/sampler/test_beam_search.py tests/unittest/_torch/sampler/test_best_of_n.py tests/unittest/_torch/sampler/test_trtllm_sampler.py +tests/unittest/_torch/speculative/test_draft_target.py +tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py +tests/unittest/_torch/speculative/test_draft_token_tree_verification.py +tests/unittest/_torch/speculative/test_dynamic_spec_decode.py tests/unittest/_torch/speculative/test_eagle3.py +tests/unittest/_torch/speculative/test_kv_cache_reuse.py +tests/unittest/_torch/speculative/test_mtp.py +tests/unittest/_torch/speculative/test_ngram.py +tests/unittest/_torch/speculative/test_save_state.py +tests/unittest/_torch/speculative/test_spec_gate.py +tests/unittest/_torch/speculative/test_torch_rejection_sampling.py +tests/unittest/_torch/speculative/test_user_provided.py tests/unittest/_torch/test_connector.py tests/unittest/_torch/test_torch_multi_arange.py tests/unittest/_torch/thop/parallel/deep_gemm_tests.py +tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py +tests/unittest/_torch/thop/parallel/test_cublas_mm.py +tests/unittest/_torch/thop/parallel/test_custom_ops.py tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py @@ -633,6 +1055,11 @@ tests/unittest/_torch/thop/parallel/test_fp8_linear.py tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py tests/unittest/_torch/thop/parallel/test_fp8_quantize.py tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py +tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py +tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py +tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py +tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py +tests/unittest/_torch/thop/parallel/test_noaux_tc.py tests/unittest/_torch/thop/parallel/test_scaled_mm.py tests/unittest/_torch/thop/parallel/test_selective_scan_op.py tests/unittest/_torch/thop/parallel/test_tinygemm2.py @@ -646,6 +1073,7 @@ tests/unittest/_torch/thop/serial/test_moe.py tests/unittest/_torch/thop/serial/test_moe_alltoall.py tests/unittest/api_stability/api_stability_core.py tests/unittest/api_stability/test_llm_api.py +tests/unittest/bindings/binding_test_utils.py tests/unittest/bindings/test_bindings_moe.py tests/unittest/bindings/test_bindings_ut.py tests/unittest/bindings/test_executor_bindings.py @@ -676,10 +1104,12 @@ tests/unittest/llmapi/apps/_test_openai_chat_guided_decoding.py tests/unittest/llmapi/apps/_test_openai_chat_harmony.py tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py tests/unittest/llmapi/apps/_test_openai_completions.py +tests/unittest/llmapi/apps/_test_openai_consistent_chat.py tests/unittest/llmapi/apps/_test_openai_lora.py tests/unittest/llmapi/apps/_test_openai_metrics.py tests/unittest/llmapi/apps/_test_openai_misc.py tests/unittest/llmapi/apps/_test_openai_mmencoder.py +tests/unittest/llmapi/apps/_test_openai_multi_chat.py tests/unittest/llmapi/apps/_test_openai_multi_gpu.py tests/unittest/llmapi/apps/_test_openai_multi_nodes.py tests/unittest/llmapi/apps/_test_openai_perf_metrics.py @@ -702,12 +1132,15 @@ tests/unittest/llmapi/run_llm.py tests/unittest/llmapi/run_llm_exit.py tests/unittest/llmapi/run_llm_with_postproc.py tests/unittest/llmapi/test_additional_model_outputs.py +tests/unittest/llmapi/test_build_cache.py tests/unittest/llmapi/test_executor.py tests/unittest/llmapi/test_gc_utils.py tests/unittest/llmapi/test_llm.py tests/unittest/llmapi/test_llm_args.py tests/unittest/llmapi/test_llm_download.py tests/unittest/llmapi/test_llm_kv_cache_events.py +tests/unittest/llmapi/test_llm_models.py +tests/unittest/llmapi/test_llm_multi_gpu.py tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py tests/unittest/llmapi/test_llm_pytorch.py tests/unittest/llmapi/test_llm_quant.py @@ -718,14 +1151,25 @@ tests/unittest/llmapi/test_reasoning_parser.py tests/unittest/llmapi/test_serialization.py tests/unittest/llmapi/test_utils.py tests/unittest/others/__init__.py +tests/unittest/others/test_builder.py tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py +tests/unittest/others/test_debugging_api.py tests/unittest/others/test_exception.py tests/unittest/others/test_export.py +tests/unittest/others/test_graph_rewriter.py +tests/unittest/others/test_kv_cache_manager.py tests/unittest/others/test_kv_cache_transceiver.py tests/unittest/others/test_kv_cache_update.py +tests/unittest/others/test_layer.py +tests/unittest/others/test_leak.py tests/unittest/others/test_mapping.py +tests/unittest/others/test_model_dtype.py +tests/unittest/others/test_module.py tests/unittest/others/test_multimodal_registry.py +tests/unittest/others/test_plugins.py +tests/unittest/others/test_precision_control.py tests/unittest/others/test_pretrained_config.py +tests/unittest/others/test_session.py tests/unittest/others/test_time_breakdown.py tests/unittest/profile_utils.py tests/unittest/scaffolding/__init__.py @@ -734,14 +1178,141 @@ tests/unittest/scaffolding/test_parallel_process.py tests/unittest/scaffolding/test_scaffolding.py tests/unittest/scaffolding/test_task_collection.py tests/unittest/scaffolding/test_worker.py +tests/unittest/test_model_runner_cpp.py tests/unittest/test_pip_install.py tests/unittest/tools/__init__.py +tests/unittest/tools/plugin_gen/__init__.py +tests/unittest/tools/plugin_gen/kernel_config.py +tests/unittest/tools/plugin_gen/test_core.py +tests/unittest/tools/plugin_gen/test_plugin_gen.py +tests/unittest/tools/plugin_gen/test_shape_infer.py tests/unittest/tools/test_prepare_dataset.py tests/unittest/tools/test_test_to_stage_mapping.py +tests/unittest/trt/__init__.py +tests/unittest/trt/attention/test_bert_attention.py +tests/unittest/trt/attention/test_gpt_attention.py +tests/unittest/trt/attention/test_gpt_attention_IFB.py +tests/unittest/trt/attention/test_gpt_attention_no_cache.py +tests/unittest/trt/attention/test_sage_attention.py +tests/unittest/trt/functional/__init__.py +tests/unittest/trt/functional/test_alibi.py +tests/unittest/trt/functional/test_allreduce_norm.py +tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py +tests/unittest/trt/functional/test_arange.py +tests/unittest/trt/functional/test_argmax.py +tests/unittest/trt/functional/test_assertion.py +tests/unittest/trt/functional/test_avg_pool2d.py +tests/unittest/trt/functional/test_cast.py +tests/unittest/trt/functional/test_conv2d.py +tests/unittest/trt/functional/test_conv3d.py +tests/unittest/trt/functional/test_cos.py +tests/unittest/trt/functional/test_cumsum.py +tests/unittest/trt/functional/test_dora.py +tests/unittest/trt/functional/test_einsum.py +tests/unittest/trt/functional/test_embedding_single_gpu.py +tests/unittest/trt/functional/test_exp.py +tests/unittest/trt/functional/test_expand.py +tests/unittest/trt/functional/test_flatten.py +tests/unittest/trt/functional/test_flip.py +tests/unittest/trt/functional/test_fp4_gemm.py +tests/unittest/trt/functional/test_fp4_gemm_ootb.py +tests/unittest/trt/functional/test_gather.py +tests/unittest/trt/functional/test_gather_nd.py +tests/unittest/trt/functional/test_geglu.py +tests/unittest/trt/functional/test_gelu.py +tests/unittest/trt/functional/test_gemm_swiglu.py +tests/unittest/trt/functional/test_group_norm.py +tests/unittest/trt/functional/test_identity.py +tests/unittest/trt/functional/test_index_select.py +tests/unittest/trt/functional/test_interpolate.py +tests/unittest/trt/functional/test_logsoftmax.py +tests/unittest/trt/functional/test_lora.py +tests/unittest/trt/functional/test_low_latency_gemm.py +tests/unittest/trt/functional/test_mamba_conv1d.py +tests/unittest/trt/functional/test_masked_scatter.py +tests/unittest/trt/functional/test_masked_select.py +tests/unittest/trt/functional/test_matmul.py +tests/unittest/trt/functional/test_meshgrid2d.py +tests/unittest/trt/functional/test_moe.py +tests/unittest/trt/functional/test_nccl.py +tests/unittest/trt/functional/test_nonzero.py +tests/unittest/trt/functional/test_outer.py +tests/unittest/trt/functional/test_pad.py +tests/unittest/trt/functional/test_permute.py +tests/unittest/trt/functional/test_pp_reduce_scatter.py +tests/unittest/trt/functional/test_quant.py +tests/unittest/trt/functional/test_rearrange.py +tests/unittest/trt/functional/test_repeat.py +tests/unittest/trt/functional/test_repeat_interleave.py +tests/unittest/trt/functional/test_rg_lru.py +tests/unittest/trt/functional/test_sample.py +tests/unittest/trt/functional/test_scatter.py +tests/unittest/trt/functional/test_scatter_nd.py +tests/unittest/trt/functional/test_select.py +tests/unittest/trt/functional/test_selective_scan.py +tests/unittest/trt/functional/test_sigmoid.py +tests/unittest/trt/functional/test_silu.py +tests/unittest/trt/functional/test_sin.py +tests/unittest/trt/functional/test_slice.py +tests/unittest/trt/functional/test_softplus.py +tests/unittest/trt/functional/test_split.py +tests/unittest/trt/functional/test_squeeze.py +tests/unittest/trt/functional/test_swiglu.py +tests/unittest/trt/functional/test_topk.py +tests/unittest/trt/functional/test_transpose.py +tests/unittest/trt/functional/test_unbind.py +tests/unittest/trt/functional/test_unsqueeze.py +tests/unittest/trt/functional/test_view.py +tests/unittest/trt/functional/test_where.py +tests/unittest/trt/model/__init__.py +tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py +tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py +tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py +tests/unittest/trt/model/redrafter/test_beams2tree.py +tests/unittest/trt/model/redrafter/test_draft_token.py +tests/unittest/trt/model/redrafter/test_draft_token_indices.py +tests/unittest/trt/model/redrafter/test_gather_beams.py +tests/unittest/trt/model/redrafter/test_mask.py +tests/unittest/trt/model/redrafter/test_packed_position_ids.py +tests/unittest/trt/model/redrafter/test_prefix_match_indices.py +tests/unittest/trt/model/redrafter/test_prepare_input.py +tests/unittest/trt/model/redrafter/test_process_logits.py +tests/unittest/trt/model/redrafter/test_top1.py +tests/unittest/trt/model/redrafter/test_unpack_gen_data.py +tests/unittest/trt/model/redrafter/test_validate.py +tests/unittest/trt/model/test_gpt.py +tests/unittest/trt/model/test_gpt_e2e.py +tests/unittest/trt/model/test_llama.py +tests/unittest/trt/model/test_mamba.py +tests/unittest/trt/model/test_mistral.py +tests/unittest/trt/model/test_nemotron_nas.py +tests/unittest/trt/model/test_phi.py +tests/unittest/trt/model/test_unet.py +tests/unittest/trt/model_api/test_model_api_multi_gpu.py +tests/unittest/trt/model_api/test_model_level_api.py +tests/unittest/trt/model_api/test_model_quantization.py +tests/unittest/trt/python_plugin/plugin_wrapper_utils.py +tests/unittest/trt/python_plugin/test_plugin_wrapper.py +tests/unittest/trt/quantization/__init__.py +tests/unittest/trt/quantization/_utils.py +tests/unittest/trt/quantization/test_fp8_quantization.py +tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py +tests/unittest/trt/quantization/test_functional.py +tests/unittest/trt/quantization/test_mode.py +tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py +tests/unittest/trt/quantization/test_qserve_gemm.py +tests/unittest/trt/quantization/test_quant.py +tests/unittest/trt/quantization/test_quant_layer.py +tests/unittest/trt/quantization/test_smooth_quant_gemm.py +tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py +tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py +tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py +tests/unittest/trt/quantization/test_weight_only_quant_matmul.py tests/unittest/utils/__init__.py tests/unittest/utils/cpp_paths.py tests/unittest/utils/llm_data.py tests/unittest/utils/runtime_defaults.py +tests/unittest/utils/test_medusa_utils.py tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py tests/unittest/utils/test_util.py tests/unittest/utils/torch_ref.py diff --git a/pyproject.toml b/pyproject.toml index 2efd23c84d48..99b65e954d00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ column_limit = 80 [tool.codespell] skip = ".git,3rdparty,triton_kernels,tests/integration/test_input_files**,**.jsonl,**.json" -ignore-words-list = "rouge,inout,atleast,strat,nd,subtile,thrid,improbe,NotIn,te,iteract,anythin,tru,Tracin,vEw,dOut,indext,asend,medias,thw" +ignore-words-list = "rouge,inout,atleast,strat,nd,subtile,thrid,improbe,NotIn,te,iteract,anythin,tru,Tracin,vEw,dOut,indext,asend,medias" [tool.autoflake] in-place = true @@ -58,6 +58,14 @@ exclude = [ ".devcontainer/make_env.py", ".github/scripts/label_community_user.py", ".github/scripts/pr_checklist_check.py", + "benchmarks/cpp/__init__.py", + "benchmarks/cpp/prepare_dataset.py", + "benchmarks/cpp/utils/__init__.py", + "benchmarks/cpp/utils/convert_nemo_dataset.py", + "benchmarks/cpp/utils/generate_rand_loras.py", + "benchmarks/cpp/utils/prepare_real_data.py", + "benchmarks/cpp/utils/prepare_synthetic_data.py", + "benchmarks/cpp/utils/utils.py", "cpp/conanfile.py", "cpp/kernels/fmha_v2/conftest.py", "cpp/kernels/fmha_v2/fmha_test.py", @@ -82,17 +90,58 @@ exclude = [ "cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py", "cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py", "cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py", + "cpp/tests/resources/scripts/build_chatglm_engines.py", + "cpp/tests/resources/scripts/build_eagle_engines.py", + "cpp/tests/resources/scripts/build_enc_dec_engines.py", + "cpp/tests/resources/scripts/build_engines_utils.py", + "cpp/tests/resources/scripts/build_gpt_engines.py", + "cpp/tests/resources/scripts/build_gptj_engines.py", + "cpp/tests/resources/scripts/build_llama_engines.py", + "cpp/tests/resources/scripts/build_mamba_engines.py", + "cpp/tests/resources/scripts/build_medusa_engines.py", + "cpp/tests/resources/scripts/build_recurrentgemma_engines.py", + "cpp/tests/resources/scripts/build_redrafter_engines.py", + "cpp/tests/resources/scripts/generate_expected_chatglm_output.py", + "cpp/tests/resources/scripts/generate_expected_eagle_output.py", + "cpp/tests/resources/scripts/generate_expected_enc_dec_output.py", + "cpp/tests/resources/scripts/generate_expected_gpt_output.py", + "cpp/tests/resources/scripts/generate_expected_gptj_output.py", + "cpp/tests/resources/scripts/generate_expected_llama_output.py", + "cpp/tests/resources/scripts/generate_expected_mamba_output.py", + "cpp/tests/resources/scripts/generate_expected_medusa_output.py", + "cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py", + "cpp/tests/resources/scripts/generate_expected_redrafter_output.py", + "cpp/tests/resources/scripts/generate_hf_gpt_output.py", "cpp/tests/resources/scripts/generate_test_lora_weights.py", + "cpp/tests/resources/scripts/io_converter.py", "docs/source/conf.py", "docs/source/helper.py", "examples/apps/chat.py", "examples/apps/fastapi_server.py", + "examples/bindings/executor/example_advanced.py", + "examples/bindings/executor/example_basic.py", + "examples/bindings/executor/example_debug.py", + "examples/bindings/executor/example_logits_processor.py", "examples/disaggregated/clients/disagg_client.py", "examples/disaggregated/slurm/benchmark/submit.py", + "examples/dora/normalize_weights.py", + "examples/eagle/convert_checkpoint.py", + "examples/eval_long_context.py", + "examples/generate_checkpoint_config.py", + "examples/generate_xgrammar_tokenizer_info.py", + "examples/hf_lora_convert.py", "examples/infinitebench/args.py", "examples/infinitebench/compute_scores.py", "examples/infinitebench/construct_synthetic_dataset.py", "examples/infinitebench/eval_utils.py", + "examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py", + "examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py", + "examples/llm-api/_tensorrt_engine/llm_inference_customize.py", + "examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py", + "examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py", + "examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py", + "examples/llm-api/_tensorrt_engine/llm_quantization.py", + "examples/llm-api/_tensorrt_engine/quickstart_example.py", "examples/llm-api/llm_guided_decoding.py", "examples/llm-api/llm_inference.py", "examples/llm-api/llm_inference_async.py", @@ -112,17 +161,122 @@ exclude = [ "examples/llm-api/quickstart_example.py", "examples/llm-api/quickstart_multimodal.py", "examples/llm-api/star_attention.py", + "examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py", "examples/longbench/eval_longbench_v1.py", + "examples/medusa/convert_checkpoint.py", + "examples/mmlu.py", + "examples/models/contrib/baichuan/convert_checkpoint.py", + "examples/models/contrib/bloom/convert_checkpoint.py", + "examples/models/contrib/chatglm-6b/tokenization_chatglm.py", + "examples/models/contrib/chatglm2-6b/tokenization_chatglm.py", + "examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py", + "examples/models/contrib/cogvlm/convert_checkpoint.py", + "examples/models/contrib/dbrx/convert_checkpoint.py", + "examples/models/contrib/deepseek_v1/__init__.py", + "examples/models/contrib/deepseek_v1/convert_checkpoint.py", + "examples/models/contrib/deepseek_v2/convert_checkpoint.py", + "examples/models/contrib/dit/convert_checkpoint.py", + "examples/models/contrib/dit/diffusion.py", + "examples/models/contrib/dit/sample.py", + "examples/models/contrib/dit/utils_modelopt.py", + "examples/models/contrib/dit/vae_decoder_trt.py", + "examples/models/contrib/falcon/convert_checkpoint.py", + "examples/models/contrib/gptj/convert_checkpoint.py", + "examples/models/contrib/gptneox/convert_checkpoint.py", + "examples/models/contrib/grok/convert_checkpoint.py", + "examples/models/contrib/mmdit/convert_checkpoint.py", + "examples/models/contrib/mmdit/sample.py", + "examples/models/contrib/mpt/convert_checkpoint.py", + "examples/models/contrib/opt/convert_checkpoint.py", + "examples/models/contrib/sdxl/build_sdxl_unet.py", + "examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py", + "examples/models/contrib/sdxl/run_sdxl.py", + "examples/models/contrib/stdit/aspect.py", + "examples/models/contrib/stdit/convert_checkpoint.py", + "examples/models/contrib/stdit/pipeline_tllm.py", + "examples/models/contrib/stdit/sample.py", + "examples/models/contrib/stdit/scheduler.py", + "examples/models/contrib/stdit/text_encoder.py", + "examples/models/contrib/stdit/utils.py", + "examples/models/contrib/stdit/vae.py", + "examples/models/contrib/stdit/video_transforms.py", + "examples/models/core/bert/__init__.py", + "examples/models/core/bert/convert_checkpoint.py", + "examples/models/core/bert/run.py", + "examples/models/core/bert/utils.py", + "examples/models/core/commandr/convert_checkpoint.py", + "examples/models/core/enc_dec/__init__.py", + "examples/models/core/enc_dec/convert_checkpoint.py", + "examples/models/core/enc_dec/helper.py", + "examples/models/core/enc_dec/run.py", + "examples/models/core/gemma/convert_checkpoint.py", + "examples/models/core/glm-4-9b/convert_checkpoint.py", + "examples/models/core/glm-4-9b/tokenization_chatglm.py", + "examples/models/core/gpt/convert_checkpoint.py", + "examples/models/core/gpt/merge_ptuning_tables.py", + "examples/models/core/gpt/nemo_lora_convert.py", + "examples/models/core/gpt/nemo_prompt_convert.py", + "examples/models/core/gpt/run_hf.py", "examples/models/core/gpt_oss/openai_chat_client_function_calling.py", + "examples/models/core/internlm2/convert_checkpoint.py", "examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py", + "examples/models/core/llama/convert_checkpoint.py", + "examples/models/core/llama/summarize_long.py", + "examples/models/core/mamba/convert_checkpoint.py", + "examples/models/core/mllama/convert_checkpoint.py", + "examples/models/core/multimodal/__init__.py", + "examples/models/core/multimodal/build_multimodal_engine.py", + "examples/models/core/multimodal/eval.py", + "examples/models/core/multimodal/run.py", + "examples/models/core/multimodal/utils.py", + "examples/models/core/nemotron_nas/calibration_utils.py", + "examples/models/core/nemotron_nas/convert_checkpoint.py", + "examples/models/core/phi/convert_checkpoint.py", + "examples/models/core/qwen/convert_checkpoint.py", + "examples/models/core/qwen2audio/run.py", + "examples/models/core/qwen2audio/run_chat.py", + "examples/models/core/qwen2audio/utils.py", + "examples/models/core/qwenvl/run.py", + "examples/models/core/qwenvl/run_chat.py", + "examples/models/core/qwenvl/show_pic.py", + "examples/models/core/qwenvl/vit_onnx_trt.py", + "examples/models/core/recurrentgemma/convert_checkpoint.py", + "examples/models/core/vit/convert_checkpoint.py", + "examples/models/core/whisper/convert_checkpoint.py", + "examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py", + "examples/models/core/whisper/run.py", + "examples/models/core/whisper/tokenizer.py", + "examples/models/core/whisper/whisper_utils.py", + "examples/ngram/run_dtm_ngram.py", + "examples/openai_triton/manual_plugin/build.py", + "examples/openai_triton/manual_plugin/fmha_triton.py", + "examples/openai_triton/manual_plugin/plugin.py", + "examples/openai_triton/manual_plugin/run.py", + "examples/openai_triton/plugin_autogen/build_engine.py", + "examples/openai_triton/plugin_autogen/kernel_config.py", + "examples/openai_triton/plugin_autogen/run_engine.py", + "examples/python_plugin/build_lookup.py", + "examples/python_plugin/plugin_lib/__init__.py", + "examples/python_plugin/plugin_lib/lookup_kernel.py", + "examples/python_plugin/plugin_lib/lookup_plugin.py", + "examples/python_plugin/run_lookup.py", + "examples/quantization/quantize.py", "examples/quantization/quantize_mixed_precision_moe.py", "examples/ray_orchestrator/llm_inference_async_ray.py", "examples/ray_orchestrator/llm_inference_distributed_ray.py", + "examples/redrafter/convert_checkpoint.py", + "examples/run.py", "examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py", "examples/scaffolding/contrib/DeepConf/run_generation.py", "examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py", "examples/scaffolding/contrib/TreeInference/run_mcts_example.py", "examples/scaffolding/contrib/TreeInference/run_tot_example.py", + "examples/scaffolding/contrib/mcp/e2b/e2bserver.py", + "examples/scaffolding/contrib/mcp/e2b/main.py", + "examples/scaffolding/contrib/mcp/mcptest.py", + "examples/scaffolding/contrib/mcp/weather/weather.py", + "examples/scaffolding/contrib/mcp/websearch/main.py", + "examples/scaffolding/contrib/mcp/websearch/websearch.py", "examples/scaffolding/run_basic_generation.py", "examples/scaffolding/run_best_of_n_with_reward.py", "examples/scaffolding/run_majority_vote_aime24.py", @@ -132,6 +286,8 @@ exclude = [ "examples/serve/openai_completion_client.py", "examples/serve/openai_completion_client_for_lora.py", "examples/serve/openai_completion_client_json_schema.py", + "examples/summarize.py", + "examples/utils.py", "examples/wide_ep/ep_load_balancer/generate_eplb_config.py", "examples/wide_ep/ep_load_balancer/report_load_statistics.py", "examples/wide_ep/ep_load_balancer/utils.py", @@ -139,10 +295,12 @@ exclude = [ "jenkins/scripts/mergeWaiveList.py", "jenkins/scripts/open_search_db.py", "jenkins/scripts/test_rerun.py", + "scripts/build_cpp_examples.py", "scripts/build_wheel.py", "scripts/check_test_list.py", "scripts/dco_check.py", "scripts/format_test_list.py", + "scripts/generate_duration.py", "scripts/generate_lock_file.py", "scripts/get_wheel_from_package.py", "scripts/git_replace.py", @@ -153,6 +311,7 @@ exclude = [ "setup.py", "tensorrt_llm/__init__.py", "tensorrt_llm/_ray_utils.py", + "tensorrt_llm/_tensorrt_engine/__init__.py", "tensorrt_llm/_torch/__init__.py", "tensorrt_llm/_torch/attention_backend/__init__.py", "tensorrt_llm/_torch/attention_backend/flashinfer.py", @@ -356,6 +515,7 @@ exclude = [ "tensorrt_llm/_torch/pyexecutor/guided_decoder.py", "tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py", "tensorrt_llm/_torch/pyexecutor/handle_logits.py", + "tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py", "tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py", "tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py", "tensorrt_llm/_torch/pyexecutor/llm_request.py", @@ -393,6 +553,11 @@ exclude = [ "tensorrt_llm/bench/benchmark/utils/asynchronous.py", "tensorrt_llm/bench/benchmark/utils/general.py", "tensorrt_llm/bench/benchmark/utils/processes.py", + "tensorrt_llm/bench/build/__init__.py", + "tensorrt_llm/bench/build/build.py", + "tensorrt_llm/bench/build/dataclasses.py", + "tensorrt_llm/bench/build/tuning.py", + "tensorrt_llm/bench/build/utils.py", "tensorrt_llm/bench/dataclasses/__init__.py", "tensorrt_llm/bench/dataclasses/configuration.py", "tensorrt_llm/bench/dataclasses/engine.py", @@ -402,9 +567,13 @@ exclude = [ "tensorrt_llm/bench/dataclasses/statistics.py", "tensorrt_llm/bench/utils/__init__.py", "tensorrt_llm/bench/utils/data.py", + "tensorrt_llm/builder.py", "tensorrt_llm/commands/__init__.py", "tensorrt_llm/commands/bench.py", + "tensorrt_llm/commands/build.py", "tensorrt_llm/commands/eval.py", + "tensorrt_llm/commands/prune.py", + "tensorrt_llm/commands/refit.py", "tensorrt_llm/commands/serve.py", "tensorrt_llm/evaluate/__init__.py", "tensorrt_llm/evaluate/cnn_dailymail.py", @@ -440,7 +609,23 @@ exclude = [ "tensorrt_llm/inputs/multimodal.py", "tensorrt_llm/inputs/registry.py", "tensorrt_llm/inputs/utils.py", + "tensorrt_llm/layers/__init__.py", + "tensorrt_llm/layers/activation.py", + "tensorrt_llm/layers/attention.py", + "tensorrt_llm/layers/cast.py", + "tensorrt_llm/layers/conv.py", + "tensorrt_llm/layers/embedding.py", + "tensorrt_llm/layers/language_adapter.py", + "tensorrt_llm/layers/linear.py", + "tensorrt_llm/layers/lora.py", + "tensorrt_llm/layers/mlp.py", + "tensorrt_llm/layers/moe.py", + "tensorrt_llm/layers/normalization.py", + "tensorrt_llm/layers/pooling.py", + "tensorrt_llm/layers/recurrent.py", + "tensorrt_llm/layers/ssm.py", "tensorrt_llm/llmapi/__init__.py", + "tensorrt_llm/llmapi/build_cache.py", "tensorrt_llm/llmapi/disagg_utils.py", "tensorrt_llm/llmapi/kv_cache_type.py", "tensorrt_llm/llmapi/llm.py", @@ -463,17 +648,179 @@ exclude = [ "tensorrt_llm/metrics/enums.py", "tensorrt_llm/models/__init__.py", "tensorrt_llm/models/automodel.py", + "tensorrt_llm/models/baichuan/__init__.py", + "tensorrt_llm/models/baichuan/config.py", + "tensorrt_llm/models/baichuan/convert.py", + "tensorrt_llm/models/baichuan/model.py", + "tensorrt_llm/models/bert/__init__.py", + "tensorrt_llm/models/bert/config.py", + "tensorrt_llm/models/bert/convert.py", + "tensorrt_llm/models/bert/model.py", + "tensorrt_llm/models/bloom/__init__.py", + "tensorrt_llm/models/bloom/model.py", + "tensorrt_llm/models/chatglm/__init__.py", + "tensorrt_llm/models/chatglm/config.py", + "tensorrt_llm/models/chatglm/convert.py", + "tensorrt_llm/models/chatglm/model.py", + "tensorrt_llm/models/clip/__init__.py", + "tensorrt_llm/models/clip/model.py", + "tensorrt_llm/models/cogvlm/__init__.py", + "tensorrt_llm/models/cogvlm/config.py", + "tensorrt_llm/models/cogvlm/convert.py", + "tensorrt_llm/models/cogvlm/model.py", + "tensorrt_llm/models/commandr/__init__.py", + "tensorrt_llm/models/commandr/config.py", + "tensorrt_llm/models/commandr/model.py", "tensorrt_llm/models/convert_utils.py", + "tensorrt_llm/models/dbrx/__init__.py", + "tensorrt_llm/models/dbrx/config.py", + "tensorrt_llm/models/dbrx/model.py", + "tensorrt_llm/models/deepseek_v1/__init__.py", + "tensorrt_llm/models/deepseek_v1/config.py", + "tensorrt_llm/models/deepseek_v1/convert.py", + "tensorrt_llm/models/deepseek_v1/model.py", + "tensorrt_llm/models/deepseek_v2/__init__.py", + "tensorrt_llm/models/deepseek_v2/config.py", + "tensorrt_llm/models/deepseek_v2/convert.py", + "tensorrt_llm/models/deepseek_v2/model.py", + "tensorrt_llm/models/dit/__init__.py", + "tensorrt_llm/models/dit/model.py", + "tensorrt_llm/models/eagle/__init__.py", + "tensorrt_llm/models/eagle/config.py", + "tensorrt_llm/models/eagle/model.py", + "tensorrt_llm/models/enc_dec/__init__.py", + "tensorrt_llm/models/enc_dec/model.py", + "tensorrt_llm/models/falcon/__init__.py", + "tensorrt_llm/models/falcon/config.py", + "tensorrt_llm/models/falcon/convert.py", + "tensorrt_llm/models/falcon/model.py", + "tensorrt_llm/models/gemma/__init__.py", + "tensorrt_llm/models/gemma/config.py", + "tensorrt_llm/models/gemma/convert.py", + "tensorrt_llm/models/gemma/model.py", + "tensorrt_llm/models/gemma/smoothquant.py", + "tensorrt_llm/models/gemma/utils/__init__.py", + "tensorrt_llm/models/gemma/utils/layers.py", + "tensorrt_llm/models/gemma/utils/modules.py", + "tensorrt_llm/models/gemma/utils/params.py", + "tensorrt_llm/models/gemma/utils/positional_embeddings.py", + "tensorrt_llm/models/gemma/utils/sampler.py", + "tensorrt_llm/models/gemma/utils/transformer.py", + "tensorrt_llm/models/gemma/weight.py", + "tensorrt_llm/models/generation_mixin.py", + "tensorrt_llm/models/gpt/__init__.py", + "tensorrt_llm/models/gpt/config.py", + "tensorrt_llm/models/gpt/convert.py", + "tensorrt_llm/models/gpt/model.py", + "tensorrt_llm/models/gptj/__init__.py", + "tensorrt_llm/models/gptj/config.py", + "tensorrt_llm/models/gptj/convert.py", + "tensorrt_llm/models/gptj/model.py", + "tensorrt_llm/models/gptneox/__init__.py", + "tensorrt_llm/models/gptneox/model.py", + "tensorrt_llm/models/grok/__init__.py", + "tensorrt_llm/models/grok/convert.py", + "tensorrt_llm/models/grok/model.py", + "tensorrt_llm/models/grok/weight.py", + "tensorrt_llm/models/llama/__init__.py", + "tensorrt_llm/models/llama/config.py", + "tensorrt_llm/models/llama/convert.py", + "tensorrt_llm/models/llama/model.py", + "tensorrt_llm/models/mamba/__init__.py", + "tensorrt_llm/models/mamba/config.py", + "tensorrt_llm/models/mamba/convert.py", + "tensorrt_llm/models/mamba/model.py", + "tensorrt_llm/models/medusa/__init__.py", + "tensorrt_llm/models/medusa/config.py", + "tensorrt_llm/models/medusa/model.py", + "tensorrt_llm/models/medusa/weight.py", + "tensorrt_llm/models/mllama/__init__.py", + "tensorrt_llm/models/mllama/config.py", + "tensorrt_llm/models/mllama/model.py", + "tensorrt_llm/models/mmdit_sd3/__init__.py", + "tensorrt_llm/models/mmdit_sd3/config.py", + "tensorrt_llm/models/mmdit_sd3/model.py", + "tensorrt_llm/models/model_weights_loader.py", "tensorrt_llm/models/modeling_utils.py", + "tensorrt_llm/models/mpt/__init__.py", + "tensorrt_llm/models/mpt/model.py", + "tensorrt_llm/models/multimodal_encoders/__init__.py", + "tensorrt_llm/models/multimodal_encoders/config.py", + "tensorrt_llm/models/multimodal_encoders/model.py", + "tensorrt_llm/models/nemotron_nas/__init__.py", + "tensorrt_llm/models/nemotron_nas/config.py", + "tensorrt_llm/models/nemotron_nas/convert.py", + "tensorrt_llm/models/nemotron_nas/layer_config.py", + "tensorrt_llm/models/nemotron_nas/model.py", + "tensorrt_llm/models/opt/__init__.py", + "tensorrt_llm/models/opt/model.py", + "tensorrt_llm/models/phi/__init__.py", + "tensorrt_llm/models/phi/config.py", + "tensorrt_llm/models/phi/convert.py", + "tensorrt_llm/models/phi/model.py", + "tensorrt_llm/models/phi3/__init__.py", + "tensorrt_llm/models/phi3/config.py", + "tensorrt_llm/models/phi3/convert.py", + "tensorrt_llm/models/phi3/model.py", + "tensorrt_llm/models/phi3/split_weights.py", + "tensorrt_llm/models/qwen/__init__.py", + "tensorrt_llm/models/qwen/config.py", + "tensorrt_llm/models/qwen/convert.py", + "tensorrt_llm/models/qwen/model.py", + "tensorrt_llm/models/qwen/utils.py", + "tensorrt_llm/models/recurrentgemma/__init__.py", + "tensorrt_llm/models/recurrentgemma/model.py", + "tensorrt_llm/models/redrafter/__init__.py", + "tensorrt_llm/models/redrafter/drafter.py", + "tensorrt_llm/models/redrafter/model.py", + "tensorrt_llm/models/redrafter/redrafter_helper.py", + "tensorrt_llm/models/stdit/__init__.py", + "tensorrt_llm/models/stdit/config.py", + "tensorrt_llm/models/stdit/model.py", + "tensorrt_llm/models/unet/__init__.py", + "tensorrt_llm/models/unet/attention.py", + "tensorrt_llm/models/unet/embeddings.py", + "tensorrt_llm/models/unet/pp/__init__.py", + "tensorrt_llm/models/unet/pp/attention.py", + "tensorrt_llm/models/unet/pp/conv2d.py", + "tensorrt_llm/models/unet/pp/groupnorm.py", + "tensorrt_llm/models/unet/pp/unet_pp.py", + "tensorrt_llm/models/unet/resnet.py", + "tensorrt_llm/models/unet/unet_2d_blocks.py", + "tensorrt_llm/models/unet/unet_2d_condition.py", + "tensorrt_llm/models/unet/weights.py", + "tensorrt_llm/network.py", + "tensorrt_llm/parameter.py", + "tensorrt_llm/plugin/__init__.py", + "tensorrt_llm/plugin/plugin.py", "tensorrt_llm/quantization/__init__.py", "tensorrt_llm/quantization/functional.py", + "tensorrt_llm/quantization/image_processing.py", + "tensorrt_llm/quantization/layers.py", "tensorrt_llm/quantization/mode.py", + "tensorrt_llm/quantization/quantize.py", + "tensorrt_llm/quantization/quantize_by_modelopt.py", "tensorrt_llm/quantization/utils/__init__.py", "tensorrt_llm/quantization/utils/fp4_utils.py", "tensorrt_llm/quantization/utils/fp8_utils.py", "tensorrt_llm/ray_stub.py", "tensorrt_llm/runtime/__init__.py", + "tensorrt_llm/runtime/enc_dec_model_runner.py", + "tensorrt_llm/runtime/generation.py", + "tensorrt_llm/runtime/kv_cache_manager.py", + "tensorrt_llm/runtime/medusa_utils.py", "tensorrt_llm/runtime/memory_pools/__init__.py", + "tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py", + "tensorrt_llm/runtime/memory_pools/pool.py", + "tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py", + "tensorrt_llm/runtime/model_runner.py", + "tensorrt_llm/runtime/model_runner_cpp.py", + "tensorrt_llm/runtime/multimodal_model_runner.py", + "tensorrt_llm/runtime/processor_wrapper/__init__.py", + "tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py", + "tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py", + "tensorrt_llm/runtime/redrafter_utils.py", + "tensorrt_llm/runtime/session.py", "tensorrt_llm/scaffolding/__init__.py", "tensorrt_llm/scaffolding/benchmark.py", "tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py", @@ -528,7 +875,12 @@ exclude = [ "tensorrt_llm/tokenizer/tokenizer.py", "tensorrt_llm/tools/__init__.py", "tensorrt_llm/tools/importlib_utils.py", + "tensorrt_llm/tools/multimodal_builder.py", + "tensorrt_llm/tools/onnx_utils.py", "tensorrt_llm/tools/plugin_gen/__init__.py", + "tensorrt_llm/tools/plugin_gen/core.py", + "tensorrt_llm/tools/plugin_gen/plugin_gen.py", + "tensorrt_llm/tools/plugin_gen/shape_infer.py", "tensorrt_llm/tools/ppl.py", "tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py", "tensorrt_llm/version.py", @@ -537,7 +889,9 @@ exclude = [ "tests/integration/defs/accuracy/accuracy_core.py", "tests/integration/defs/accuracy/scripts/collect_evaluated_accuracies.py", "tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py", + "tests/integration/defs/accuracy/test_cli_flow.py", "tests/integration/defs/accuracy/test_disaggregated_serving.py", + "tests/integration/defs/accuracy/test_llm_api.py", "tests/integration/defs/accuracy/test_llm_api_autodeploy.py", "tests/integration/defs/accuracy/test_llm_api_pytorch.py", "tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py", @@ -546,29 +900,63 @@ exclude = [ "tests/integration/defs/conftest.py", "tests/integration/defs/cpp/conftest.py", "tests/integration/defs/cpp/cpp_common.py", + "tests/integration/defs/cpp/test_e2e.py", "tests/integration/defs/cpp/test_multi_gpu.py", "tests/integration/defs/cpp/test_unit_tests.py", + "tests/integration/defs/deterministic/mixtral_deterministic.py", + "tests/integration/defs/deterministic/test_mixtral_deterministic.py", "tests/integration/defs/disaggregated/test_auto_scaling.py", "tests/integration/defs/disaggregated/test_disaggregated.py", "tests/integration/defs/disaggregated/test_disaggregated_etcd.py", "tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py", "tests/integration/defs/disaggregated/test_workers.py", + "tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py", "tests/integration/defs/examples/run_llm_quickstart_atexit.py", "tests/integration/defs/examples/serve/test_serve.py", "tests/integration/defs/examples/serve/test_serve_negative.py", "tests/integration/defs/examples/test_ad_guided_decoding.py", + "tests/integration/defs/examples/test_bert.py", + "tests/integration/defs/examples/test_bindings.py", + "tests/integration/defs/examples/test_chatglm.py", + "tests/integration/defs/examples/test_commandr.py", + "tests/integration/defs/examples/test_draft_target_model.py", + "tests/integration/defs/examples/test_eagle.py", + "tests/integration/defs/examples/test_enc_dec.py", + "tests/integration/defs/examples/test_exaone.py", + "tests/integration/defs/examples/test_gemma.py", "tests/integration/defs/examples/test_gpt.py", + "tests/integration/defs/examples/test_gptj.py", + "tests/integration/defs/examples/test_granite.py", + "tests/integration/defs/examples/test_internlm.py", + "tests/integration/defs/examples/test_llama.py", "tests/integration/defs/examples/test_llm_api_with_mpi.py", + "tests/integration/defs/examples/test_mamba.py", + "tests/integration/defs/examples/test_medusa.py", + "tests/integration/defs/examples/test_mistral.py", + "tests/integration/defs/examples/test_mixtral.py", + "tests/integration/defs/examples/test_multimodal.py", + "tests/integration/defs/examples/test_nemotron.py", + "tests/integration/defs/examples/test_nemotron_nas.py", + "tests/integration/defs/examples/test_ngram.py", + "tests/integration/defs/examples/test_openai.py", "tests/integration/defs/examples/test_phi.py", + "tests/integration/defs/examples/test_qwen.py", + "tests/integration/defs/examples/test_qwen2audio.py", + "tests/integration/defs/examples/test_qwenvl.py", "tests/integration/defs/examples/test_ray.py", + "tests/integration/defs/examples/test_recurrentgemma.py", + "tests/integration/defs/examples/test_redrafter.py", + "tests/integration/defs/examples/test_whisper.py", "tests/integration/defs/llmapi/__init__.py", "tests/integration/defs/llmapi/_run_llmapi_llm.py", "tests/integration/defs/llmapi/test_llm_api_connector.py", "tests/integration/defs/llmapi/test_llm_api_qa.py", + "tests/integration/defs/llmapi/test_llm_e2e.py", "tests/integration/defs/llmapi/test_llm_examples.py", "tests/integration/defs/local_venv.py", "tests/integration/defs/perf/__init__.py", "tests/integration/defs/perf/allowed_configs.py", + "tests/integration/defs/perf/build.py", "tests/integration/defs/perf/create_perf_comparison_report.py", "tests/integration/defs/perf/data.py", "tests/integration/defs/perf/data_export.py", @@ -589,21 +977,38 @@ exclude = [ "tests/integration/defs/test_fmha.py", "tests/integration/defs/test_list_parser.py", "tests/integration/defs/test_list_validation.py", + "tests/integration/defs/test_mlpf_results.py", "tests/integration/defs/test_sanity.py", "tests/integration/defs/test_unittests.py", "tests/integration/defs/triton_server/__init__.py", + "tests/integration/defs/triton_server/build_engines.py", "tests/integration/defs/triton_server/common.py", "tests/integration/defs/triton_server/conftest.py", + "tests/integration/defs/triton_server/local_venv.py", + "tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py", + "tests/integration/defs/triton_server/runner_interface.py", "tests/integration/defs/triton_server/test_list_parser.py", + "tests/integration/defs/triton_server/test_triton.py", + "tests/integration/defs/triton_server/test_triton_llm.py", + "tests/integration/defs/triton_server/test_triton_memleak.py", + "tests/integration/defs/triton_server/test_triton_multi_node.py", + "tests/integration/defs/triton_server/test_triton_rcca.py", "tests/integration/defs/triton_server/trt_test_alternative.py", "tests/integration/defs/trt_test_alternative.py", "tests/integration/defs/utils/__init__.py", "tests/integration/defs/utils/periodic_junit.py", "tests/integration/defs/utils/timeout_manager.py", "tests/microbenchmarks/all_reduce.py", + "tests/microbenchmarks/build_time_benchmark.py", + "tests/microbenchmarks/build_time_dashboard.py", "tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py", "tests/scripts/allreduce_perf/allreduce_perf_viz.py", "tests/scripts/iteration_log_parser.py", + "tests/scripts/perf-sanity/parse_benchmark_results.py", + "tests/scripts/perf-sanity/run_benchmark_serve.py", + "tests/unittest/_torch/attention/sparse/test_dsa_indexer.py", + "tests/unittest/_torch/attention/sparse/test_flash_mla.py", + "tests/unittest/_torch/attention/sparse/test_rocketkv.py", "tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py", "tests/unittest/_torch/attention/test_attention.py", "tests/unittest/_torch/attention/test_attention_mla.py", @@ -624,6 +1029,7 @@ exclude = [ "tests/unittest/_torch/misc/test_virtual_memory.py", "tests/unittest/_torch/modeling/test_modeling_bert.py", "tests/unittest/_torch/modeling/test_modeling_clip.py", + "tests/unittest/_torch/modeling/test_modeling_exaone4.py", "tests/unittest/_torch/modeling/test_modeling_gemma3.py", "tests/unittest/_torch/modeling/test_modeling_gpt_oss.py", "tests/unittest/_torch/modeling/test_modeling_llama.py", @@ -647,6 +1053,8 @@ exclude = [ "tests/unittest/_torch/modules/test_moe_routing.py", "tests/unittest/_torch/modules/test_rotary_embedding.py", "tests/unittest/_torch/modules/test_triton_linear.py", + "tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py", + "tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py", "tests/unittest/_torch/multi_gpu/test_allreduce.py", "tests/unittest/_torch/multi_gpu/test_alltoall.py", "tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py", @@ -673,10 +1081,24 @@ exclude = [ "tests/unittest/_torch/sampler/test_beam_search.py", "tests/unittest/_torch/sampler/test_best_of_n.py", "tests/unittest/_torch/sampler/test_trtllm_sampler.py", + "tests/unittest/_torch/speculative/test_draft_target.py", + "tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py", + "tests/unittest/_torch/speculative/test_draft_token_tree_verification.py", + "tests/unittest/_torch/speculative/test_dynamic_spec_decode.py", "tests/unittest/_torch/speculative/test_eagle3.py", + "tests/unittest/_torch/speculative/test_kv_cache_reuse.py", + "tests/unittest/_torch/speculative/test_mtp.py", + "tests/unittest/_torch/speculative/test_ngram.py", + "tests/unittest/_torch/speculative/test_save_state.py", + "tests/unittest/_torch/speculative/test_spec_gate.py", + "tests/unittest/_torch/speculative/test_torch_rejection_sampling.py", + "tests/unittest/_torch/speculative/test_user_provided.py", "tests/unittest/_torch/test_connector.py", "tests/unittest/_torch/test_torch_multi_arange.py", "tests/unittest/_torch/thop/parallel/deep_gemm_tests.py", + "tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py", + "tests/unittest/_torch/thop/parallel/test_cublas_mm.py", + "tests/unittest/_torch/thop/parallel/test_custom_ops.py", "tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py", "tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py", "tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py", @@ -690,6 +1112,11 @@ exclude = [ "tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py", "tests/unittest/_torch/thop/parallel/test_fp8_quantize.py", "tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py", + "tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py", + "tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py", + "tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py", + "tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py", + "tests/unittest/_torch/thop/parallel/test_noaux_tc.py", "tests/unittest/_torch/thop/parallel/test_scaled_mm.py", "tests/unittest/_torch/thop/parallel/test_selective_scan_op.py", "tests/unittest/_torch/thop/parallel/test_tinygemm2.py", @@ -703,6 +1130,7 @@ exclude = [ "tests/unittest/_torch/thop/serial/test_moe_alltoall.py", "tests/unittest/api_stability/api_stability_core.py", "tests/unittest/api_stability/test_llm_api.py", + "tests/unittest/bindings/binding_test_utils.py", "tests/unittest/bindings/test_bindings_moe.py", "tests/unittest/bindings/test_bindings_ut.py", "tests/unittest/bindings/test_executor_bindings.py", @@ -733,10 +1161,12 @@ exclude = [ "tests/unittest/llmapi/apps/_test_openai_chat_harmony.py", "tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py", "tests/unittest/llmapi/apps/_test_openai_completions.py", + "tests/unittest/llmapi/apps/_test_openai_consistent_chat.py", "tests/unittest/llmapi/apps/_test_openai_lora.py", "tests/unittest/llmapi/apps/_test_openai_metrics.py", "tests/unittest/llmapi/apps/_test_openai_misc.py", "tests/unittest/llmapi/apps/_test_openai_mmencoder.py", + "tests/unittest/llmapi/apps/_test_openai_multi_chat.py", "tests/unittest/llmapi/apps/_test_openai_multi_gpu.py", "tests/unittest/llmapi/apps/_test_openai_multi_nodes.py", "tests/unittest/llmapi/apps/_test_openai_perf_metrics.py", @@ -759,12 +1189,15 @@ exclude = [ "tests/unittest/llmapi/run_llm_exit.py", "tests/unittest/llmapi/run_llm_with_postproc.py", "tests/unittest/llmapi/test_additional_model_outputs.py", + "tests/unittest/llmapi/test_build_cache.py", "tests/unittest/llmapi/test_executor.py", "tests/unittest/llmapi/test_gc_utils.py", "tests/unittest/llmapi/test_llm.py", "tests/unittest/llmapi/test_llm_args.py", "tests/unittest/llmapi/test_llm_download.py", "tests/unittest/llmapi/test_llm_kv_cache_events.py", + "tests/unittest/llmapi/test_llm_models.py", + "tests/unittest/llmapi/test_llm_multi_gpu.py", "tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py", "tests/unittest/llmapi/test_llm_pytorch.py", "tests/unittest/llmapi/test_llm_quant.py", @@ -775,14 +1208,25 @@ exclude = [ "tests/unittest/llmapi/test_serialization.py", "tests/unittest/llmapi/test_utils.py", "tests/unittest/others/__init__.py", + "tests/unittest/others/test_builder.py", "tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py", + "tests/unittest/others/test_debugging_api.py", "tests/unittest/others/test_exception.py", "tests/unittest/others/test_export.py", + "tests/unittest/others/test_graph_rewriter.py", + "tests/unittest/others/test_kv_cache_manager.py", "tests/unittest/others/test_kv_cache_transceiver.py", "tests/unittest/others/test_kv_cache_update.py", + "tests/unittest/others/test_layer.py", + "tests/unittest/others/test_leak.py", "tests/unittest/others/test_mapping.py", + "tests/unittest/others/test_model_dtype.py", + "tests/unittest/others/test_module.py", "tests/unittest/others/test_multimodal_registry.py", + "tests/unittest/others/test_plugins.py", + "tests/unittest/others/test_precision_control.py", "tests/unittest/others/test_pretrained_config.py", + "tests/unittest/others/test_session.py", "tests/unittest/others/test_time_breakdown.py", "tests/unittest/profile_utils.py", "tests/unittest/scaffolding/__init__.py", @@ -791,14 +1235,141 @@ exclude = [ "tests/unittest/scaffolding/test_scaffolding.py", "tests/unittest/scaffolding/test_task_collection.py", "tests/unittest/scaffolding/test_worker.py", + "tests/unittest/test_model_runner_cpp.py", "tests/unittest/test_pip_install.py", "tests/unittest/tools/__init__.py", + "tests/unittest/tools/plugin_gen/__init__.py", + "tests/unittest/tools/plugin_gen/kernel_config.py", + "tests/unittest/tools/plugin_gen/test_core.py", + "tests/unittest/tools/plugin_gen/test_plugin_gen.py", + "tests/unittest/tools/plugin_gen/test_shape_infer.py", "tests/unittest/tools/test_prepare_dataset.py", "tests/unittest/tools/test_test_to_stage_mapping.py", + "tests/unittest/trt/__init__.py", + "tests/unittest/trt/attention/test_bert_attention.py", + "tests/unittest/trt/attention/test_gpt_attention.py", + "tests/unittest/trt/attention/test_gpt_attention_IFB.py", + "tests/unittest/trt/attention/test_gpt_attention_no_cache.py", + "tests/unittest/trt/attention/test_sage_attention.py", + "tests/unittest/trt/functional/__init__.py", + "tests/unittest/trt/functional/test_alibi.py", + "tests/unittest/trt/functional/test_allreduce_norm.py", + "tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py", + "tests/unittest/trt/functional/test_arange.py", + "tests/unittest/trt/functional/test_argmax.py", + "tests/unittest/trt/functional/test_assertion.py", + "tests/unittest/trt/functional/test_avg_pool2d.py", + "tests/unittest/trt/functional/test_cast.py", + "tests/unittest/trt/functional/test_conv2d.py", + "tests/unittest/trt/functional/test_conv3d.py", + "tests/unittest/trt/functional/test_cos.py", + "tests/unittest/trt/functional/test_cumsum.py", + "tests/unittest/trt/functional/test_dora.py", + "tests/unittest/trt/functional/test_einsum.py", + "tests/unittest/trt/functional/test_embedding_single_gpu.py", + "tests/unittest/trt/functional/test_exp.py", + "tests/unittest/trt/functional/test_expand.py", + "tests/unittest/trt/functional/test_flatten.py", + "tests/unittest/trt/functional/test_flip.py", + "tests/unittest/trt/functional/test_fp4_gemm.py", + "tests/unittest/trt/functional/test_fp4_gemm_ootb.py", + "tests/unittest/trt/functional/test_gather.py", + "tests/unittest/trt/functional/test_gather_nd.py", + "tests/unittest/trt/functional/test_geglu.py", + "tests/unittest/trt/functional/test_gelu.py", + "tests/unittest/trt/functional/test_gemm_swiglu.py", + "tests/unittest/trt/functional/test_group_norm.py", + "tests/unittest/trt/functional/test_identity.py", + "tests/unittest/trt/functional/test_index_select.py", + "tests/unittest/trt/functional/test_interpolate.py", + "tests/unittest/trt/functional/test_logsoftmax.py", + "tests/unittest/trt/functional/test_lora.py", + "tests/unittest/trt/functional/test_low_latency_gemm.py", + "tests/unittest/trt/functional/test_mamba_conv1d.py", + "tests/unittest/trt/functional/test_masked_scatter.py", + "tests/unittest/trt/functional/test_masked_select.py", + "tests/unittest/trt/functional/test_matmul.py", + "tests/unittest/trt/functional/test_meshgrid2d.py", + "tests/unittest/trt/functional/test_moe.py", + "tests/unittest/trt/functional/test_nccl.py", + "tests/unittest/trt/functional/test_nonzero.py", + "tests/unittest/trt/functional/test_outer.py", + "tests/unittest/trt/functional/test_pad.py", + "tests/unittest/trt/functional/test_permute.py", + "tests/unittest/trt/functional/test_pp_reduce_scatter.py", + "tests/unittest/trt/functional/test_quant.py", + "tests/unittest/trt/functional/test_rearrange.py", + "tests/unittest/trt/functional/test_repeat.py", + "tests/unittest/trt/functional/test_repeat_interleave.py", + "tests/unittest/trt/functional/test_rg_lru.py", + "tests/unittest/trt/functional/test_sample.py", + "tests/unittest/trt/functional/test_scatter.py", + "tests/unittest/trt/functional/test_scatter_nd.py", + "tests/unittest/trt/functional/test_select.py", + "tests/unittest/trt/functional/test_selective_scan.py", + "tests/unittest/trt/functional/test_sigmoid.py", + "tests/unittest/trt/functional/test_silu.py", + "tests/unittest/trt/functional/test_sin.py", + "tests/unittest/trt/functional/test_slice.py", + "tests/unittest/trt/functional/test_softplus.py", + "tests/unittest/trt/functional/test_split.py", + "tests/unittest/trt/functional/test_squeeze.py", + "tests/unittest/trt/functional/test_swiglu.py", + "tests/unittest/trt/functional/test_topk.py", + "tests/unittest/trt/functional/test_transpose.py", + "tests/unittest/trt/functional/test_unbind.py", + "tests/unittest/trt/functional/test_unsqueeze.py", + "tests/unittest/trt/functional/test_view.py", + "tests/unittest/trt/functional/test_where.py", + "tests/unittest/trt/model/__init__.py", + "tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py", + "tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py", + "tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py", + "tests/unittest/trt/model/redrafter/test_beams2tree.py", + "tests/unittest/trt/model/redrafter/test_draft_token.py", + "tests/unittest/trt/model/redrafter/test_draft_token_indices.py", + "tests/unittest/trt/model/redrafter/test_gather_beams.py", + "tests/unittest/trt/model/redrafter/test_mask.py", + "tests/unittest/trt/model/redrafter/test_packed_position_ids.py", + "tests/unittest/trt/model/redrafter/test_prefix_match_indices.py", + "tests/unittest/trt/model/redrafter/test_prepare_input.py", + "tests/unittest/trt/model/redrafter/test_process_logits.py", + "tests/unittest/trt/model/redrafter/test_top1.py", + "tests/unittest/trt/model/redrafter/test_unpack_gen_data.py", + "tests/unittest/trt/model/redrafter/test_validate.py", + "tests/unittest/trt/model/test_gpt.py", + "tests/unittest/trt/model/test_gpt_e2e.py", + "tests/unittest/trt/model/test_llama.py", + "tests/unittest/trt/model/test_mamba.py", + "tests/unittest/trt/model/test_mistral.py", + "tests/unittest/trt/model/test_nemotron_nas.py", + "tests/unittest/trt/model/test_phi.py", + "tests/unittest/trt/model/test_unet.py", + "tests/unittest/trt/model_api/test_model_api_multi_gpu.py", + "tests/unittest/trt/model_api/test_model_level_api.py", + "tests/unittest/trt/model_api/test_model_quantization.py", + "tests/unittest/trt/python_plugin/plugin_wrapper_utils.py", + "tests/unittest/trt/python_plugin/test_plugin_wrapper.py", + "tests/unittest/trt/quantization/__init__.py", + "tests/unittest/trt/quantization/_utils.py", + "tests/unittest/trt/quantization/test_fp8_quantization.py", + "tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py", + "tests/unittest/trt/quantization/test_functional.py", + "tests/unittest/trt/quantization/test_mode.py", + "tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py", + "tests/unittest/trt/quantization/test_qserve_gemm.py", + "tests/unittest/trt/quantization/test_quant.py", + "tests/unittest/trt/quantization/test_quant_layer.py", + "tests/unittest/trt/quantization/test_smooth_quant_gemm.py", + "tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py", + "tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py", + "tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py", + "tests/unittest/trt/quantization/test_weight_only_quant_matmul.py", "tests/unittest/utils/__init__.py", "tests/unittest/utils/cpp_paths.py", "tests/unittest/utils/llm_data.py", "tests/unittest/utils/runtime_defaults.py", + "tests/unittest/utils/test_medusa_utils.py", "tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py", "tests/unittest/utils/test_util.py", "tests/unittest/utils/torch_ref.py", diff --git a/requirements-dev.txt b/requirements-dev.txt index 57d64fb0436c..8c6766a590eb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,4 @@ -r requirements.txt -# VisualGen LPIPS goldens were recorded with this version. Install it before -# pytest collection so already-imported Diffusers modules match the package files. -diffusers==0.38.0 boto3 einops lpips diff --git a/requirements.txt b/requirements.txt index d07491d3a454..2274d391fe75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ pandas h5py==3.12.1 StrEnum sentencepiece>=0.1.99 +tensorrt~=10.16.1 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.12.0a0. torch>=2.11.0,<=2.13.0a0 torchvision @@ -29,8 +30,6 @@ nvidia-modelopt[torch]~=0.37.0 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.30.4 # torch 2.11.0+cu130 depends on nvidia-nccl-cu13==2.28.9 nvidia-nccl-cu13>=2.28.9,<=2.30.4 -# NcclEP uses nccl4py's nccl.ep package without changing the NCCL wheel constraint. -nccl4py>=0.3.1,<0.4 nvidia-cuda-nvrtc transformers==5.5.4 prometheus_client @@ -57,7 +56,7 @@ ordered-set peft>=0.18.1,<0.19.0 patchelf einops -flashinfer-python==0.6.15 +flashinfer-python==0.6.14 xgrammar==0.1.32 llguidance==0.7.29 jsonschema @@ -74,9 +73,6 @@ tiktoken blobfile openai-harmony==0.0.4 nvidia-cutlass-dsl[cu13]==4.5.0; python_version >= "3.10" -nvidia-matmul-heuristics==0.1.0.27; python_version >= "3.10" # analytic GEMM heuristics for CuTe DSL autotuner tactic pruning -quack-kernels>=0.2.10; python_version >= "3.10" # required for MinimaxM3 MSA -jinja2 # required for MinimaxM3 MSA plotly numexpr partial_json_parser diff --git a/ruff-legacy-baseline.json b/ruff-legacy-baseline.json index eae44689bafc..ee17a68362c2 100644 --- a/ruff-legacy-baseline.json +++ b/ruff-legacy-baseline.json @@ -1,12 +1,22 @@ { "_meta": { "generated_by": "scripts/legacy_utils.py lint-update-violations", - "total_violations": 2096, - "total_files": 224 + "total_violations": 2917, + "total_files": 303 }, ".github/scripts/label_community_user.py": { "D212": 1 }, + "benchmarks/cpp/utils/convert_nemo_dataset.py": { + "E741": 1 + }, + "benchmarks/cpp/utils/prepare_real_data.py": { + "D202": 1, + "D205": 1, + "D410": 7, + "D411": 8, + "D415": 1 + }, "cpp/kernels/fmha_v2/setup.py": { "E712": 27, "E731": 1 @@ -70,6 +80,23 @@ "D212": 1, "D300": 1 }, + "examples/bindings/executor/example_advanced.py": { + "E402": 1 + }, + "examples/bindings/executor/example_logits_processor.py": { + "D200": 1, + "D212": 1, + "D415": 1 + }, + "examples/dora/normalize_weights.py": { + "D200": 3, + "D202": 1, + "D205": 1, + "D212": 4 + }, + "examples/eval_long_context.py": { + "E711": 2 + }, "examples/infinitebench/compute_scores.py": { "D200": 1, "D205": 1, @@ -117,10 +144,111 @@ "D212": 1, "F821": 1 }, + "examples/mmlu.py": { + "D205": 1, + "D415": 1, + "E741": 1 + }, + "examples/models/contrib/chatglm-6b/tokenization_chatglm.py": { + "D205": 4, + "D210": 4, + "D212": 7, + "D301": 2, + "D415": 4 + }, + "examples/models/contrib/chatglm2-6b/tokenization_chatglm.py": { + "D205": 1, + "D210": 3, + "D212": 3, + "D415": 3 + }, + "examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py": { + "D205": 1, + "D210": 3, + "D212": 3, + "D415": 3 + }, + "examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py": { + "D202": 1, + "D205": 6, + "D212": 10, + "D414": 1, + "D415": 2 + }, + "examples/models/core/multimodal/run.py": { + "E731": 1 + }, + "examples/models/core/qwen2audio/run.py": { + "D200": 1, + "D212": 1, + "D415": 1, + "E722": 1 + }, + "examples/models/core/qwen2audio/run_chat.py": { + "E722": 1 + }, + "examples/models/core/qwenvl/run.py": { + "E722": 1 + }, + "examples/models/core/qwenvl/run_chat.py": { + "E722": 1 + }, + "examples/ngram/run_dtm_ngram.py": { + "D200": 1, + "D205": 1, + "D212": 2, + "D415": 2, + "E741": 3 + }, + "examples/openai_triton/manual_plugin/build.py": { + "D202": 1, + "D205": 1, + "D212": 1, + "D300": 1 + }, + "examples/openai_triton/manual_plugin/fmha_triton.py": { + "D205": 1, + "D212": 1, + "D415": 1 + }, + "examples/openai_triton/manual_plugin/plugin.py": { + "D202": 1, + "D205": 1, + "D212": 1, + "D300": 1 + }, + "examples/openai_triton/plugin_autogen/build_engine.py": { + "D202": 2, + "D205": 2, + "D212": 2, + "D300": 2 + }, + "examples/openai_triton/plugin_autogen/kernel_config.py": { + "F403": 1, + "F405": 22 + }, + "examples/python_plugin/plugin_lib/lookup_plugin.py": { + "E731": 1 + }, "examples/quantization/quantize_mixed_precision_moe.py": { "E741": 1, "F601": 2 }, + "examples/run.py": { + "E711": 4 + }, + "examples/scaffolding/contrib/mcp/e2b/e2bserver.py": { + "D202": 1, + "D205": 1, + "D411": 1 + }, + "examples/scaffolding/contrib/mcp/websearch/websearch.py": { + "D205": 1, + "D415": 1 + }, + "examples/summarize.py": { + "E741": 1 + }, "jenkins/scripts/open_search_db.py": { "D205": 1, "D212": 7 @@ -129,6 +257,9 @@ "D212": 1, "E741": 1 }, + "scripts/check_test_list.py": { + "D212": 1 + }, "scripts/dco_check.py": { "D212": 1 }, @@ -161,21 +292,39 @@ "D212": 1, "E402": 20 }, + "tensorrt_llm/_torch/attention_backend/sparse/dsa.py": { + "F821": 2 + }, "tensorrt_llm/_torch/attention_backend/sparse/kernel.py": { "E731": 3 }, "tensorrt_llm/_torch/attention_backend/sparse/rocket.py": { "E712": 2 }, + "tensorrt_llm/_torch/attention_backend/sparse/utils.py": { + "F821": 4 + }, "tensorrt_llm/_torch/attention_backend/trtllm.py": { "E712": 1 }, + "tensorrt_llm/_torch/attention_backend/utils.py": { + "F821": 2 + }, + "tensorrt_llm/_torch/compilation/patterns/ar_residual_norm.py": { + "E402": 1 + }, "tensorrt_llm/_torch/compilation/patterns/residual_add_norm.py": { "E402": 1 }, "tensorrt_llm/_torch/compilation/piecewise_optimizer.py": { "E711": 2 }, + "tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py": { + "E741": 11 + }, + "tensorrt_llm/_torch/custom_ops/torch_custom_ops.py": { + "F821": 2 + }, "tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py": { "E741": 3 }, @@ -183,6 +332,10 @@ "E712": 1, "E731": 1 }, + "tensorrt_llm/_torch/model_config.py": { + "F811": 1, + "F821": 4 + }, "tensorrt_llm/_torch/models/checkpoints/auto_mapper.py": { "F821": 1 }, @@ -213,6 +366,12 @@ "tensorrt_llm/_torch/models/modeling_nemotron.py": { "E731": 1 }, + "tensorrt_llm/_torch/models/modeling_qwen2vl.py": { + "F811": 1 + }, + "tensorrt_llm/_torch/models/modeling_qwen3_next.py": { + "F821": 1 + }, "tensorrt_llm/_torch/models/modeling_siglip.py": { "F811": 1 }, @@ -220,6 +379,9 @@ "E731": 1, "F821": 5 }, + "tensorrt_llm/_torch/modules/attention.py": { + "E731": 2 + }, "tensorrt_llm/_torch/modules/fused_moe/deep_ep_utils.py": { "F821": 2 }, @@ -253,6 +415,18 @@ "tensorrt_llm/_torch/modules/mamba/ssd_state_passing.py": { "E731": 1 }, + "tensorrt_llm/_torch/pyexecutor/_util.py": { + "F811": 1 + }, + "tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py": { + "F821": 1 + }, + "tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py": { + "F821": 1 + }, + "tensorrt_llm/_torch/pyexecutor/model_engine.py": { + "F821": 1 + }, "tensorrt_llm/_torch/pyexecutor/model_loader.py": { "F811": 1 }, @@ -260,15 +434,24 @@ "E712": 1, "E721": 1 }, + "tensorrt_llm/_torch/pyexecutor/resource_manager.py": { + "F821": 2 + }, "tensorrt_llm/_torch/pyexecutor/seq_slot_manager.py": { "F821": 1 }, "tensorrt_llm/_torch/speculative/auto_heuristic.py": { "F821": 1 }, + "tensorrt_llm/_torch/speculative/interface.py": { + "F821": 1 + }, "tensorrt_llm/_torch/speculative/ngram.py": { "E741": 1 }, + "tensorrt_llm/_torch/speculative/speculation_gate.py": { + "F821": 1 + }, "tensorrt_llm/_utils.py": { "E722": 1, "F821": 1 @@ -281,6 +464,18 @@ "D411": 1, "D415": 1 }, + "tensorrt_llm/bench/build/build.py": { + "D205": 1, + "D210": 2, + "D411": 1 + }, + "tensorrt_llm/bench/build/dataclasses.py": { + "D210": 3 + }, + "tensorrt_llm/bench/build/tuning.py": { + "D205": 2, + "D210": 2 + }, "tensorrt_llm/bench/dataclasses/reporting.py": { "D200": 1, "D212": 1 @@ -340,7 +535,9 @@ "D202": 1, "D205": 1, "D208": 3, + "D210": 1, "D212": 1, + "D300": 1, "E722": 1 }, "tensorrt_llm/executor/ray_executor.py": { @@ -381,6 +578,7 @@ }, "tensorrt_llm/executor/rpc_proxy.py": { "D205": 1, + "D212": 1, "D415": 1 }, "tensorrt_llm/executor/rpc_worker.py": { @@ -391,6 +589,24 @@ "D300": 6, "F811": 1 }, + "tensorrt_llm/functional.py": { + "D200": 42, + "D202": 6, + "D205": 20, + "D207": 1, + "D208": 6, + "D210": 2, + "D212": 142, + "D214": 3, + "D300": 144, + "D301": 2, + "D411": 8, + "D415": 9 + }, + "tensorrt_llm/inputs/evs.py": { + "D205": 1, + "D212": 2 + }, "tensorrt_llm/inputs/multimodal.py": { "D415": 1 }, @@ -411,11 +627,11 @@ "E731": 2 }, "tensorrt_llm/llmapi/disagg_utils.py": { - "D205": 1, + "D205": 2, "D210": 1, - "D212": 1, - "D403": 1, - "D415": 1, + "D212": 2, + "D403": 2, + "D415": 2, "F822": 2 }, "tensorrt_llm/llmapi/llm.py": { @@ -423,7 +639,8 @@ "D205": 4 }, "tensorrt_llm/llmapi/llm_args.py": { - "D205": 10 + "D205": 11, + "D212": 1 }, "tensorrt_llm/llmapi/llm_utils.py": { "D200": 1 @@ -440,7 +657,11 @@ }, "tensorrt_llm/llmapi/mpi_session.py": { "D200": 1, - "D205": 1, + "D205": 2, + "D210": 7, + "D212": 3, + "D300": 10, + "D411": 1, "D415": 1 }, "tensorrt_llm/llmapi/reasoning_parser.py": { @@ -482,6 +703,36 @@ "E731": 1, "F821": 1 }, + "tensorrt_llm/models/modeling_utils.py": { + "D200": 3, + "D202": 1, + "D205": 4, + "D208": 4, + "D210": 3, + "D212": 2, + "D300": 5, + "D415": 4, + "E721": 2, + "E722": 1 + }, + "tensorrt_llm/quantization/functional.py": { + "D205": 4, + "D212": 4, + "D300": 4, + "D411": 2, + "D415": 2 + }, + "tensorrt_llm/quantization/quantize_by_modelopt.py": { + "D200": 1, + "D205": 1, + "D208": 2, + "D212": 2, + "D300": 1, + "D415": 2, + "E722": 1, + "F601": 1, + "F821": 2 + }, "tensorrt_llm/quantization/utils/fp4_utils.py": { "D200": 2, "D202": 1, @@ -540,9 +791,18 @@ "tensorrt_llm/scaffolding/result.py": { "F821": 1 }, + "tensorrt_llm/scaffolding/scaffolding_llm.py": { + "PLE0302": 1 + }, "tensorrt_llm/scaffolding/task.py": { "F821": 1 }, + "tensorrt_llm/scaffolding/task_collection.py": { + "F821": 1 + }, + "tensorrt_llm/scaffolding/worker.py": { + "PLE0302": 1 + }, "tensorrt_llm/serve/disagg_auto_scaling.py": { "D205": 2, "D212": 2 @@ -559,14 +819,19 @@ "F811": 1 }, "tensorrt_llm/serve/openai_server.py": { + "D205": 1, + "D212": 2, "F821": 3 }, "tensorrt_llm/serve/responses_utils.py": { "D200": 2, - "D212": 7 + "D205": 4, + "D212": 11 }, "tensorrt_llm/serve/router.py": { "D205": 1, + "D212": 1, + "D300": 1, "D415": 7 }, "tensorrt_llm/serve/scripts/benchmark_dataset.py": { @@ -631,10 +896,20 @@ "D403": 9, "D415": 11 }, + "tests/integration/defs/accuracy/accuracy_core.py": { + "D205": 1, + "D212": 1 + }, "tests/integration/defs/accuracy/test_disaggregated_serving.py": { "D212": 1, "F601": 1 }, + "tests/integration/defs/accuracy/test_llm_api_autodeploy.py": { + "D205": 1, + "D209": 1, + "D415": 1, + "F811": 2 + }, "tests/integration/defs/accuracy/test_llm_api_pytorch.py": { "D212": 1, "D300": 3, @@ -646,9 +921,9 @@ "D205": 3, "D210": 2, "D212": 5, - "D300": 6, - "D403": 5, - "D415": 11 + "D300": 7, + "D403": 6, + "D415": 12 }, "tests/integration/defs/conftest.py": { "D200": 3, @@ -657,9 +932,9 @@ "D209": 1, "D212": 5, "D214": 1, - "D300": 91, + "D300": 92, "D403": 51, - "D415": 91, + "D415": 92, "E722": 2, "E741": 1 }, @@ -671,7 +946,8 @@ "E741": 2 }, "tests/integration/defs/disaggregated/test_disaggregated.py": { - "D205": 2, + "D205": 3, + "D209": 1, "D212": 5, "D415": 1 }, @@ -691,17 +967,79 @@ "D415": 15, "E722": 1 }, + "tests/integration/defs/examples/test_bert.py": { + "D300": 1, + "D415": 1 + }, + "tests/integration/defs/examples/test_bindings.py": { + "D300": 1, + "D415": 1 + }, + "tests/integration/defs/examples/test_chatglm.py": { + "D300": 1 + }, + "tests/integration/defs/examples/test_commandr.py": { + "D300": 1 + }, "tests/integration/defs/examples/test_gpt.py": { "D202": 3, "D300": 5, "D403": 1, "D415": 4 }, + "tests/integration/defs/examples/test_granite.py": { + "D202": 1, + "D300": 1 + }, + "tests/integration/defs/examples/test_internlm.py": { + "D300": 1, + "D415": 1 + }, + "tests/integration/defs/examples/test_llama.py": { + "D202": 2, + "D300": 4, + "D403": 2, + "D415": 2 + }, + "tests/integration/defs/examples/test_mamba.py": { + "D300": 2, + "D415": 2 + }, + "tests/integration/defs/examples/test_mistral.py": { + "D202": 1 + }, + "tests/integration/defs/examples/test_mixtral.py": { + "D300": 1, + "D403": 1 + }, + "tests/integration/defs/examples/test_multimodal.py": { + "D202": 1, + "D300": 2, + "D415": 2 + }, + "tests/integration/defs/examples/test_openai.py": { + "D300": 3, + "D403": 1, + "D415": 3 + }, "tests/integration/defs/examples/test_phi.py": { "D202": 1, "D300": 2, "D415": 2 }, + "tests/integration/defs/examples/test_qwen.py": { + "D202": 1, + "D300": 2, + "D403": 1 + }, + "tests/integration/defs/examples/test_qwen2audio.py": { + "D300": 2, + "D415": 1 + }, + "tests/integration/defs/examples/test_qwenvl.py": { + "D300": 1, + "D415": 1 + }, "tests/integration/defs/llmapi/test_llm_api_qa.py": { "D200": 1, "D212": 1, @@ -821,6 +1159,16 @@ "E741": 2, "F811": 1 }, + "tests/integration/defs/test_mlpf_results.py": { + "D200": 1, + "D205": 1, + "D208": 4, + "D212": 2, + "D415": 2 + }, + "tests/integration/defs/triton_server/common.py": { + "E402": 1 + }, "tests/integration/defs/triton_server/conftest.py": { "D200": 3, "D202": 1, @@ -867,6 +1215,16 @@ "tests/integration/defs/utils/timeout_manager.py": { "D212": 9 }, + "tests/microbenchmarks/build_time_benchmark.py": { + "D200": 1, + "D300": 1, + "D415": 1 + }, + "tests/microbenchmarks/build_time_dashboard.py": { + "D200": 1, + "D300": 1, + "D415": 1 + }, "tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py": { "D212": 1, "E712": 1 @@ -887,7 +1245,11 @@ "D415": 6 }, "tests/unittest/bindings/test_executor_bindings.py": { - "E712": 51 + "D202": 1, + "E712": 54, + "F403": 2, + "F405": 3, + "F811": 1 }, "tests/unittest/conftest.py": { "D200": 3, @@ -967,6 +1329,10 @@ "D205": 1, "D212": 1 }, + "tests/unittest/llmapi/test_llm_utils.py": { + "F403": 2, + "F405": 14 + }, "tests/unittest/llmapi/test_mpi_session.py": { "D200": 1, "D212": 1, @@ -979,6 +1345,12 @@ "tests/unittest/others/test_kv_cache_transceiver.py": { "D212": 1 }, + "tests/unittest/others/test_leak.py": { + "D200": 1, + "D210": 1, + "D300": 1, + "D415": 1 + }, "tests/unittest/others/test_time_breakdown.py": { "D212": 1, "D415": 1 diff --git a/ruff-legacy.toml b/ruff-legacy.toml index 39a722295b41..2f9d1bd00745 100644 --- a/ruff-legacy.toml +++ b/ruff-legacy.toml @@ -18,6 +18,14 @@ include = [ ".devcontainer/make_env.py", ".github/scripts/label_community_user.py", ".github/scripts/pr_checklist_check.py", + "benchmarks/cpp/__init__.py", + "benchmarks/cpp/prepare_dataset.py", + "benchmarks/cpp/utils/__init__.py", + "benchmarks/cpp/utils/convert_nemo_dataset.py", + "benchmarks/cpp/utils/generate_rand_loras.py", + "benchmarks/cpp/utils/prepare_real_data.py", + "benchmarks/cpp/utils/prepare_synthetic_data.py", + "benchmarks/cpp/utils/utils.py", "cpp/conanfile.py", "cpp/kernels/fmha_v2/conftest.py", "cpp/kernels/fmha_v2/fmha_test.py", @@ -42,17 +50,58 @@ include = [ "cpp/tensorrt_llm/deep_ep/strip_nvshmem_helper.py", "cpp/tensorrt_llm/kernels/cutlass_kernels/python/generate_kernels.py", "cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/copy_cu.py", + "cpp/tests/resources/scripts/build_chatglm_engines.py", + "cpp/tests/resources/scripts/build_eagle_engines.py", + "cpp/tests/resources/scripts/build_enc_dec_engines.py", + "cpp/tests/resources/scripts/build_engines_utils.py", + "cpp/tests/resources/scripts/build_gpt_engines.py", + "cpp/tests/resources/scripts/build_gptj_engines.py", + "cpp/tests/resources/scripts/build_llama_engines.py", + "cpp/tests/resources/scripts/build_mamba_engines.py", + "cpp/tests/resources/scripts/build_medusa_engines.py", + "cpp/tests/resources/scripts/build_recurrentgemma_engines.py", + "cpp/tests/resources/scripts/build_redrafter_engines.py", + "cpp/tests/resources/scripts/generate_expected_chatglm_output.py", + "cpp/tests/resources/scripts/generate_expected_eagle_output.py", + "cpp/tests/resources/scripts/generate_expected_enc_dec_output.py", + "cpp/tests/resources/scripts/generate_expected_gpt_output.py", + "cpp/tests/resources/scripts/generate_expected_gptj_output.py", + "cpp/tests/resources/scripts/generate_expected_llama_output.py", + "cpp/tests/resources/scripts/generate_expected_mamba_output.py", + "cpp/tests/resources/scripts/generate_expected_medusa_output.py", + "cpp/tests/resources/scripts/generate_expected_recurrentgemma_output.py", + "cpp/tests/resources/scripts/generate_expected_redrafter_output.py", + "cpp/tests/resources/scripts/generate_hf_gpt_output.py", "cpp/tests/resources/scripts/generate_test_lora_weights.py", + "cpp/tests/resources/scripts/io_converter.py", "docs/source/conf.py", "docs/source/helper.py", "examples/apps/chat.py", "examples/apps/fastapi_server.py", + "examples/bindings/executor/example_advanced.py", + "examples/bindings/executor/example_basic.py", + "examples/bindings/executor/example_debug.py", + "examples/bindings/executor/example_logits_processor.py", "examples/disaggregated/clients/disagg_client.py", "examples/disaggregated/slurm/benchmark/submit.py", + "examples/dora/normalize_weights.py", + "examples/eagle/convert_checkpoint.py", + "examples/eval_long_context.py", + "examples/generate_checkpoint_config.py", + "examples/generate_xgrammar_tokenizer_info.py", + "examples/hf_lora_convert.py", "examples/infinitebench/args.py", "examples/infinitebench/compute_scores.py", "examples/infinitebench/construct_synthetic_dataset.py", "examples/infinitebench/eval_utils.py", + "examples/llm-api/_tensorrt_engine/llm_eagle2_decoding.py", + "examples/llm-api/_tensorrt_engine/llm_eagle_decoding.py", + "examples/llm-api/_tensorrt_engine/llm_inference_customize.py", + "examples/llm-api/_tensorrt_engine/llm_inference_kv_events.py", + "examples/llm-api/_tensorrt_engine/llm_lookahead_decoding.py", + "examples/llm-api/_tensorrt_engine/llm_medusa_decoding.py", + "examples/llm-api/_tensorrt_engine/llm_quantization.py", + "examples/llm-api/_tensorrt_engine/quickstart_example.py", "examples/llm-api/llm_guided_decoding.py", "examples/llm-api/llm_inference.py", "examples/llm-api/llm_inference_async.py", @@ -72,17 +121,122 @@ include = [ "examples/llm-api/quickstart_example.py", "examples/llm-api/quickstart_multimodal.py", "examples/llm-api/star_attention.py", + "examples/llm-eval/lm-eval-harness/lm_eval_tensorrt_llm.py", "examples/longbench/eval_longbench_v1.py", + "examples/medusa/convert_checkpoint.py", + "examples/mmlu.py", + "examples/models/contrib/baichuan/convert_checkpoint.py", + "examples/models/contrib/bloom/convert_checkpoint.py", + "examples/models/contrib/chatglm-6b/tokenization_chatglm.py", + "examples/models/contrib/chatglm2-6b/tokenization_chatglm.py", + "examples/models/contrib/chatglm3-6b-32k/tokenization_chatglm.py", + "examples/models/contrib/cogvlm/convert_checkpoint.py", + "examples/models/contrib/dbrx/convert_checkpoint.py", + "examples/models/contrib/deepseek_v1/__init__.py", + "examples/models/contrib/deepseek_v1/convert_checkpoint.py", + "examples/models/contrib/deepseek_v2/convert_checkpoint.py", + "examples/models/contrib/dit/convert_checkpoint.py", + "examples/models/contrib/dit/diffusion.py", + "examples/models/contrib/dit/sample.py", + "examples/models/contrib/dit/utils_modelopt.py", + "examples/models/contrib/dit/vae_decoder_trt.py", + "examples/models/contrib/falcon/convert_checkpoint.py", + "examples/models/contrib/gptj/convert_checkpoint.py", + "examples/models/contrib/gptneox/convert_checkpoint.py", + "examples/models/contrib/grok/convert_checkpoint.py", + "examples/models/contrib/mmdit/convert_checkpoint.py", + "examples/models/contrib/mmdit/sample.py", + "examples/models/contrib/mpt/convert_checkpoint.py", + "examples/models/contrib/opt/convert_checkpoint.py", + "examples/models/contrib/sdxl/build_sdxl_unet.py", + "examples/models/contrib/sdxl/pipeline_stable_diffusion_xl.py", + "examples/models/contrib/sdxl/run_sdxl.py", + "examples/models/contrib/stdit/aspect.py", + "examples/models/contrib/stdit/convert_checkpoint.py", + "examples/models/contrib/stdit/pipeline_tllm.py", + "examples/models/contrib/stdit/sample.py", + "examples/models/contrib/stdit/scheduler.py", + "examples/models/contrib/stdit/text_encoder.py", + "examples/models/contrib/stdit/utils.py", + "examples/models/contrib/stdit/vae.py", + "examples/models/contrib/stdit/video_transforms.py", + "examples/models/core/bert/__init__.py", + "examples/models/core/bert/convert_checkpoint.py", + "examples/models/core/bert/run.py", + "examples/models/core/bert/utils.py", + "examples/models/core/commandr/convert_checkpoint.py", + "examples/models/core/enc_dec/__init__.py", + "examples/models/core/enc_dec/convert_checkpoint.py", + "examples/models/core/enc_dec/helper.py", + "examples/models/core/enc_dec/run.py", + "examples/models/core/gemma/convert_checkpoint.py", + "examples/models/core/glm-4-9b/convert_checkpoint.py", + "examples/models/core/glm-4-9b/tokenization_chatglm.py", + "examples/models/core/gpt/convert_checkpoint.py", + "examples/models/core/gpt/merge_ptuning_tables.py", + "examples/models/core/gpt/nemo_lora_convert.py", + "examples/models/core/gpt/nemo_prompt_convert.py", + "examples/models/core/gpt/run_hf.py", "examples/models/core/gpt_oss/openai_chat_client_function_calling.py", + "examples/models/core/internlm2/convert_checkpoint.py", "examples/models/core/kimi_k2/kimi_k2_tool_calling_example.py", + "examples/models/core/llama/convert_checkpoint.py", + "examples/models/core/llama/summarize_long.py", + "examples/models/core/mamba/convert_checkpoint.py", + "examples/models/core/mllama/convert_checkpoint.py", + "examples/models/core/multimodal/__init__.py", + "examples/models/core/multimodal/build_multimodal_engine.py", + "examples/models/core/multimodal/eval.py", + "examples/models/core/multimodal/run.py", + "examples/models/core/multimodal/utils.py", + "examples/models/core/nemotron_nas/calibration_utils.py", + "examples/models/core/nemotron_nas/convert_checkpoint.py", + "examples/models/core/phi/convert_checkpoint.py", + "examples/models/core/qwen/convert_checkpoint.py", + "examples/models/core/qwen2audio/run.py", + "examples/models/core/qwen2audio/run_chat.py", + "examples/models/core/qwen2audio/utils.py", + "examples/models/core/qwenvl/run.py", + "examples/models/core/qwenvl/run_chat.py", + "examples/models/core/qwenvl/show_pic.py", + "examples/models/core/qwenvl/vit_onnx_trt.py", + "examples/models/core/recurrentgemma/convert_checkpoint.py", + "examples/models/core/vit/convert_checkpoint.py", + "examples/models/core/whisper/convert_checkpoint.py", + "examples/models/core/whisper/distil_whisper/convert_from_distil_whisper.py", + "examples/models/core/whisper/run.py", + "examples/models/core/whisper/tokenizer.py", + "examples/models/core/whisper/whisper_utils.py", + "examples/ngram/run_dtm_ngram.py", + "examples/openai_triton/manual_plugin/build.py", + "examples/openai_triton/manual_plugin/fmha_triton.py", + "examples/openai_triton/manual_plugin/plugin.py", + "examples/openai_triton/manual_plugin/run.py", + "examples/openai_triton/plugin_autogen/build_engine.py", + "examples/openai_triton/plugin_autogen/kernel_config.py", + "examples/openai_triton/plugin_autogen/run_engine.py", + "examples/python_plugin/build_lookup.py", + "examples/python_plugin/plugin_lib/__init__.py", + "examples/python_plugin/plugin_lib/lookup_kernel.py", + "examples/python_plugin/plugin_lib/lookup_plugin.py", + "examples/python_plugin/run_lookup.py", + "examples/quantization/quantize.py", "examples/quantization/quantize_mixed_precision_moe.py", "examples/ray_orchestrator/llm_inference_async_ray.py", "examples/ray_orchestrator/llm_inference_distributed_ray.py", + "examples/redrafter/convert_checkpoint.py", + "examples/run.py", "examples/scaffolding/contrib/AsyncGeneration/stream_generation_controller.py", "examples/scaffolding/contrib/DeepConf/run_generation.py", "examples/scaffolding/contrib/Dynasor/scaffolding_dynasor_run.py", "examples/scaffolding/contrib/TreeInference/run_mcts_example.py", "examples/scaffolding/contrib/TreeInference/run_tot_example.py", + "examples/scaffolding/contrib/mcp/e2b/e2bserver.py", + "examples/scaffolding/contrib/mcp/e2b/main.py", + "examples/scaffolding/contrib/mcp/mcptest.py", + "examples/scaffolding/contrib/mcp/weather/weather.py", + "examples/scaffolding/contrib/mcp/websearch/main.py", + "examples/scaffolding/contrib/mcp/websearch/websearch.py", "examples/scaffolding/run_basic_generation.py", "examples/scaffolding/run_best_of_n_with_reward.py", "examples/scaffolding/run_majority_vote_aime24.py", @@ -92,6 +246,8 @@ include = [ "examples/serve/openai_completion_client.py", "examples/serve/openai_completion_client_for_lora.py", "examples/serve/openai_completion_client_json_schema.py", + "examples/summarize.py", + "examples/utils.py", "examples/wide_ep/ep_load_balancer/generate_eplb_config.py", "examples/wide_ep/ep_load_balancer/report_load_statistics.py", "examples/wide_ep/ep_load_balancer/utils.py", @@ -99,10 +255,12 @@ include = [ "jenkins/scripts/mergeWaiveList.py", "jenkins/scripts/open_search_db.py", "jenkins/scripts/test_rerun.py", + "scripts/build_cpp_examples.py", "scripts/build_wheel.py", "scripts/check_test_list.py", "scripts/dco_check.py", "scripts/format_test_list.py", + "scripts/generate_duration.py", "scripts/generate_lock_file.py", "scripts/get_wheel_from_package.py", "scripts/git_replace.py", @@ -113,6 +271,7 @@ include = [ "setup.py", "tensorrt_llm/__init__.py", "tensorrt_llm/_ray_utils.py", + "tensorrt_llm/_tensorrt_engine/__init__.py", "tensorrt_llm/_torch/__init__.py", "tensorrt_llm/_torch/attention_backend/__init__.py", "tensorrt_llm/_torch/attention_backend/flashinfer.py", @@ -316,6 +475,7 @@ include = [ "tensorrt_llm/_torch/pyexecutor/guided_decoder.py", "tensorrt_llm/_torch/pyexecutor/handle_additional_outputs.py", "tensorrt_llm/_torch/pyexecutor/handle_logits.py", + "tensorrt_llm/_torch/pyexecutor/kv_cache_connector.py", "tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py", "tensorrt_llm/_torch/pyexecutor/layerwise_nvtx_marker.py", "tensorrt_llm/_torch/pyexecutor/llm_request.py", @@ -353,6 +513,11 @@ include = [ "tensorrt_llm/bench/benchmark/utils/asynchronous.py", "tensorrt_llm/bench/benchmark/utils/general.py", "tensorrt_llm/bench/benchmark/utils/processes.py", + "tensorrt_llm/bench/build/__init__.py", + "tensorrt_llm/bench/build/build.py", + "tensorrt_llm/bench/build/dataclasses.py", + "tensorrt_llm/bench/build/tuning.py", + "tensorrt_llm/bench/build/utils.py", "tensorrt_llm/bench/dataclasses/__init__.py", "tensorrt_llm/bench/dataclasses/configuration.py", "tensorrt_llm/bench/dataclasses/engine.py", @@ -362,9 +527,13 @@ include = [ "tensorrt_llm/bench/dataclasses/statistics.py", "tensorrt_llm/bench/utils/__init__.py", "tensorrt_llm/bench/utils/data.py", + "tensorrt_llm/builder.py", "tensorrt_llm/commands/__init__.py", "tensorrt_llm/commands/bench.py", + "tensorrt_llm/commands/build.py", "tensorrt_llm/commands/eval.py", + "tensorrt_llm/commands/prune.py", + "tensorrt_llm/commands/refit.py", "tensorrt_llm/commands/serve.py", "tensorrt_llm/evaluate/__init__.py", "tensorrt_llm/evaluate/cnn_dailymail.py", @@ -400,7 +569,23 @@ include = [ "tensorrt_llm/inputs/multimodal.py", "tensorrt_llm/inputs/registry.py", "tensorrt_llm/inputs/utils.py", + "tensorrt_llm/layers/__init__.py", + "tensorrt_llm/layers/activation.py", + "tensorrt_llm/layers/attention.py", + "tensorrt_llm/layers/cast.py", + "tensorrt_llm/layers/conv.py", + "tensorrt_llm/layers/embedding.py", + "tensorrt_llm/layers/language_adapter.py", + "tensorrt_llm/layers/linear.py", + "tensorrt_llm/layers/lora.py", + "tensorrt_llm/layers/mlp.py", + "tensorrt_llm/layers/moe.py", + "tensorrt_llm/layers/normalization.py", + "tensorrt_llm/layers/pooling.py", + "tensorrt_llm/layers/recurrent.py", + "tensorrt_llm/layers/ssm.py", "tensorrt_llm/llmapi/__init__.py", + "tensorrt_llm/llmapi/build_cache.py", "tensorrt_llm/llmapi/disagg_utils.py", "tensorrt_llm/llmapi/kv_cache_type.py", "tensorrt_llm/llmapi/llm.py", @@ -423,17 +608,179 @@ include = [ "tensorrt_llm/metrics/enums.py", "tensorrt_llm/models/__init__.py", "tensorrt_llm/models/automodel.py", + "tensorrt_llm/models/baichuan/__init__.py", + "tensorrt_llm/models/baichuan/config.py", + "tensorrt_llm/models/baichuan/convert.py", + "tensorrt_llm/models/baichuan/model.py", + "tensorrt_llm/models/bert/__init__.py", + "tensorrt_llm/models/bert/config.py", + "tensorrt_llm/models/bert/convert.py", + "tensorrt_llm/models/bert/model.py", + "tensorrt_llm/models/bloom/__init__.py", + "tensorrt_llm/models/bloom/model.py", + "tensorrt_llm/models/chatglm/__init__.py", + "tensorrt_llm/models/chatglm/config.py", + "tensorrt_llm/models/chatglm/convert.py", + "tensorrt_llm/models/chatglm/model.py", + "tensorrt_llm/models/clip/__init__.py", + "tensorrt_llm/models/clip/model.py", + "tensorrt_llm/models/cogvlm/__init__.py", + "tensorrt_llm/models/cogvlm/config.py", + "tensorrt_llm/models/cogvlm/convert.py", + "tensorrt_llm/models/cogvlm/model.py", + "tensorrt_llm/models/commandr/__init__.py", + "tensorrt_llm/models/commandr/config.py", + "tensorrt_llm/models/commandr/model.py", "tensorrt_llm/models/convert_utils.py", + "tensorrt_llm/models/dbrx/__init__.py", + "tensorrt_llm/models/dbrx/config.py", + "tensorrt_llm/models/dbrx/model.py", + "tensorrt_llm/models/deepseek_v1/__init__.py", + "tensorrt_llm/models/deepseek_v1/config.py", + "tensorrt_llm/models/deepseek_v1/convert.py", + "tensorrt_llm/models/deepseek_v1/model.py", + "tensorrt_llm/models/deepseek_v2/__init__.py", + "tensorrt_llm/models/deepseek_v2/config.py", + "tensorrt_llm/models/deepseek_v2/convert.py", + "tensorrt_llm/models/deepseek_v2/model.py", + "tensorrt_llm/models/dit/__init__.py", + "tensorrt_llm/models/dit/model.py", + "tensorrt_llm/models/eagle/__init__.py", + "tensorrt_llm/models/eagle/config.py", + "tensorrt_llm/models/eagle/model.py", + "tensorrt_llm/models/enc_dec/__init__.py", + "tensorrt_llm/models/enc_dec/model.py", + "tensorrt_llm/models/falcon/__init__.py", + "tensorrt_llm/models/falcon/config.py", + "tensorrt_llm/models/falcon/convert.py", + "tensorrt_llm/models/falcon/model.py", + "tensorrt_llm/models/gemma/__init__.py", + "tensorrt_llm/models/gemma/config.py", + "tensorrt_llm/models/gemma/convert.py", + "tensorrt_llm/models/gemma/model.py", + "tensorrt_llm/models/gemma/smoothquant.py", + "tensorrt_llm/models/gemma/utils/__init__.py", + "tensorrt_llm/models/gemma/utils/layers.py", + "tensorrt_llm/models/gemma/utils/modules.py", + "tensorrt_llm/models/gemma/utils/params.py", + "tensorrt_llm/models/gemma/utils/positional_embeddings.py", + "tensorrt_llm/models/gemma/utils/sampler.py", + "tensorrt_llm/models/gemma/utils/transformer.py", + "tensorrt_llm/models/gemma/weight.py", + "tensorrt_llm/models/generation_mixin.py", + "tensorrt_llm/models/gpt/__init__.py", + "tensorrt_llm/models/gpt/config.py", + "tensorrt_llm/models/gpt/convert.py", + "tensorrt_llm/models/gpt/model.py", + "tensorrt_llm/models/gptj/__init__.py", + "tensorrt_llm/models/gptj/config.py", + "tensorrt_llm/models/gptj/convert.py", + "tensorrt_llm/models/gptj/model.py", + "tensorrt_llm/models/gptneox/__init__.py", + "tensorrt_llm/models/gptneox/model.py", + "tensorrt_llm/models/grok/__init__.py", + "tensorrt_llm/models/grok/convert.py", + "tensorrt_llm/models/grok/model.py", + "tensorrt_llm/models/grok/weight.py", + "tensorrt_llm/models/llama/__init__.py", + "tensorrt_llm/models/llama/config.py", + "tensorrt_llm/models/llama/convert.py", + "tensorrt_llm/models/llama/model.py", + "tensorrt_llm/models/mamba/__init__.py", + "tensorrt_llm/models/mamba/config.py", + "tensorrt_llm/models/mamba/convert.py", + "tensorrt_llm/models/mamba/model.py", + "tensorrt_llm/models/medusa/__init__.py", + "tensorrt_llm/models/medusa/config.py", + "tensorrt_llm/models/medusa/model.py", + "tensorrt_llm/models/medusa/weight.py", + "tensorrt_llm/models/mllama/__init__.py", + "tensorrt_llm/models/mllama/config.py", + "tensorrt_llm/models/mllama/model.py", + "tensorrt_llm/models/mmdit_sd3/__init__.py", + "tensorrt_llm/models/mmdit_sd3/config.py", + "tensorrt_llm/models/mmdit_sd3/model.py", + "tensorrt_llm/models/model_weights_loader.py", "tensorrt_llm/models/modeling_utils.py", + "tensorrt_llm/models/mpt/__init__.py", + "tensorrt_llm/models/mpt/model.py", + "tensorrt_llm/models/multimodal_encoders/__init__.py", + "tensorrt_llm/models/multimodal_encoders/config.py", + "tensorrt_llm/models/multimodal_encoders/model.py", + "tensorrt_llm/models/nemotron_nas/__init__.py", + "tensorrt_llm/models/nemotron_nas/config.py", + "tensorrt_llm/models/nemotron_nas/convert.py", + "tensorrt_llm/models/nemotron_nas/layer_config.py", + "tensorrt_llm/models/nemotron_nas/model.py", + "tensorrt_llm/models/opt/__init__.py", + "tensorrt_llm/models/opt/model.py", + "tensorrt_llm/models/phi/__init__.py", + "tensorrt_llm/models/phi/config.py", + "tensorrt_llm/models/phi/convert.py", + "tensorrt_llm/models/phi/model.py", + "tensorrt_llm/models/phi3/__init__.py", + "tensorrt_llm/models/phi3/config.py", + "tensorrt_llm/models/phi3/convert.py", + "tensorrt_llm/models/phi3/model.py", + "tensorrt_llm/models/phi3/split_weights.py", + "tensorrt_llm/models/qwen/__init__.py", + "tensorrt_llm/models/qwen/config.py", + "tensorrt_llm/models/qwen/convert.py", + "tensorrt_llm/models/qwen/model.py", + "tensorrt_llm/models/qwen/utils.py", + "tensorrt_llm/models/recurrentgemma/__init__.py", + "tensorrt_llm/models/recurrentgemma/model.py", + "tensorrt_llm/models/redrafter/__init__.py", + "tensorrt_llm/models/redrafter/drafter.py", + "tensorrt_llm/models/redrafter/model.py", + "tensorrt_llm/models/redrafter/redrafter_helper.py", + "tensorrt_llm/models/stdit/__init__.py", + "tensorrt_llm/models/stdit/config.py", + "tensorrt_llm/models/stdit/model.py", + "tensorrt_llm/models/unet/__init__.py", + "tensorrt_llm/models/unet/attention.py", + "tensorrt_llm/models/unet/embeddings.py", + "tensorrt_llm/models/unet/pp/__init__.py", + "tensorrt_llm/models/unet/pp/attention.py", + "tensorrt_llm/models/unet/pp/conv2d.py", + "tensorrt_llm/models/unet/pp/groupnorm.py", + "tensorrt_llm/models/unet/pp/unet_pp.py", + "tensorrt_llm/models/unet/resnet.py", + "tensorrt_llm/models/unet/unet_2d_blocks.py", + "tensorrt_llm/models/unet/unet_2d_condition.py", + "tensorrt_llm/models/unet/weights.py", + "tensorrt_llm/network.py", + "tensorrt_llm/parameter.py", + "tensorrt_llm/plugin/__init__.py", + "tensorrt_llm/plugin/plugin.py", "tensorrt_llm/quantization/__init__.py", "tensorrt_llm/quantization/functional.py", + "tensorrt_llm/quantization/image_processing.py", + "tensorrt_llm/quantization/layers.py", "tensorrt_llm/quantization/mode.py", + "tensorrt_llm/quantization/quantize.py", + "tensorrt_llm/quantization/quantize_by_modelopt.py", "tensorrt_llm/quantization/utils/__init__.py", "tensorrt_llm/quantization/utils/fp4_utils.py", "tensorrt_llm/quantization/utils/fp8_utils.py", "tensorrt_llm/ray_stub.py", "tensorrt_llm/runtime/__init__.py", + "tensorrt_llm/runtime/enc_dec_model_runner.py", + "tensorrt_llm/runtime/generation.py", + "tensorrt_llm/runtime/kv_cache_manager.py", + "tensorrt_llm/runtime/medusa_utils.py", "tensorrt_llm/runtime/memory_pools/__init__.py", + "tensorrt_llm/runtime/memory_pools/memory_pools_allocator.py", + "tensorrt_llm/runtime/memory_pools/pool.py", + "tensorrt_llm/runtime/memory_pools/pools_kv_cache_manager.py", + "tensorrt_llm/runtime/model_runner.py", + "tensorrt_llm/runtime/model_runner_cpp.py", + "tensorrt_llm/runtime/multimodal_model_runner.py", + "tensorrt_llm/runtime/processor_wrapper/__init__.py", + "tensorrt_llm/runtime/processor_wrapper/mllama_processor_wrapper.py", + "tensorrt_llm/runtime/processor_wrapper/processor_wrapper.py", + "tensorrt_llm/runtime/redrafter_utils.py", + "tensorrt_llm/runtime/session.py", "tensorrt_llm/scaffolding/__init__.py", "tensorrt_llm/scaffolding/benchmark.py", "tensorrt_llm/scaffolding/contrib/AsyncGeneration/stream_generation.py", @@ -488,7 +835,12 @@ include = [ "tensorrt_llm/tokenizer/tokenizer.py", "tensorrt_llm/tools/__init__.py", "tensorrt_llm/tools/importlib_utils.py", + "tensorrt_llm/tools/multimodal_builder.py", + "tensorrt_llm/tools/onnx_utils.py", "tensorrt_llm/tools/plugin_gen/__init__.py", + "tensorrt_llm/tools/plugin_gen/core.py", + "tensorrt_llm/tools/plugin_gen/plugin_gen.py", + "tensorrt_llm/tools/plugin_gen/shape_infer.py", "tensorrt_llm/tools/ppl.py", "tensorrt_llm/tools/profiler/nsys_profile_tools/gputrc2graph.py", "tensorrt_llm/version.py", @@ -497,7 +849,9 @@ include = [ "tests/integration/defs/accuracy/accuracy_core.py", "tests/integration/defs/accuracy/scripts/collect_evaluated_accuracies.py", "tests/integration/defs/accuracy/scripts/compute_theta_and_thresholds.py", + "tests/integration/defs/accuracy/test_cli_flow.py", "tests/integration/defs/accuracy/test_disaggregated_serving.py", + "tests/integration/defs/accuracy/test_llm_api.py", "tests/integration/defs/accuracy/test_llm_api_autodeploy.py", "tests/integration/defs/accuracy/test_llm_api_pytorch.py", "tests/integration/defs/accuracy/test_llm_api_pytorch_ray.py", @@ -506,29 +860,63 @@ include = [ "tests/integration/defs/conftest.py", "tests/integration/defs/cpp/conftest.py", "tests/integration/defs/cpp/cpp_common.py", + "tests/integration/defs/cpp/test_e2e.py", "tests/integration/defs/cpp/test_multi_gpu.py", "tests/integration/defs/cpp/test_unit_tests.py", + "tests/integration/defs/deterministic/mixtral_deterministic.py", + "tests/integration/defs/deterministic/test_mixtral_deterministic.py", "tests/integration/defs/disaggregated/test_auto_scaling.py", "tests/integration/defs/disaggregated/test_disaggregated.py", "tests/integration/defs/disaggregated/test_disaggregated_etcd.py", "tests/integration/defs/disaggregated/test_disaggregated_single_gpu.py", "tests/integration/defs/disaggregated/test_workers.py", + "tests/integration/defs/examples/run_llm_fp8_quant_llama_70b.py", "tests/integration/defs/examples/run_llm_quickstart_atexit.py", "tests/integration/defs/examples/serve/test_serve.py", "tests/integration/defs/examples/serve/test_serve_negative.py", "tests/integration/defs/examples/test_ad_guided_decoding.py", + "tests/integration/defs/examples/test_bert.py", + "tests/integration/defs/examples/test_bindings.py", + "tests/integration/defs/examples/test_chatglm.py", + "tests/integration/defs/examples/test_commandr.py", + "tests/integration/defs/examples/test_draft_target_model.py", + "tests/integration/defs/examples/test_eagle.py", + "tests/integration/defs/examples/test_enc_dec.py", + "tests/integration/defs/examples/test_exaone.py", + "tests/integration/defs/examples/test_gemma.py", "tests/integration/defs/examples/test_gpt.py", + "tests/integration/defs/examples/test_gptj.py", + "tests/integration/defs/examples/test_granite.py", + "tests/integration/defs/examples/test_internlm.py", + "tests/integration/defs/examples/test_llama.py", "tests/integration/defs/examples/test_llm_api_with_mpi.py", + "tests/integration/defs/examples/test_mamba.py", + "tests/integration/defs/examples/test_medusa.py", + "tests/integration/defs/examples/test_mistral.py", + "tests/integration/defs/examples/test_mixtral.py", + "tests/integration/defs/examples/test_multimodal.py", + "tests/integration/defs/examples/test_nemotron.py", + "tests/integration/defs/examples/test_nemotron_nas.py", + "tests/integration/defs/examples/test_ngram.py", + "tests/integration/defs/examples/test_openai.py", "tests/integration/defs/examples/test_phi.py", + "tests/integration/defs/examples/test_qwen.py", + "tests/integration/defs/examples/test_qwen2audio.py", + "tests/integration/defs/examples/test_qwenvl.py", "tests/integration/defs/examples/test_ray.py", + "tests/integration/defs/examples/test_recurrentgemma.py", + "tests/integration/defs/examples/test_redrafter.py", + "tests/integration/defs/examples/test_whisper.py", "tests/integration/defs/llmapi/__init__.py", "tests/integration/defs/llmapi/_run_llmapi_llm.py", "tests/integration/defs/llmapi/test_llm_api_connector.py", "tests/integration/defs/llmapi/test_llm_api_qa.py", + "tests/integration/defs/llmapi/test_llm_e2e.py", "tests/integration/defs/llmapi/test_llm_examples.py", "tests/integration/defs/local_venv.py", "tests/integration/defs/perf/__init__.py", "tests/integration/defs/perf/allowed_configs.py", + "tests/integration/defs/perf/build.py", "tests/integration/defs/perf/create_perf_comparison_report.py", "tests/integration/defs/perf/data.py", "tests/integration/defs/perf/data_export.py", @@ -549,21 +937,38 @@ include = [ "tests/integration/defs/test_fmha.py", "tests/integration/defs/test_list_parser.py", "tests/integration/defs/test_list_validation.py", + "tests/integration/defs/test_mlpf_results.py", "tests/integration/defs/test_sanity.py", "tests/integration/defs/test_unittests.py", "tests/integration/defs/triton_server/__init__.py", + "tests/integration/defs/triton_server/build_engines.py", "tests/integration/defs/triton_server/common.py", "tests/integration/defs/triton_server/conftest.py", + "tests/integration/defs/triton_server/local_venv.py", + "tests/integration/defs/triton_server/rcca/bug_4323566/inflight_batcher_llm_client_with_end_id.py", + "tests/integration/defs/triton_server/runner_interface.py", "tests/integration/defs/triton_server/test_list_parser.py", + "tests/integration/defs/triton_server/test_triton.py", + "tests/integration/defs/triton_server/test_triton_llm.py", + "tests/integration/defs/triton_server/test_triton_memleak.py", + "tests/integration/defs/triton_server/test_triton_multi_node.py", + "tests/integration/defs/triton_server/test_triton_rcca.py", "tests/integration/defs/triton_server/trt_test_alternative.py", "tests/integration/defs/trt_test_alternative.py", "tests/integration/defs/utils/__init__.py", "tests/integration/defs/utils/periodic_junit.py", "tests/integration/defs/utils/timeout_manager.py", "tests/microbenchmarks/all_reduce.py", + "tests/microbenchmarks/build_time_benchmark.py", + "tests/microbenchmarks/build_time_dashboard.py", "tests/scripts/allreduce_perf/allreduce_heuristic_code_gen.py", "tests/scripts/allreduce_perf/allreduce_perf_viz.py", "tests/scripts/iteration_log_parser.py", + "tests/scripts/perf-sanity/parse_benchmark_results.py", + "tests/scripts/perf-sanity/run_benchmark_serve.py", + "tests/unittest/_torch/attention/sparse/test_dsa_indexer.py", + "tests/unittest/_torch/attention/sparse/test_flash_mla.py", + "tests/unittest/_torch/attention/sparse/test_rocketkv.py", "tests/unittest/_torch/attention/sparse/test_sparse_mla_forward.py", "tests/unittest/_torch/attention/test_attention.py", "tests/unittest/_torch/attention/test_attention_mla.py", @@ -584,6 +989,7 @@ include = [ "tests/unittest/_torch/misc/test_virtual_memory.py", "tests/unittest/_torch/modeling/test_modeling_bert.py", "tests/unittest/_torch/modeling/test_modeling_clip.py", + "tests/unittest/_torch/modeling/test_modeling_exaone4.py", "tests/unittest/_torch/modeling/test_modeling_gemma3.py", "tests/unittest/_torch/modeling/test_modeling_gpt_oss.py", "tests/unittest/_torch/modeling/test_modeling_llama.py", @@ -607,6 +1013,8 @@ include = [ "tests/unittest/_torch/modules/test_moe_routing.py", "tests/unittest/_torch/modules/test_rotary_embedding.py", "tests/unittest/_torch/modules/test_triton_linear.py", + "tests/unittest/_torch/modules/tests_lora_modules/test_lora_attention_pytorch_flow_vs_trt.py", + "tests/unittest/_torch/modules/tests_lora_modules/test_lora_plugin_vs_lora_op.py", "tests/unittest/_torch/multi_gpu/test_allreduce.py", "tests/unittest/_torch/multi_gpu/test_alltoall.py", "tests/unittest/_torch/multi_gpu/test_ar_residual_norm.py", @@ -633,10 +1041,24 @@ include = [ "tests/unittest/_torch/sampler/test_beam_search.py", "tests/unittest/_torch/sampler/test_best_of_n.py", "tests/unittest/_torch/sampler/test_trtllm_sampler.py", + "tests/unittest/_torch/speculative/test_draft_target.py", + "tests/unittest/_torch/speculative/test_draft_token_tree_sampling.py", + "tests/unittest/_torch/speculative/test_draft_token_tree_verification.py", + "tests/unittest/_torch/speculative/test_dynamic_spec_decode.py", "tests/unittest/_torch/speculative/test_eagle3.py", + "tests/unittest/_torch/speculative/test_kv_cache_reuse.py", + "tests/unittest/_torch/speculative/test_mtp.py", + "tests/unittest/_torch/speculative/test_ngram.py", + "tests/unittest/_torch/speculative/test_save_state.py", + "tests/unittest/_torch/speculative/test_spec_gate.py", + "tests/unittest/_torch/speculative/test_torch_rejection_sampling.py", + "tests/unittest/_torch/speculative/test_user_provided.py", "tests/unittest/_torch/test_connector.py", "tests/unittest/_torch/test_torch_multi_arange.py", "tests/unittest/_torch/thop/parallel/deep_gemm_tests.py", + "tests/unittest/_torch/thop/parallel/test_causal_conv1d_op.py", + "tests/unittest/_torch/thop/parallel/test_cublas_mm.py", + "tests/unittest/_torch/thop/parallel/test_custom_ops.py", "tests/unittest/_torch/thop/parallel/test_dsv3_fused_a_gemm.py", "tests/unittest/_torch/thop/parallel/test_dsv3_router_gemm.py", "tests/unittest/_torch/thop/parallel/test_finegrained_mixed_dtype_gemm.py", @@ -650,6 +1072,11 @@ include = [ "tests/unittest/_torch/thop/parallel/test_fp8_per_tensor_scale_tllmg_gemm.py", "tests/unittest/_torch/thop/parallel/test_fp8_quantize.py", "tests/unittest/_torch/thop/parallel/test_fp8_rowwise_linear.py", + "tests/unittest/_torch/thop/parallel/test_fused_qk_norm_rope.py", + "tests/unittest/_torch/thop/parallel/test_logits_bitmask_op.py", + "tests/unittest/_torch/thop/parallel/test_mamba2_chunk_ss_update.py", + "tests/unittest/_torch/thop/parallel/test_mamba_conv1d_op.py", + "tests/unittest/_torch/thop/parallel/test_noaux_tc.py", "tests/unittest/_torch/thop/parallel/test_scaled_mm.py", "tests/unittest/_torch/thop/parallel/test_selective_scan_op.py", "tests/unittest/_torch/thop/parallel/test_tinygemm2.py", @@ -663,6 +1090,7 @@ include = [ "tests/unittest/_torch/thop/serial/test_moe_alltoall.py", "tests/unittest/api_stability/api_stability_core.py", "tests/unittest/api_stability/test_llm_api.py", + "tests/unittest/bindings/binding_test_utils.py", "tests/unittest/bindings/test_bindings_moe.py", "tests/unittest/bindings/test_bindings_ut.py", "tests/unittest/bindings/test_executor_bindings.py", @@ -693,10 +1121,12 @@ include = [ "tests/unittest/llmapi/apps/_test_openai_chat_harmony.py", "tests/unittest/llmapi/apps/_test_openai_chat_multimodal.py", "tests/unittest/llmapi/apps/_test_openai_completions.py", + "tests/unittest/llmapi/apps/_test_openai_consistent_chat.py", "tests/unittest/llmapi/apps/_test_openai_lora.py", "tests/unittest/llmapi/apps/_test_openai_metrics.py", "tests/unittest/llmapi/apps/_test_openai_misc.py", "tests/unittest/llmapi/apps/_test_openai_mmencoder.py", + "tests/unittest/llmapi/apps/_test_openai_multi_chat.py", "tests/unittest/llmapi/apps/_test_openai_multi_gpu.py", "tests/unittest/llmapi/apps/_test_openai_multi_nodes.py", "tests/unittest/llmapi/apps/_test_openai_perf_metrics.py", @@ -719,12 +1149,15 @@ include = [ "tests/unittest/llmapi/run_llm_exit.py", "tests/unittest/llmapi/run_llm_with_postproc.py", "tests/unittest/llmapi/test_additional_model_outputs.py", + "tests/unittest/llmapi/test_build_cache.py", "tests/unittest/llmapi/test_executor.py", "tests/unittest/llmapi/test_gc_utils.py", "tests/unittest/llmapi/test_llm.py", "tests/unittest/llmapi/test_llm_args.py", "tests/unittest/llmapi/test_llm_download.py", "tests/unittest/llmapi/test_llm_kv_cache_events.py", + "tests/unittest/llmapi/test_llm_models.py", + "tests/unittest/llmapi/test_llm_multi_gpu.py", "tests/unittest/llmapi/test_llm_multi_gpu_pytorch.py", "tests/unittest/llmapi/test_llm_pytorch.py", "tests/unittest/llmapi/test_llm_quant.py", @@ -735,14 +1168,25 @@ include = [ "tests/unittest/llmapi/test_serialization.py", "tests/unittest/llmapi/test_utils.py", "tests/unittest/others/__init__.py", + "tests/unittest/others/test_builder.py", "tests/unittest/others/test_convert_spec_decoding_mask_to_packed_mask.py", + "tests/unittest/others/test_debugging_api.py", "tests/unittest/others/test_exception.py", "tests/unittest/others/test_export.py", + "tests/unittest/others/test_graph_rewriter.py", + "tests/unittest/others/test_kv_cache_manager.py", "tests/unittest/others/test_kv_cache_transceiver.py", "tests/unittest/others/test_kv_cache_update.py", + "tests/unittest/others/test_layer.py", + "tests/unittest/others/test_leak.py", "tests/unittest/others/test_mapping.py", + "tests/unittest/others/test_model_dtype.py", + "tests/unittest/others/test_module.py", "tests/unittest/others/test_multimodal_registry.py", + "tests/unittest/others/test_plugins.py", + "tests/unittest/others/test_precision_control.py", "tests/unittest/others/test_pretrained_config.py", + "tests/unittest/others/test_session.py", "tests/unittest/others/test_time_breakdown.py", "tests/unittest/profile_utils.py", "tests/unittest/scaffolding/__init__.py", @@ -751,14 +1195,141 @@ include = [ "tests/unittest/scaffolding/test_scaffolding.py", "tests/unittest/scaffolding/test_task_collection.py", "tests/unittest/scaffolding/test_worker.py", + "tests/unittest/test_model_runner_cpp.py", "tests/unittest/test_pip_install.py", "tests/unittest/tools/__init__.py", + "tests/unittest/tools/plugin_gen/__init__.py", + "tests/unittest/tools/plugin_gen/kernel_config.py", + "tests/unittest/tools/plugin_gen/test_core.py", + "tests/unittest/tools/plugin_gen/test_plugin_gen.py", + "tests/unittest/tools/plugin_gen/test_shape_infer.py", "tests/unittest/tools/test_prepare_dataset.py", "tests/unittest/tools/test_test_to_stage_mapping.py", + "tests/unittest/trt/__init__.py", + "tests/unittest/trt/attention/test_bert_attention.py", + "tests/unittest/trt/attention/test_gpt_attention.py", + "tests/unittest/trt/attention/test_gpt_attention_IFB.py", + "tests/unittest/trt/attention/test_gpt_attention_no_cache.py", + "tests/unittest/trt/attention/test_sage_attention.py", + "tests/unittest/trt/functional/__init__.py", + "tests/unittest/trt/functional/test_alibi.py", + "tests/unittest/trt/functional/test_allreduce_norm.py", + "tests/unittest/trt/functional/test_allreduce_prepost_residual_norm.py", + "tests/unittest/trt/functional/test_arange.py", + "tests/unittest/trt/functional/test_argmax.py", + "tests/unittest/trt/functional/test_assertion.py", + "tests/unittest/trt/functional/test_avg_pool2d.py", + "tests/unittest/trt/functional/test_cast.py", + "tests/unittest/trt/functional/test_conv2d.py", + "tests/unittest/trt/functional/test_conv3d.py", + "tests/unittest/trt/functional/test_cos.py", + "tests/unittest/trt/functional/test_cumsum.py", + "tests/unittest/trt/functional/test_dora.py", + "tests/unittest/trt/functional/test_einsum.py", + "tests/unittest/trt/functional/test_embedding_single_gpu.py", + "tests/unittest/trt/functional/test_exp.py", + "tests/unittest/trt/functional/test_expand.py", + "tests/unittest/trt/functional/test_flatten.py", + "tests/unittest/trt/functional/test_flip.py", + "tests/unittest/trt/functional/test_fp4_gemm.py", + "tests/unittest/trt/functional/test_fp4_gemm_ootb.py", + "tests/unittest/trt/functional/test_gather.py", + "tests/unittest/trt/functional/test_gather_nd.py", + "tests/unittest/trt/functional/test_geglu.py", + "tests/unittest/trt/functional/test_gelu.py", + "tests/unittest/trt/functional/test_gemm_swiglu.py", + "tests/unittest/trt/functional/test_group_norm.py", + "tests/unittest/trt/functional/test_identity.py", + "tests/unittest/trt/functional/test_index_select.py", + "tests/unittest/trt/functional/test_interpolate.py", + "tests/unittest/trt/functional/test_logsoftmax.py", + "tests/unittest/trt/functional/test_lora.py", + "tests/unittest/trt/functional/test_low_latency_gemm.py", + "tests/unittest/trt/functional/test_mamba_conv1d.py", + "tests/unittest/trt/functional/test_masked_scatter.py", + "tests/unittest/trt/functional/test_masked_select.py", + "tests/unittest/trt/functional/test_matmul.py", + "tests/unittest/trt/functional/test_meshgrid2d.py", + "tests/unittest/trt/functional/test_moe.py", + "tests/unittest/trt/functional/test_nccl.py", + "tests/unittest/trt/functional/test_nonzero.py", + "tests/unittest/trt/functional/test_outer.py", + "tests/unittest/trt/functional/test_pad.py", + "tests/unittest/trt/functional/test_permute.py", + "tests/unittest/trt/functional/test_pp_reduce_scatter.py", + "tests/unittest/trt/functional/test_quant.py", + "tests/unittest/trt/functional/test_rearrange.py", + "tests/unittest/trt/functional/test_repeat.py", + "tests/unittest/trt/functional/test_repeat_interleave.py", + "tests/unittest/trt/functional/test_rg_lru.py", + "tests/unittest/trt/functional/test_sample.py", + "tests/unittest/trt/functional/test_scatter.py", + "tests/unittest/trt/functional/test_scatter_nd.py", + "tests/unittest/trt/functional/test_select.py", + "tests/unittest/trt/functional/test_selective_scan.py", + "tests/unittest/trt/functional/test_sigmoid.py", + "tests/unittest/trt/functional/test_silu.py", + "tests/unittest/trt/functional/test_sin.py", + "tests/unittest/trt/functional/test_slice.py", + "tests/unittest/trt/functional/test_softplus.py", + "tests/unittest/trt/functional/test_split.py", + "tests/unittest/trt/functional/test_squeeze.py", + "tests/unittest/trt/functional/test_swiglu.py", + "tests/unittest/trt/functional/test_topk.py", + "tests/unittest/trt/functional/test_transpose.py", + "tests/unittest/trt/functional/test_unbind.py", + "tests/unittest/trt/functional/test_unsqueeze.py", + "tests/unittest/trt/functional/test_view.py", + "tests/unittest/trt/functional/test_where.py", + "tests/unittest/trt/model/__init__.py", + "tests/unittest/trt/model/eagle/test_decode_draft_tokens_plugin.py", + "tests/unittest/trt/model/eagle/test_prepare_drafter_inputs_plugin.py", + "tests/unittest/trt/model/eagle/test_sample_accept_draft_tokens_plugin.py", + "tests/unittest/trt/model/redrafter/test_beams2tree.py", + "tests/unittest/trt/model/redrafter/test_draft_token.py", + "tests/unittest/trt/model/redrafter/test_draft_token_indices.py", + "tests/unittest/trt/model/redrafter/test_gather_beams.py", + "tests/unittest/trt/model/redrafter/test_mask.py", + "tests/unittest/trt/model/redrafter/test_packed_position_ids.py", + "tests/unittest/trt/model/redrafter/test_prefix_match_indices.py", + "tests/unittest/trt/model/redrafter/test_prepare_input.py", + "tests/unittest/trt/model/redrafter/test_process_logits.py", + "tests/unittest/trt/model/redrafter/test_top1.py", + "tests/unittest/trt/model/redrafter/test_unpack_gen_data.py", + "tests/unittest/trt/model/redrafter/test_validate.py", + "tests/unittest/trt/model/test_gpt.py", + "tests/unittest/trt/model/test_gpt_e2e.py", + "tests/unittest/trt/model/test_llama.py", + "tests/unittest/trt/model/test_mamba.py", + "tests/unittest/trt/model/test_mistral.py", + "tests/unittest/trt/model/test_nemotron_nas.py", + "tests/unittest/trt/model/test_phi.py", + "tests/unittest/trt/model/test_unet.py", + "tests/unittest/trt/model_api/test_model_api_multi_gpu.py", + "tests/unittest/trt/model_api/test_model_level_api.py", + "tests/unittest/trt/model_api/test_model_quantization.py", + "tests/unittest/trt/python_plugin/plugin_wrapper_utils.py", + "tests/unittest/trt/python_plugin/test_plugin_wrapper.py", + "tests/unittest/trt/quantization/__init__.py", + "tests/unittest/trt/quantization/_utils.py", + "tests/unittest/trt/quantization/test_fp8_quantization.py", + "tests/unittest/trt/quantization/test_fp8_rowwise_gemm.py", + "tests/unittest/trt/quantization/test_functional.py", + "tests/unittest/trt/quantization/test_mode.py", + "tests/unittest/trt/quantization/test_moe_weight_only_quant_matmul.py", + "tests/unittest/trt/quantization/test_qserve_gemm.py", + "tests/unittest/trt/quantization/test_quant.py", + "tests/unittest/trt/quantization/test_quant_layer.py", + "tests/unittest/trt/quantization/test_smooth_quant_gemm.py", + "tests/unittest/trt/quantization/test_smooth_quant_layer_norm.py", + "tests/unittest/trt/quantization/test_smooth_quant_rms_norm.py", + "tests/unittest/trt/quantization/test_weight_only_groupwise_quant_matmul.py", + "tests/unittest/trt/quantization/test_weight_only_quant_matmul.py", "tests/unittest/utils/__init__.py", "tests/unittest/utils/cpp_paths.py", "tests/unittest/utils/llm_data.py", "tests/unittest/utils/runtime_defaults.py", + "tests/unittest/utils/test_medusa_utils.py", "tests/unittest/utils/test_prebuilt_whl_cpp_extensions.py", "tests/unittest/utils/test_util.py", "tests/unittest/utils/torch_ref.py", diff --git a/scripts/attribution/scan/metadata/msa.yml b/scripts/attribution/scan/metadata/msa.yml deleted file mode 100644 index ab212fc1b664..000000000000 --- a/scripts/attribution/scan/metadata/msa.yml +++ /dev/null @@ -1,5 +0,0 @@ -name: msa -description: MiniMax Sparse Attention (fmha_sm100) kernels for SM100 sparse attention -source: submodule -directory_matches: -- 3rdparty/MSA diff --git a/scripts/build_cpp_examples.py b/scripts/build_cpp_examples.py new file mode 100644 index 000000000000..cb2591acfea2 --- /dev/null +++ b/scripts/build_cpp_examples.py @@ -0,0 +1,88 @@ +import argparse +import contextlib +import logging +import os +import platform +import shutil +import subprocess +from os import PathLike +from pathlib import Path + + +@contextlib.contextmanager +def working_directory(path: PathLike): + """Changes working directory and returns to previous on exit.""" + prev_cwd = Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(prev_cwd) + + +def build_cpp_examples(build_dir: PathLike, trt_dir: PathLike, + enable_multi_device: str, loglevel: int) -> None: + logging.basicConfig(level=loglevel, + format='%(asctime)s - %(levelname)s - %(message)s') + # Convert input paths to pathlib.Path objects + build_dir = Path(build_dir) + trt_dir = Path(trt_dir) + + assert trt_dir.is_dir() + + def cmake_parse(path: PathLike) -> str: + return str(path).replace("\\", "/") + + # Remove the build directory if it exists + if build_dir.exists(): + logging.info(f"Removed directory: {build_dir}") + shutil.rmtree(build_dir) + + # Create the build directory + build_dir.mkdir(parents=True, exist_ok=True) + + # Change to the build directory + with working_directory(build_dir): + # Run CMake with the specified TensorRT directories + generator = ["-GNinja"] if platform.system() == "Windows" else [] + generate_command = [ + 'cmake', + '-S', + '..', + '-B', + '.', + f'-DTensorRT_ROOT={cmake_parse(trt_dir)}', + f'-DENABLE_MULTI_DEVICE={enable_multi_device}', + ] + generator + logging.info(f"Executing {generate_command}") + subprocess.run(generate_command, check=True) + + # Build the project using make + build_command = ["cmake", "--build", ".", "--config", "Release"] + logging.info(f"Executing {build_command}") + subprocess.run(build_command, check=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Build C++ examples') + parser.add_argument('--build-dir', + default='examples/cpp/executor/build', + help='Build directory path') + parser.add_argument('--trt-dir', + default='/usr/local/tensorrt', + help='TensorRT directory path') + parser.add_argument('--enable-multi-device', + default='ON', + help='Enable multi device support (requires MPI)') + parser.add_argument('-v', + '--verbose', + help="verbose", + action="store_const", + dest="loglevel", + const=logging.DEBUG, + default=logging.INFO) + cli = parser.parse_args() + + args = vars(cli) + print(args) # Log on Jenkins instance. + build_cpp_examples(**args) diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py index 462b74d8e511..263f061d14b6 100755 --- a/scripts/build_wheel.py +++ b/scripts/build_wheel.py @@ -264,18 +264,9 @@ def setup_conan(scripts_dir, venv_python): return venv_conan -def _fmha_generation_stamp(fmha_v2_cu_dir: Path) -> Path: - # Written as the last step of generate_fmha_cu; its absence means a - # previous generation was interrupted and the directory contents cannot - # be trusted (the bare directory exists from the moment generation - # starts). - return fmha_v2_cu_dir / ".generation_complete" - - def generate_fmha_cu(project_dir, venv_python): fmha_v2_cu_dir = project_dir / "cpp/tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmha_v2_cu" fmha_v2_cu_dir.mkdir(parents=True, exist_ok=True) - _fmha_generation_stamp(fmha_v2_cu_dir).unlink(missing_ok=True) fmha_v2_dir = project_dir / "cpp/kernels/fmha_v2" @@ -325,11 +316,6 @@ def move_if_updated(src, dst): move_if_updated(cu_file, dst_file) generated_files.add(str(dst_file.resolve())) - if not generated_files: - raise RuntimeError( - f"FMHA generation produced no *_sm*.cu files in {fmha_v2_cu_dir}; " - "generation may have failed silently.") - # Remove extra files for root, _, files in os.walk(fmha_v2_cu_dir): for file in files: @@ -337,8 +323,6 @@ def move_if_updated(src, dst): if file_path not in generated_files: os.remove(file_path) - _fmha_generation_stamp(fmha_v2_cu_dir).touch() - def create_cuda_stub_links(cuda_stub_dir: str, missing_libs: list[str]) -> str: """ @@ -505,6 +489,7 @@ def main(*, job_count: int = None, extra_cmake_vars: Sequence[str] = tuple(), extra_make_targets: str = "", + trt_root: str = '/usr/local/tensorrt', nccl_root: str = None, nixl_root: str = None, mooncake_root: str = None, @@ -520,6 +505,7 @@ def main(*, install: bool = False, skip_building_wheel: bool = False, linking_install_binary: bool = False, + benchmarks: bool = False, micro_benchmarks: bool = False, nvtx: bool = False, skip_stubs: bool = False, @@ -556,6 +542,21 @@ def main(*, no_venv, yes=yes) + # Ensure base TRT is installed (check inside the venv) + try: + check_output([str(venv_python), "-m", "pip", "show", "tensorrt"]) + except CalledProcessError: + error_msg = "TensorRT was not installed properly." + if on_windows: + error_msg += ( + " Please download the TensorRT zip file manually," + " install it and relaunch build_wheel.py." + " See https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html#installing-zip for more details." + ) + else: + error_msg += f" Please install tensorrt into the venv using \"`{venv_python}` -m pip install tensorrt\" and relaunch build_wheel.py" + raise RuntimeError(error_msg) + if cuda_architectures is not None: if "70-real" in cuda_architectures: raise RuntimeError("Volta architecture is deprecated support.") @@ -611,6 +612,9 @@ def main(*, # Don't include duplicate conditions cmake_def_args.extend(set(extra_cmake_vars)) + if trt_root is not None: + cmake_def_args.append(f"-DTensorRT_ROOT={trt_root}") + if nccl_root is not None: cmake_def_args.append(f"-DNCCL_ROOT={nccl_root}") @@ -666,7 +670,7 @@ def main(*, "-- BOLT: Forcing NVRTC_DYNAMIC_LINKING=ON (static NVIDIA libs lack relocations)" ) - targets = ["tensorrt_llm"] + targets = ["tensorrt_llm", "nvinfer_plugin_tensorrt_llm"] if cpp_only: build_pyt = "OFF" @@ -683,6 +687,9 @@ def main(*, build_deep_gemm = "ON" build_flash_mla = "ON" + if benchmarks: + targets.append("benchmarks") + if micro_benchmarks: targets.append("micro_benchmarks") build_micro_benchmarks = "ON" @@ -691,11 +698,13 @@ def main(*, disable_nvtx = "OFF" if nvtx else "ON" + if not on_windows: + targets.append("executorWorker") + source_dir = get_source_dir() fmha_v2_cu_dir = project_dir / "cpp/tensorrt_llm/kernels/contextFusedMultiHeadAttention/fmha_v2_cu" - if (clean or generate_fmha - or not _fmha_generation_stamp(fmha_v2_cu_dir).exists()): + if clean or generate_fmha or not fmha_v2_cu_dir.exists(): generate_fmha_cu(project_dir, venv_python) with working_directory(build_dir): @@ -923,11 +932,18 @@ def copy_resolving_symlink(src_path, dst_path): lib_dir / "tensorrt_llm.dll") install_file(build_dir / f"tensorrt_llm/thop/th_common.dll", lib_dir / "th_common.dll") + install_file( + build_dir / f"tensorrt_llm/plugins/nvinfer_plugin_tensorrt_llm.dll", + lib_dir / "nvinfer_plugin_tensorrt_llm.dll") else: install_file(build_dir / "tensorrt_llm/libtensorrt_llm.so", lib_dir / "libtensorrt_llm.so") install_file(build_dir / "tensorrt_llm/thop/libth_common.so", lib_dir / "libth_common.so") + install_file( + build_dir / + "tensorrt_llm/plugins/libnvinfer_plugin_tensorrt_llm.so", + lib_dir / "libnvinfer_plugin_tensorrt_llm.so") if os.path.exists( build_dir / "tensorrt_llm/executor/cache_transmission/ucx_utils/libtensorrt_llm_ucx_wrapper.so" @@ -1012,9 +1028,23 @@ def copy_resolving_symlink(src_path, dst_path): clear_folder(deep_gemm_dir) deep_gemm_dir.rmdir() + bin_dir = pkg_dir / "bin" + if bin_dir.exists(): + clear_folder(bin_dir) + bin_dir.mkdir(parents=True, exist_ok=True) + + if not on_windows: + install_file(build_dir / "tensorrt_llm/executor_worker/executorWorker", + bin_dir / "executorWorker") + scripts_dir = pkg_dir / "scripts" if scripts_dir.exists(): clear_folder(scripts_dir) + scripts_dir.mkdir(parents=True, exist_ok=True) + + if not on_windows: + install_file(project_dir / "docker/common/install_tensorrt.sh", + scripts_dir / "install_tensorrt.sh") if not cpp_only: @@ -1243,6 +1273,10 @@ def add_arguments(parser: ArgumentParser): help="Additional make targets to build. Example: \"target_1 target_2\"", nargs="+", default=[]) + parser.add_argument( + "--trt_root", + default="/usr/local/tensorrt", + help="Directory containing TensorRT headers and libraries") parser.add_argument("--nccl_root", help="Directory containing NCCL headers and libraries") parser.add_argument("--nixl_root", @@ -1279,6 +1313,9 @@ def add_arguments(parser: ArgumentParser): help= "Install the built binary by creating symbolic links instead of copying files" ) + parser.add_argument("--benchmarks", + action="store_true", + help="Build the benchmarks for the C++ runtime") parser.add_argument("--micro_benchmarks", action="store_true", help="Build the micro benchmarks for C++ components") diff --git a/scripts/check_auto_deploy_imports.py b/scripts/check_auto_deploy_imports.py index 3d8477cf42a6..1f6335173fc1 100755 --- a/scripts/check_auto_deploy_imports.py +++ b/scripts/check_auto_deploy_imports.py @@ -15,7 +15,7 @@ instead. These rules keep auto_deploy's source tree portable: it can be copied -verbatim into the standalone ``paragraf`` package without rewriting any +verbatim into the standalone ``llmc`` package without rewriting any in-package import statements. """ diff --git a/scripts/generate_config_database_tests.py b/scripts/generate_config_database_tests.py index 74c7050cd89f..10fdfbc66e26 100644 --- a/scripts/generate_config_database_tests.py +++ b/scripts/generate_config_database_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -71,17 +71,12 @@ def generate_server_name(recipe: Recipe) -> str: """Generate a unique server name from recipe.""" model_slug = recipe.model.replace("/", "_").replace("-", "_").replace(".", "_") name = f"{model_slug}_{recipe.isl}_{recipe.osl}_conc{recipe.concurrency}_gpu{recipe.num_gpus}" - if recipe.profile: - name = f"{name}_{recipe.profile}" return name def generate_client_name(recipe: Recipe) -> str: """Generate client config name.""" - name = f"con{recipe.concurrency}_isl{recipe.isl}_osl{recipe.osl}" - if recipe.profile: - name = f"{name}_{recipe.profile}" - return name + return f"con{recipe.concurrency}_isl{recipe.isl}_osl{recipe.osl}" def recipe_to_server_config(recipe: Recipe, llm_api_config: dict) -> dict: diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index fc73c29d8d90..70fa8145c55c 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -32,8 +32,6 @@ from examples.configs.database.database import ( # noqa: E402 CURATED_LIST_PATH, DATABASE_LIST_PATH, - PROFILE_DISPLAY_NAMES, - PROFILE_ORDER, CuratedRecipeList, RecipeList, assign_profile, @@ -92,9 +90,9 @@ "display_name": "Kimi-K2-Thinking (NVFP4)", "url": "https://huggingface.co/nvidia/Kimi-K2-Thinking-NVFP4", }, - "MiniMaxAI/MiniMax-M3-MXFP8": { - "display_name": "MiniMax-M3 (MXFP8)", - "url": "https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8", + "MiniMaxAI/MiniMax-M3": { + "display_name": "MiniMax-M3 (BF16)", + "url": "https://huggingface.co/MiniMaxAI/MiniMax-M3", }, } @@ -116,9 +114,6 @@ class RecipeRow: config_filename: str config_github_url: str config_raw_url: str - profile: str | None - validated_trtllm_commit: str | None - validated_trtllm_version: str | None @dataclass(frozen=True) @@ -172,7 +167,7 @@ def build_curated_rows(yaml_path: Path) -> list[CuratedRow]: return rows -def build_rows(yaml_path: Path) -> list[RecipeRow]: +def build_rows(yaml_path) -> list[RecipeRow]: recipe_list = RecipeList.from_yaml(Path(yaml_path)) model_groups = defaultdict(lambda: defaultdict(list)) @@ -194,7 +189,7 @@ def build_rows(yaml_path: Path) -> list[RecipeRow]: for key in sorted_keys: entries = subgroups[key] - entries.sort(key=lambda x: (x.concurrency, PROFILE_ORDER.get(x.profile, -1))) + entries.sort(key=lambda x: x.concurrency) for idx, entry in enumerate(entries): gpu = entry.gpu @@ -205,11 +200,7 @@ def build_rows(yaml_path: Path) -> list[RecipeRow]: conc = entry.concurrency config_path = entry.config_path - performance_profile = ( - PROFILE_DISPLAY_NAMES[entry.profile] - if entry.profile - else assign_profile(len(entries), idx, conc) - ) + profile = assign_profile(len(entries), idx, conc) command = f"trtllm-serve {model} --config ${{TRTLLM_DIR}}/{config_path}" @@ -233,23 +224,18 @@ def build_rows(yaml_path: Path) -> list[RecipeRow]: concurrency=conc, config_path=config_path, gpu_display=gpu_display, - performance_profile=performance_profile, + performance_profile=profile, command=command, config_filename=config_filename, config_github_url=config_github_url, config_raw_url=config_raw_url, - profile=entry.profile, - validated_trtllm_commit=entry.validated_trtllm_commit, - validated_trtllm_version=entry.validated_trtllm_version, ) ) return rows -def generate_json( - yaml_path: Path, output_file: Path, curated_yaml_path: Path | None = None -) -> None: +def generate_json(yaml_path: Path, output_file: Path, curated_yaml_path: Path | None = None): rows = build_rows(yaml_path) source_path = Path(yaml_path) @@ -280,9 +266,7 @@ def generate_json( payload = { "source": source, "models": models, - "entries": [ - {key: value for key, value in asdict(row).items() if value is not None} for row in rows - ], + "entries": [asdict(r) for r in rows], "curated_entries": curated_entries, } diff --git a/scripts/generate_duration.py b/scripts/generate_duration.py new file mode 100644 index 000000000000..fec429c305e6 --- /dev/null +++ b/scripts/generate_duration.py @@ -0,0 +1,75 @@ +import argparse +import glob +import json +import os + +# Parse command-line arguments +parser = argparse.ArgumentParser(description="Generate test duration file.") +parser.add_argument( + "--duration-file", + type=str, + default="new_test_duration.json", + help="Path to the output duration file (default: new_test_duration.json)") +parser.add_argument( + "--cluster", + type=str, + default=None, + help="Cluster name (e.g. 'aws_dfw'). When set, writes " + "tests/integration/defs/.test_durations_ relative to the " + "repo root instead of --duration-file.") +args = parser.parse_args() + +# Define the directory containing the test result folders +TEST_RESULTS_DIR = os.getcwd() + +# Define the output file paths +FULL_RESULT_LOG = "full_result.log" +if args.cluster: + _repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + NEW_TEST_DURATION = os.path.join(_repo_root, "tests", "integration", "defs", + f".test_durations_{args.cluster}") +else: + NEW_TEST_DURATION = args.duration_file + +# Step 1: Prepare full_result.log +with open(FULL_RESULT_LOG, 'w') as full_result_file: + print(f"TEST_RESULTS_DIR: {TEST_RESULTS_DIR}") + for report_csv in glob.glob(os.path.join(TEST_RESULTS_DIR, '*/report.csv')): + print(f"Processing {report_csv}...") + with open(report_csv, 'r') as csv_file: + for line in csv_file: + if 'passed' in line: + full_result_file.write(line) + +# Step 2: Generate new_test_duration.json +test_durations = {} + +# Read the full_result.log file line by line +with open(FULL_RESULT_LOG, 'r') as file: + for line in file: + # Extract the first column and the last column + columns = line.strip().split(',') + first_column = columns[0] + last_column = columns[-1] + + # Remove from left to first '/' in the first column + test_name = first_column.split('/', 1)[-1] + # Replace \"\" with \" and ]\" with ] in case we got these in names from report.csv + # which will broken the json parse + test_name = test_name.replace(']\"', ']').replace('\"\"', '\"') + + try: + last_column = float(last_column) + except ValueError: + print( + f"Warning: Could not convert {last_column} to float. Skipping.") + continue + + # Add to the test duration dictionary + test_durations[test_name] = last_column + +# Write the test durations to the new test duration file +with open(NEW_TEST_DURATION, 'w') as file: + json.dump(test_durations, file, indent=3) + +print(f"Test durations have been written to {NEW_TEST_DURATION}") diff --git a/scripts/get_wheel_from_package.py b/scripts/get_wheel_from_package.py index f8dc16652361..cb604482c27b 100644 --- a/scripts/get_wheel_from_package.py +++ b/scripts/get_wheel_from_package.py @@ -78,11 +78,27 @@ def get_wheel_from_package(arch, artifact_path, timeout): build_dir = llm_root / "build" build_dir.mkdir(parents=True, exist_ok=True) + benchmarks_dir = llm_root / "cpp" / "build" / "benchmarks" + benchmarks_dir.mkdir(parents=True, exist_ok=True) + wheel_files = glob.glob(str(tmp_dir / "tensorrt_llm*.whl")) for wheel_file in wheel_files: shutil.move(wheel_file, str(build_dir)) print(f"Moved wheel file: {wheel_file} -> {build_dir}") + benchmark_files = [ + "bertBenchmark", "gptManagerBenchmark", "disaggServerBenchmark" + ] + + for benchmark in benchmark_files: + src_path = tmp_dir / "benchmarks" / "cpp" / benchmark + if src_path.exists(): + dst_path = benchmarks_dir / benchmark + shutil.copy2(src_path, dst_path) + print(f"Copied benchmark file: {src_path} -> {dst_path}") + else: + print(f"Warning: Benchmark file not found: {src_path}") + shutil.rmtree(tmp_dir) if os.path.exists(tarfile_name): diff --git a/scripts/test_to_stage_mapping.py b/scripts/test_to_stage_mapping.py index 24f84c458f51..04626131b658 100644 --- a/scripts/test_to_stage_mapping.py +++ b/scripts/test_to_stage_mapping.py @@ -47,7 +47,7 @@ def _load_tests_file(path: str) -> List[str]: # Regex to parse Jenkins stage configurations from Groovy files -# Matches patterns like: "Stage-Name": ["platform", "yaml_file", split_id, split_count, gpu_count, node_count, runWithSbatch] +# Matches patterns like: "Stage-Name": ["platform", "yaml_file", split_id, split_count, gpu_count] # # Pattern breakdown: # "(?P[^"]+)" - Captures stage name in quotes (group 'stage') @@ -56,12 +56,10 @@ def _load_tests_file(path: str) -> List[str]: # "[^"]+" - Matches platform string in quotes (ignored) # ,\s* - Matches comma with optional whitespace # "(?P[^"]+)" - Captures yaml filename in quotes (group 'yml') -# (?:,\s*(?:\d+|true|false))* - Matches zero or more comma-separated numbers or -# booleans (split_id, split_count, gpu_count, node_count, runWithSbatch) +# (?:,\s*\d+)* - Matches zero or more comma-separated numbers (split_id, split_count, gpu_count) # \s*\] - Matches closing bracket with optional whitespace _STAGE_RE = re.compile( - r'"(?P[^"]+)"\s*:\s*\["[^"]+",\s*"(?P[^"]+)"(?:,\s*(?:\d+|true|false))*\s*\]' -) + r'"(?P[^"]+)"\s*:\s*\["[^"]+",\s*"(?P[^"]+)"(?:,\s*\d+)*\s*\]') def _extract_terms(entry): @@ -87,8 +85,6 @@ def _parse_stage_mapping(path): yaml_to_stages = defaultdict(list) with open(path, 'r') as f: for line in f: - if line.lstrip().startswith('//'): - continue m = _STAGE_RE.search(line) if m: stage = m.group('stage') diff --git a/security_scanning/docs/poetry.lock b/security_scanning/docs/poetry.lock index 3eb5d3e3c43b..9f572444dc2f 100644 --- a/security_scanning/docs/poetry.lock +++ b/security_scanning/docs/poetry.lock @@ -828,14 +828,14 @@ files = [ [[package]] name = "soupsieve" -version = "2.9.1" +version = "2.8.4" description = "A modern CSS selector implementation for Beautiful Soup." optional = false -python-versions = ">=3.10" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "soupsieve-2.9.1-py3-none-any.whl", hash = "sha256:4f4477399246b7a0c720a88ca2454b11cd6bb9ae4c9d170140786e916776c14c"}, - {file = "soupsieve-2.9.1.tar.gz", hash = "sha256:c33e6605bbc71dd628b00c632d58ae607c22bade247e52553928f83bbb75b4ba"}, + {file = "soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65"}, + {file = "soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e"}, ] [[package]] @@ -1025,14 +1025,14 @@ test = ["flake8", "mypy", "pytest"] [[package]] name = "sphinxcontrib-mermaid" -version = "2.1.0" +version = "2.0.3" description = "Mermaid diagrams in your Sphinx-powered docs" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "sphinxcontrib_mermaid-2.1.0-py3-none-any.whl", hash = "sha256:417cd144ec4b28852f46ba653f02ce8e538881c812111671a4c30344e87f2112"}, - {file = "sphinxcontrib_mermaid-2.1.0.tar.gz", hash = "sha256:13c5f9ac395cb6abf403eca34e228dc9fb3a30c9d960dbf3e40e9a8cef969549"}, + {file = "sphinxcontrib_mermaid-2.0.3-py3-none-any.whl", hash = "sha256:f001ed36a55c108f6221a2d656a441c487ee30651b54db72b7c752a20c7a66e8"}, + {file = "sphinxcontrib_mermaid-2.0.3.tar.gz", hash = "sha256:a6865ef6b65b225c5403a3170de63a04a07227cada11a4a71a6b87b4f9ed185a"}, ] [package.dependencies] @@ -1198,4 +1198,4 @@ packaging = ">=24.0" [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "7af684263a6aff0d081aa130bca302515fac11c985e02a678c589f234cd97194" +content-hash = "6229137ad4354a8ad47953bd64635fd210251b2104fc49baad0d65705acc73cc" diff --git a/security_scanning/docs/pyproject.toml b/security_scanning/docs/pyproject.toml index 7bddd72b848b..30472f07fd21 100644 --- a/security_scanning/docs/pyproject.toml +++ b/security_scanning/docs/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "sphinx-copybutton (>=0.5.2,<0.6.0)", "autodoc-pydantic (>=2.2.0,<3.0.0)", "sphinx-togglebutton (>=0.4.5,<0.5.0)", - "sphinxcontrib-mermaid (>=2.1.0,<3.0.0)" + "sphinxcontrib-mermaid (>=2.0.3,<3.0.0)" ] diff --git a/security_scanning/examples/apps/poetry.lock b/security_scanning/examples/apps/poetry.lock index ab1377d046d2..cbf25147aac4 100644 --- a/security_scanning/examples/apps/poetry.lock +++ b/security_scanning/examples/apps/poetry.lock @@ -278,14 +278,14 @@ files = [ [[package]] name = "openai" -version = "2.46.0" +version = "2.45.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "openai-2.46.0-py3-none-any.whl", hash = "sha256:672381db55efb3a1e2610f29304c130cccdd0b319bace4d492b2443cb64c1e7c"}, - {file = "openai-2.46.0.tar.gz", hash = "sha256:0421e0735ac41451cad894af4cddf0435bfbf8cbc538ac0e15b3c062f2ddc06a"}, + {file = "openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777"}, + {file = "openai-2.45.0.tar.gz", hash = "sha256:10d34ca9c5643bce775852fddbfc172505cb1d4de1ccd101696c3ecff358765d"}, ] [package.dependencies] @@ -474,14 +474,14 @@ files = [ [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] diff --git a/security_scanning/examples/auto_deploy/poetry.lock b/security_scanning/examples/auto_deploy/poetry.lock index 2861ddc525f9..e8a3dc2f2f0c 100644 --- a/security_scanning/examples/auto_deploy/poetry.lock +++ b/security_scanning/examples/auto_deploy/poetry.lock @@ -110,131 +110,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -543,14 +543,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.6.0" +version = "1.5.6" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, + {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, ] [[package]] @@ -799,14 +799,14 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -1030,30 +1030,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "(sys_platform == \"linux\" or sys_platform == \"win32\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -1108,14 +1116,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1976,7 +1984,6 @@ description = "Fast numerical expression evaluator for NumPy" optional = false python-versions = ">=3.10" groups = ["main"] -markers = "python_version == \"3.10\"" files = [ {file = "numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8"}, {file = "numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a"}, @@ -2040,66 +2047,6 @@ files = [ [package.dependencies] numpy = ">=1.23.0" -[[package]] -name = "numexpr" -version = "2.14.2" -description = "Fast numerical expression evaluator for NumPy" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "numexpr-2.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2aa65ddc2243f19c6915f34ee0978b4a2df20f297230a793c4ee6d55f3472599"}, - {file = "numexpr-2.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf959e6df6cb603611c034b6cba7b03a361be0ad0b80b73f163fab95f5ccbb7f"}, - {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d534ecb456a4ae3995f99c8a5deb469bfff05d4ec610a7885c175c881d12f710"}, - {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f41170e9d0dbba76851e35d80cfa9f4ca5fe78628c5bf24d941cf3364940ab7a"}, - {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6acafb2fdbeaaa6681a8f1a1d8b3f7dcd33704baace7057b950754b258be7c43"}, - {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7ca9e71195b36cc7aeafe97347549e1e1c1e889ff700238782ef6447651ec26d"}, - {file = "numexpr-2.14.2-cp311-cp311-win32.whl", hash = "sha256:779129d50974e7d6d6581d322f75b8f8375e96215b6861a2d5460347997ef649"}, - {file = "numexpr-2.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:2f132777d7d425471c458af5617e023402f13f5006301eacf8a1a6e7118ea70c"}, - {file = "numexpr-2.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:f1de5c88515ed9fbcad42699a0e2b5821b4d0f0adb0da6fb7e009e5cb19d8493"}, - {file = "numexpr-2.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:606ceaf5722e295ef965ca591736fc26d9e5f13ad950a479e64cead1947f8a3d"}, - {file = "numexpr-2.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:790da022539fe7c37dc893acf530a91c2ca6964d7ba11f464131383729d058f3"}, - {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:327be9ee62251c173236dc620147ff2d0e732a32f5bad918d78a10082f502f63"}, - {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6a5d8fc7016bf6f6e1808b011510aa7c3bd75ec1407f7650874ec591db59f5e"}, - {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4b1ff261c3e69c4c59578d3a9ca6132603619d38ae1abe73325563bed3b9bbaf"}, - {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b8384592c49cb15a91caa54e2cd84d1ce18edb7af030bb76cd29b52e5dc155d"}, - {file = "numexpr-2.14.2-cp312-cp312-win32.whl", hash = "sha256:41cdeacf1b4e51c1143983ea61fcee68139ca47222b55a9265b4fa73826c4260"}, - {file = "numexpr-2.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:8fc55d14bcf17b3fe69213bea14f999451892b4690717008c66f2edfd6a085ce"}, - {file = "numexpr-2.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:806a4471310fe20aa7cb1b2816a6f5e508073a1ad1c2e18041b83e57066fad6a"}, - {file = "numexpr-2.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0741efbd75c284e709b0fd430c85c31982b44c9962922ba8a9cbbea1bf413321"}, - {file = "numexpr-2.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92b00c78664070e3af155c6be713a0a5d75d598647ce32a5609adb79a8f961d3"}, - {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:149ab5744a5222f07b1d60455c4021c754d395e44938944ac7c7c2495f7feb54"}, - {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd2f5882a66a7792aa6614c68831aa20085b499d41422aedd001080624ebb14c"}, - {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:375d8bee15be42dab22100a0a3de05fe6689a2de853eca012858768a9a7e02ab"}, - {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c1ffaf805d8636c3f95d0996517ecf9684c9ac62d768030ca78d1d00af2b3504"}, - {file = "numexpr-2.14.2-cp313-cp313-win32.whl", hash = "sha256:449a57fb9d38de136e742b1fc429572b42f29778f1d695c3fe50ffec9d3c9a71"}, - {file = "numexpr-2.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:dd905922d7dce457947d54b84c7ac345cef37332b724445e159a5a1a2080ce2b"}, - {file = "numexpr-2.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:b02738853b9b5b8a995f6c680f8f6ef33e8f419395b8fa380e38690495fdb911"}, - {file = "numexpr-2.14.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:76e87c7bd70d721ce4d418e81f4fb7ecf9e7e67d7cea8102527b07fd3d3facf9"}, - {file = "numexpr-2.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:939c89f613b814e64bb568859397dc9f99b219c3ef681a72fb99a86e435262f9"}, - {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b20c1c55aba7812ff2f2c6a50006425d02282fabb1eaf8d75fe638ffcf6deb02"}, - {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bac00898930f962f360c3d763a8e2273fc931f65a1759ff1bf64b3cf13d65aee"}, - {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:022e61a3d5dbf5807746264b62126d1c2c24057ad90052478a4d4482ab2555c2"}, - {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1d4593e2c6fa060cd7441e8b6ef25c16321a6be2144b3c82d1e00885f1fb6e94"}, - {file = "numexpr-2.14.2-cp314-cp314-win32.whl", hash = "sha256:66f3b125b1104241322811de87918724d6709bf082dc0703722d0cecb7b29e82"}, - {file = "numexpr-2.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:ef576a1cded27ba2f3129bc3c42df452a1c498072680d560793f98b0024cd7e6"}, - {file = "numexpr-2.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8274c51ae1842948f3ae7fe6951a23dcf4ddcbeeaff3737e978e7740b754662d"}, - {file = "numexpr-2.14.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:f3526699350f94c6277fb16863773a1af9defd95a6f78bbd69b1f0338fd94756"}, - {file = "numexpr-2.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:91e7928435f14fcb351c0157000bce65122b897cc8b0df6bcc48251f25850a6d"}, - {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c66925deb968f0b5280f723e2bb5918c11e6be2ca60e9e1530006286ab44031d"}, - {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a404c9a55902572eec810068d06b79a7c99e96f0400f5a7d73f39dff5ec5e371"}, - {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:44dc6b1dfa9abcbfc9917297f0d2af7c87c16b6ecd45747a8e70f54399a3a2f9"}, - {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93233040f4bed3bce5abb0c2d20aeb1074511f29cbaa9c14828f86bcfa44d321"}, - {file = "numexpr-2.14.2-cp314-cp314t-win32.whl", hash = "sha256:2aceefa08f8f86317fa6e8fe9f6dc20d24ab8365d715be4a26306acf406d2dbe"}, - {file = "numexpr-2.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:cd684ac9daa539fcdac3437678834797b29d7780cfaad71111745132d466d51f"}, - {file = "numexpr-2.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:2ef72de3d3dd466cb0c435cae7141c99b0f8091b1eae9d03dcb38690f56c3f79"}, - {file = "numexpr-2.14.2.tar.gz", hash = "sha256:e7144e83ea9e581f2273e0304f15836736c4e470e2bd2e378ce617662a1ca278"}, -] - -[package.dependencies] -numpy = ">=1.26.0" - [[package]] name = "numpy" version = "2.2.6" @@ -3223,126 +3170,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -4150,14 +4097,14 @@ pyyaml = ["pyyaml"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -4206,14 +4153,14 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.17.0,<1.19)", "pre-commit", "pytest [[package]] name = "transformers" -version = "5.14.1" +version = "5.13.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, - {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, + {file = "transformers-5.13.1-py3-none-any.whl", hash = "sha256:53f0ea8aa397e29244c2377ba981bcaf0c87adcf44fbdd447ef6306522afcacd"}, + {file = "transformers-5.13.1.tar.gz", hash = "sha256:1e2452d6778a7482158df5d5dacf6bf775d5b2fdcfce33caaf7f6b0e5f3e3397"}, ] [package.dependencies] @@ -4229,29 +4176,29 @@ typer = "*" [package.extras] accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] chat-template = ["jinja2 (>=3.1.0)"] codecarbon = ["codecarbon (>=2.8.1)"] deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] docs = ["hf-doc-builder"] integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] num2words = ["num2words"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] timm = ["timm (>=1.0.23)"] torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] @@ -4310,14 +4257,14 @@ test = ["packaging", "pytest (>=6.0.1)", "python-dateutil (>=2.8.0,<3.0.0)", "py [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -4652,116 +4599,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock index 144c4bc70db3..b03fdf0b61c4 100644 --- a/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock +++ b/security_scanning/examples/llm-eval/lm-eval-harness/poetry.lock @@ -59,131 +59,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -464,14 +464,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.6.0" +version = "1.5.6" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, + {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, ] [[package]] @@ -678,14 +678,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -885,30 +885,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "(sys_platform == \"linux\" or sys_platform == \"win32\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -963,14 +971,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1746,7 +1754,6 @@ description = "Fast numerical expression evaluator for NumPy" optional = false python-versions = ">=3.10" groups = ["main"] -markers = "python_version == \"3.10\"" files = [ {file = "numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8"}, {file = "numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a"}, @@ -1810,66 +1817,6 @@ files = [ [package.dependencies] numpy = ">=1.23.0" -[[package]] -name = "numexpr" -version = "2.14.2" -description = "Fast numerical expression evaluator for NumPy" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "numexpr-2.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2aa65ddc2243f19c6915f34ee0978b4a2df20f297230a793c4ee6d55f3472599"}, - {file = "numexpr-2.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf959e6df6cb603611c034b6cba7b03a361be0ad0b80b73f163fab95f5ccbb7f"}, - {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d534ecb456a4ae3995f99c8a5deb469bfff05d4ec610a7885c175c881d12f710"}, - {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f41170e9d0dbba76851e35d80cfa9f4ca5fe78628c5bf24d941cf3364940ab7a"}, - {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6acafb2fdbeaaa6681a8f1a1d8b3f7dcd33704baace7057b950754b258be7c43"}, - {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7ca9e71195b36cc7aeafe97347549e1e1c1e889ff700238782ef6447651ec26d"}, - {file = "numexpr-2.14.2-cp311-cp311-win32.whl", hash = "sha256:779129d50974e7d6d6581d322f75b8f8375e96215b6861a2d5460347997ef649"}, - {file = "numexpr-2.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:2f132777d7d425471c458af5617e023402f13f5006301eacf8a1a6e7118ea70c"}, - {file = "numexpr-2.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:f1de5c88515ed9fbcad42699a0e2b5821b4d0f0adb0da6fb7e009e5cb19d8493"}, - {file = "numexpr-2.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:606ceaf5722e295ef965ca591736fc26d9e5f13ad950a479e64cead1947f8a3d"}, - {file = "numexpr-2.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:790da022539fe7c37dc893acf530a91c2ca6964d7ba11f464131383729d058f3"}, - {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:327be9ee62251c173236dc620147ff2d0e732a32f5bad918d78a10082f502f63"}, - {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6a5d8fc7016bf6f6e1808b011510aa7c3bd75ec1407f7650874ec591db59f5e"}, - {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4b1ff261c3e69c4c59578d3a9ca6132603619d38ae1abe73325563bed3b9bbaf"}, - {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b8384592c49cb15a91caa54e2cd84d1ce18edb7af030bb76cd29b52e5dc155d"}, - {file = "numexpr-2.14.2-cp312-cp312-win32.whl", hash = "sha256:41cdeacf1b4e51c1143983ea61fcee68139ca47222b55a9265b4fa73826c4260"}, - {file = "numexpr-2.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:8fc55d14bcf17b3fe69213bea14f999451892b4690717008c66f2edfd6a085ce"}, - {file = "numexpr-2.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:806a4471310fe20aa7cb1b2816a6f5e508073a1ad1c2e18041b83e57066fad6a"}, - {file = "numexpr-2.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0741efbd75c284e709b0fd430c85c31982b44c9962922ba8a9cbbea1bf413321"}, - {file = "numexpr-2.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92b00c78664070e3af155c6be713a0a5d75d598647ce32a5609adb79a8f961d3"}, - {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:149ab5744a5222f07b1d60455c4021c754d395e44938944ac7c7c2495f7feb54"}, - {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd2f5882a66a7792aa6614c68831aa20085b499d41422aedd001080624ebb14c"}, - {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:375d8bee15be42dab22100a0a3de05fe6689a2de853eca012858768a9a7e02ab"}, - {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c1ffaf805d8636c3f95d0996517ecf9684c9ac62d768030ca78d1d00af2b3504"}, - {file = "numexpr-2.14.2-cp313-cp313-win32.whl", hash = "sha256:449a57fb9d38de136e742b1fc429572b42f29778f1d695c3fe50ffec9d3c9a71"}, - {file = "numexpr-2.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:dd905922d7dce457947d54b84c7ac345cef37332b724445e159a5a1a2080ce2b"}, - {file = "numexpr-2.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:b02738853b9b5b8a995f6c680f8f6ef33e8f419395b8fa380e38690495fdb911"}, - {file = "numexpr-2.14.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:76e87c7bd70d721ce4d418e81f4fb7ecf9e7e67d7cea8102527b07fd3d3facf9"}, - {file = "numexpr-2.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:939c89f613b814e64bb568859397dc9f99b219c3ef681a72fb99a86e435262f9"}, - {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b20c1c55aba7812ff2f2c6a50006425d02282fabb1eaf8d75fe638ffcf6deb02"}, - {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bac00898930f962f360c3d763a8e2273fc931f65a1759ff1bf64b3cf13d65aee"}, - {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:022e61a3d5dbf5807746264b62126d1c2c24057ad90052478a4d4482ab2555c2"}, - {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1d4593e2c6fa060cd7441e8b6ef25c16321a6be2144b3c82d1e00885f1fb6e94"}, - {file = "numexpr-2.14.2-cp314-cp314-win32.whl", hash = "sha256:66f3b125b1104241322811de87918724d6709bf082dc0703722d0cecb7b29e82"}, - {file = "numexpr-2.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:ef576a1cded27ba2f3129bc3c42df452a1c498072680d560793f98b0024cd7e6"}, - {file = "numexpr-2.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8274c51ae1842948f3ae7fe6951a23dcf4ddcbeeaff3737e978e7740b754662d"}, - {file = "numexpr-2.14.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:f3526699350f94c6277fb16863773a1af9defd95a6f78bbd69b1f0338fd94756"}, - {file = "numexpr-2.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:91e7928435f14fcb351c0157000bce65122b897cc8b0df6bcc48251f25850a6d"}, - {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c66925deb968f0b5280f723e2bb5918c11e6be2ca60e9e1530006286ab44031d"}, - {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a404c9a55902572eec810068d06b79a7c99e96f0400f5a7d73f39dff5ec5e371"}, - {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:44dc6b1dfa9abcbfc9917297f0d2af7c87c16b6ecd45747a8e70f54399a3a2f9"}, - {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93233040f4bed3bce5abb0c2d20aeb1074511f29cbaa9c14828f86bcfa44d321"}, - {file = "numexpr-2.14.2-cp314-cp314t-win32.whl", hash = "sha256:2aceefa08f8f86317fa6e8fe9f6dc20d24ab8365d715be4a26306acf406d2dbe"}, - {file = "numexpr-2.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:cd684ac9daa539fcdac3437678834797b29d7780cfaad71111745132d466d51f"}, - {file = "numexpr-2.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:2ef72de3d3dd466cb0c435cae7141c99b0f8091b1eae9d03dcb38690f56c3f79"}, - {file = "numexpr-2.14.2.tar.gz", hash = "sha256:e7144e83ea9e581f2273e0304f15836736c4e470e2bd2e378ce617662a1ca278"}, -] - -[package.dependencies] -numpy = ">=1.26.0" - [[package]] name = "numpy" version = "2.2.6" @@ -3042,126 +2989,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -3934,14 +3881,14 @@ pyyaml = ["pyyaml"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -3974,14 +3921,14 @@ dev = ["twine"] [[package]] name = "transformers" -version = "5.14.1" +version = "5.13.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, - {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, + {file = "transformers-5.13.1-py3-none-any.whl", hash = "sha256:53f0ea8aa397e29244c2377ba981bcaf0c87adcf44fbdd447ef6306522afcacd"}, + {file = "transformers-5.13.1.tar.gz", hash = "sha256:1e2452d6778a7482158df5d5dacf6bf775d5b2fdcfce33caaf7f6b0e5f3e3397"}, ] [package.dependencies] @@ -3997,29 +3944,29 @@ typer = "*" [package.extras] accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] chat-template = ["jinja2 (>=3.1.0)"] codecarbon = ["codecarbon (>=2.8.1)"] deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] docs = ["hf-doc-builder"] integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] num2words = ["num2words"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] timm = ["timm (>=1.0.23)"] torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] @@ -4078,14 +4025,14 @@ test = ["packaging", "pytest (>=6.0.1)", "python-dateutil (>=2.8.0,<3.0.0)", "py [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -4347,116 +4294,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock index 061acc014d73..0871b91e2b22 100644 --- a/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm-6b/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1774,126 +1782,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2095,14 +2103,14 @@ blobfile = ["blobfile (>=3)"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2356,116 +2364,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock index 061acc014d73..0871b91e2b22 100644 --- a/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm2-6b/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1774,126 +1782,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2095,14 +2103,14 @@ blobfile = ["blobfile (>=3)"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2356,116 +2364,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock index 061acc014d73..0871b91e2b22 100644 --- a/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock +++ b/security_scanning/examples/models/contrib/chatglm3-6b-32k/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1774,126 +1782,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2095,14 +2103,14 @@ blobfile = ["blobfile (>=3)"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2356,116 +2364,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock index a97f66ae7a9a..ba30259eb5f4 100644 --- a/security_scanning/examples/models/contrib/hyperclovax/poetry.lock +++ b/security_scanning/examples/models/contrib/hyperclovax/poetry.lock @@ -97,14 +97,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.6.0" +version = "1.5.6" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, + {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, ] [[package]] @@ -199,14 +199,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -263,30 +263,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -341,14 +349,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1282,14 +1290,14 @@ scipy = ["scipy"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/internlm/poetry.lock b/security_scanning/examples/models/contrib/internlm/poetry.lock index 7c3431c948e4..66374311f86f 100644 --- a/security_scanning/examples/models/contrib/internlm/poetry.lock +++ b/security_scanning/examples/models/contrib/internlm/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1756,126 +1764,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2003,14 +2011,14 @@ files = [ [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2264,116 +2272,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/jais/poetry.lock b/security_scanning/examples/models/contrib/jais/poetry.lock index d533be390fbe..0f548f4c56af 100644 --- a/security_scanning/examples/models/contrib/jais/poetry.lock +++ b/security_scanning/examples/models/contrib/jais/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1756,126 +1764,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2003,14 +2011,14 @@ files = [ [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2264,116 +2272,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/skywork/poetry.lock b/security_scanning/examples/models/contrib/skywork/poetry.lock index d533be390fbe..0f548f4c56af 100644 --- a/security_scanning/examples/models/contrib/skywork/poetry.lock +++ b/security_scanning/examples/models/contrib/skywork/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1756,126 +1764,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2003,14 +2011,14 @@ files = [ [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2264,116 +2272,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/contrib/smaug/poetry.lock b/security_scanning/examples/models/contrib/smaug/poetry.lock index d533be390fbe..0f548f4c56af 100644 --- a/security_scanning/examples/models/contrib/smaug/poetry.lock +++ b/security_scanning/examples/models/contrib/smaug/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1756,126 +1764,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2003,14 +2011,14 @@ files = [ [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2264,116 +2272,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/core/mixtral/poetry.lock b/security_scanning/examples/models/core/mixtral/poetry.lock index 9098a1cf6042..90ddd0e3ce42 100644 --- a/security_scanning/examples/models/core/mixtral/poetry.lock +++ b/security_scanning/examples/models/core/mixtral/poetry.lock @@ -196,14 +196,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.6.0" +version = "1.5.6" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, + {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, ] [[package]] @@ -261,14 +261,14 @@ sanitizer = ["nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\ [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -313,30 +313,38 @@ tqdm = ["tqdm"] [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\" and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -1020,126 +1028,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -1344,14 +1352,14 @@ pyyaml = ["pyyaml"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/core/nemotron/poetry.lock b/security_scanning/examples/models/core/nemotron/poetry.lock index bd7844944998..6b9f6e06bcde 100644 --- a/security_scanning/examples/models/core/nemotron/poetry.lock +++ b/security_scanning/examples/models/core/nemotron/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1756,126 +1764,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -1931,14 +1939,14 @@ files = [ [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2192,116 +2200,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/core/qwen2audio/poetry.lock b/security_scanning/examples/models/core/qwen2audio/poetry.lock index 9638d51df1a0..ca13c5686007 100644 --- a/security_scanning/examples/models/core/qwen2audio/poetry.lock +++ b/security_scanning/examples/models/core/qwen2audio/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -523,14 +523,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -730,30 +730,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -808,14 +816,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1831,126 +1839,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2267,14 +2275,14 @@ testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2288,14 +2296,14 @@ telegram = ["envwrap", "requests"] [[package]] name = "transformers" -version = "5.14.1" +version = "5.13.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, - {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, + {file = "transformers-5.13.1-py3-none-any.whl", hash = "sha256:53f0ea8aa397e29244c2377ba981bcaf0c87adcf44fbdd447ef6306522afcacd"}, + {file = "transformers-5.13.1.tar.gz", hash = "sha256:1e2452d6778a7482158df5d5dacf6bf775d5b2fdcfce33caaf7f6b0e5f3e3397"}, ] [package.dependencies] @@ -2311,29 +2319,29 @@ typer = "*" [package.extras] accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] chat-template = ["jinja2 (>=3.1.0)"] codecarbon = ["codecarbon (>=2.8.1)"] deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] docs = ["hf-doc-builder"] integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] num2words = ["num2words"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] timm = ["timm (>=1.0.23)"] torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] @@ -2356,14 +2364,14 @@ transformers = ">=4.26.1" [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -2614,116 +2622,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/models/core/qwenvl/poetry.lock b/security_scanning/examples/models/core/qwenvl/poetry.lock index 6e4c4391ba5f..e639fd0c6925 100644 --- a/security_scanning/examples/models/core/qwenvl/poetry.lock +++ b/security_scanning/examples/models/core/qwenvl/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -420,14 +420,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.6.0" +version = "1.5.6" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, + {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, ] [[package]] @@ -623,14 +623,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -830,30 +830,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "(sys_platform == \"linux\" or sys_platform == \"win32\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -908,14 +916,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -2488,126 +2496,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -3061,14 +3069,14 @@ scipy = ["scipy"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -3082,14 +3090,14 @@ telegram = ["envwrap", "requests"] [[package]] name = "transformers" -version = "5.14.1" +version = "5.13.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, - {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, + {file = "transformers-5.13.1-py3-none-any.whl", hash = "sha256:53f0ea8aa397e29244c2377ba981bcaf0c87adcf44fbdd447ef6306522afcacd"}, + {file = "transformers-5.13.1.tar.gz", hash = "sha256:1e2452d6778a7482158df5d5dacf6bf775d5b2fdcfce33caaf7f6b0e5f3e3397"}, ] [package.dependencies] @@ -3105,29 +3113,29 @@ typer = "*" [package.extras] accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] chat-template = ["jinja2 (>=3.1.0)"] codecarbon = ["codecarbon (>=2.8.1)"] deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] docs = ["hf-doc-builder"] integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] num2words = ["num2words"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] timm = ["timm (>=1.0.23)"] torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] @@ -3178,14 +3186,14 @@ tutorials = ["matplotlib", "pandas", "tabulate"] [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -3436,116 +3444,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/ngram/poetry.lock b/security_scanning/examples/ngram/poetry.lock index 4056b73df8e6..53e70e9e8011 100644 --- a/security_scanning/examples/ngram/poetry.lock +++ b/security_scanning/examples/ngram/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -499,14 +499,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -706,30 +706,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -784,14 +792,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1771,126 +1779,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -1946,14 +1954,14 @@ files = [ [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2207,116 +2215,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/quantization/poetry.lock b/security_scanning/examples/quantization/poetry.lock index 121b56a9bddd..8a057abb99d1 100644 --- a/security_scanning/examples/quantization/poetry.lock +++ b/security_scanning/examples/quantization/poetry.lock @@ -26,131 +26,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -475,14 +475,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -682,30 +682,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -760,14 +768,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1801,126 +1809,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2165,14 +2173,14 @@ testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2186,14 +2194,14 @@ telegram = ["envwrap", "requests"] [[package]] name = "transformers" -version = "5.14.1" +version = "5.13.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, - {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, + {file = "transformers-5.13.1-py3-none-any.whl", hash = "sha256:53f0ea8aa397e29244c2377ba981bcaf0c87adcf44fbdd447ef6306522afcacd"}, + {file = "transformers-5.13.1.tar.gz", hash = "sha256:1e2452d6778a7482158df5d5dacf6bf775d5b2fdcfce33caaf7f6b0e5f3e3397"}, ] [package.dependencies] @@ -2209,29 +2217,29 @@ typer = "*" [package.extras] accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] chat-template = ["jinja2 (>=3.1.0)"] codecarbon = ["codecarbon (>=2.8.1)"] deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] docs = ["hf-doc-builder"] integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] num2words = ["num2words"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] timm = ["timm (>=1.0.23)"] torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] @@ -2254,14 +2262,14 @@ transformers = ">=4.26.1" [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -2512,116 +2520,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/ray_orchestrator/poetry.lock b/security_scanning/examples/ray_orchestrator/poetry.lock index 2e4c4e3441f7..3442d2db7973 100644 --- a/security_scanning/examples/ray_orchestrator/poetry.lock +++ b/security_scanning/examples/ray_orchestrator/poetry.lock @@ -14,131 +14,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -572,14 +572,14 @@ files = [ [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -724,14 +724,14 @@ files = [ [[package]] name = "google-api-core" -version = "2.32.0" +version = "2.31.0" description = "Google API client core library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_api_core-2.32.0-py3-none-any.whl", hash = "sha256:ae1f0d58a6c8869350bf469f8eb3092e7f8c494a942d9525494afb6c162b0904"}, - {file = "google_api_core-2.32.0.tar.gz", hash = "sha256:2b33aad226b19272458c46abfe5c5a38d9531ece0c44502129a1463ce83674ac"}, + {file = "google_api_core-2.31.0-py3-none-any.whl", hash = "sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab"}, + {file = "google_api_core-2.31.0.tar.gz", hash = "sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2"}, ] [package.dependencies] @@ -747,14 +747,14 @@ grpc = ["grpcio (>=1.41.0,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version > [[package]] name = "google-auth" -version = "2.56.2" +version = "2.56.0" description = "Google Authentication Library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_auth-2.56.2-py3-none-any.whl", hash = "sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6"}, - {file = "google_auth-2.56.2.tar.gz", hash = "sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051"}, + {file = "google_auth-2.56.0-py3-none-any.whl", hash = "sha256:6e88c10217e07a92bfd01cac8ee99e32ccfb08414c3102e6c5b8d58f37a0d1e0"}, + {file = "google_auth-2.56.0.tar.gz", hash = "sha256:f90fa030b569a92654b9d690665a073841df33d57487be53db583a9a0867a553"}, ] [package.dependencies] @@ -1177,14 +1177,14 @@ files = [ [[package]] name = "opentelemetry-api" -version = "1.44.0" +version = "1.43.0" description = "OpenTelemetry Python API" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef"}, - {file = "opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a"}, + {file = "opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd"}, + {file = "opentelemetry_api-1.43.0.tar.gz", hash = "sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1"}, ] [package.dependencies] @@ -1192,31 +1192,31 @@ typing-extensions = ">=4.5.0" [[package]] name = "opentelemetry-exporter-prometheus" -version = "0.65b0" +version = "0.64b0" description = "Prometheus Metric Exporter for OpenTelemetry" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_exporter_prometheus-0.65b0-py3-none-any.whl", hash = "sha256:3b3d24b586d0ad9712c7b52b7d19c8a9dfbb318b9b284121b5f95e90ed019367"}, - {file = "opentelemetry_exporter_prometheus-0.65b0.tar.gz", hash = "sha256:2777cbf41c403c119e10f418fce5d645c956b47f673a2ee120285d1d0c6df2d5"}, + {file = "opentelemetry_exporter_prometheus-0.64b0-py3-none-any.whl", hash = "sha256:9979a15f8d007d442bc7a6e16f4cbde5e8e0c5e99689887ceb5f33da251f0655"}, + {file = "opentelemetry_exporter_prometheus-0.64b0.tar.gz", hash = "sha256:96fec79be9527cb9dc994d7e663051df35161eb936fe2d41954725e4595abbc1"}, ] [package.dependencies] opentelemetry-api = ">=1.12,<2.0" -opentelemetry-sdk = ">=1.44.0,<1.45.0" +opentelemetry-sdk = ">=1.43.0,<1.44.0" prometheus-client = ">=0.5.0,<1.0.0" [[package]] name = "opentelemetry-proto" -version = "1.44.0" +version = "1.43.0" description = "OpenTelemetry Python Proto" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56"}, - {file = "opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3"}, + {file = "opentelemetry_proto-1.43.0-py3-none-any.whl", hash = "sha256:c58f1f7ef84bc7dc2834016c0c37fe0081dde7ca9f6339be1970fbf9cdaaa90d"}, + {file = "opentelemetry_proto-1.43.0.tar.gz", hash = "sha256:224778df17e1f3fafeaaa21d874236ca5f6ffc2f86e0899298ec7351aac27924"}, ] [package.dependencies] @@ -1224,38 +1224,38 @@ protobuf = ">=5.0,<8.0" [[package]] name = "opentelemetry-sdk" -version = "1.44.0" +version = "1.43.0" description = "OpenTelemetry Python SDK" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad"}, - {file = "opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b"}, + {file = "opentelemetry_sdk-1.43.0-py3-none-any.whl", hash = "sha256:d1323a547c1ce69d6a069a17a44b7da82bb8b332051ecb074041f87642c86823"}, + {file = "opentelemetry_sdk-1.43.0.tar.gz", hash = "sha256:d8187c81c162df9913e4003dd6485f7390d9a24fc17026ec7387b8b8218b08e9"}, ] [package.dependencies] -opentelemetry-api = "1.44.0" -opentelemetry-semantic-conventions = "0.65b0" +opentelemetry-api = "1.43.0" +opentelemetry-semantic-conventions = "0.64b0" typing-extensions = ">=4.5.0" [package.extras] -file-configuration = ["opentelemetry-configuration (==0.65b0)"] +file-configuration = ["jsonschema (>=4.0)", "pyyaml (>=6.0)"] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.65b0" +version = "0.64b0" description = "OpenTelemetry Semantic Conventions" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb"}, - {file = "opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60"}, + {file = "opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6"}, + {file = "opentelemetry_semantic_conventions-0.64b0.tar.gz", hash = "sha256:72f76fb2d1582d9d033dd1fcd84532e961e6ff3d90d24ba6fabc72975a83864c"}, ] [package.dependencies] -opentelemetry-api = "1.44.0" +opentelemetry-api = "1.43.0" typing-extensions = ">=4.5.0" [[package]] @@ -1272,14 +1272,14 @@ files = [ [[package]] name = "platformdirs" -version = "4.11.0" +version = "4.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74"}, - {file = "platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0"}, + {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, + {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, ] [[package]] @@ -1684,20 +1684,24 @@ typing-extensions = ">=4.14.1" [[package]] name = "python-discovery" -version = "1.5.0" +version = "1.4.4" description = "Python interpreter discovery" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "python_discovery-1.5.0-py3-none-any.whl", hash = "sha256:70c4fc61b4e7404e44f01d6fc44a715c4d685ca6cea83d295922f05891877c98"}, - {file = "python_discovery-1.5.0.tar.gz", hash = "sha256:3e014c6327154d3dda27939a9a0dc9c5c000439f1906d3f303b48f984bd2ecef"}, + {file = "python_discovery-1.4.4-py3-none-any.whl", hash = "sha256:abebe9120b43453b68c908acfb1e72a19d1a959ed2cb620ad38fc57d08056dbe"}, + {file = "python_discovery-1.4.4.tar.gz", hash = "sha256:5cad33982d412c1f3ffb8f9ca4ea292c9680bca3942451d30b69c37fce53a4a3"}, ] [package.dependencies] filelock = ">=3.15.4" platformdirs = ">=4.3.6,<5" +[package.extras] +docs = ["furo (>=2025.12.19)", "sphinx (>=9.1)", "sphinx-autodoc-typehints (>=3.6.3)", "sphinxcontrib-mermaid (>=2)", "sphinxcontrib-towncrier (>=0.4)", "towncrier (>=25.8)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.5.4)", "pytest (>=8.3.5)", "pytest-mock (>=3.14)", "setuptools (>=75.1)"] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1783,30 +1787,30 @@ files = [ [[package]] name = "ray" -version = "2.56.1" +version = "2.56.0" description = "Ray provides a simple, universal API for building distributed applications." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ray-2.56.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:58b8c037a9f2b7b7866439cb6fe19d452ba3961f2b62d0a247dc3adf2ca45265"}, - {file = "ray-2.56.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:7005a13785c7478c3d183b6f1e7a344c069c4c4fb187707c4fc31ca8c84d8d9f"}, - {file = "ray-2.56.1-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:49aa1f8aa1799a6b63adc8e4edd600b949b2bb648461abedcbcccd56c8ef0f82"}, - {file = "ray-2.56.1-cp310-cp310-win_amd64.whl", hash = "sha256:ad87fa57c5c69c77edab14820226868842648de7181894dba4fb1e874a0ab27e"}, - {file = "ray-2.56.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5eebd776cd461edebc5874d2dfba3005147b6b45645d8a16a6549311c5efffea"}, - {file = "ray-2.56.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:4df45f33cc176b11c69191efdcfb4aceaeea22fecfa784f6485b6504ef8c8b26"}, - {file = "ray-2.56.1-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:e7003a47a42ef2ad33ec0b34dc5b6afb03f63fe59465e6f4c8f6d05492d9e4a6"}, - {file = "ray-2.56.1-cp311-cp311-win_amd64.whl", hash = "sha256:c102fb09e82c2e10828d5c21cfb744e27bca690b287c7d0a77d0d883c0c86907"}, - {file = "ray-2.56.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:44bc0000c5bfad85b2ff6e0ef91e95f901d1a2d2fdd72f94f08a046eb494cd61"}, - {file = "ray-2.56.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:8fdd6b096215906cf1f9acdc7898c9d6140606f2d27245778b8385a9f19e6cb0"}, - {file = "ray-2.56.1-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:e5d3173696831134c76bd09451dfe95c32d72c271253b0d6b09d2df9994aa660"}, - {file = "ray-2.56.1-cp312-cp312-win_amd64.whl", hash = "sha256:8052573ee5ef8c4fdd7aeb6a257c80542e69c48f3f6d117101f95c970ffdc7e2"}, - {file = "ray-2.56.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:93dedab658334af81877b6ed840c4e5f85e2e0b1b3641b20eaaee37cad835cc6"}, - {file = "ray-2.56.1-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:7fdc47de4e230f0db7c6c668a9e161f864dac548bd32230986e0b0c36e386eb5"}, - {file = "ray-2.56.1-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:81f2db202cc31bc3f5c4acf9ef154d10f1f39ece602d9d2d3108875c49bf01c3"}, - {file = "ray-2.56.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:76f6268a669c1d9910f1d7b73903de3ede9850ed94d3e28318d495559bd37d0e"}, - {file = "ray-2.56.1-cp314-cp314-manylinux2014_aarch64.whl", hash = "sha256:ea372c7f95b14f1f76f0bdd2abc9602c56e88bc30699d2228f78133e7749b2f1"}, - {file = "ray-2.56.1-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:ae91fe578fadea38c13a208a0fec27e1ced238ccdf5fff95ef3e30ac3071dec9"}, + {file = "ray-2.56.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f34b2345a47ad144292c1b34eeba2ed8d556078f7bd118d1adf2090d5199c843"}, + {file = "ray-2.56.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:15ea7ac36bfa3961c1eb2b2a099ed7dcf892f001f462920b6ec379ccafb038b0"}, + {file = "ray-2.56.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:58be75df2d4a6a85b5e514e4d3261760fe21cc09ba974ce22cc08a8e4e07c449"}, + {file = "ray-2.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:b837c5a905647c9b6a4f55061c2782437da24009e4eb4781db1c9c4badc84d0b"}, + {file = "ray-2.56.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:a9ad4e26941eb2f8dbd494ad07f9f2227143164c6114132b26b23ad4f20b1c6f"}, + {file = "ray-2.56.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:aea655831d25084cb343002a8e67a77b6aa552ddb776a65461d49f62884f096a"}, + {file = "ray-2.56.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:c5bf1a4384c0e2aa4420c95474b734d064cac354b0f00760be02d886afe96ca4"}, + {file = "ray-2.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e57781685bb4332edf8a7cfb1135ff53d50002b6a0504006e68365bcd26aa7c"}, + {file = "ray-2.56.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:684a427c50745989e92a332343f0812c93b8506f71c768b95b1eefc113492699"}, + {file = "ray-2.56.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:e1fd03c6ecc5fe4c31466569e41ce0a4faf26fb930798c9d1f1eb1f405a687c8"}, + {file = "ray-2.56.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:78ef34a71383c1fcf335e531e0e590867857fce9069f06ed351be6ce7a58fc50"}, + {file = "ray-2.56.0-cp312-cp312-win_amd64.whl", hash = "sha256:a8fc809dab6fc07cf05d45ea93a776c852c990512eb1fac0e3d15819fe6df10a"}, + {file = "ray-2.56.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:992047f50473b5bfea74c8f528f999968e0b4bc735af23ad476a0f4e04741aea"}, + {file = "ray-2.56.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:a0e9cfe92c88ab74abca23923c15a592f49bc7617ffdda4190daca7785a7c4f6"}, + {file = "ray-2.56.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:54d2725f8b65d9615c933fec5ec62e54a67b8f2e14286026fb014606253670ef"}, + {file = "ray-2.56.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:f38e03b77c53e3d94091aedb84b14efe7ee5b581d85b7d13925066bcd48c44a2"}, + {file = "ray-2.56.0-cp314-cp314-manylinux2014_aarch64.whl", hash = "sha256:c3a16d43d75283a3d64fa1d904a3adaf3f526f3f508f447505b8bb8dc70bad6c"}, + {file = "ray-2.56.0-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:73edb6fb5fd05481b1f358ac2e8a4c7f6a031d89f9b823d25a587ddb7529070f"}, ] [package.dependencies] @@ -1839,10 +1843,10 @@ virtualenv = {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, marke adag = ["cupy-cuda12x ; sys_platform != \"darwin\""] air = ["aiohttp (>=3.13.3)", "aiohttp_cors", "colorful", "fastapi (>=0.133.0)", "fsspec", "grpcio (>=1.42.0)", "mmh3", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=17.0.0)", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "requests", "smart_open", "starlette (>=1.0.1)", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] all = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi (>=0.133.0)", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "mmh3", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=17.0.0)", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "pyyaml", "requests", "scipy", "smart_open", "starlette (>=1.0.1)", "taskiq", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] -all-cpp = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi (>=0.133.0)", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "mmh3", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=17.0.0)", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "pyyaml", "ray-cpp (==2.56.1)", "requests", "scipy", "smart_open", "starlette (>=1.0.1)", "taskiq", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +all-cpp = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi (>=0.133.0)", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "mmh3", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=17.0.0)", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "pyyaml", "ray-cpp (==2.56.0)", "requests", "scipy", "smart_open", "starlette (>=1.0.1)", "taskiq", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] cgraph = ["cupy-cuda12x ; sys_platform != \"darwin\""] client = ["grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\""] -cpp = ["ray-cpp (==2.56.1)"] +cpp = ["ray-cpp (==2.56.0)"] data = ["fsspec", "numpy (>=1.20)", "pandas (>=2.2.3)", "pyarrow (>=17.0.0)"] default = ["aiohttp (>=3.13.3)", "aiohttp_cors", "colorful", "grpcio (>=1.42.0)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "requests", "smart_open", "virtualenv (>=20.0.24,!=20.21.1)"] llm = ["aiohttp (>=3.13.3)", "aiohttp_cors", "async-timeout ; python_version < \"3.11\"", "colorful", "fastapi (>=0.133.0)", "fsspec", "grpcio (>=1.42.0)", "hf_transfer", "jsonref (>=1.1.0)", "jsonschema", "meson", "mmh3", "ninja", "nixl (==1.1.0)", "nixl-cu13 (==1.1.0)", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas (>=2.2.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=17.0.0)", "pybind11", "pydantic (>=2.13.0,<3) ; python_version >= \"3.14\"", "pydantic (>=2.5.0,<3) ; python_version < \"3.14\"", "requests", "smart_open", "starlette (>=1.0.1)", "typer", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "vllm[audio] (==0.22.0)", "watchfiles"] @@ -2160,14 +2164,14 @@ files = [ [[package]] name = "smart-open" -version = "8.0.1" +version = "8.0.0" description = "Utils for streaming large files (S3, HDFS, GCS, SFTP, Azure Blob Storage, gzip, bz2, zst...)" optional = false python-versions = "<4.0,>=3.10" groups = ["main"] files = [ - {file = "smart_open-8.0.1-py3-none-any.whl", hash = "sha256:3e97f90e92a952cb57863dfe132082c400a52eeeb27c067692fb51dbcc5b0089"}, - {file = "smart_open-8.0.1.tar.gz", hash = "sha256:18b1c4496003c6902be17c15f032b5c319f307c89c6ae9e6b028b508bed8b2cf"}, + {file = "smart_open-8.0.0-py3-none-any.whl", hash = "sha256:ff4f395c9e86f23e27771dc4ba756ad4bd145f181859a782c50d64168485761b"}, + {file = "smart_open-8.0.0.tar.gz", hash = "sha256:5a2008d60688bd3b33c52e2ef666d3c60cf956e73e215de8c7b242cf56fdd1b2"}, ] [package.dependencies] @@ -2232,14 +2236,14 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" -version = "21.7.0" +version = "21.6.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "virtualenv-21.7.0-py3-none-any.whl", hash = "sha256:a8370c1c5530fbabf955e40b8fbbc68a431648b10f9433faa587db30a06e51dd"}, - {file = "virtualenv-21.7.0.tar.gz", hash = "sha256:7f9519b9432ff11b6e1a3e94061664efc2ff99ea21780e3cf4f6bd0a5da8b37c"}, + {file = "virtualenv-21.6.1-py3-none-any.whl", hash = "sha256:afe991df855715a2b2f60edfcc0107ef95a79fdfd8cb4cdaa71603d1c12e463b"}, + {file = "virtualenv-21.6.1.tar.gz", hash = "sha256:15f978b7cd329f24855ff4a0c4b4899cc7678589f49adbdcbbb4d3232e641128"}, ] [package.dependencies] @@ -2354,116 +2358,116 @@ dev = ["pytest", "setuptools"] [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] @@ -2474,4 +2478,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "e8827da46bd551d6f12e3b6f66cd75e4fdb3716ef47dd8160170552c64b142d6" +content-hash = "0d340bb8bd7c6f0d2b6e1b59b8633a4a890c075680cfaa3e97d1615acff94789" diff --git a/security_scanning/examples/ray_orchestrator/pyproject.toml b/security_scanning/examples/ray_orchestrator/pyproject.toml index 86dd96fb56ab..6724fffb5f98 100644 --- a/security_scanning/examples/ray_orchestrator/pyproject.toml +++ b/security_scanning/examples/ray_orchestrator/pyproject.toml @@ -7,7 +7,7 @@ authors = [ ] requires-python = ">=3.10,<3.13" dependencies = [ - "ray[default] (>=2.56.1,<3.0.0)" + "ray[default] (>=2.56.0,<3.0.0)" ] diff --git a/security_scanning/examples/serve/poetry.lock b/security_scanning/examples/serve/poetry.lock index a0a47a0ee2b1..9763fed10c4d 100644 --- a/security_scanning/examples/serve/poetry.lock +++ b/security_scanning/examples/serve/poetry.lock @@ -768,14 +768,14 @@ files = [ [[package]] name = "colorlog" -version = "6.11.0" +version = "6.10.1" description = "Add colours to the output of Python's logging module." optional = false python-versions = ">=3.6" groups = ["main"] files = [ - {file = "colorlog-6.11.0-py3-none-any.whl", hash = "sha256:f1e27d75aa2cb138f3f640c0e305b65b680ccbef6ecc034eba7e03494ffcd2a1"}, - {file = "colorlog-6.11.0.tar.gz", hash = "sha256:9d90fb53fa906c8970c18fbe46506bae1fb5f86b513b8f867db37e4ace9be7ae"}, + {file = "colorlog-6.10.1-py3-none-any.whl", hash = "sha256:2d7e8348291948af66122cff006c9f8da6255d224e7cf8e37d8de2df3bad8c9c"}, + {file = "colorlog-6.10.1.tar.gz", hash = "sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321"}, ] [package.dependencies] @@ -1012,14 +1012,14 @@ tests = ["pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "cyclopts" -version = "4.22.1" +version = "4.21.0" description = "Intuitive, easy CLIs based on type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cyclopts-4.22.1-py3-none-any.whl", hash = "sha256:9b614e231075aee9849c0bfd78f7611ab7adf417f16af5b9e42b9ed6e18c17d1"}, - {file = "cyclopts-4.22.1.tar.gz", hash = "sha256:49cd3779da7113a96ac5c23b151aa61ac9ae1b4b1fe813594d207ca843c97892"}, + {file = "cyclopts-4.21.0-py3-none-any.whl", hash = "sha256:ded3ddb15b0c815f44d245011fc4cdd5a4809a3bb8202869e9e02195a87c0e18"}, + {file = "cyclopts-4.21.0.tar.gz", hash = "sha256:477c18c791c924cca4836f79fce000a7bae45f551e340d9e1654e102c6d9ab9d"}, ] [package.dependencies] @@ -1189,14 +1189,14 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.139.2" +version = "0.139.0" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "fastapi-0.139.2-py3-none-any.whl", hash = "sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c"}, - {file = "fastapi-0.139.2.tar.gz", hash = "sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e"}, + {file = "fastapi-0.139.0-py3-none-any.whl", hash = "sha256:cf15e1e9e667ddb0ad63811e60bd11390d1aac838ca4a7a23f421807b2308189"}, + {file = "fastapi-0.139.0.tar.gz", hash = "sha256:99ab7b2d92223c76d6cf10757ab3f89d45b38267fc20b2a136cf02f6beac3145"}, ] [package.dependencies] @@ -1231,14 +1231,14 @@ dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4 [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -1641,30 +1641,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -1779,14 +1787,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -2312,65 +2320,65 @@ dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setup [[package]] name = "matplotlib" -version = "3.11.1" +version = "3.11.0" description = "Python plotting package" optional = false python-versions = ">=3.11" groups = ["main"] markers = "python_version >= \"3.11\"" files = [ - {file = "matplotlib-3.11.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b7cf158e7add54a8d51ac9b5a84abd6d4e13ed4951b4f25f1c5139f41c2addb2"}, - {file = "matplotlib-3.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d2ace7273b9a5061a3b420918a16fae1f2dc5dfee1abcc13aba71b5d94b1820c"}, - {file = "matplotlib-3.11.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee55e9041211bf84302ab55ec3965df18dd90ae19f8b58332a7feaf208bfe83"}, - {file = "matplotlib-3.11.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f4bdeea33a8d15a071dbfe6d119451b1d719c733ac666d65357082901a9099"}, - {file = "matplotlib-3.11.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b4c78ceb2f11bcac7389d305cda17aeb1f4586a857854ab5780bd3dd8dbfc407"}, - {file = "matplotlib-3.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:7f33a781e12b1e53b278deb2f5373c2e55ec4f10727be3440c0cfb5cda9f944f"}, - {file = "matplotlib-3.11.1-cp311-cp311-win_arm64.whl", hash = "sha256:67e4c3cd578c65ebd81bdc09a1b6592ceafee6dfafe116dc85dfcb647b5bbb18"}, - {file = "matplotlib-3.11.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e15ef41507f3d525f46154ac9e3ae785dacde9f20e593a25de8986267892ef74"}, - {file = "matplotlib-3.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21a67b961a6d597bca54fae826cd20695ba4a6e4d05424a08da6e13e3176fd6b"}, - {file = "matplotlib-3.11.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba8f811b8ddfac493734d6af0b2dff96919d0c28ca0d641858dab4262777c6ea"}, - {file = "matplotlib-3.11.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c52f7ad20ef476806ed212380b1d54d20310c8b86bdc2c9a68b51f0024a44472"}, - {file = "matplotlib-3.11.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b14eb22961fe865efb0e4ff167e333e428908b00115a8d800ccb65ee108e481"}, - {file = "matplotlib-3.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:88a2a27dd9691ae448dfae4b26f59036be90c3c28757edd3553a29559d00859f"}, - {file = "matplotlib-3.11.1-cp312-cp312-win_arm64.whl", hash = "sha256:480194afceca4df2f137c2721227d3cba67121fbf4397b69cee7f83714b0a58a"}, - {file = "matplotlib-3.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6771b0cd7838c6a857a7209814158c0ad09bfef878db3033dd82d70ad101f191"}, - {file = "matplotlib-3.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2abdee5ffa2fe11b2d19f7a5c63b785fb7c28cc46c7bc1814156341d9d1a33e1"}, - {file = "matplotlib-3.11.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0a19dcf73406d3746d25a5ed42d713604c9a3e024d129b102852b0d941cb9f3"}, - {file = "matplotlib-3.11.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7389b77ed2ab0552f46d9a90b81b7b8e6dfcdc42adc36c37a0865799843e0e3e"}, - {file = "matplotlib-3.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c90be0b73568da4f662afac580956a76e308437e641b4a45aa08925eeb67d95f"}, - {file = "matplotlib-3.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:68408341f2312836fbbdf6b3c78047f65b2d8752f5fd221c3e72d348f5b34f8b"}, - {file = "matplotlib-3.11.1-cp313-cp313-win_arm64.whl", hash = "sha256:0c1f44890d435c1b4ef52f701ad5828cb450ea97bcc83918fda6be74965d6cd2"}, - {file = "matplotlib-3.11.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:5e510088c27a89d53580a752f959146893563e63c330e161d159b0fee652af6f"}, - {file = "matplotlib-3.11.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:1524e2bdd48a93557aa47ddcfe9c225dfdd57d5a01a5c49128c20f0632980ee1"}, - {file = "matplotlib-3.11.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:11664c551345553db92e61cae6cf1376f138f8c47cafdf13b64b18f3e3e9e464"}, - {file = "matplotlib-3.11.1-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e1f8922ba31959cf6a9dfb51be64b7f7bc582801a3957dc0c2f3afcd3537adf"}, - {file = "matplotlib-3.11.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83235693abde86e5e0129998f80ee39fc7f58e6d56a88fafb28a9278833e9d5f"}, - {file = "matplotlib-3.11.1-cp313-cp313t-win_amd64.whl", hash = "sha256:9a076f4fc5cdc43fdf510f5981418d25c2db4973418d9f22d8bb3dc8045ada78"}, - {file = "matplotlib-3.11.1-cp313-cp313t-win_arm64.whl", hash = "sha256:216fbb93a74add02ddb4cb38ef5348f59ac00b3e84567eaf16598772d40e150a"}, - {file = "matplotlib-3.11.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:30c492d4ba9448595b6fd8708c6725963f8148e25c0d8842948da5b05f0ee8d3"}, - {file = "matplotlib-3.11.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ac104be2768ffdd8655db9e71b768cbb45f2b9aa7b450cf1595e8f65d3822319"}, - {file = "matplotlib-3.11.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6be943cb68bc6660ead58c55b3aa6366cba2ef7feb06460fbcce32360376f19f"}, - {file = "matplotlib-3.11.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5af0dcda57d471440a7b5b623e70e0a61003518443d9098f211a96ecfbbc25be"}, - {file = "matplotlib-3.11.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3d3fd84082b1afbd9398466c81309e20045be20d48fe0fb18c43504d164cbbb2"}, - {file = "matplotlib-3.11.1-cp314-cp314-win_amd64.whl", hash = "sha256:9601a1e90be21e4884c53b4f3dc3ee0544654946f9975258d691f1c2e2f119c6"}, - {file = "matplotlib-3.11.1-cp314-cp314-win_arm64.whl", hash = "sha256:ae30c6109848ac0f9fa36c5d6270938487614c47ba31860bd5361266dabc5685"}, - {file = "matplotlib-3.11.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dadfe80797174e2984aae3be0b77594a3c72d2c0a40fbd4a0de48d2728caf3ae"}, - {file = "matplotlib-3.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:89b193b255f4f6f7948dbcee3691f4f341ab05d9a8874a67b45ddb4182922eda"}, - {file = "matplotlib-3.11.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191163532cdefcb1571ca38a6d7e6474baccde64495783e6ba47aa07ec4b9bbb"}, - {file = "matplotlib-3.11.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fdf1c818ab05d0e74002091ddaf414478a3a449ec9d51c8976d45be7e3a01e2"}, - {file = "matplotlib-3.11.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b937b9dba5f5f6c1e31c47abe2186c865c0914fd18f2ce0dfc39c9adcef5951d"}, - {file = "matplotlib-3.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f2912f647f3fbe1ccf085f91e213936f9101bead81a5e670565b1f1b3712f4fb"}, - {file = "matplotlib-3.11.1-cp314-cp314t-win_arm64.whl", hash = "sha256:54d47b8ae8b579633a3902ca5b4ad6c1e132a5626d64447b2e22a66394e79987"}, - {file = "matplotlib-3.11.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:427258425f9a3fc4ed79a91f9e9b9aaf5a82cb6571e85dc14063cc6fbb993741"}, - {file = "matplotlib-3.11.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1ac697e591c11b6ad04679a73c2d2f9980fe9d9f0311fb414a2e329706343dfb"}, - {file = "matplotlib-3.11.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4b9ac2f1f607ecda2af90a5232beee2af7582fce1cc30c4b6a1b012dc21ee99"}, - {file = "matplotlib-3.11.1.tar.gz", hash = "sha256:69647db5746941c793d6e445a4cd349323ffb87d9cc958c2ad84a659b4832d30"}, + {file = "matplotlib-3.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f857524b442f0f36e641868ce2171aafa88cb0bc0644f4e1d8a5df9b32649fef"}, + {file = "matplotlib-3.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:57baa92fdc82948ed716eae6d2579d4d6f40965cd8d2f416755b4a72580a3233"}, + {file = "matplotlib-3.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630eee0e67d35cce2019a0e670719f4816e3b86aff0fa72729f6c69786fceb45"}, + {file = "matplotlib-3.11.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5106c444d0bf966eee2853548c03772af4ab7199118e086c62fbac8ccb07c055"}, + {file = "matplotlib-3.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4d7aea652b58e686444079be3376ef546bffa1eee9b9bb9c472b9fcf6cf410d3"}, + {file = "matplotlib-3.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:70a5b3e9a5dab708c0f039709ae7c68d5b4d254e291ef76492cdba230c8bb5e4"}, + {file = "matplotlib-3.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:3d68266213e73823ac3be90615bab0cf31f88851e114cdb1dd25dacf3b01e1a7"}, + {file = "matplotlib-3.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:06b5872e9cf11adc8f589ded3ce11bc3e1061ad498259664fabc1f6615beb918"}, + {file = "matplotlib-3.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0515d495124be3124340e59f164d901ed4484e2246a5b74cfa483cac3b80bd97"}, + {file = "matplotlib-3.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be5f93a1d21981bfb802ded0d77a0caa92d4342a47d45754fac77e314a506344"}, + {file = "matplotlib-3.11.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41635d7909d19e52e924a521dde6d8f670b0f53ab1d0e8c331fa831554f681d1"}, + {file = "matplotlib-3.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94f5000f67ca9faa300863ea17f8bce9175cb67b88bec4bc7780502d53dd7c9e"}, + {file = "matplotlib-3.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ac6f1ef39f3d0f9e2463303013094992cdbe0f85f43bc54155bc472b2042768e"}, + {file = "matplotlib-3.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:9dd11fb612ce7bc60b1de5b4fc87ff959d22317b5de42aabf392f66f97af22eb"}, + {file = "matplotlib-3.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6ce3b839b34ae1f430b4616893a2945a2999debaa7e94e7e29a2a8bbf286f7b5"}, + {file = "matplotlib-3.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373db8f91214e8ccaf35ac833cc1dd59dd961e148bbd55dd027141591dde1313"}, + {file = "matplotlib-3.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be152b7570324dc8d01574cc9474dd2d803237acf528bcbb5b211fa347461a09"}, + {file = "matplotlib-3.11.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:126f256df600652d7e4b394cf3164ff75210a00038f287c95a012a6f58d0e83f"}, + {file = "matplotlib-3.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:03acfeddf87b0dddb11b081ef7740ad445a3ca8bcb6b8e3011b08f2cf802b75c"}, + {file = "matplotlib-3.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:ab3722f04f3ff34c23b5012c5873d2894174e06c3822fcdac3610965a5ac7d06"}, + {file = "matplotlib-3.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:c945824670fb8915b4ac879e5e61f3c58e0913022f70a0de4c082b17372f8771"}, + {file = "matplotlib-3.11.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3489c3dc487669b4a980bc3068f87856de7a1564248d3f6c629efb2a58b03f24"}, + {file = "matplotlib-3.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6a98f5476ce784a50ce09998f4ae1e6a9f25043cef8a480c98949902eda74620"}, + {file = "matplotlib-3.11.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:565af866fd63e4bd3f987d580afe27c44c2552a3b3305f4ecbb85133601ea6f3"}, + {file = "matplotlib-3.11.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b3e64dea5062c570f04358e2711859f3531b459f29516274fbad889079e4f3"}, + {file = "matplotlib-3.11.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:942b37c5db1899610bd1543ce8e13e4ecff9a4633e7f63bb6aa9205d2644ebd1"}, + {file = "matplotlib-3.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:c08e649a6313e1291e713623b97a38e5bb4aa580b2a100a94a3309bc6b9c8eb3"}, + {file = "matplotlib-3.11.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2746cd2c113742ff6ce37a864c5ac5fd7aa644568f445e66166e457ac78e40e0"}, + {file = "matplotlib-3.11.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3338e3e3de128cf50d0d2fb92a122815daf9c755bd882a474343c05f8fd7ec79"}, + {file = "matplotlib-3.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:25c2e5455efd8d99f41fb79871a31feb7d301569642e332ec58d72cfe9282bc3"}, + {file = "matplotlib-3.11.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9695457a467ff86d23f35037a43deb6f1134dd6d3e2ac8ce1e2087cff09ffb9"}, + {file = "matplotlib-3.11.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19c16c61dea63b3582918503e6b294193961261d9daa806d4ae2151f1ad05430"}, + {file = "matplotlib-3.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2d72ea8b7924f3cb955e61518d21e43b3df1e6c8a793b480a0c1214f185d30ba"}, + {file = "matplotlib-3.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:1c02da0a629dfa9debf52725ea06866b74c1fb70a895bae05e4493d34074f9f2"}, + {file = "matplotlib-3.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:aa55d73b3117d4b07f959cd9eb6f69b375d8df3414139c479388e551aa5d999d"}, + {file = "matplotlib-3.11.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a9d8c6e7cd2f0ddf11d8d92e520dd1d9d2abb0cf6ac8831e338666c81e905847"}, + {file = "matplotlib-3.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:be050fcf32f729eda99f7f75a80bf67612ce16ab9ac1c23a387dcaede95cb70e"}, + {file = "matplotlib-3.11.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfabef0230d0697aa0d717385194dd41162e00207a68bf4abf94c2bf4c27dca0"}, + {file = "matplotlib-3.11.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644db30e759199443493ac5e5caec24fdb775a8f6123021f85ba47c4133c3cb"}, + {file = "matplotlib-3.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:15b0d160079cb10699a0e98b5989c70677b2df7cacdc62af67c30f2facec46d9"}, + {file = "matplotlib-3.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:446307e6b04b57b1f1239e228a1ec2af0d589a1008cebc3dfa3f5441d095cfb6"}, + {file = "matplotlib-3.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:652fb5696271d4c50f196d22a5ff4f8e4444c74f847423570d7dc0aa2bbd0159"}, + {file = "matplotlib-3.11.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:81ae77077a1e16d37a5b61096ccb07c8d90a99b518fa8256b8f21578932f2f62"}, + {file = "matplotlib-3.11.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ddef37840695f5eef65f9f070fe2d2f510f584c2156203f9f622a5b0584efffd"}, + {file = "matplotlib-3.11.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cf662e5ac5707658cb931e19972c4bd99f7b4f8b7bf79d3c821d239fa6b71e64"}, + {file = "matplotlib-3.11.0.tar.gz", hash = "sha256:68c0c7be01b30dcca3638934f7f591df73401235cbdbf0d1ab1c71e7db7f8b57"}, ] [package.dependencies] contourpy = ">=1.0.1" cycler = ">=0.10" -fonttools = ">=4.28.2" +fonttools = ">=4.22.0" kiwisolver = ">=1.3.1" numpy = ">=1.25" packaging = ">=20.0" @@ -3247,14 +3255,14 @@ xmp = ["defusedxml"] [[package]] name = "platformdirs" -version = "4.11.0" +version = "4.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74"}, - {file = "platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0"}, + {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, + {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, ] [[package]] @@ -4032,126 +4040,126 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -5061,14 +5069,14 @@ full = ["httpx (>=0.27.0,<0.29.0)", "httpx2 (>=2.0.0)", "itsdangerous", "jinja2" [[package]] name = "starlette-compress" -version = "1.8.0" +version = "1.7.1" description = "Compression middleware for Starlette - supporting ZStd, Brotli, and GZip" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "starlette_compress-1.8.0-py3-none-any.whl", hash = "sha256:3c991e21d31c7f845a004055db9ae3c7e79ed59cac34c8444e9e7e689456b623"}, - {file = "starlette_compress-1.8.0.tar.gz", hash = "sha256:4dc0077468a2276b6dfd1b4f946fb61b5465a41c996c010486248e65af15b4b6"}, + {file = "starlette_compress-1.7.1-py3-none-any.whl", hash = "sha256:cd229d64f93789f90137bc08391ca946639812f514c9f5db72ef232687753cea"}, + {file = "starlette_compress-1.7.1.tar.gz", hash = "sha256:f4df7aa6b0029ec5c4ae960040cd5d375563a4d3f7fc134bd108ebc0ed61536c"}, ] [package.dependencies] @@ -5275,14 +5283,14 @@ files = [ [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -5296,14 +5304,14 @@ telegram = ["envwrap", "requests"] [[package]] name = "transformers" -version = "5.14.1" +version = "5.13.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, - {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, + {file = "transformers-5.13.1-py3-none-any.whl", hash = "sha256:53f0ea8aa397e29244c2377ba981bcaf0c87adcf44fbdd447ef6306522afcacd"}, + {file = "transformers-5.13.1.tar.gz", hash = "sha256:1e2452d6778a7482158df5d5dacf6bf775d5b2fdcfce33caaf7f6b0e5f3e3397"}, ] [package.dependencies] @@ -5319,29 +5327,29 @@ typer = "*" [package.extras] accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] chat-template = ["jinja2 (>=3.1.0)"] codecarbon = ["codecarbon (>=2.8.1)"] deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] docs = ["hf-doc-builder"] integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] num2words = ["num2words"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] timm = ["timm (>=1.0.23)"] torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] @@ -5350,14 +5358,14 @@ vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -5651,121 +5659,121 @@ anyio = ">=3.0.0" [[package]] name = "websockets" -version = "16.1.1" +version = "16.1" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "websockets-16.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d"}, - {file = "websockets-16.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731"}, - {file = "websockets-16.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4"}, - {file = "websockets-16.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb"}, - {file = "websockets-16.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838"}, - {file = "websockets-16.1.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87"}, - {file = "websockets-16.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3"}, - {file = "websockets-16.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4"}, - {file = "websockets-16.1.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3"}, - {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b"}, - {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d"}, - {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d"}, - {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165"}, - {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc"}, - {file = "websockets-16.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a"}, - {file = "websockets-16.1.1-cp310-cp310-win32.whl", hash = "sha256:7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9"}, - {file = "websockets-16.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f"}, - {file = "websockets-16.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c"}, - {file = "websockets-16.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a"}, - {file = "websockets-16.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22"}, - {file = "websockets-16.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2"}, - {file = "websockets-16.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01"}, - {file = "websockets-16.1.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0"}, - {file = "websockets-16.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29"}, - {file = "websockets-16.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512"}, - {file = "websockets-16.1.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3"}, - {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57"}, - {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3"}, - {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648"}, - {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d"}, - {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be"}, - {file = "websockets-16.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81"}, - {file = "websockets-16.1.1-cp311-cp311-win32.whl", hash = "sha256:2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57"}, - {file = "websockets-16.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a"}, - {file = "websockets-16.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00"}, - {file = "websockets-16.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b"}, - {file = "websockets-16.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175"}, - {file = "websockets-16.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1"}, - {file = "websockets-16.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15"}, - {file = "websockets-16.1.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa"}, - {file = "websockets-16.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab"}, - {file = "websockets-16.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847"}, - {file = "websockets-16.1.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428"}, - {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf"}, - {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751"}, - {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f"}, - {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2"}, - {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383"}, - {file = "websockets-16.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3"}, - {file = "websockets-16.1.1-cp312-cp312-win32.whl", hash = "sha256:1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747"}, - {file = "websockets-16.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7"}, - {file = "websockets-16.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ab59169ace05dcb49a1d4118f0bde139557adf45091bd85747e36bf5de984dd1"}, - {file = "websockets-16.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5e3b7d601f6f84156b08cc4a5e541c2b50ad7b36cfc302b657a12477c904a5df"}, - {file = "websockets-16.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cd2ca96a082a36964aca83e992f72abeb61b7306c1a6cba4c7d06a7b93750cac"}, - {file = "websockets-16.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f5d497865f05bb222cab7016c6034542e84e5f29f49c6fd3f4939cda7197b5b8"}, - {file = "websockets-16.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bae954c382e013d5ea5b190d2830526bfa45ad121c326da0049b8c769f185db6"}, - {file = "websockets-16.1.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e09f753a169951eb4f28c2c774f71069304f66e7277e0f5a2892423599cfa854"}, - {file = "websockets-16.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:024193f8551a2b0eafbdd160911012c4e6c228c28430c84433253299a9e42d6a"}, - {file = "websockets-16.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:aabe464bfd13bd25f4821faf111da6fefdc389f870265a53105580e45b0a2e49"}, - {file = "websockets-16.1.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a28fcbc9b6baf54a2e23f8655f308e4ccc6afdd7266f8fe7954f320dcda0f785"}, - {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:79eace538c6a97e96d0d03d4f9d314f9677f5ed85a8a984992ffd90b13cb8a56"}, - {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:496af849a472b531f758dbd4d61338f5000538cb1a7b3d20d9d32a264517f509"}, - {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5283810d2646741a0d8da2aa733d6aefa0545809afccb2a5d105a26bc45125f1"}, - {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:4e3b680b1e0a27457e727a0d572fd81dffa87b6dbf8b228ab57da64f7d85aead"}, - {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:69159730a823dde3ea8d08783e8d47ef135a6d7e8d44eb127e32b321c9db8e3e"}, - {file = "websockets-16.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ed5bb271084b46530ee2ddc0410537a9961152c5ccba2fc98c5276d992ccba87"}, - {file = "websockets-16.1.1-cp313-cp313-win32.whl", hash = "sha256:cfb70b4eb56cac4da0a83588f3ad50d46beb0690391082f3d4e2d488c70b68ea"}, - {file = "websockets-16.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:d9531d9cbeac99af6f038fb1bc351403531f7d634a2c2e10e2f7c854c6ed5b68"}, - {file = "websockets-16.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:443aefe96b7fdb132e2a70806cca1f2af49bb3f28e47abcd7c2e9dcf4d8fa1b8"}, - {file = "websockets-16.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6456ff333092d509127d75a638cb411afae8ff17f092635015d1902efec8a293"}, - {file = "websockets-16.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fce6c48559c86d1ac3632ccb1bebc7d5442fbe79bd9bb0e40379ee54be2a4051"}, - {file = "websockets-16.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:92b820d345f7a3fc7b8163949ee92df910f290c3fc517b3d5301c78065adafe1"}, - {file = "websockets-16.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a606d9c24035242a3e256e9d5b77ed9cd6bccfcb7cf993e5ca3c0f6f68fb6a7"}, - {file = "websockets-16.1.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:414e596c75f74e0994084694189d7dc9229fb278e33064d6784b73ffbba3ca31"}, - {file = "websockets-16.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:536676848fc5961aca9d20389951f59169508f765637a172403dc5434d722fa0"}, - {file = "websockets-16.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:97fd3a0e8b53efa41970ac1dff3d8cf0d2884cadeb4caaf95db7ad1526926ee3"}, - {file = "websockets-16.1.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7b1b19636af86a3c7995d4d028dbe376f39b4bf31541146f9c123582a6c94562"}, - {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:41c8e77f17294c0ac18008a7309b99b34ee72247ef10b6dff4c3f8b5ac29896b"}, - {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9f63bcef7f4b02b06b35fc01c93b96c43b5e88e1e8868676caacf493d5a31f3a"}, - {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:dab9eb87869da2d6ed3af3f3adf28414baae6ec9d4df355ffc18889132f3436c"}, - {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:43e3a9fdd7cbf7ba6040c31fae0faf84ca1474fef777c4e37912f1540f854499"}, - {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:056ae37939ed7e9974f364f5864e76e49182622d8f9751ac1903c0d09b013985"}, - {file = "websockets-16.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a0eadbbf2c30f01efa58e1f110eb6fa293261f6b0b1aa38f7f48707107690af9"}, - {file = "websockets-16.1.1-cp314-cp314-win32.whl", hash = "sha256:195c978b065fa40910582464f99d6b15c8b314c68e0546549a55ed83f4735328"}, - {file = "websockets-16.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:4e8d01cc3bcae7bbf8167f944aeafefed590fae5693552bba9794a9df68371cc"}, - {file = "websockets-16.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0ffd3031ea8bda8d61762e84220186105ba3b748b3c8da2ae4f7816fac03e573"}, - {file = "websockets-16.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:84a2cef8deffbd9ab8ee0ea546a2a6a7030c28f44e6cdd4547dbfeb489eb8999"}, - {file = "websockets-16.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3df13f73af9b3b38ab1195eb299ecb67a4330c911c97ae04043ff74085728abe"}, - {file = "websockets-16.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:23253dd5bcae3f9aaee0a1d30967a8dbd52e5d3cff93a2e5b84df57b77d4750d"}, - {file = "websockets-16.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c1c5705e314449e3308872fe084b8571ce078ee4fc55a98a769bdefe5917392"}, - {file = "websockets-16.1.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:69e52d175a0a7d1e13b4b67ad41c560b7d98e8c6f6126eb0bda496c784faf8c7"}, - {file = "websockets-16.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f79c89b5eb034d1722938a891916582f8f7f503f58ca22518a63c3f2cd18499"}, - {file = "websockets-16.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39f2a024af5c345ffe8fcf1ee18c049c024c94df393bb09b044a6917c77bde43"}, - {file = "websockets-16.1.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:952303a7318d4cbe1011400839bb2051c9f84fa0a35923267f5daba34b15d458"}, - {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:249116b4a76063d930a46391ad56e135c286e4562a18309029fc2c73f4ed4c62"}, - {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:61922544a0587a13fd3f53e4c0e5e606510c7b0d9d22c8444e5fae22a06b38cb"}, - {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:46dcaa042cd1de6c59e7d9269fa63ff7572b6df40510600b678f0826b3c7af51"}, - {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:38565aca3e01ea8734e578fb2118dade0ecb0250533f29e22b8d1a7a196cf4d0"}, - {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:42f599f4d48c7e1a3338fdaac3acd075be3b3cf02d4b274f3bf2767aedd3d217"}, - {file = "websockets-16.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dcc04fedf83effaeb9cce98abc9469bb1b42ef85f03e01c8c1f4438ef7555737"}, - {file = "websockets-16.1.1-cp314-cp314t-win32.whl", hash = "sha256:8483c2096363120eea8b07c06ae7304d520f686665fffd4811fad423930a65d7"}, - {file = "websockets-16.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:bcce07e23e5769375158f5efdcdafa8d5cd014b93c6683865b840ed65b96f231"}, - {file = "websockets-16.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869"}, - {file = "websockets-16.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9"}, - {file = "websockets-16.1.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e"}, - {file = "websockets-16.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d"}, - {file = "websockets-16.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5"}, - {file = "websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3"}, - {file = "websockets-16.1.1.tar.gz", hash = "sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57"}, + {file = "websockets-16.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de72a9c611178b15557d98eabd3101c9663c4d68938510478a6d162f99afd213"}, + {file = "websockets-16.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:37b0e4d726ffea3776670092d3d13e1cb605076f036a695fd1259de0d9b9fe02"}, + {file = "websockets-16.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:00d50c0a27098fcb7ab47b3d99a1b1159b534dbcd959fbf05113ebc37e5f927b"}, + {file = "websockets-16.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1acb698bff1da1782b31aebd8d7a24d7d05453964abcd7d03dbf6e25893908e8"}, + {file = "websockets-16.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc2c453f3b5f99c56b16e233aad5299860558487d26adb2ed27a00c14ca24b8c"}, + {file = "websockets-16.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1a9f08a0728b0835f1c6abe1d9b746ab3de49b7336a0e1919cf96be1e76273eb"}, + {file = "websockets-16.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a089979d6173b27af18026c8d8b0077f83669a9169174482c4651e9f5739a5b6"}, + {file = "websockets-16.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a3c18dba232ec2b92a68579c9fed8ff5a18f853d1e09fc0b6ca3159e94f689fe"}, + {file = "websockets-16.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c1eb7df4170d5068892a8834fb5c07b9552353deb0dbeb0bff3820481ae4792"}, + {file = "websockets-16.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c522bd48e625b6d557aa228967258d6d3da031c4cc21d3352fb302479aa9ba0a"}, + {file = "websockets-16.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:d106396927a7f00b0f3a69215c3357f87bf0bca6844247121f7e8291e826a3b1"}, + {file = "websockets-16.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d71bed12909b8039955536e192867d02d76cd3797cedfd0facf822e7668636c3"}, + {file = "websockets-16.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:9c1cf6f9a936b030b5bed0e800c5ee32069338129084546baf5ff5014dc62fa9"}, + {file = "websockets-16.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:3fd3e6a7af2c8fcdcf4ffbeaf7f54a567b91a83267204187797f31faaa2a4efa"}, + {file = "websockets-16.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dddd27175bf640acae5561fa79b77e8ec71fc445816200523e5c19b6a556fb72"}, + {file = "websockets-16.1-cp310-cp310-win32.whl", hash = "sha256:cce36c80b3f2fede7942f1756d3d885fa6fa086766c8c1bcf00695ab80f0d51a"}, + {file = "websockets-16.1-cp310-cp310-win_amd64.whl", hash = "sha256:115fc4695b94bb855995b23fb1abcb66099a5995575d3d5bc5605a616c58d0eb"}, + {file = "websockets-16.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a9b1d7a63cba8e6b9b77e499a81eab29d31100298d090ad4507d1048c0b9cae0"}, + {file = "websockets-16.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bedbc5efeb96621aa2921d2d92608246691399418cac22acba427eb11877ea1f"}, + {file = "websockets-16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fd847ab82133015afe65d778e7966ab42dba16bd7ad2e5b8a7918db6539f3f94"}, + {file = "websockets-16.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e2fb33ccb16ee40a95cc676d7b0ff451a9a2632f11a0dbc2e666326892b2e1de"}, + {file = "websockets-16.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f15b6d9ea9c2eaf6ccab964a082b09bfa6634a495bb0c2e9e7ee6943f58976"}, + {file = "websockets-16.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:638cf57c48b4ad8ac1ff1e453f4f97db2426b690ddc111e6da96b27b4a340bc3"}, + {file = "websockets-16.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c1c85f61bc9d5eac57ce705d848dc2d2ce3680638300bf4e1da7d749e2cf4ce"}, + {file = "websockets-16.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:eeab6d27f51c7e579023c971f5e6dff200deadf01faf6831beaecd32052dfaef"}, + {file = "websockets-16.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2ed64e5a97b0b97a0b66e18bfe281317a75fbbd5afe692f939ea8d14a4292f2c"}, + {file = "websockets-16.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9b3b021d0ed4bc16eea9775f62c9fa71acdacba0fc790b38581754dedf29ca60"}, + {file = "websockets-16.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:6eb604a4167f0a0d53c2243dfc667a29f0b43c3436057184e070bb82a1000fa2"}, + {file = "websockets-16.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9a3f125e44c3e34d61d111652e608e0f5b85ce08c225c8d56ad0eb822fa40030"}, + {file = "websockets-16.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8fdf0b00d0d1f30d1f06a92cab46fe542eec3eb302a7aee7163f142d0780f216"}, + {file = "websockets-16.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:67b56828712f5fa7852de4c0265c28827311a657a4d275b7312ed0d1a918bee4"}, + {file = "websockets-16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39c7e7730be33b8f0cd6f0aa8e8c82f9cdd1813f159765e073b2ece65f4824b5"}, + {file = "websockets-16.1-cp311-cp311-win32.whl", hash = "sha256:c54fe94fb2f11e11b48920c5f971e298cec73ac35db56efe57a49db63dfc95d4"}, + {file = "websockets-16.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9f4fb9ae8b802e55609685db98382d48fd3feb1397804e1e774968dea0f28c7"}, + {file = "websockets-16.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b6aa3f7ad345cf3862c21f4fbf2ef5e14d911348476c2845e137c091fe3a3f0b"}, + {file = "websockets-16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b43fcfb521ac2f34ba80b7b8ea16303e4ad82dd8af667bf40839ad3a5d37b164"}, + {file = "websockets-16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2bd3e12cd9afbe2baedae0b1eeade8ba64329b60fe2f9abdc966bd10fd2c2ef5"}, + {file = "websockets-16.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f41979c8623df9bd30d949d82010a8fda5c56ff12cd8508a5b7272b6d4b53a"}, + {file = "websockets-16.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a24d1f35aef07d794a16c853c688e74956c50239bec37b4f2de080056046419b"}, + {file = "websockets-16.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0c64c024ddf7a35331b21fcddb562a039c275d2c82e8c2d12939e7da23997270"}, + {file = "websockets-16.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c3e99757f5baafe20fc598e202ea6f5b0b265186ad38d0a17bd8beca16296955"}, + {file = "websockets-16.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:353f3bc6e058ac1ccab4b3588e8598837a8c04cfc8351233e6d523be675d844c"}, + {file = "websockets-16.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0352f5b38b40e857b6428d468fa21dbb4dd4a567d933c26d9831b4efe1b92f43"}, + {file = "websockets-16.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70bd789afab579602968c39f21cb925466505f3edff22f0ae852bca54978a4f9"}, + {file = "websockets-16.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d0fb4b46f121eccd539353baebd1083a8767a9a351109453d1d1caecd1ba40c2"}, + {file = "websockets-16.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c14b6634af01541e4efe2954fd8f263386f7aa6d37c01e55dd8109fd17661452"}, + {file = "websockets-16.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a58532c49a851bcb481e58c1be23b315c17fe2fbbed509d75aeea12f543d2c15"}, + {file = "websockets-16.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4e969170c3b08e1d8dabd990fef1fa702c4233aeaabec33f871806e444f6a0e4"}, + {file = "websockets-16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ff9b000064b88787ba9f7a3cb2af2b68a658ca5aad76458a46469e7124b678a0"}, + {file = "websockets-16.1-cp312-cp312-win32.whl", hash = "sha256:b9f5d83f80f4d7c4bba6d97f3755ac05850c784dce0fd2ab371c4e41172f53ff"}, + {file = "websockets-16.1-cp312-cp312-win_amd64.whl", hash = "sha256:6852c9f653966c16109d3b6f31181fd734f7914927e3f0fa1117af7a18c9aa21"}, + {file = "websockets-16.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:b0232ed141cec3df2af5a3959a071c51f40036336b0d37e17faf9ef52fc73e47"}, + {file = "websockets-16.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a71b73d143991714144e159f767b698f03c4a70b8a65ae1733b650cff488045b"}, + {file = "websockets-16.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:187323204c3b2fc465e8fc2609e60437c521790cb9c1acb49c4c452a33e57f37"}, + {file = "websockets-16.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9dba74233c8c3ce368850818c98354dad2570f57231b3fd3bd00d7aa57628881"}, + {file = "websockets-16.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63339bc8c63c86a463177775cb7c677691f5bcfac7b3b2f01b286d42acd41600"}, + {file = "websockets-16.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:23e545ea8ae4263e37cdfd4e22a217f519e48e432728bc461185bbf585f38a83"}, + {file = "websockets-16.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2237081454846fb40403a80ba86d82e2038b9c45865ab96af0abe7d002a91045"}, + {file = "websockets-16.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5f5218de1ed047385ca53744caba9435d65f75d008364970a3fae95a05812cf9"}, + {file = "websockets-16.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:75c98e3920039d0edff03b74478ada504b7ce3a1bc406db2cabfca84320f7baf"}, + {file = "websockets-16.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1facd189d8190af30487a55b4c3688484dd50801628a3b5b2ccd26db08e67057"}, + {file = "websockets-16.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:cc0c6a6eef613c7da32d4fb068f82ef834b58134f6a16b54e6c1e5bf9529ab3d"}, + {file = "websockets-16.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ad9411eded8988b879be6038206698bf7106c85a78f642c004485bcb95be17eb"}, + {file = "websockets-16.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:cd68f0914f3b64694895bc5e9b14e8b447e41d7bf5ffaf989bb8dcb5e2dfdce7"}, + {file = "websockets-16.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fef2debfe7f7ebdda12176f26166f95b7af17af05ba06150fcf889032e0213e9"}, + {file = "websockets-16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3cd6c9b798218798f4bb7b2e71c38f0e744bb94ca537b13376f88019d46384d"}, + {file = "websockets-16.1-cp313-cp313-win32.whl", hash = "sha256:84c170c6869633536921e4474b1cce7254c0c9b0053ef5725f966cee47e718e4"}, + {file = "websockets-16.1-cp313-cp313-win_amd64.whl", hash = "sha256:bef52d327d70fa75dad93ee61ea2cb1d1489aca9f35c188833563f5a3b4df0a5"}, + {file = "websockets-16.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f881fca0a45dd6789939bd6637cd98169b92f1c3fdc78262f2cb9ec2cb1f324e"}, + {file = "websockets-16.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:30c379d5b207d3a7f0ba4c2e4602a895b0bcc63fb5f5371a4ae7fbddb03b672b"}, + {file = "websockets-16.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:98ab58a4faa72b46da0127ccc1931dcbfc0985b0778892300a092185910c4cbe"}, + {file = "websockets-16.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e9c4e369fc181b2d41a99e01477215cecdc8546a39f7d41a59cc0a7065a0b09"}, + {file = "websockets-16.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0704df094b2d5fa7f6f410925a594c2a5c9a09167731a76292e5410934208209"}, + {file = "websockets-16.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b22b1f4950f6ab7126623329c3b47b3b90a14c05db517f2db2a026ad6c928352"}, + {file = "websockets-16.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1ae4a686a662964a6671069f84f7f908cc3475e782227726b0c622c715962105"}, + {file = "websockets-16.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:856bdd638f8277f86465057bfdd4da097c73058fb0f9d2bd5baea29e2bf2d367"}, + {file = "websockets-16.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9003a1fde1c21a322a3ca3fa0c4bda8c639da81dbc925162766086643b05ba87"}, + {file = "websockets-16.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e947b1f5fdab045174306e3916785bf3ed537648acc1549827c08c33b10953"}, + {file = "websockets-16.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5dd0e666b5931c0509cf65714686a1c5126771e663a79ac5d40da4f58b1f9502"}, + {file = "websockets-16.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a0285df7925657ad65a65fb8dc330808bce082827538fd50ef45fa12d1fc5bca"}, + {file = "websockets-16.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:82d1c2cab3c133e9d059b3a5420bed9376bd30e21c185c63dda4ddadf6ddda47"}, + {file = "websockets-16.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c39907f1eaf11f6277def65aa02d68f30576b693d0c1ca332aafa3caa723ac6d"}, + {file = "websockets-16.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:45c5ea55446171949eb99fd34b771ceddd511ca21958d40d0197ced33159e5ee"}, + {file = "websockets-16.1-cp314-cp314-win32.whl", hash = "sha256:b8ef8b1c8d6bd029a475ac432e730fba2dfd456715d26c473e2a82291024b99c"}, + {file = "websockets-16.1-cp314-cp314-win_amd64.whl", hash = "sha256:7358ff21632b5d062707f73e859c824f1c3807e73d8ca25e71caca7c4cdcf145"}, + {file = "websockets-16.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d0f38f4c3e9b359e257c339c2cc1967ccaeedb102e57c1c986bdce4bf4f32268"}, + {file = "websockets-16.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3c3d2cbd1602593bad49bd86fa3fbb25407d87a3b4bf8857c0ac5ac4914e1901"}, + {file = "websockets-16.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:36069b74671e7e667f48a7484249f84c45a825a134c8b1bdc01875d0daa10d79"}, + {file = "websockets-16.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:587f83c2ce8a5d628e166384d77fa7f0ac69b9007d515ab442123e6615aa8da3"}, + {file = "websockets-16.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6db7972d52bc1b66cefe2246902e256cbaebc9ba8a45eac09343d7eb6671b2"}, + {file = "websockets-16.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e7d6014888a0632e1ed7a4095248bb3095232999447f2d83bfb1900987dd9ed9"}, + {file = "websockets-16.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9cb074d150e4ad2a77aa8a332c2be85f3f64f2681519d2570c1225c12c9821ff"}, + {file = "websockets-16.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d19c9067e1fe9490f974bffbc0e443b80a7674c5efb4980c429cc00771f07c5a"}, + {file = "websockets-16.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d440ff0c6c7469ad59c0a412c383c235935b43635e89425e3f6a0c36de90c31b"}, + {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8613129a2533f08de24505e69a3e403cedaadae49abdb043c4d170ca71b7e4bd"}, + {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:a5bf9c23f197b4ec88290fd5463f33db67362a1bb10f85fc2e8e7627f0ddab97"}, + {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:520b0fd0395f075febb283c76755af724ab9fd19dffa4f3bfd18cb4e622790a3"}, + {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:7143aa09a67e1c013be44e81a88dfe90fc6244198ab86c7edd064152cf619805"}, + {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:7acb811fad08e611755800d1560e395c67e11a6bd563598ea6abb319afb86938"}, + {file = "websockets-16.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c5cf88e3faa2f7931bc6baeee7599c97656a3f6ac7f831f4fccba233e141783a"}, + {file = "websockets-16.1-cp314-cp314t-win32.whl", hash = "sha256:589f8842521c8307684ce0b40ce4ad70c5e0aa46484c6f1225a94ef4b8970341"}, + {file = "websockets-16.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c0e0857c30bbbc2bb5c30687508f0b7ec19aa026cd9f2ff8424d0fee42dcc07"}, + {file = "websockets-16.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7289d899c79e763e6221c8dcb8959361cb43274418538d7c7ad16a43b01d12f9"}, + {file = "websockets-16.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:e22e9e3719f5131bd62da4db63c8da63eb8c91cc99e16c1cbd122f130e1ae07a"}, + {file = "websockets-16.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:83bdabafef431247e6b11a9aab8a0893fd8e82e1ed95b32e0373625b03ffce4a"}, + {file = "websockets-16.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b8d13ceabc5c60995f201b5211d76876e17e68706ebf5d3bc666b32eefff1a6"}, + {file = "websockets-16.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:81495f9c0085361c582efbc3207fb877174cfe03370f17d9cd70624404aa526f"}, + {file = "websockets-16.1-py3-none-any.whl", hash = "sha256:c5149dfe490ec7e5ee5dbf624c642fb725f93a5575c7f00ab594ca9eddb8dd81"}, + {file = "websockets-16.1.tar.gz", hash = "sha256:299468cbe42e2b9981134c7c51d99387d8a7bf562b00183b3eec53f882846dad"}, ] [[package]] @@ -5985,116 +5993,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/examples/trtllm-eval/poetry.lock b/security_scanning/examples/trtllm-eval/poetry.lock index 39f0bed379db..27cf9cafed09 100644 --- a/security_scanning/examples/trtllm-eval/poetry.lock +++ b/security_scanning/examples/trtllm-eval/poetry.lock @@ -59,131 +59,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -464,14 +464,14 @@ all = ["cuda-toolkit (==13.*)", "cuda-toolkit[cufile] (==13.*) ; sys_platform == [[package]] name = "cuda-pathfinder" -version = "1.6.0" +version = "1.5.6" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] markers = "platform_system == \"Linux\"" files = [ - {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, + {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, ] [[package]] @@ -678,14 +678,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -885,30 +885,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "(sys_platform == \"linux\" or sys_platform == \"win32\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"x86_64\") and (platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or sys_platform == \"linux\") and (platform_machine == \"aarch64\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\")" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -963,14 +971,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -1748,7 +1756,6 @@ description = "Fast numerical expression evaluator for NumPy" optional = false python-versions = ">=3.10" groups = ["main"] -markers = "python_version == \"3.10\"" files = [ {file = "numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8"}, {file = "numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a"}, @@ -1812,66 +1819,6 @@ files = [ [package.dependencies] numpy = ">=1.23.0" -[[package]] -name = "numexpr" -version = "2.14.2" -description = "Fast numerical expression evaluator for NumPy" -optional = false -python-versions = ">=3.11" -groups = ["main"] -markers = "python_version >= \"3.11\"" -files = [ - {file = "numexpr-2.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2aa65ddc2243f19c6915f34ee0978b4a2df20f297230a793c4ee6d55f3472599"}, - {file = "numexpr-2.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf959e6df6cb603611c034b6cba7b03a361be0ad0b80b73f163fab95f5ccbb7f"}, - {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d534ecb456a4ae3995f99c8a5deb469bfff05d4ec610a7885c175c881d12f710"}, - {file = "numexpr-2.14.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f41170e9d0dbba76851e35d80cfa9f4ca5fe78628c5bf24d941cf3364940ab7a"}, - {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6acafb2fdbeaaa6681a8f1a1d8b3f7dcd33704baace7057b950754b258be7c43"}, - {file = "numexpr-2.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7ca9e71195b36cc7aeafe97347549e1e1c1e889ff700238782ef6447651ec26d"}, - {file = "numexpr-2.14.2-cp311-cp311-win32.whl", hash = "sha256:779129d50974e7d6d6581d322f75b8f8375e96215b6861a2d5460347997ef649"}, - {file = "numexpr-2.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:2f132777d7d425471c458af5617e023402f13f5006301eacf8a1a6e7118ea70c"}, - {file = "numexpr-2.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:f1de5c88515ed9fbcad42699a0e2b5821b4d0f0adb0da6fb7e009e5cb19d8493"}, - {file = "numexpr-2.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:606ceaf5722e295ef965ca591736fc26d9e5f13ad950a479e64cead1947f8a3d"}, - {file = "numexpr-2.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:790da022539fe7c37dc893acf530a91c2ca6964d7ba11f464131383729d058f3"}, - {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:327be9ee62251c173236dc620147ff2d0e732a32f5bad918d78a10082f502f63"}, - {file = "numexpr-2.14.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6a5d8fc7016bf6f6e1808b011510aa7c3bd75ec1407f7650874ec591db59f5e"}, - {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4b1ff261c3e69c4c59578d3a9ca6132603619d38ae1abe73325563bed3b9bbaf"}, - {file = "numexpr-2.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b8384592c49cb15a91caa54e2cd84d1ce18edb7af030bb76cd29b52e5dc155d"}, - {file = "numexpr-2.14.2-cp312-cp312-win32.whl", hash = "sha256:41cdeacf1b4e51c1143983ea61fcee68139ca47222b55a9265b4fa73826c4260"}, - {file = "numexpr-2.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:8fc55d14bcf17b3fe69213bea14f999451892b4690717008c66f2edfd6a085ce"}, - {file = "numexpr-2.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:806a4471310fe20aa7cb1b2816a6f5e508073a1ad1c2e18041b83e57066fad6a"}, - {file = "numexpr-2.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0741efbd75c284e709b0fd430c85c31982b44c9962922ba8a9cbbea1bf413321"}, - {file = "numexpr-2.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92b00c78664070e3af155c6be713a0a5d75d598647ce32a5609adb79a8f961d3"}, - {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:149ab5744a5222f07b1d60455c4021c754d395e44938944ac7c7c2495f7feb54"}, - {file = "numexpr-2.14.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd2f5882a66a7792aa6614c68831aa20085b499d41422aedd001080624ebb14c"}, - {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:375d8bee15be42dab22100a0a3de05fe6689a2de853eca012858768a9a7e02ab"}, - {file = "numexpr-2.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c1ffaf805d8636c3f95d0996517ecf9684c9ac62d768030ca78d1d00af2b3504"}, - {file = "numexpr-2.14.2-cp313-cp313-win32.whl", hash = "sha256:449a57fb9d38de136e742b1fc429572b42f29778f1d695c3fe50ffec9d3c9a71"}, - {file = "numexpr-2.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:dd905922d7dce457947d54b84c7ac345cef37332b724445e159a5a1a2080ce2b"}, - {file = "numexpr-2.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:b02738853b9b5b8a995f6c680f8f6ef33e8f419395b8fa380e38690495fdb911"}, - {file = "numexpr-2.14.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:76e87c7bd70d721ce4d418e81f4fb7ecf9e7e67d7cea8102527b07fd3d3facf9"}, - {file = "numexpr-2.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:939c89f613b814e64bb568859397dc9f99b219c3ef681a72fb99a86e435262f9"}, - {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b20c1c55aba7812ff2f2c6a50006425d02282fabb1eaf8d75fe638ffcf6deb02"}, - {file = "numexpr-2.14.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bac00898930f962f360c3d763a8e2273fc931f65a1759ff1bf64b3cf13d65aee"}, - {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:022e61a3d5dbf5807746264b62126d1c2c24057ad90052478a4d4482ab2555c2"}, - {file = "numexpr-2.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1d4593e2c6fa060cd7441e8b6ef25c16321a6be2144b3c82d1e00885f1fb6e94"}, - {file = "numexpr-2.14.2-cp314-cp314-win32.whl", hash = "sha256:66f3b125b1104241322811de87918724d6709bf082dc0703722d0cecb7b29e82"}, - {file = "numexpr-2.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:ef576a1cded27ba2f3129bc3c42df452a1c498072680d560793f98b0024cd7e6"}, - {file = "numexpr-2.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8274c51ae1842948f3ae7fe6951a23dcf4ddcbeeaff3737e978e7740b754662d"}, - {file = "numexpr-2.14.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:f3526699350f94c6277fb16863773a1af9defd95a6f78bbd69b1f0338fd94756"}, - {file = "numexpr-2.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:91e7928435f14fcb351c0157000bce65122b897cc8b0df6bcc48251f25850a6d"}, - {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c66925deb968f0b5280f723e2bb5918c11e6be2ca60e9e1530006286ab44031d"}, - {file = "numexpr-2.14.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a404c9a55902572eec810068d06b79a7c99e96f0400f5a7d73f39dff5ec5e371"}, - {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:44dc6b1dfa9abcbfc9917297f0d2af7c87c16b6ecd45747a8e70f54399a3a2f9"}, - {file = "numexpr-2.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93233040f4bed3bce5abb0c2d20aeb1074511f29cbaa9c14828f86bcfa44d321"}, - {file = "numexpr-2.14.2-cp314-cp314t-win32.whl", hash = "sha256:2aceefa08f8f86317fa6e8fe9f6dc20d24ab8365d715be4a26306acf406d2dbe"}, - {file = "numexpr-2.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:cd684ac9daa539fcdac3437678834797b29d7780cfaad71111745132d466d51f"}, - {file = "numexpr-2.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:2ef72de3d3dd466cb0c435cae7141c99b0f8091b1eae9d03dcb38690f56c3f79"}, - {file = "numexpr-2.14.2.tar.gz", hash = "sha256:e7144e83ea9e581f2273e0304f15836736c4e470e2bd2e378ce617662a1ca278"}, -] - -[package.dependencies] -numpy = ">=1.26.0" - [[package]] name = "numpy" version = "2.2.6" @@ -3044,126 +2991,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -3936,14 +3883,14 @@ pyyaml = ["pyyaml"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -3976,14 +3923,14 @@ dev = ["twine"] [[package]] name = "transformers" -version = "5.14.1" +version = "5.13.1" description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725"}, - {file = "transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173"}, + {file = "transformers-5.13.1-py3-none-any.whl", hash = "sha256:53f0ea8aa397e29244c2377ba981bcaf0c87adcf44fbdd447ef6306522afcacd"}, + {file = "transformers-5.13.1.tar.gz", hash = "sha256:1e2452d6778a7482158df5d5dacf6bf775d5b2fdcfce33caaf7f6b0e5f3e3397"}, ] [package.dependencies] @@ -3999,29 +3946,29 @@ typer = "*" [package.extras] accelerate = ["accelerate (>=1.1.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.11.5)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] chat-template = ["jinja2 (>=3.1.0)"] codecarbon = ["codecarbon (>=2.8.1)"] deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.11.5)", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "kernels (>=0.15.2,<0.16)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] docs = ["hf-doc-builder"] integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.15.2,<0.16)", "optuna", "ray[tune] (>=2.7.0)"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] kernels = ["kernels (>=0.15.2,<0.16)"] -mistral-common = ["mistral-common[image] (>=1.11.5)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] num2words = ["num2words"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.11.5)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.2)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] timm = ["timm (>=1.0.23)"] torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] @@ -4080,14 +4027,14 @@ test = ["packaging", "pytest (>=6.0.1)", "python-dateutil (>=2.8.0,<3.0.0)", "py [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -4349,116 +4296,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] diff --git a/security_scanning/metadata.json b/security_scanning/metadata.json index 170ba429305d..0f981c210761 100644 --- a/security_scanning/metadata.json +++ b/security_scanning/metadata.json @@ -1,4 +1,4 @@ { - "commit_hash": "8e2816bf675687ce2803247884ab53af5b00b0c3", - "timestamp": "2026-07-22T02:48:45Z" + "commit_hash": "13f87162059f11e87f1e4049fe13a4f5b50fa9c9", + "timestamp": "2026-07-14T02:49:56Z" } diff --git a/security_scanning/poetry.lock b/security_scanning/poetry.lock index baa7bc30b6d0..15a9bb849265 100644 --- a/security_scanning/poetry.lock +++ b/security_scanning/poetry.lock @@ -60,131 +60,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -530,14 +530,14 @@ urllib3 = ">=2" [[package]] name = "build" -version = "1.5.0" +version = "1.5.1" description = "A simple, correct Python build frontend" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "build-1.5.0-py3-none-any.whl", hash = "sha256:13f3eecb844759ab66efec90ca17639bbf14dc06cb2fdf37a9010322d9c50a6f"}, - {file = "build-1.5.0.tar.gz", hash = "sha256:302c22c3ba2a0fd5f3911918651341ebb3896176cbdec15bd421f80b1afc7647"}, + {file = "build-1.5.1-py3-none-any.whl", hash = "sha256:f1a58fe2e5af5b0238a07b9e70207492c79ddebbdb1ad954fc86d62a56be3e0d"}, + {file = "build-1.5.1.tar.gz", hash = "sha256:94e17f1db803ab22f46049376c44c8437c52090f0dfdf1adc43df56542d644fb"}, ] [package.dependencies] @@ -550,7 +550,7 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [package.extras] keyring = ["keyring"] uv = ["uv (>=0.1.18)"] -virtualenv = ["virtualenv (>=20.17) ; python_version >= \"3.10\" and python_version < \"3.14\"", "virtualenv (>=20.31) ; python_version >= \"3.14\""] +virtualenv = ["virtualenv (>=20.36.1)"] [[package]] name = "cache-dit" @@ -1004,13 +1004,13 @@ cu13 = ["cuda-bindings[all] (==13.*)", "cuda-toolkit (==13.*)"] [[package]] name = "cuda-pathfinder" -version = "1.6.0" +version = "1.5.6" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, + {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, ] [[package]] @@ -1327,14 +1327,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -1364,23 +1364,21 @@ dev = ["pytest", "pytest-xdist", "ruff"] [[package]] name = "flashinfer-python" -version = "0.6.15" +version = "0.6.14" description = "FlashInfer: Kernel Library for LLM Serving" optional = false python-versions = "<4.0,>=3.10" groups = ["main"] files = [ - {file = "flashinfer_python-0.6.15-py3-none-any.whl", hash = "sha256:da6c339e14db4831ade0d593324f02907d44bd4b86b640faad4727d9e089477b"}, - {file = "flashinfer_python-0.6.15.tar.gz", hash = "sha256:2a3f1ed47129f9ac9505a26a8f12cadefc0f27d3104fb623ae281032f49eae5f"}, + {file = "flashinfer_python-0.6.14-py3-none-any.whl", hash = "sha256:d124369346a3d48eac67e31c42f7a3c813bcc0abc10e2e36db413b7b3dfd97df"}, + {file = "flashinfer_python-0.6.14.tar.gz", hash = "sha256:f4da8b5e005601784e85e0dcaa3389f908ee2d32c2560142d67124ab10e4a070"}, ] [package.dependencies] apache-tvm-ffi = ">=0.1.6,<0.1.8 || >0.1.8,<0.1.8.post0 || >0.1.8.post0,<0.2" click = "*" -cuda-python = ">=12.0" cuda-tile = ">=1.4.0" einops = "*" -nccl4py = ">=0.3.1" ninja = "*" numpy = "*" nvidia-cudnn-frontend = ">=1.13.0" @@ -1395,6 +1393,7 @@ tqdm = "*" [package.extras] cu12 = ["nvidia-cutlass-dsl (>=4.5.0)"] cu13 = ["nvidia-cutlass-dsl[cu13] (>=4.5.0)"] +nvep = ["cuda-python (>=13.0)"] [[package]] name = "frozenlist" @@ -1731,30 +1730,38 @@ numpy = ">=1.19.3" [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -2548,14 +2555,14 @@ typing = ["mypy", "typing_extensions ; python_version < \"3.8\""] [[package]] name = "mistral-common" -version = "1.11.6" +version = "1.11.5" description = "Mistral-common is a library of common utilities for Mistral AI." optional = false python-versions = "<3.15,>=3.10.0" groups = ["main"] files = [ - {file = "mistral_common-1.11.6-py3-none-any.whl", hash = "sha256:d719534ae1079070bc3a37ffb4976862f85db7a6c6549085832357b46ff89f4b"}, - {file = "mistral_common-1.11.6.tar.gz", hash = "sha256:35ed428e0e886808f0c048adac56843ce77d6cee13c1b54a045ad8d11e77c756"}, + {file = "mistral_common-1.11.5-py3-none-any.whl", hash = "sha256:7c1b09f43a589027315840bfd6f3528d5abf520eed701f8d8b7a922d6e5b855e"}, + {file = "mistral_common-1.11.5.tar.gz", hash = "sha256:ef8c03ad8359fa1386d66ee08d534d8f4a65a6955c9b24bd5caa2e005066cdec"}, ] [package.dependencies] @@ -3067,39 +3074,6 @@ pyspark-connect = ["pyspark[connect] (>=3.5.0)"] sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] -[[package]] -name = "nccl4py" -version = "0.3.1" -description = "NCCL4Py: Python bindings for NCCL" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "nccl4py-0.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:15517f1255a84cf3f48d36e406ff925afa40425925a1d68a7672621286488e0a"}, - {file = "nccl4py-0.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3115d8471cf4c207720c624174a0c023d8ef9893452310e6447dbc78a9e04ee"}, - {file = "nccl4py-0.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8e247d3a2b6e0253567322d518ec94320caa92649475416abceefaf0bc2db71"}, - {file = "nccl4py-0.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7945acf8c0224f727f87db40eb3a6060ad7809256b0f37dde6b94d9d8fa6b65c"}, - {file = "nccl4py-0.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0b1bab08b374ba21bb36612710866173e703c06dc197ab13b1e093436ac27ce"}, - {file = "nccl4py-0.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f96117a0aed13744d2636760962f1cb45be9138846023b69c5a8053e531cc76"}, - {file = "nccl4py-0.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50068bb4e6f60dd831b2d394a9789f08a6f4346f0e51a9d717536868a8fdd398"}, - {file = "nccl4py-0.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fd7862777777162f4735b85472951d91b8847cd7010d8f60bfefe58c7285583"}, - {file = "nccl4py-0.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13db9f786c7919ed1df7079c03acbe49eb569d5625fd5cec2344075f37b5e140"}, - {file = "nccl4py-0.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b684a3ac083fd76bf1e57e69e76575502bcd2aface23e2dad7a88461ce8916d"}, - {file = "nccl4py-0.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa09f12a93e0eb7b2dbbffbcb0a3daeeb3c211f2433cc6cc896faeec85ea4b2e"}, - {file = "nccl4py-0.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c58bf4db2eb27636587b3ce899fe47d72c16bf1eeb76c5e1c9bb1feaf57048f5"}, -] - -[package.dependencies] -"cuda.core" = ">=1.0,<2.0" -cuda-pathfinder = ">=1.5.4,<2.0.0" -numpy = "*" -packaging = "*" -typing_extensions = {version = "*", markers = "python_version < \"3.13\""} - -[package.extras] -cu12 = ["cuda-bindings (>=12.0,<13.0)", "nvidia-cutlass-dsl (>=4.5.2,<5.0)", "nvidia-nccl-cu12"] -cu13 = ["cuda-bindings (>=13.0,<14.0)", "nvidia-cutlass-dsl[cu13] (>=4.5.2,<5.0)", "nvidia-nccl-cu13"] - [[package]] name = "networkx" version = "3.4.2" @@ -3188,6 +3162,76 @@ files = [ llvmlite = "==0.48.*" numpy = ">=1.22,<2.5" +[[package]] +name = "numexpr" +version = "2.14.1" +description = "Fast numerical expression evaluator for NumPy" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8"}, + {file = "numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a"}, + {file = "numexpr-2.14.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:955c92b064f9074d2970cf3138f5e3b965be673b82024962ed526f39bc25a920"}, + {file = "numexpr-2.14.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75440c54fc01e130396650fdf307aa9d41a67dc06ddbfb288971b591c13a395b"}, + {file = "numexpr-2.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dde9fa47ed319e1e1728940a539df3cb78326b7754bc7c6ab3152afc91808f9b"}, + {file = "numexpr-2.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76db0bc6267e591ab9c4df405ffb533598e4c88239db7338d11ae9e4b368a85a"}, + {file = "numexpr-2.14.1-cp310-cp310-win32.whl", hash = "sha256:0d1dcbdc4d0374c0d523cee2f94f06b001623cbc1fd163612841017a3495427c"}, + {file = "numexpr-2.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:823cd82c8e7937981339f634e7a9c6a92cb2d0b9d0a5cf627a5e394fffc05377"}, + {file = "numexpr-2.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2d03fcb4644a12f70a14d74006f72662824da5b6128bf1bcd10cc3ed80e64c34"}, + {file = "numexpr-2.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2773ee1133f77009a1fc2f34fe236f3d9823779f5f75450e183137d49f00499f"}, + {file = "numexpr-2.14.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebe4980f9494b9f94d10d2e526edc29e72516698d3bf95670ba79415492212a4"}, + {file = "numexpr-2.14.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a381e5e919a745c9503bcefffc1c7f98c972c04ec58fc8e999ed1a929e01ba6"}, + {file = "numexpr-2.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d08856cfc1b440eb1caaa60515235369654321995dd68eb9377577392020f6cb"}, + {file = "numexpr-2.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03130afa04edf83a7b590d207444f05a00363c9b9ea5d81c0f53b1ea13fad55a"}, + {file = "numexpr-2.14.1-cp311-cp311-win32.whl", hash = "sha256:db78fa0c9fcbaded3ae7453faf060bd7a18b0dc10299d7fcd02d9362be1213ed"}, + {file = "numexpr-2.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:e9b2f957798c67a2428be96b04bce85439bed05efe78eb78e4c2ca43737578e7"}, + {file = "numexpr-2.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ebae0ab18c799b0e6b8c5a8d11e1fa3848eb4011271d99848b297468a39430"}, + {file = "numexpr-2.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:47041f2f7b9e69498fb311af672ba914a60e6e6d804011caacb17d66f639e659"}, + {file = "numexpr-2.14.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d686dfb2c1382d9e6e0ee0b7647f943c1886dba3adbf606c625479f35f1956c1"}, + {file = "numexpr-2.14.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee6d4fbbbc368e6cdd0772734d6249128d957b3b8ad47a100789009f4de7083"}, + {file = "numexpr-2.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3a2839efa25f3c8d4133252ea7342d8f81226c7c4dda81f97a57e090b9d87a48"}, + {file = "numexpr-2.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9f9137f1351b310436662b5dc6f4082a245efa8950c3b0d9008028df92fefb9b"}, + {file = "numexpr-2.14.1-cp312-cp312-win32.whl", hash = "sha256:36f8d5c1bd1355df93b43d766790f9046cccfc1e32b7c6163f75bcde682cda07"}, + {file = "numexpr-2.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:fdd886f4b7dbaf167633ee396478f0d0aa58ea2f9e7ccc3c6431019623e8d68f"}, + {file = "numexpr-2.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:09078ba73cffe94745abfbcc2d81ab8b4b4e9d7bfbbde6cac2ee5dbf38eee222"}, + {file = "numexpr-2.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dce0b5a0447baa7b44bc218ec2d7dcd175b8eee6083605293349c0c1d9b82fb6"}, + {file = "numexpr-2.14.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:06855053de7a3a8425429bd996e8ae3c50b57637ad3e757e0fa0602a7874be30"}, + {file = "numexpr-2.14.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f9366d23a2e991fd5a8b5e61a17558f028ba86158a4552f8f239b005cdf83c"}, + {file = "numexpr-2.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c5f1b1605695778896534dfc6e130d54a65cd52be7ed2cd0cfee3981fd676bf5"}, + {file = "numexpr-2.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a4ba71db47ea99c659d88ee6233fa77b6dc83392f1d324e0c90ddf617ae3f421"}, + {file = "numexpr-2.14.1-cp313-cp313-win32.whl", hash = "sha256:638dce8320f4a1483d5ca4fda69f60a70ed7e66be6e68bc23fb9f1a6b78a9e3b"}, + {file = "numexpr-2.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:9fdcd4735121658a313f878fd31136d1bfc6a5b913219e7274e9fca9f8dac3bb"}, + {file = "numexpr-2.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:557887ad7f5d3c2a40fd7310e50597045a68e66b20a77b3f44d7bc7608523b4b"}, + {file = "numexpr-2.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:af111c8fe6fc55d15e4c7cab11920fc50740d913636d486545b080192cd0ad73"}, + {file = "numexpr-2.14.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33265294376e7e2ae4d264d75b798a915d2acf37b9dd2b9405e8b04f84d05cfc"}, + {file = "numexpr-2.14.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83647d846d3eeeb9a9255311236135286728b398d0d41d35dedb532dca807fe9"}, + {file = "numexpr-2.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6e575fd3ad41ddf3355d0c7ef6bd0168619dc1779a98fe46693cad5e95d25e6e"}, + {file = "numexpr-2.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:67ea4771029ce818573b1998f5ca416bd255156feea017841b86176a938f7d19"}, + {file = "numexpr-2.14.1-cp313-cp313t-win32.whl", hash = "sha256:15015d47d3d1487072d58c0e7682ef2eb608321e14099c39d52e2dd689483611"}, + {file = "numexpr-2.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:94c711f6d8f17dfb4606842b403699603aa591ab9f6bf23038b488ea9cfb0f09"}, + {file = "numexpr-2.14.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ede79f7ff06629f599081de644546ce7324f1581c09b0ac174da88a470d39c21"}, + {file = "numexpr-2.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2eac7a5a2f70b3768c67056445d1ceb4ecd9b853c8eda9563823b551aeaa5082"}, + {file = "numexpr-2.14.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aedf38d4c0c19d3cecfe0334c3f4099fb496f54c146223d30fa930084bc8574"}, + {file = "numexpr-2.14.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439ec4d57b853792ebe5456e3160312281c3a7071ecac5532ded3278ede614de"}, + {file = "numexpr-2.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e23b87f744e04e302d82ac5e2189ae20a533566aec76a46885376e20b0645bf8"}, + {file = "numexpr-2.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:44f84e0e5af219dbb62a081606156420815890e041b87252fbcea5df55214c4c"}, + {file = "numexpr-2.14.1-cp314-cp314-win32.whl", hash = "sha256:1f1a5e817c534539351aa75d26088e9e1e0ef1b3a6ab484047618a652ccc4fc3"}, + {file = "numexpr-2.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:587c41509bc373dfb1fe6086ba55a73147297247bedb6d588cda69169fc412f2"}, + {file = "numexpr-2.14.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ec368819502b64f190c3f71be14a304780b5935c42aae5bf22c27cc2cbba70b5"}, + {file = "numexpr-2.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7e87f6d203ac57239de32261c941e9748f9309cbc0da6295eabd0c438b920d3a"}, + {file = "numexpr-2.14.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd72d8c2a165fe45ea7650b16eb8cc1792a94a722022006bb97c86fe51fd2091"}, + {file = "numexpr-2.14.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70d80fcb418a54ca208e9a38e58ddc425c07f66485176b261d9a67c7f2864f73"}, + {file = "numexpr-2.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:edea2f20c2040df8b54ee8ca8ebda63de9545b2112872466118e9df4d0ae99f3"}, + {file = "numexpr-2.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:790447be6879a6c51b9545f79612d24c9ea0a41d537a84e15e6a8ddef0b6268e"}, + {file = "numexpr-2.14.1-cp314-cp314t-win32.whl", hash = "sha256:538961096c2300ea44240209181e31fae82759d26b51713b589332b9f2a4117e"}, + {file = "numexpr-2.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:a40b350cd45b4446076fa11843fa32bbe07024747aeddf6d467290bf9011b392"}, + {file = "numexpr-2.14.1.tar.gz", hash = "sha256:4be00b1086c7b7a5c32e31558122b7b80243fe098579b170967da83f3152b48b"}, +] + +[package.dependencies] +numpy = ">=1.23.0" + [[package]] name = "numpy" version = "2.2.6" @@ -3536,18 +3580,6 @@ numpy = "*" nvidia-cutlass-dsl-libs-base = "4.5.0" typing-extensions = "*" -[[package]] -name = "nvidia-matmul-heuristics" -version = "0.1.0.27" -description = "NVIDIA Matmul Heuristics" -optional = false -python-versions = ">=3.9.0" -groups = ["main"] -files = [ - {file = "nvidia_matmul_heuristics-0.1.0.27-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4107ae54308d9262a6b14c30982c45c720b9c5f4196bf93dac64f6739ed8f8d9"}, - {file = "nvidia_matmul_heuristics-0.1.0.27-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:475cd174aad5cf5b5828cc894b07fa96ad7af5c681a2631c077ebf482c92e4ad"}, -] - [[package]] name = "nvidia-ml-py" version = "13.610.43" @@ -3765,14 +3797,14 @@ onnx = ">=1.14.0" [[package]] name = "openai" -version = "2.46.0" +version = "2.45.0" description = "The official Python library for the openai API" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "openai-2.46.0-py3-none-any.whl", hash = "sha256:672381db55efb3a1e2610f29304c130cccdd0b319bace4d492b2443cb64c1e7c"}, - {file = "openai-2.46.0.tar.gz", hash = "sha256:0421e0735ac41451cad894af4cddf0435bfbf8cbc538ac0e15b3c062f2ddc06a"}, + {file = "openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777"}, + {file = "openai-2.45.0.tar.gz", hash = "sha256:10d34ca9c5643bce775852fddbfc172505cb1d4de1ccd101696c3ecff358765d"}, ] [package.dependencies] @@ -4249,14 +4281,14 @@ xmp = ["defusedxml"] [[package]] name = "platformdirs" -version = "4.11.0" +version = "4.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74"}, - {file = "platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0"}, + {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, + {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, ] [[package]] @@ -5261,126 +5293,126 @@ typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -6253,14 +6285,14 @@ test = ["pytest"] [[package]] name = "sse-starlette" -version = "3.4.6" +version = "3.4.5" description = "SSE plugin for Starlette" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "sse_starlette-3.4.6-py3-none-any.whl", hash = "sha256:56217ab4c9a9f9c5db7b21e08732d3e7c2b807f45231ad23de0551a24c4a41f6"}, - {file = "sse_starlette-3.4.6.tar.gz", hash = "sha256:725f8a1bd6d26ae1b2c9610c0ef5065dfdd496f3988d28adcf8c4b49dc25c627"}, + {file = "sse_starlette-3.4.5-py3-none-any.whl", hash = "sha256:e71bad53323f65573c3864a6c3bd0c1eb6e5f092b2e48082b0c35927d19ca296"}, + {file = "sse_starlette-3.4.5.tar.gz", hash = "sha256:83072538bc211a2f68b7b0422226c4af3e9b62e106e07034664b832ca019842a"}, ] [package.dependencies] @@ -6343,6 +6375,80 @@ files = [ [package.extras] widechars = ["wcwidth"] +[[package]] +name = "tensorrt" +version = "10.16.1.11" +description = "TensorRT Metapackage" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "tensorrt-10.16.1.11.tar.gz", hash = "sha256:5c31ef98e1a1b53197acc600327d6b1e4abb503024119a2e66f21a5654ea3996"}, +] + +[package.dependencies] +tensorrt_cu13 = "10.16.1.11" + +[[package]] +name = "tensorrt-cu13" +version = "10.16.1.11" +description = "A high performance deep learning inference library" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "tensorrt_cu13-10.16.1.11.tar.gz", hash = "sha256:5d34203a92f38851b150c4eddd32b9b55c01fd40fc4d19bff83e8dfef1d8f69e"}, +] + +[package.dependencies] +tensorrt_cu13_bindings = "10.16.1.11" +tensorrt_cu13_libs = "10.16.1.11" + +[package.extras] +numpy = ["numpy"] + +[[package]] +name = "tensorrt-cu13-bindings" +version = "10.16.1.11" +description = "A high performance deep learning inference library" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "tensorrt_cu13_bindings-10.16.1.11-cp310-none-manylinux_2_28_x86_64.whl", hash = "sha256:c8e171511a01a2678cb3e00ca4792bc1d7ea6c48da9781ab127970f6ad1edc4a"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp310-none-manylinux_2_35_aarch64.whl", hash = "sha256:9a7330fecbc0fff40ff227d0f2ee2d2d5f6fd55aafdd9fe0364e4e06fa4a9612"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp310-none-win_amd64.whl", hash = "sha256:631c7f0c979d740696a6daf1c17f1fd128863939cc3720282cc53d21cdd05e33"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp311-none-manylinux_2_28_x86_64.whl", hash = "sha256:f986d47980d042cc126506a9b84586e55ad13737a7a46826fe45457c04d66294"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp311-none-manylinux_2_35_aarch64.whl", hash = "sha256:6fa40558659d027ef6d71b475f012e0531f3d69c7b27b4a2b92779cc6de24a9a"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp311-none-win_amd64.whl", hash = "sha256:483a58b8cf3a7e97d66d7245084d08f9ee9cd19d09210305ed43b34fbb721f7c"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp312-none-manylinux_2_28_x86_64.whl", hash = "sha256:2fb87ac24a5e5f42e43f283c16d1b661dc72ef18568fd847ce2080b7db9ca232"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp312-none-manylinux_2_35_aarch64.whl", hash = "sha256:80d0cea41cb695e73a442ab5a0c1fb023730c488caa93ee3ca9c49d57314821d"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp312-none-win_amd64.whl", hash = "sha256:70ca2d73301ff427104d7986c26d5fa56f6a91578242d20e82b37c139be14ec0"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp313-none-manylinux_2_28_x86_64.whl", hash = "sha256:da17c222ba99f46e77e1dab568cd17c4652b0432d4fae074e4c79285356aa0ac"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp313-none-manylinux_2_35_aarch64.whl", hash = "sha256:2cba1722377323516a46ed15859deaf03f20ab2a47e63a89d69b3898d110fcac"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp313-none-win_amd64.whl", hash = "sha256:44b44154027e5e9e39d74e14b453704af50ba774b3775704f6d40408c8457526"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp38-none-manylinux_2_28_x86_64.whl", hash = "sha256:74ad6eb30fc6db3cc72eeacf7e989f887905d89eba92dcefaa0bf7310efba707"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp38-none-manylinux_2_35_aarch64.whl", hash = "sha256:50ea9129ac563d9bcfafb495b8135710107da795c664f499d1072d39ab09e5fb"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp38-none-win_amd64.whl", hash = "sha256:3fd27a1441173257eaa20b41562981a0bbcd1914bce4fab22a5feae1d352facb"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp39-none-manylinux_2_28_x86_64.whl", hash = "sha256:fbbd3ccfcb794c7aafec6900d6eefe289d783d89c065d8af57f9d11724307e51"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp39-none-manylinux_2_35_aarch64.whl", hash = "sha256:cc085106269123f6fa75998095671e43717df55289971cee68a1984a3c14f2bc"}, + {file = "tensorrt_cu13_bindings-10.16.1.11-cp39-none-win_amd64.whl", hash = "sha256:2ab1dd2297046ae3d4a88f0c5a5b0601bb03928b558a72f158d605fbfd0bb285"}, +] + +[package.extras] +numpy = ["numpy"] + +[[package]] +name = "tensorrt-cu13-libs" +version = "10.16.1.11" +description = "TensorRT Libraries" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "tensorrt_cu13_libs-10.16.1.11.tar.gz", hash = "sha256:0e86cd02321b7258c2521495a7d8f0591852e6966ed7e43cfe33640c737495a3"}, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -6661,14 +6767,14 @@ test = ["pytest", "torchvision (>=0.15)", "transformers"] [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -6766,14 +6872,14 @@ tutorials = ["matplotlib", "pandas", "tabulate"] [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -6938,14 +7044,14 @@ files = [ [[package]] name = "xdsl" -version = "0.69.0" +version = "0.68.0" description = "xDSL" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "xdsl-0.69.0-py3-none-any.whl", hash = "sha256:40445d6a63d8539ba9527ffea30e5c47138fd9fa19b5f2c4eae978b5450ce76b"}, - {file = "xdsl-0.69.0.tar.gz", hash = "sha256:cc365cc1c746bd5cbf07f348e6c99e2da85a814f75ee44a91cb9caa20ea3c166"}, + {file = "xdsl-0.68.0-py3-none-any.whl", hash = "sha256:9f28d722a1828df77b71128a5a725adbaddafcf6e20deb4de85a55cb53d8e90c"}, + {file = "xdsl-0.68.0.tar.gz", hash = "sha256:e0331528e02bbd6baab529e32a1c942518d37294e14710ab141086e9d77af25f"}, ] [package.dependencies] @@ -6954,9 +7060,9 @@ ordered-set = "4.1" typing-extensions = ">=4.7,<5" [package.extras] -dev = ["coverage (<8)", "filecheck (==1.0.3)", "ipykernel", "lit (<19)", "marimo (>=0.23,<0.24)", "nbconvert (>=7.7.2,<8)", "nbval (<0.12)", "prek (>=0.4.0,<0.5.0)", "pyright (==1.1.411)", "pytest (<9.2)", "pytest-asyncio", "pytest-cov", "ruff (==0.15.20)", "sympy (==1.14)", "textual-dev (==1.8)", "toml (<0.11)"] +dev = ["coverage (<8)", "filecheck (==1.0.3)", "ipykernel", "lit (<19)", "marimo (>=0.23,<0.24)", "nbconvert (>=7.7.2,<8)", "nbval (<0.12)", "prek (>=0.4.0,<0.5.0)", "pyright (==1.1.410)", "pytest (<9.2)", "pytest-asyncio", "pytest-cov", "ruff (==0.15.20)", "sympy (==1.14)", "textual-dev (==1.8)", "toml (<0.11)"] gui = ["pyclip (==0.7)", "textual (>=8,<9)"] -heir = ["heir-py (==2026.7.1)"] +heir = ["heir-py (==2026.6.5.dev0)"] llvm = ["llvmlite (>=0.47.0,<0.48.0)"] [[package]] @@ -7215,116 +7321,116 @@ files = [ [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] @@ -7355,4 +7461,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "26e270bc32d800f75b2db3240b3694e707588bbaa9d7e89d13b767a0d7e42bda" +content-hash = "a174d9a387dfc1bff0d82e796d3d0887a7ac50390600b6d7dd77594494fd0cdd" diff --git a/security_scanning/pyproject.toml b/security_scanning/pyproject.toml index 89479e19fd2b..7bb0d1e674c8 100644 --- a/security_scanning/pyproject.toml +++ b/security_scanning/pyproject.toml @@ -9,7 +9,7 @@ license = "Apache-2.0" requires-python = ">=3.10,<3.13" dependencies = [ "accelerate (>=1.7.0)", - "build (>=1.5.0,<2.0.0)", + "build (>=1.5.1,<2.0.0)", "colored (>=2.3.2,<3.0.0)", "cuda-python (>=13)", "diffusers (>=0.37.1)", @@ -21,7 +21,7 @@ dependencies = [ "onnx (>=1.21.0)", "onnx-graphsurgeon (>=0.5.2)", "graphviz (>=0.21,<0.22)", - "openai (>=2.46.0,<3.0.0)", + "openai (>=2.45.0,<3.0.0)", "polygraphy (>=0.50.3,<0.51.0)", "psutil (>=7.2.2,<8.0.0)", "nvidia-ml-py (>=13)", @@ -29,9 +29,10 @@ dependencies = [ "h5py (==3.12.1)", "strenum (>=0.4.15,<0.5.0)", "sentencepiece (>=0.1.99)", - "torch (>=2.11.0,<=2.13.0a0)", + "tensorrt (>=10.16.1,<10.17.0)", + "torch (>=2.11.0,<=2.12.0a0)", "nvidia-modelopt[torch] (>=0.37.0,<0.38.0)", - "nvidia-nccl-cu13 (>=2.28.9,<=2.30.4)", + "nvidia-nccl-cu13 (>=2.28.9,<=2.29.7)", "transformers (==5.5.4)", "prometheus-client (>=0.25.0,<0.26.0)", "prometheus-fastapi-instrumentator (>=8.0.2,<9.0.0)", @@ -55,7 +56,7 @@ dependencies = [ "peft (>=0.18.1,<0.19.0)", "patchelf (>=0.17.2.4,<0.18.0.0)", "einops (>=0.8.2,<0.9.0)", - "flashinfer-python (==0.6.15)", + "flashinfer-python (==0.6.14)", "xgrammar (==0.1.32)", "llguidance (==0.7.29)", "jsonschema (>=4.26.0,<5.0.0)", @@ -71,8 +72,8 @@ dependencies = [ "blobfile (>=3.2.0,<4.0.0)", "openai-harmony (==0.0.4)", "nvidia-cutlass-dsl[cu13] (==4.5.0)", - "nvidia-matmul-heuristics (==0.1.0.27)", "plotly (>=6.9.0,<7.0.0)", + "numexpr (>=2.14.1,<3.0.0)", "partial-json-parser (>=0.2.1.1.post7,<0.3.0.0)", "mcp (>=1.28.1,<2.0.0)", "torch-c-dlpack-ext (==0.1.3)", diff --git a/security_scanning/triton_backend/poetry.lock b/security_scanning/triton_backend/poetry.lock index 9e35520685a7..0dcbd7dcf144 100644 --- a/security_scanning/triton_backend/poetry.lock +++ b/security_scanning/triton_backend/poetry.lock @@ -14,131 +14,131 @@ files = [ [[package]] name = "aiohttp" -version = "3.14.2" +version = "3.14.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4"}, - {file = "aiohttp-3.14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796"}, - {file = "aiohttp-3.14.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0"}, - {file = "aiohttp-3.14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91"}, - {file = "aiohttp-3.14.2-cp310-cp310-win32.whl", hash = "sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a"}, - {file = "aiohttp-3.14.2-cp310-cp310-win_arm64.whl", hash = "sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd"}, - {file = "aiohttp-3.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7"}, - {file = "aiohttp-3.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e"}, - {file = "aiohttp-3.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f"}, - {file = "aiohttp-3.14.2-cp311-cp311-win32.whl", hash = "sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7"}, - {file = "aiohttp-3.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034"}, - {file = "aiohttp-3.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384"}, - {file = "aiohttp-3.14.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a"}, - {file = "aiohttp-3.14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc"}, - {file = "aiohttp-3.14.2-cp312-cp312-win32.whl", hash = "sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf"}, - {file = "aiohttp-3.14.2-cp312-cp312-win_arm64.whl", hash = "sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea"}, - {file = "aiohttp-3.14.2-cp313-cp313-android_21_x86_64.whl", hash = "sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014"}, - {file = "aiohttp-3.14.2-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f"}, - {file = "aiohttp-3.14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853"}, - {file = "aiohttp-3.14.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31"}, - {file = "aiohttp-3.14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d"}, - {file = "aiohttp-3.14.2-cp313-cp313-win32.whl", hash = "sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd"}, - {file = "aiohttp-3.14.2-cp313-cp313-win_arm64.whl", hash = "sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30"}, - {file = "aiohttp-3.14.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f"}, - {file = "aiohttp-3.14.2-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb"}, - {file = "aiohttp-3.14.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803"}, - {file = "aiohttp-3.14.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d"}, - {file = "aiohttp-3.14.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320"}, - {file = "aiohttp-3.14.2-cp314-cp314-win32.whl", hash = "sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b"}, - {file = "aiohttp-3.14.2-cp314-cp314-win_arm64.whl", hash = "sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569"}, - {file = "aiohttp-3.14.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1"}, - {file = "aiohttp-3.14.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3"}, - {file = "aiohttp-3.14.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win32.whl", hash = "sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49"}, - {file = "aiohttp-3.14.2-cp314-cp314t-win_arm64.whl", hash = "sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900"}, - {file = "aiohttp-3.14.2.tar.gz", hash = "sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491"}, + {file = "aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb"}, + {file = "aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d"}, + {file = "aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966"}, + {file = "aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df"}, + {file = "aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f"}, + {file = "aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730"}, + {file = "aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85"}, + {file = "aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3"}, + {file = "aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5"}, + {file = "aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d"}, + {file = "aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c"}, + {file = "aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087"}, + {file = "aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3"}, + {file = "aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271"}, + {file = "aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178"}, + {file = "aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe"}, + {file = "aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da"}, + {file = "aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451"}, + {file = "aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345"}, + {file = "aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588"}, + {file = "aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a"}, + {file = "aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15"}, + {file = "aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba"}, + {file = "aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004"}, + {file = "aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602"}, + {file = "aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95"}, + {file = "aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444"}, + {file = "aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719"}, + {file = "aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340"}, + {file = "aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3"}, + {file = "aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d"}, + {file = "aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9"}, + {file = "aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6"}, + {file = "aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035"}, ] [package.dependencies] @@ -578,13 +578,13 @@ cu13 = ["cuda-bindings[all] (==13.*)"] [[package]] name = "cuda-pathfinder" -version = "1.6.0" +version = "1.5.6" description = "Pathfinder for CUDA components" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51"}, + {file = "cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0"}, ] [[package]] @@ -627,14 +627,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.32.0" +version = "3.29.7" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3"}, - {file = "filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402"}, + {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, + {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, ] [[package]] @@ -1153,30 +1153,38 @@ files = [ [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.5.1" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" files = [ - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b"}, - {file = "hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380"}, - {file = "hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e"}, - {file = "hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799"}, - {file = "hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c"}, - {file = "hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f"}, - {file = "hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65"}, - {file = "hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d"}, - {file = "hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577"}, + {file = "hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947"}, + {file = "hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283"}, + {file = "hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff"}, + {file = "hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f"}, + {file = "hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9"}, + {file = "hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6"}, + {file = "hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e"}, + {file = "hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350"}, + {file = "hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6"}, + {file = "hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5"}, + {file = "hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e"}, + {file = "hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6"}, ] [package.extras] @@ -1231,14 +1239,14 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "1.24.0" +version = "1.23.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.10.0" groups = ["main"] files = [ - {file = "huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59"}, - {file = "huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5"}, + {file = "huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2"}, + {file = "huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88"}, ] [package.dependencies] @@ -2107,126 +2115,126 @@ files = [ [[package]] name = "regex" -version = "2026.7.19" +version = "2026.7.10" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, - {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, - {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, - {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, - {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, - {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, - {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, - {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, - {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, - {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, - {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, - {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, - {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, - {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, - {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, - {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, - {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, - {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, - {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, - {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, - {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, - {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, - {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, - {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, - {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, - {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, - {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, - {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, - {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, - {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, - {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, - {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, - {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, - {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, - {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, - {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, - {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, - {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, - {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, - {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, - {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, - {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, - {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, - {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:799a369bdab91dcf0eb424ebd7aa9650897025ce22f729248d8f2c72002c4daa"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f0192e5f1cfc70e3cb35347135dd02e7497b3e7d83e378aa226d8b3e53a93f19"}, + {file = "regex-2026.7.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:221f2771cb780186b94bbf125a151bbeb242fa1a971da6ad59d7b0370f19de9a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2fb1f7a2deb4ca3ddebbae6b93905d21480a3b4e11de28d79d9fb0d316fcf8"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f98ef73a13791a387d5c841416ad7f52040ae5caf10bcf46fa12bd2b3d63745"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9a094ed44a22f9da497453137c3118b531fd783866ab524b0b0fc146e7395e1d"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53bbbd6c610489700f7110db1d85f3623924c3f7c760f987eca033867360788a"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87b776cf2890e356e4ab104b9df846e169da3eb5b0f110975547091f4e51854e"}, + {file = "regex-2026.7.10-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab39d2c967aae3b48a412bff9cdbe7cd7559cd1e277599aceaeada7bc82b7200"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b56416091bfd7a429f958f69aaf6823c517be9a49cb5bf1daa3767ce8bf8095e"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:617e8f10472e34a8477931f978ff3a88d46ae2ba0e41927e580b933361f60948"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31fa17378b29519bfd0a1b8ba4e9c10cf0baf1cf4099b39b0689429e7dc2c795"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c363de7c0339d39341b6181839ed32509820b85ef506deafcf2e7e43baadab4"}, + {file = "regex-2026.7.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed7c886a2fcbf14493ceaf9579394b33521730c161ebb8dad7db9c3e9fcab1a8"}, + {file = "regex-2026.7.10-cp310-cp310-win32.whl", hash = "sha256:b04583e8867136ae66353fa274f45121ab3ec3166dc45aaff3655a5db90d9f0e"}, + {file = "regex-2026.7.10-cp310-cp310-win_amd64.whl", hash = "sha256:e21e888a6b471b2bb1cdd4247e8d86632672232f29be583e7eafaa5f4634d34c"}, + {file = "regex-2026.7.10-cp310-cp310-win_arm64.whl", hash = "sha256:081acf191b4d614d573a56cab69f948b6864daa5e3cc69f209ee92e26e454c2f"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:66d2c35587cd601c95965d5c0415058ba5cfd6ffbab7624ce198bd967102b341"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28a0973eeffff4292f5a7ee498ab65d5e94ee8cc9cea364239251eb4a260a0f1"}, + {file = "regex-2026.7.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8331484450b3894298bef8abecce532171ff6ac60b71f999eed10f2c01941a8a"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0639b2488b775a0109f55a5a2172deebdedb4b6c5ab0d48c90b43cbf5de58d17"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be4223af640d0aa04c05db81d5d96ada3ead9c09187d892fd37f4f97829480be"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3c75d57a00109255e60bc9c623b6ececaf7905eaab845c79f036670ed4750a2"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:724ee9379568658ec06362cf24325c5315cc5a67f61dfe585bfeff58300a355b"}, + {file = "regex-2026.7.10-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:732c19e5828eb287d01edb83b2eb87f283ba8e5fc3441c732709d3e8cbd14aaa"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:982d07727c809b42a3968785354f11c3728414e4e90af0754345b431b2c32561"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4574feca202f8c470bf678aed8b5d89df04aaf8dc677f3b83d92825051301c0f"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:80151ca5bfc6c4524186b3e08b499e97319b2001fc265ed2d4fc12c0d5692cdf"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bb52e10e453b5493afe1f7702a2973bc10f4dd8901c0f2ed869ffaa3f8319296"}, + {file = "regex-2026.7.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e37aba1994d73b4944053ab65a15f313bd5c28c885dd7f0d494a11749d89db6e"}, + {file = "regex-2026.7.10-cp311-cp311-win32.whl", hash = "sha256:6cbedeb5112f59dbd169385459b9943310bdd241c6966c19c5f6e2295055c93a"}, + {file = "regex-2026.7.10-cp311-cp311-win_amd64.whl", hash = "sha256:b1963ec5ba4d52788fb0eac6aca6eb8040e8e318c7e47ebbdfc09440c802919c"}, + {file = "regex-2026.7.10-cp311-cp311-win_arm64.whl", hash = "sha256:3750c42d47712e362158a04d0fd80131f73a55e8c715b2885442a0ff6f9fc3fc"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7252b48b0c60100095088fbeb281fca9a4fcf678a4e04b1c520c3f8613c952c4"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:da6ef4cb8d457aab0482b50120136ae94238aaa421863eaa7d599759742c72d6"}, + {file = "regex-2026.7.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe7ff456c22725c9d9017f7a2a7df2b51af6df77314176760b22e2d05278e181"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3463a5f26be513a49e4d497debcf1b252a2db7b92c77d89621aa90b83d2dd38"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:948dfc62683a6947b9b486c4598d8f6e3ecc542478b6767b87d52be68aeb55c6"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c2cbd385d82f63bb35edb60b09b08abad3619bd0a4a492ae59e55afaf98e1b9d"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6222cafe00e072bb2b8f14142cd969637411fbc4dd3b1d73a90a3b817fa046f"}, + {file = "regex-2026.7.10-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:65ee5d1ac3cd541325f5ac92625b1c1505f4d171520dd931bda7952895c5321a"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa34473fbcc108fea403074f3f45091461b18b2047d136f16ffaa4c65ad46a68"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d028d189d8f38d7ff292f22187c0df37f2317f554d2ed9a2908ada330af57c0"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:396ea70e4ea1f19571940add3bad9fd3eb6a19dc610d0d01f692bc1ba0c10cb4"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ebbf0d83ed5271991d666e54bb6c90ac2c55fb2ef3a88740c6af85dc85de2402"}, + {file = "regex-2026.7.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58a4571b2a093f6f6ee4fd281faa8ebf645abcf575f758173ea2605c7a1e1ecb"}, + {file = "regex-2026.7.10-cp312-cp312-win32.whl", hash = "sha256:eac1207936555aa691ce32df1432b478f2729d54e6d93a1f4db9215bcd8eb47d"}, + {file = "regex-2026.7.10-cp312-cp312-win_amd64.whl", hash = "sha256:ecae626449d00db8c08f8f1fc00047a32d6d7eb5402b3976f5c3fda2b80a7a4f"}, + {file = "regex-2026.7.10-cp312-cp312-win_arm64.whl", hash = "sha256:87794549a3f5c1c2bdfba2380c1bf87b931e375f4133d929da44f95e396bf5fe"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4db009b4fc533d79af3e841d6c8538730423f82ea8508e353a3713725de7901c"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b96341cb29a3faa5db05aff29c77d141d827414f145330e5d8846892119351c1"}, + {file = "regex-2026.7.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14d27f6bd04beb01f6a25a1153d73e58c290fd45d92ba56af1bb44199fd1010d"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e6b6a11bf898cca3ce7bfaa17b646901107f3975677fbd5097f36e5eb5641983"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:234f8e0d65cf1df9becadae98648f74030ee85a8f12edcb5eb0f60a22a602197"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:91b916d495db3e1b473c7c8e68733beec4dce8e487442db61764fff94f59740e"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0d4ccf70b1d13711242de0ba78967db5c35d12ac408378c70e06295c3f6644"}, + {file = "regex-2026.7.10-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c622f4c638a725c39abcb2e680b1bd592663c83b672a4ed350a17f806d75618e"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:41a47c2b28d9421e2509a4583a22510dc31d83212fcf38e1508a7013140f71a8"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:13fba679fe035037e9d5286620f88bbfd105df4d5fcd975942edd282ab986775"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8e26a075fa9945b9e44a3d02cc83d776c3b76bb1ff4b133bbfa620d5650131da"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0834c84ae8750ae1c4cede59b0afd4d2f775be958e11b18a3eea24ed9d0d9f1"}, + {file = "regex-2026.7.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64722a5031aeace7f6c8d5ea9a9b22d9368af0d6e8fa532585da8158549ea963"}, + {file = "regex-2026.7.10-cp313-cp313-win32.whl", hash = "sha256:74ae61d8573ecd51b5eeee7be2218e4c56e99c14fa8fcf97cf7519611d4be92e"}, + {file = "regex-2026.7.10-cp313-cp313-win_amd64.whl", hash = "sha256:5e792367e5f9b4ffb8cad93f1beaa91837056b94da98aa5c65a0db0c1b474927"}, + {file = "regex-2026.7.10-cp313-cp313-win_arm64.whl", hash = "sha256:82ab8330e7e2e416c2d42fcec67f02c242393b8681014750d4b70b3f158e1f08"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:2b93eafd92c4128bab2f93500e8912cc9ecb3d3765f6685b902c6820d0909b6b"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3f03b92fb6ec739df042e45b06423fc717ecf0063e07ffe2897f7b2d5735e1e8"}, + {file = "regex-2026.7.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:bb5aab464a0c5e03a97abad5bdf54517061ebbf72340d576e99ff661a42575cc"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fadb07dbe36a541283ff454b1a268afd54b077d917043f2e1e5615372cb5f200"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:21150500b970b12202879dfd82e7fd809d8e853140fff84d08e57a90cf1e154e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a68b637451d64ba30ed8ae125c973fa834cc2d37dfa7f154c2b479015d477ba8"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e23458d8903e33e7d27196d7a311523dc4e2f4137a5f34e4dbd30c8d37ff33e"}, + {file = "regex-2026.7.10-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae27622c094558e519abf3242cf4272db961d12c5c9a9ffb7a1b44b2627d5c6"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ee877b6d78f9dff1da94fef51ae8cf9cce0967e043fdcc864c40b85cf293c192"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:2c66a8a1969cfd506d1e203c0005fd0fc3fe6efc83c945606566b6f9611d4851"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2bc350e1c5fa250f30ab0c3e38e5cfdffcd82cb8af224df69955cab4e3003812"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:53f54993b462f3f91fea0f2076b46deb6619a5f45d70dbd1f543f789d8b900ef"}, + {file = "regex-2026.7.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cfcec18f7da682c4e2d82112829ce906569cb8d69fa6c26f3a50dfbed5ceb682"}, + {file = "regex-2026.7.10-cp313-cp313t-win32.whl", hash = "sha256:a2d6d30be35ddd70ce0f8ee259a4c25f24d6d689a45a5ac440f03e6bcc5a21d1"}, + {file = "regex-2026.7.10-cp313-cp313t-win_amd64.whl", hash = "sha256:c57b6ad3f7a1bdd101b2966f29dc161adf49727b1e8d3e1e89db2eda8a75c344"}, + {file = "regex-2026.7.10-cp313-cp313t-win_arm64.whl", hash = "sha256:3d8ef9df02c8083c7b4b855e3cb87c8e0ebbcfea088d98c7a886aaefdf88d837"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:39f81d1fdf594446495f2f4edd8e62d8eda0f7a802c77ac596dc8448ad4cc5ca"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:441edc66a54063f8269d1494fc8474d06605e71e8a918f4bcfd079ebda4ce042"}, + {file = "regex-2026.7.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cfeb11990f59e59a0df26c648f0adfcbf27be77241250636f5769eb08db662be"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:460176b2db044a292baaee6891106566739657877af89a251cded228689015a6"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9dc55698737aca028848bde418d6c51d74f2a5fd44872d3c8b56b626729adb89"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d3e10779f60c000213a5b53f518824bd07b3dc119333b26d70c6be1c27b5c794"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38a5926601aaccf379512746b86eb0ac1d29121f6c776dac6ac5b31077432f2c"}, + {file = "regex-2026.7.10-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a72ecf5bfd3fc8d57927f7e3ded2487e144472f39010c3acaec3f6f3ff53f361"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d50714405845c1010c871098558cfe5718fe39d2a2fab5f95c8863caeb7a82b3"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ec1c44cf9bd22079aac37a07cb49a29ced9050ab5bddf24e50aba298f1e34d90"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9e9aaef25a40d1f1e1bbb1d0eb0190c4a64a7a1750f7eb67b8399bed6f4fd2a6"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e54e088dc64dd2766014e7cfe5f8bc45399400fd486816e494f93e3f0f55da06"}, + {file = "regex-2026.7.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:834271b1ff2cfa1f67fcd65a48bf11d11e9ab837e21bf79ce554efb648599ae8"}, + {file = "regex-2026.7.10-cp314-cp314-win32.whl", hash = "sha256:f988a1cec68058f71a38471813fba9e87dffe855582682e8a10e40ece12567a2"}, + {file = "regex-2026.7.10-cp314-cp314-win_amd64.whl", hash = "sha256:2129e4a5e86f26926982d883dff815056f2e98220fdf630e59f961b578a26c43"}, + {file = "regex-2026.7.10-cp314-cp314-win_arm64.whl", hash = "sha256:9cd5b6805396157b4cf993a6940cbb8663161f29b4df2458c1c9991f099299c5"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:103e8f3acc3dcede88c0331c8612766bdcfc47c9250c5477f0e10e0550b9da49"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:538ddb143f5ca085e372def17ef3ed9d74b50ad7fc431bd85dc50a9af1a7076f"}, + {file = "regex-2026.7.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6e3448e86b05ce87d4eb50f9c680860830f3b32493660b39f43957d6263e2eba"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eab9d3f981c423afd1a61db055cfe83553c3f6455949e334db04722469dd0a2"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:177f930af3ad72e1045f8877540e0c43a38f7d328cf05f31963d0bd5f7ecf067"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd3b6d97beb39afb412f2c79522b9e099463c31f4c49ab8347c5a2ca3531c478"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8679f0652a183d93da646fcec8da8228db0be40d1595da37e6d74c2dc8c4713c"}, + {file = "regex-2026.7.10-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:494b19a5805438aeb582de99f9d97603d8fd48e6f4cc74d0088bb292b4da3b70"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0911e34151a5429d0325dae538ba9851ec0b62426bdfd613060cda8f1c36ec7f"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b862572b7a5f5ed47d2ba5921e63bf8d9e3b682f859d8f11e0e5ca46f7e82173"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:3f361215e000d68a4aff375106637b83c80be36091d83ee5107ad3b32bd73f48"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4533af6099543db32ef26abc2b2f824781d4eebb309ab9296150fd1a0c7eb07d"}, + {file = "regex-2026.7.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:668ab85105361d0200e3545bec198a1acfc6b0aeb5fff8897647a826e5a171be"}, + {file = "regex-2026.7.10-cp314-cp314t-win32.whl", hash = "sha256:dd7715817a187edd7e2a2390908757f7ba42148e59cad755fb8ee1160c628eca"}, + {file = "regex-2026.7.10-cp314-cp314t-win_amd64.whl", hash = "sha256:78712d4954234df5ca24fdadb65a2ab034213f0cdfde376c272f9fc5e09866bb"}, + {file = "regex-2026.7.10-cp314-cp314t-win_arm64.whl", hash = "sha256:749b92640e1970e881fdf22a411d74bf9d049b154f4ef7232eeb9a90dd8be7f3"}, + {file = "regex-2026.7.10.tar.gz", hash = "sha256:1050fedf0a8a92e843971120c2f57c3a99bea86c0dfa1d63a9fac053fe54b135"}, ] [[package]] @@ -2391,14 +2399,14 @@ dev = ["bitsandbytes", "blobfile", "cmake (>=3.19.0,<4.0.0)", "diskcache", "expe [[package]] name = "tqdm" -version = "4.69.0" +version = "4.68.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "tqdm-4.69.0-py3-none-any.whl", hash = "sha256:9979978912be667a6ef21fd5d8abf54e324e63d82f7f43c360792ebc2bc4e622"}, - {file = "tqdm-4.69.0.tar.gz", hash = "sha256:700c5e85dcd5f009dd6222588a29180a193a748247a5d855b4d67db93d79a53b"}, + {file = "tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2"}, + {file = "tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520"}, ] [package.dependencies] @@ -2499,14 +2507,14 @@ perf-analyzer = ["perf-analyzer"] [[package]] name = "typer" -version = "0.27.0" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1"}, - {file = "typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] @@ -2547,116 +2555,116 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "yarl" -version = "1.24.5" +version = "1.24.2" description = "Yet another URL library" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88f50c94e21a0a7f14042c015b0eba1881af78562e7bf007e0033e624da59750"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6efbccc3d7f75d5b03105172a8dc86d82ba4da86817952529dd93185f4a88be2"}, - {file = "yarl-1.24.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0ebfaffe1a16cb72141c8e09f18cc76856dbe58639f393a4f2b26e474b96b871"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ac73abdc7ab75610f95a8fd994c6457e87752b02a63987e188f937a1fc180f0"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4d97a951a81039050e45f04e96689b58b8243fa5e62aa14fe67cb6075300885e"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fe7b7bb170daccbba19ad33012d2b15f1e7942296fd4d45fc1b79013da8cc0f2"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89a1bbb58e0e3f7a283653d854b1e95d65e5cfd4af224dac5f02629ec1a3e621"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fa5e51397466ea7e98de493fa2ff1b8193cfef8a7b0f9b4842f92d342df0dba"}, - {file = "yarl-1.24.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4103b77b8a8225e413107d2349b65eb3c1c52627b5cc5c3c4c1c6a798b218950"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f9f3e9c8a9ecffa57bef8fb4fa19e5fa4d2d8307cf6bac5b1fca5e5860f4ba00"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c0ebc836c47a6477e182169c6a476fc691d12b518894bf7dd2572f0d59f1c7ed"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:96d30286dd02679e32a39aa8f0b7498fc847fcda46cfc09df5513e82ce252440"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:fd8c81f346b58f45818d09ea11db69a8d5fd34a224b79871f6d44f12cd7977b1"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5c55256dee8f4b27bfbf636c8363383c7c8db7890c7cba5217d7bd5f5f21dab6"}, - {file = "yarl-1.24.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f4d8cf085a4c6a40fb97ea0f46938a8df43c85d31f9d45e2a8867ea9293790d"}, - {file = "yarl-1.24.5-cp310-cp310-win_amd64.whl", hash = "sha256:240cbec09667c1fed4c6cd0060b9ec57332427d7441289a2ed8875dc9fb2b224"}, - {file = "yarl-1.24.5-cp310-cp310-win_arm64.whl", hash = "sha256:8a6987eaad834cb32dd57d9d582225f0054a5d1af706ccfbbdba735af4927e13"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a"}, - {file = "yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e"}, - {file = "yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077"}, - {file = "yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd"}, - {file = "yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25"}, - {file = "yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec"}, - {file = "yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9"}, - {file = "yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce"}, - {file = "yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba"}, - {file = "yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b"}, - {file = "yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb"}, - {file = "yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16"}, - {file = "yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144"}, - {file = "yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4"}, - {file = "yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740"}, - {file = "yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d"}, - {file = "yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9"}, - {file = "yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5"}, - {file = "yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4"}, - {file = "yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad"}, - {file = "yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba"}, - {file = "yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a"}, - {file = "yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6"}, - {file = "yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047"}, - {file = "yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104"}, - {file = "yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688"}, - {file = "yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7"}, - {file = "yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0"}, + {file = "yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90"}, + {file = "yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53"}, + {file = "yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342"}, + {file = "yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4"}, + {file = "yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c"}, + {file = "yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003"}, + {file = "yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b"}, + {file = "yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5"}, + {file = "yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45"}, + {file = "yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2"}, + {file = "yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c"}, + {file = "yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c"}, + {file = "yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1"}, + {file = "yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad"}, + {file = "yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607"}, + {file = "yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617"}, + {file = "yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056"}, + {file = "yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992"}, + {file = "yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656"}, + {file = "yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630"}, + {file = "yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208"}, + {file = "yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b"}, + {file = "yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8"}, + {file = "yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0"}, + {file = "yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761"}, + {file = "yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf"}, + {file = "yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe"}, + {file = "yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd"}, + {file = "yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215"}, + {file = "yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d"}, + {file = "yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9"}, + {file = "yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8"}, ] [package.dependencies] @@ -2739,4 +2747,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "5bd084cbcf8afceb839ea1f0a8f0500a95c66d59c772bf028b514d68f275790e" +content-hash = "5caec91a6dcd3e66c173ee2cb35f0c5563e0531aae1f701d98e15e50e839ff07" diff --git a/security_scanning/triton_backend/pyproject.toml b/security_scanning/triton_backend/pyproject.toml index f23e1fef69fa..43476e749d04 100644 --- a/security_scanning/triton_backend/pyproject.toml +++ b/security_scanning/triton_backend/pyproject.toml @@ -7,7 +7,7 @@ authors = [ ] requires-python = ">=3.10,<3.13" dependencies = [ - "regex (>=2026.7.19,<2027.0.0)", + "regex (>=2026.7.10,<2027.0.0)", "fire (>=0.7.1,<0.8.0)", "tritonclient[all] (>=2.70.0,<3.0.0)", "transformers (==5.5.4)", diff --git a/setup.py b/setup.py index e71819c66fb9..7a3b7d2af8f7 100644 --- a/setup.py +++ b/setup.py @@ -14,9 +14,6 @@ # limitations under the License. import os import platform -import re -import subprocess -import sys from pathlib import Path from setuptools import find_packages, setup @@ -73,23 +70,6 @@ def get_version(): if version is None: raise RuntimeError(f"Could not set version from {version_file}") - # For develop / editable installs (`pip install -e .` or - # `python setup.py develop`), append the git commit hash as a PEP 440 - # local version segment so the installed package is identifiable, - # e.g. "1.3.0rc21+58d8964d13". - is_develop = any(arg in sys.argv for arg in ("develop", "editable_wheel")) - if is_develop: - try: - commit = subprocess.check_output( - ["git", "rev-parse", "--short=10", "HEAD"], - cwd=Path(__file__).resolve().parent, - stderr=subprocess.DEVNULL).decode().strip() - except (subprocess.CalledProcessError, FileNotFoundError, OSError): - commit = "" - commit = re.sub(r"[^A-Za-z0-9.]", "", commit) - if commit: - version = f"{version}+{commit}" - return version @@ -129,7 +109,6 @@ def has_ext_modules(self): devel_deps, _ = parse_requirements( Path("requirements-dev-windows.txt" if on_windows else "requirements-dev.txt")) -mx_deps = ["modelexpress==0.4.1"] constraints_file = Path("constraints.txt") if constraints_file.exists(): constraints, _ = parse_requirements(constraints_file) @@ -137,13 +116,15 @@ def has_ext_modules(self): if on_windows: package_data = [ - 'libs/th_common.dll', 'libs/tensorrt_llm.dll', 'bindings.*.pyd', - "include/**/*" + 'libs/th_common.dll', 'libs/tensorrt_llm.dll', + 'libs/nvinfer_plugin_tensorrt_llm.dll', 'bindings.*.pyd', "include/**/*" ] else: package_data = [ + 'bin/executorWorker', 'libs/libtensorrt_llm.so', 'libs/libth_common.so', + 'libs/libnvinfer_plugin_tensorrt_llm.so', 'libs/libtensorrt_llm_ucx_wrapper.so', 'libs/libdecoder_attention_0.so', 'libs/libtensorrt_llm_nixl_wrapper.so', @@ -166,6 +147,7 @@ def has_ext_modules(self): 'deep_gemm/include/**/*', 'deep_gemm/*.py', 'deep_gemm_cpp_tllm.*.so', + 'scripts/install_tensorrt.sh', 'flash_mla/LICENSE', 'flash_mla/*.py', 'flash_mla_cpp_tllm.*.so', @@ -181,11 +163,14 @@ def has_ext_modules(self): package_data += [ 'bindings/*.pyi', 'bindings/**/*.pyi', + 'tools/plugin_gen/templates/*', + 'bench/build/benchmark_config.yml', 'evaluate/lm_eval_tasks/**/*', "_torch/auto_deploy/config/*.yaml", # Include CUDA source for fused MoE align extension so runtime JIT can find it in wheels '_torch/auto_deploy/custom_ops/fused_moe/moe_align_kernel.cu', '_torch/auto_deploy/custom_ops/fused_moe/triton_fused_moe_configs/*', + '_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/**/*.so', 'usage/schemas/*.json', ] @@ -421,9 +406,6 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str], # internal absolute imports (e.g., "from triton_kernels.foo import bar") work. packages += find_packages(include=["triton_kernels", "triton_kernels.*"]) -msa_package_dir = {"fmha_sm100": "3rdparty/MSA/python/fmha_sm100"} -packages += ["fmha_sm100"] - # https://setuptools.pypa.io/en/latest/references/keywords.html setup( name='tensorrt_llm', @@ -438,7 +420,6 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str], url="https://github.com/NVIDIA/TensorRT-LLM", download_url="https://github.com/NVIDIA/TensorRT-LLM/tags", packages=packages, - package_dir=msa_package_dir, exclude_package_data=exclude_package_data, # TODO Add windows support for python bindings. classifiers=[ @@ -451,17 +432,8 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str], license="Apache License 2.0", keywords="nvidia tensorrt deeplearning inference", package_data={ - 'tensorrt_llm': - package_data, + 'tensorrt_llm': package_data, 'triton_kernels': ['LICENSE', 'VERSION', 'README.md'], - 'fmha_sm100': [ - '*.py', - 'csrc/**/*', - 'cute/**/*', - 'cutlass/include/**/*', - 'cutlass/tools/util/include/**/*', - 'cutlass/LICENSE.txt', - ], }, license_files=get_license(), entry_points={ @@ -474,7 +446,10 @@ def extract_from_precompiled(precompiled_location: str, package_data: list[str], scripts=['tensorrt_llm/llmapi/trtllm-llmapi-launch'], extras_require={ "devel": devel_deps, - "mx": mx_deps, + # MX remains prototype-only and is intentionally not declared as an + # optional package extra until its external dependency completes OSS + # allowlist onboarding. Keep install instructions in docs/PR text + # rather than packaging metadata. }, zip_safe=True, install_requires=required_deps, diff --git a/tensorrt_llm/__init__.py b/tensorrt_llm/__init__.py index d707ca0e0694..91e5f98a500d 100644 --- a/tensorrt_llm/__init__.py +++ b/tensorrt_llm/__init__.py @@ -104,6 +104,24 @@ def _setup_vendored_triton_kernels(): # ImportError: libc10.so: cannot open shared object file: No such file or directory import torch # noqa + +def _preload_tensorrt_libs(): + """Preload the TensorRT libraries needed by the bindings extension. + + The C++ runtime still links against the TensorRT libraries until it is + decoupled from TensorRT. Importing the tensorrt package loads libnvinfer + from the tensorrt_libs wheel for environments where it is not on the + system loader path; without it, importing tensorrt_llm.bindings raises + ImportError: libnvinfer.so.10: cannot open shared object file. + """ + try: + import tensorrt # noqa: F401 + except ImportError: + pass + + +_preload_tensorrt_libs() + import tensorrt_llm._torch.models as torch_models import tensorrt_llm.math_utils as math_utils import tensorrt_llm.models as models diff --git a/tensorrt_llm/_common.py b/tensorrt_llm/_common.py index d7feeededb9f..00bac35cc11b 100644 --- a/tensorrt_llm/_common.py +++ b/tensorrt_llm/_common.py @@ -18,6 +18,7 @@ import platform import threading import time +from functools import wraps from pathlib import Path import torch @@ -88,3 +89,27 @@ def _print_stacks(): print_stacks_thread.start() logger.info("TensorRT LLM inited.") + + +# TODO: dead on the Python side (no remaining @_is_building users); the IS_BUILDING +# env var is only read by the C++ isBuilding() in the TensorRT plugins. Remove this +# together with that C++ half in the C++ decouple step. +class _BuildingFlag: + def __enter__(self): + os.environ["IS_BUILDING"] = "1" + + def __exit__(self, type, value, tb): + del os.environ["IS_BUILDING"] + + +def _is_building(f): + """Use this to decorate functions which are called during engine building/refitting process, + otherwise, the plugin registration will fail. + """ + + @wraps(f) + def decorated(*args, **kwargs): + with _BuildingFlag(): + return f(*args, **kwargs) + + return decorated diff --git a/tensorrt_llm/_deprecation.py b/tensorrt_llm/_deprecation.py new file mode 100644 index 000000000000..1d436040ef9a --- /dev/null +++ b/tensorrt_llm/_deprecation.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Legacy-workflow warnings for the TensorRT engine-build path. + +The TensorRT engine-build workflow (convert_checkpoint.py -> trtllm-build -> +run.py) is a legacy path. The PyTorch backend (trtllm-serve / LLM API) is the +recommended approach for new projects. + +This module provides a shared warning function that can be called from legacy +scripts and internal chokepoints to inform users about the recommended +migration path. +""" + +import warnings + +_DEPRECATION_DOCS_URL = "https://nvidia.github.io/TensorRT-LLM/quick-start-guide.html" + +_warned: set = set() + + +def emit_engine_arch_deprecation(caller_name: str) -> None: + """Emit a one-time FutureWarning for legacy engine-architecture usage. + + Each unique *caller_name* triggers the warning at most once per process, + so hot paths like ``builder.build()`` don't spam the console. + + Args: + caller_name: Human-readable identifier for the caller + (e.g., ``"convert_checkpoint.py"``, ``"trtllm-build"``, + ``"builder.build()"``). + """ + if caller_name in _warned: + return + _warned.add(caller_name) + + warnings.warn( + f"\n{'=' * 70}\n" + f"LEGACY WARNING: {caller_name}\n" + f"{'=' * 70}\n" + f"This is part of the legacy TensorRT engine-build workflow.\n" + f"New projects should use the PyTorch backend instead.\n\n" + f" # Serve a model (recommended):\n" + f" trtllm-serve \n\n" + f" # Python API:\n" + f" from tensorrt_llm import LLM\n" + f" llm = LLM(model='')\n" + f" output = llm.generate(['Hello, how are you?'])\n\n" + f"Documentation: {_DEPRECATION_DOCS_URL}\n" + f"{'=' * 70}\n", + FutureWarning, + stacklevel=2, + ) diff --git a/tensorrt_llm/_torch/attention_backend/flashinfer.py b/tensorrt_llm/_torch/attention_backend/flashinfer.py index 733d477009cb..0cdbdff5761a 100644 --- a/tensorrt_llm/_torch/attention_backend/flashinfer.py +++ b/tensorrt_llm/_torch/attention_backend/flashinfer.py @@ -1,17 +1,3 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import functools import math import os @@ -221,10 +207,6 @@ class FlashInferWrappers: repr=False) host_decode_block_tables: Optional[torch.Tensor] = field(default=None, repr=False) - # Remember the previously populated rectangle so a smaller subsequent batch can clear stale rows - # and columns before narrowing future updates. - decode_block_table_active_rows: int = field(default=0, repr=False) - decode_block_table_active_width: int = field(default=0, repr=False) @dataclass(kw_only=True) @@ -648,7 +630,7 @@ def _post_init_with_buffers(self, buffers) -> None: self._host_pool_indices: Dict[int, torch.Tensor] = {} self._host_paged_kv_indices: Optional[torch.Tensor] = None self._host_paged_kv_indptr_decode: Optional[torch.Tensor] = None - self._max_num_blocks_per_seq = 0 + self._max_num_blocks = 0 # VSWA (Variable Sliding Window Attention): models with per-layer # max_attention_window create separate V2 pool groups with independent @@ -668,39 +650,30 @@ def _post_init_with_buffers(self, buffers) -> None: ) # Maximum block count across ALL pools: sizes the VSWA pool - # buffers below. Computed for every model, not just VSWA — - # non-VSWA managers have a single pool, so this stays - # blocks_in_primary_pool for them. + # buffers below and bounds the per-request width of the + # persistent trtllm-gen decode block tables (a request can + # never reference more blocks than its pool holds). Computed + # for every model, not just VSWA — non-VSWA managers have a + # single pool, so this stays blocks_in_primary_pool for them. max_num_blocks = blocks_in_primary_pool if hasattr(self.kv_cache_manager, 'layer_offsets'): for lid in self.kv_cache_manager.layer_offsets: lbuf = self.kv_cache_manager.get_buffers(lid) if lbuf is not None: max_num_blocks = max(max_num_blocks, lbuf.shape[0]) - self._max_num_blocks_per_seq = self.kv_cache_manager.max_blocks_per_seq - - # Layers may share one page-index list only when they are in the - # same pool AND have the same page-index scale: VSWA splits pools, - # and per-layer geometry (e.g. Gemma4 sliding/global head_dim) - # splits scales even when all windows collapse to max_seq_len and - # is_vswa is False. Guarded on V2-specific attributes (V1 managers - # lack the per-pool infrastructure). - mgr = self.kv_cache_manager - get_scale = getattr(mgr, 'get_layer_page_index_scale', None) - layer_space: Dict[int, int] = {} - if hasattr(mgr, 'layer_to_pool_mapping_dict') and get_scale: - space_ids = {} - for layer_idx in getattr(mgr, 'layer_offsets', {}): - layer_offset = mgr.layer_offsets[layer_idx] - key = (mgr.layer_to_pool_mapping_dict[layer_offset], - get_scale(layer_idx)) - space_ids.setdefault(key, len(space_ids)) - layer_space[layer_idx] = space_ids[key] - if layer_space and (getattr(mgr, 'is_vswa', False) - or len(set(layer_space.values())) > 1): + self._max_num_blocks = max_num_blocks + + # Detect VSWA: check if the manager has multiple pools. + # Guard on layer_to_pool_mapping_dict which is V2-specific — V1 + # managers also expose is_vswa but lack the per-pool infrastructure. + if (getattr(self.kv_cache_manager, 'is_vswa', False) and hasattr( + self.kv_cache_manager, 'layer_to_pool_mapping_dict')): + mgr = self.kv_cache_manager self._vswa_layer_to_pool = {} self._vswa_pool_to_rep_layer: Dict[int, int] = {} - for layer_idx, pool_id in layer_space.items(): + for layer_idx in getattr(mgr, 'layer_offsets', {}): + layer_offset = mgr.layer_offsets[layer_idx] + pool_id = mgr.layer_to_pool_mapping_dict[layer_offset] self._vswa_layer_to_pool[layer_idx] = pool_id if pool_id not in self._vswa_pool_to_rep_layer: self._vswa_pool_to_rep_layer[pool_id] = layer_idx @@ -1005,9 +978,9 @@ def _build_decode_block_tables( gen_num_blocks = np.asarray(self.num_blocks[self.num_contexts:], dtype=np.int64) max_n = int(gen_num_blocks.max()) - if max_n > self._max_num_blocks_per_seq: - # A request can never reference more than max_blocks_per_seq; - # defensive guard for inconsistent metadata. + if max_n > self._max_num_blocks: + # A request can never reference more blocks than any pool + # holds; defensive guard for inconsistent metadata. return None block_tables = wrappers.decode_block_tables if (self.is_cuda_graph and block_tables is not None @@ -1019,12 +992,12 @@ def _build_decode_block_tables( if self.is_cuda_graph: # Allocated once at capture warmup; full capacity width so # replays never need a wider table. - width = self._max_num_blocks_per_seq + width = self._max_num_blocks else: # Eager path replans (and re-reads the table) every step, # so the buffer may grow geometrically as sequences do. width = min(max(64, 1 << (max_n - 1).bit_length()), - self._max_num_blocks_per_seq) + self._max_num_blocks) try: block_tables = torch.zeros((self.max_num_requests, width), dtype=torch.int32, @@ -1055,10 +1028,6 @@ def _build_decode_block_tables( # completed. block_tables[:num_gens, :max_n].copy_( host_block_tables[:num_gens, :max_n], non_blocking=True) - # Seed the extent used by `prepare()` to clear entries if the first graph replay has fewer - # requests or a narrower block table. - wrappers.decode_block_table_active_rows = num_gens - wrappers.decode_block_table_active_width = max_n return block_tables[:num_gens] def _clean_cached_plans(self, *, defer_plan: bool): @@ -1330,7 +1299,7 @@ def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: if (self.is_cuda_graph and self._vswa_layer_to_pool is not None and self._vswa_pool_indices_cache is not None and self.num_generations > 0): - decode_blocks = num_blocks[self.num_contexts:] + decode_blocks = self.num_blocks[self.num_contexts:] head_dim_to_pool = getattr(self, '_vswa_head_dim_to_pool', None) for plan_params, wrappers in self._plan_params_to_wrappers.items(): if plan_params.attention_mask_data is not None: @@ -1343,62 +1312,34 @@ def _to_int32_tensor(arr: np.ndarray) -> torch.Tensor: if head_dim_to_pool else None) if pool_id is None: continue + pool_buf = self._vswa_pool_indices_cache[pool_id] batch_size, table_width = block_tables.shape rows = min(batch_size, self.num_generations) - num_blocks_per_row = decode_blocks[:rows] - active_width = min(int(num_blocks_per_row.max()), table_width) - # Include the previous live extent once when the batch or its longest sequence - # shrinks. The zero-filled copy below clears entries that are no longer live; the - # next update can shrink. - update_rows = min( - batch_size, - max(rows, wrappers.decode_block_table_active_rows)) - update_width = min( - table_width, - max(active_width, wrappers.decode_block_table_active_width)) - host_block_tables = wrappers.host_decode_block_tables - if (host_block_tables is None - or host_block_tables.size(0) < update_rows - or host_block_tables.size(1) < update_width): - # Grow geometrically to avoid reallocating whenever the longest sequence - # acquires one more KV-cache block. - host_width = min( - max(64, 1 << (update_width - 1).bit_length()), - table_width) - host_block_tables = torch.zeros( - (self.max_num_requests, host_width), - dtype=torch.int32, - pin_memory=prefer_pinned()) - wrappers.host_decode_block_tables = host_block_tables - - # Build only the live (or previously live) rectangle on the host. This preserves - # padded-row zeroing when batch size shrinks without launching full-capacity - # `arange`, `gather`, `where`, `zero`, and copy kernels on every decode step. - host_table = host_block_tables[:update_rows, :update_width] - host_table.zero_() - host_pool_indices = self._host_pool_indices[pool_id] - # Pool indices are flattened as context blocks followed by each generation request's - # blocks. - source_offset = self.num_context_blocks - # This loop scales with the number of generation requests. Vectorizing ragged - # rows would require padded mask or index tensor, so keep contiguous row copies - # until profiling identifies this as a CPU bottleneck. - for row, num_blocks_for_row in enumerate(num_blocks_per_row): - num_blocks_for_row = int(num_blocks_for_row) - copy_width = min(num_blocks_for_row, table_width) - host_table[row, :copy_width].copy_( - host_pool_indices[source_offset:source_offset + - copy_width]) - # Advance by the uncropped count so the next request starts at the correct - # offset even if this row hit `table_width`. - source_offset += num_blocks_for_row - # Keep the graph-captured device address stable and transfer only the compact - # rectangle prepared in pinned host memory. - block_tables[:update_rows, :update_width].copy_( - host_block_tables[:update_rows, :update_width], - non_blocking=True) - wrappers.decode_block_table_active_rows = rows - wrappers.decode_block_table_active_width = active_width + # Vectorized equivalent of a per-request copy loop: row i + # gets pool_buf[offset + row_starts[i] :] for its first + # min(num_blocks_per_row[i], table_width) columns, zero- + # padded — one gather + where instead of ~batch slice + # copies per pool per step. + num_blocks_per_row = torch.tensor( + decode_blocks[:rows], + dtype=torch.int64).to(device=block_tables.device, + non_blocking=True) + row_starts = torch.cumsum(num_blocks_per_row, + dim=0) - num_blocks_per_row + columns = torch.arange(table_width, + dtype=torch.int64, + device=block_tables.device) + mask = columns.unsqueeze(0) < num_blocks_per_row.clamp( + max=table_width).unsqueeze(1) + source_indices = (self.num_context_blocks + + row_starts.unsqueeze(1) + + columns.unsqueeze(0)).clamp( + max=pool_buf.numel() - 1) + new_block_tables = torch.zeros_like(block_tables) + new_block_tables[:rows] = torch.where( + mask, pool_buf[source_indices.reshape(-1)].view( + rows, table_width), new_block_tables[:rows]) + block_tables.copy_(new_block_tables) kv_lens_buf = getattr(decode_wrapper, '_kv_lens_buffer', None) if kv_lens_buf is not None: decode_kv_lens = _to_int32_tensor( diff --git a/tensorrt_llm/_torch/attention_backend/fmha/__init__.py b/tensorrt_llm/_torch/attention_backend/fmha/__init__.py index b2cd1e75ec7e..1c3981abcf91 100644 --- a/tensorrt_llm/_torch/attention_backend/fmha/__init__.py +++ b/tensorrt_llm/_torch/attention_backend/fmha/__init__.py @@ -16,7 +16,6 @@ from .fallback import FallbackFmha from .flashinfer_trtllm_gen import FlashInferTrtllmGenFmha from .interface import Fmha -from .msa_sparse_gqa import MsaSparseGqaFmha from .phased import FmhaParams, PhasedFmha from .registry import DEFAULT_FMHA_LIBS, FMHA_LIBS, FmhaCls, get_enabled_fmha_lib_classes @@ -28,7 +27,6 @@ "Fmha", "FmhaCls", "FmhaParams", - "MsaSparseGqaFmha", "PhasedFmha", "get_enabled_fmha_lib_classes", ] diff --git a/tensorrt_llm/_torch/attention_backend/fmha/fallback.py b/tensorrt_llm/_torch/attention_backend/fmha/fallback.py index 9c081bf19dfc..8f1da5dc36f9 100644 --- a/tensorrt_llm/_torch/attention_backend/fmha/fallback.py +++ b/tensorrt_llm/_torch/attention_backend/fmha/fallback.py @@ -98,7 +98,6 @@ def forward( spec_decoding_bl_tree_mask_offset=metadata.spec_decoding_bl_tree_mask_offset, spec_decoding_bl_tree_mask=metadata.spec_decoding_bl_tree_mask, spec_decoding_target_max_draft_tokens=metadata.max_total_draft_tokens, - force_prepare_spec_dec_tree_mask=metadata.force_prepare_spec_dec_tree_mask, spec_bl_tree_first_sparse_mask_offset_kv=metadata.spec_bl_tree_first_sparse_mask_offset_kv, num_sparse_topk=metadata.num_sparse_topk, flash_mla_tile_scheduler_metadata=metadata.flash_mla_tile_scheduler_metadata, diff --git a/tensorrt_llm/_torch/attention_backend/fmha/flashinfer_trtllm_gen.py b/tensorrt_llm/_torch/attention_backend/fmha/flashinfer_trtllm_gen.py index ff2002c36d57..21e67006fc95 100644 --- a/tensorrt_llm/_torch/attention_backend/fmha/flashinfer_trtllm_gen.py +++ b/tensorrt_llm/_torch/attention_backend/fmha/flashinfer_trtllm_gen.py @@ -67,21 +67,26 @@ ) -_MULTI_CTAS_KV_COUNTER_ALIGNMENT = 8 +def _clear_multi_ctas_kv_counter_workspace( + fmha_workspace: torch.Tensor, + num_heads: int, + max_num_requests: int, + multi_processor_count: Optional[int], +) -> None: + counter_size = _get_multi_ctas_kv_counter_size( + num_heads, + max_num_requests, + multi_processor_count, + ) + fmha_workspace.flatten().narrow(0, 0, counter_size).zero_() def _get_multi_ctas_kv_counter_size( num_heads: int, max_num_requests: int, - multi_processor_count: int, + multi_processor_count: Optional[int], ) -> int: - num_counters = max(num_heads * max_num_requests, multi_processor_count) - aligned_num_counters = ( - (num_counters + _MULTI_CTAS_KV_COUNTER_ALIGNMENT - 1) - // _MULTI_CTAS_KV_COUNTER_ALIGNMENT - * _MULTI_CTAS_KV_COUNTER_ALIGNMENT - ) - return aligned_num_counters * torch.int32.itemsize + return max(num_heads * max_num_requests, multi_processor_count or 0) * torch.int32.itemsize def _get_bmm1_scale_log2(bmm1_scale: torch.Tensor) -> torch.Tensor: @@ -94,7 +99,6 @@ def _trtllm_gen_batch_decode_with_kv_cache( query: torch.Tensor, kv_pool: torch.Tensor, workspace_buffer: torch.Tensor, - multi_ctas_kv_counter_buffer: torch.Tensor, block_tables: torch.Tensor, seq_lens: torch.Tensor, max_seq_len: int, @@ -134,7 +138,6 @@ def _trtllm_gen_batch_decode_with_kv_cache( kv_pool, kv_pool, workspace_buffer, - multi_ctas_kv_counter_buffer, block_tables, seq_lens, decode_max_q_len, @@ -166,7 +169,6 @@ def _trtllm_gen_batch_context_with_kv_cache( query: torch.Tensor, kv_pool: torch.Tensor, workspace_buffer: torch.Tensor, - multi_ctas_kv_counter_buffer: torch.Tensor, block_tables: torch.Tensor, seq_lens: torch.Tensor, max_q_len: int, @@ -197,7 +199,6 @@ def _trtllm_gen_batch_context_with_kv_cache( kv_pool, kv_pool, workspace_buffer, - multi_ctas_kv_counter_buffer, block_tables, seq_lens, max_q_len, @@ -419,7 +420,6 @@ def __init__(self, attn: "TrtllmAttention"): # Lazily set on the first forward() call from the query device. self._multi_processor_count: Optional[int] = None - self._multi_ctas_kv_counter_buffer: Optional[torch.Tensor] = None def _get_total_num_blocks(self, meta: "TrtllmAttentionMetadata") -> int: kv_cache_manager = meta.kv_cache_manager @@ -802,28 +802,6 @@ def prepare_workspace( if self._multi_processor_count is None: self._multi_processor_count = self._get_multi_processor_count(q.device) - required_counter_size = _get_multi_ctas_kv_counter_size( - attn.num_heads, - metadata.max_num_requests, - self._multi_processor_count, - ) - counter_buffer = self._multi_ctas_kv_counter_buffer - if ( - counter_buffer is None - or counter_buffer.device != q.device - or counter_buffer.numel() * counter_buffer.element_size() < required_counter_size - ): - if metadata.is_cuda_graph and torch.cuda.is_current_stream_capturing(): - raise RuntimeError( - "The trtllm-gen multi-CTA KV counter buffer must be allocated " - "before CUDA graph capture." - ) - self._multi_ctas_kv_counter_buffer = torch.zeros( - required_counter_size, - dtype=torch.uint8, - device=q.device, - ) - num_tokens = q.size(0) attention_input_type = forward_args.attention_input_type is_gen_only = attention_input_type == AttentionInputType.generation_only @@ -857,12 +835,6 @@ def prepare_workspace( required_workspace_numel = math.ceil(required_workspace_size / workspace.element_size()) workspace.resize_((required_workspace_numel,)) - def _get_multi_ctas_kv_counter_buffer(self) -> torch.Tensor: - counter_buffer = self._multi_ctas_kv_counter_buffer - if counter_buffer is None: - raise RuntimeError("The trtllm-gen multi-CTA KV counter buffer is not initialized.") - return counter_buffer - @staticmethod def _compute_window_left( cyclic_attention_window_size: int, @@ -989,7 +961,6 @@ def run_context( q_processed, # query kv_pool, # kv_pool fmha_workspace, # workspace_buffer - self._get_multi_ctas_kv_counter_buffer(), # multi_ctas_kv_counter_buffer block_tables, # block_tables params.sequence_lengths, # seq_lens max_q_len, # max_q_len @@ -1121,6 +1092,22 @@ def run_generation( params.is_cross, # is_cross ) + # FIXME: Flashinfer trtllm-gen API doesn't support a separate + # multi CTAs counter buffer. We have to clear a small buffer + # before trtllm_gen_batch_decode_with_kv_cache. + # + # We must also avoid clearing the workspace only when it is + # resized. The warmup phase may have already cached the workspace + # pointer; if the capture phase skips the zeroing step, the + # CUDA graph will not include the counter initialization. We + # have already verified—specifically in the context of the GPTOSS-20B + # test graph replay scenario—that this skipping logic is unsafe. + # + # https://github.com/flashinfer-ai/flashinfer/issues/3433 + _clear_multi_ctas_kv_counter_workspace( + fmha_workspace, attn.num_heads, meta.max_num_requests, self._multi_processor_count + ) + q_len_per_req = None if is_multi_token_gen else params.input_seq_length decode_max_q_len = max_q_len if is_multi_token_gen else None decode_cu_seqlens = cu_seqlens if is_multi_token_gen else None @@ -1144,7 +1131,6 @@ def run_generation( q_processed, # query kv_pool, # kv_pool fmha_workspace, # workspace_buffer - self._get_multi_ctas_kv_counter_buffer(), # multi_ctas_kv_counter_buffer block_tables, # block_tables params.sequence_lengths, # seq_lens max_kv_len, # max_seq_len @@ -1247,6 +1233,9 @@ def run_mla_generation( bmm1_scale = 1.0 / (attn.q_scaling * math.sqrt(qk_nope_head_dim + qk_rope_head_dim)) bmm2_scale = 1.0 workspace_buffer = params.workspace.view(-1, 4) + _clear_multi_ctas_kv_counter_workspace( + workspace_buffer, attn.num_heads, meta.max_num_requests, self._multi_processor_count + ) flashinfer.mla.trtllm_batch_decode_with_kv_cache_mla( query, # query @@ -1268,5 +1257,4 @@ def run_mla_generation( "trtllm-gen", # backend True, # is_var_seq self.USE_SHARED_PAGED_KV_IDX, # uses_shared_paged_kv_idx - multi_ctas_kv_counter_buffer=self._get_multi_ctas_kv_counter_buffer(), ) diff --git a/tensorrt_llm/_torch/attention_backend/fmha/msa_sparse_gqa.py b/tensorrt_llm/_torch/attention_backend/fmha/msa_sparse_gqa.py deleted file mode 100644 index 0bbb1ad1509a..000000000000 --- a/tensorrt_llm/_torch/attention_backend/fmha/msa_sparse_gqa.py +++ /dev/null @@ -1,237 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Block-sparse GQA FMHA backed by MSA's fmha_sm100 kernel. - -MsaSparseGqaFmha wraps the fmha_sm100 paged sparse GQA kernel and -participates in the standard TrtllmAttention.forward dispatch loop. The -owning MiniMax-M3 MSA attention layer runs an MsaIndexer to select the -per-query KV blocks and publishes them on forward_args.sparse_prediction; -this class attends over them. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -import torch - -from tensorrt_llm._utils import is_sm_100f - -from .interface import Fmha - -if TYPE_CHECKING: - from tensorrt_llm._torch.attention_backend.interface import AttentionForwardArgs - from tensorrt_llm._torch.attention_backend.trtllm import ( - TrtllmAttention, - TrtllmAttentionMetadata, - ) - - -def run_msa_sparse_gqa( - q: torch.Tensor, - k_paged: torch.Tensor, - v_paged: torch.Tensor, - kv_block_indexes: Optional[torch.Tensor] = None, - *, - kv_indices: torch.Tensor, - sm_scale: float, - qo_lens_cpu: Optional[torch.Tensor] = None, - kv_lens_cpu: Optional[torch.Tensor] = None, - qo_offset_cpu: Optional[torch.Tensor] = None, - causal: bool = True, - head_dim: int = 128, - plan: Optional[tuple] = None, - out: Optional[torch.Tensor] = None, - use_fp8: bool = False, -) -> None: - """Run fmha_sm100 paged GQA (plan/run split). - - `kv_block_indexes`: if set, sparse top-k mode (fixed `kv_block_num=topk`); - if None, dense mode attending all pages in `kv_indices`. - `plan`: prebuilt execution plan; if None, built inline from the CPU length - tensors (eager prefill/tests vs. CUDA-graph decode). - `out`: destination buffer the kernel writes in place. - `use_fp8`: FP8 KV cache. The caller must pass FP8 `q` to match the FP8 paged - K/V, since the kernel variant shares one dtype across q/k/v. Also selects the - FP8 AOT kernels for an inline sparse-prefill plan; no-op for the decode planner. - """ - from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_utils import require_msa_module - - fmha_sm100 = require_msa_module() - - if q.dim() != 3: - raise ValueError( - f"MsaSparseGqaFmha expects q [total_q, num_qo_heads, head_dim]; got {tuple(q.shape)}." - ) - if q.shape[-1] != head_dim: - raise NotImplementedError( - f"MsaSparseGqaFmha supports head_dim={head_dim}; got {q.shape[-1]}." - ) - if k_paged.dim() != 4 or v_paged.dim() != 4: - raise ValueError( - "MsaSparseGqaFmha expects paged KV [num_pages, num_kv_heads, page_size, head_dim]; " - f"got k={tuple(k_paged.shape)}, v={tuple(v_paged.shape)}." - ) - if k_paged.shape != v_paged.shape: - raise ValueError( - f"MsaSparseGqaFmha requires k and v to share shape; " - f"got k={tuple(k_paged.shape)}, v={tuple(v_paged.shape)}." - ) - - if plan is None: - # kv_block_num is planned only for the sparse (block-indexed) path; - # dense paged GQA leaves it unset and attends the full page table. - kv_block_num = int(kv_block_indexes.shape[-1]) if kv_block_indexes is not None else -1 - plan = fmha_sm100.fmha_sm100_plan( - qo_lens_cpu, - kv_lens_cpu, - int(q.shape[1]), # num query heads. - num_kv_heads=int(k_paged.shape[1]), - qo_offset=qo_offset_cpu, - page_size=int(k_paged.shape[2]), - kv_block_num=kv_block_num, - causal=causal, - num_kv_splits=1, - use_fp8_kvcache=use_fp8, - ) - fmha_sm100.fmha_sm100( - q, - k_paged, - v_paged, - plan, - kv_indices=kv_indices, - kv_block_indexes=kv_block_indexes, - out=out, - sm_scale=sm_scale, - output_maxscore=False, - ) - - -def run_msa_paged_gqa( - attn: "TrtllmAttention", - q: torch.Tensor, - k: Optional[torch.Tensor], - v: Optional[torch.Tensor], - metadata: "TrtllmAttentionMetadata", - output: torch.Tensor, - *, - kv_block_indexes: Optional[torch.Tensor], - plan: Optional[tuple], -) -> None: - """Write the new-token main K/V, then run paged GQA into output in place. - - Shared by the sparse layers (kv_block_indexes is the per-query top-k table, - with the sparse plan) and the dense layers (kv_block_indexes None, with the - dense plan, attending the full page table). fmha_sm100 reads the paged cache - directly, so the new-token K/V must be resident before the run. - """ - from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_utils import ( - msa_paged_kv, - write_msa_main_kv, - ) - - layer_idx = attn.layer_idx - head_dim = attn.head_dim - kv_cache_manager = metadata.kv_cache_manager - num_tokens = int(q.shape[0]) - if k is not None and v is not None: - write_msa_main_kv( - kv_cache_manager, layer_idx, metadata.msa_out_cache_loc[:num_tokens], k, v - ) - - q_view = q.view(num_tokens, attn.num_heads, head_dim) - out_view = output.view(num_tokens, attn.num_heads, head_dim) - k_paged, v_paged = msa_paged_kv(kv_cache_manager, layer_idx) - sm_scale = (head_dim**-0.5) / float(attn.q_scaling) - - # The fmha_sm100 variant is chosen from q.dtype and shares one dtype across - # q/k/v, so q must be FP8 to match an FP8 paged K/V. MiniMax-M3 has no - # KV-cache scales, so the scale is 1.0 and this is a plain E4M3 cast. - use_fp8 = k_paged.dtype == torch.float8_e4m3fn - if use_fp8: - q_view = q_view.to(torch.float8_e4m3fn) - - run_msa_sparse_gqa( - q_view, - k_paged, - v_paged, - kv_block_indexes, - kv_indices=metadata.msa_kv_indices, - sm_scale=sm_scale, - qo_lens_cpu=metadata.msa_qo_lens_cpu, - kv_lens_cpu=metadata.msa_kv_lens_cpu, - qo_offset_cpu=metadata.msa_qo_offset_cpu, - causal=True, - head_dim=head_dim, - plan=plan, - out=out_view, - use_fp8=use_fp8, - ) - - -class MsaSparseGqaFmha(Fmha): - """SM100 paged GQA FMHA powered by MSA's fmha_sm100 kernel. - - Handles every MiniMax-M3 MSA layer. Sparse layers pass the indexer's - selected KV block indices on forward_args.sparse_prediction.sparse_attn_indices - and attend those blocks; dense layers leave the indices None and attend the - full page table. - - Inherits Fmha rather than PhasedFmha: fmha_sm100 takes a single plan and - the selected block indices span the whole batch, so it handles a mixed - context and generation batch in one call and there is no - context/generation split from PhasedFmha to reuse. Requires head_dim 128 - and 4-D HND paged K/V. - """ - - @classmethod - def is_available(cls, attn: Optional["TrtllmAttention"] = None) -> bool: - # fmha_sm100 runs only on the SM100 family and ships in the MSA git - # submodule, so it is unavailable off SM100 or without the package. - # Imported lazily because the minimax_m3 package init imports the trtllm - # attention classes, which a module-scope import here would cycle with. - from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_utils import ( - msa_package_available, - ) - - if not is_sm_100f() or not msa_package_available(): - return False - # Only the MiniMax-M3 MSA layer uses this library. Matching the lowered - # sparse algorithm lets the base create_fmha_libs add it to that layer - # alone, so no create_fmha_libs override is needed. - return attn.sparse_params is not None and attn.sparse_params.algorithm == "minimax_m3" - - def forward( - self, - q: torch.Tensor, - k: Optional[torch.Tensor], - v: Optional[torch.Tensor], - metadata: "TrtllmAttentionMetadata", - forward_args: "AttentionForwardArgs", - ) -> None: - output = forward_args.output - if output is None: - raise RuntimeError(f"{type(self).__name__} requires an output buffer.") - - # Sparse layers attend the per-query top-k blocks with the sparse plan; - # dense layers leave the indices None and attend the full page table - # with the dense plan. - kv_block_indexes = forward_args.sparse_prediction.sparse_attn_indices - plan = ( - metadata.msa_decode_gqa_plan - if kv_block_indexes is not None - else metadata.msa_decode_dense_plan - ) - run_msa_paged_gqa( - self.attn, - q, - k, - v, - metadata, - output, - kv_block_indexes=kv_block_indexes, - plan=plan, - ) - - -__all__ = ["MsaSparseGqaFmha"] diff --git a/tensorrt_llm/_torch/attention_backend/fmha/registry.py b/tensorrt_llm/_torch/attention_backend/fmha/registry.py index 2b3ef3fc7ff3..97467657e9b5 100644 --- a/tensorrt_llm/_torch/attention_backend/fmha/registry.py +++ b/tensorrt_llm/_torch/attention_backend/fmha/registry.py @@ -22,24 +22,10 @@ FmhaCls: TypeAlias = type[Fmha] - -def init_fmha_libs() -> dict[str, "FmhaCls"]: - """Build the ordered FMHA library registry. - - Backend classes are imported inside this factory rather than at module - scope, so backends can import trtllm attention classes at module scope - without an import cycle. - """ - from .msa_sparse_gqa import MsaSparseGqaFmha - - return { - "msa_sparse_gqa": MsaSparseGqaFmha, - "flashinfer_trtllm_gen": FlashInferTrtllmGenFmha, - "fallback": FallbackFmha, - } - - -FMHA_LIBS: dict[str, FmhaCls] = init_fmha_libs() +FMHA_LIBS: dict[str, FmhaCls] = { + "flashinfer_trtllm_gen": FlashInferTrtllmGenFmha, + "fallback": FallbackFmha, +} DEFAULT_FMHA_LIBS: tuple[str, ...] = tuple(FMHA_LIBS) @@ -92,5 +78,4 @@ def get_enabled_fmha_lib_classes() -> list[FmhaCls]: "FMHA_LIBS", "FmhaCls", "get_enabled_fmha_lib_classes", - "init_fmha_libs", ] diff --git a/tensorrt_llm/_torch/attention_backend/interface.py b/tensorrt_llm/_torch/attention_backend/interface.py index fec6ba4e6b92..fc6cfdd50222 100644 --- a/tensorrt_llm/_torch/attention_backend/interface.py +++ b/tensorrt_llm/_torch/attention_backend/interface.py @@ -411,14 +411,10 @@ def create_cuda_graph_metadata(self, return cuda_graph_metadata def prepare_for_spec_dec(self, *fields) -> None: - assert len(self._saved_tensors) == 0, ( - "prepare_for_spec_dec called while fields " - f"{list(self._saved_tensors)} are still saved; a previous " - "forward likely raised between prepare_for_spec_dec and " - "restore_from_spec_dec") + assert len(self._saved_tensors) == 0 for f in fields: v = getattr(self, f) - assert isinstance(v, torch.Tensor), f"{f} is not a torch.Tensor" + assert isinstance(v, torch.Tensor) self._saved_tensors[f] = v setattr(self, f, v.clone()) @@ -427,11 +423,6 @@ def restore_from_spec_dec(self) -> None: setattr(self, f, v) self._saved_tensors.clear() - @property - def has_spec_dec_saved_state(self) -> bool: - """True when prepare_for_spec_dec state has not been restored yet.""" - return bool(self._saved_tensors) - def update_spec_dec_param( self, batch_size, diff --git a/tensorrt_llm/_torch/attention_backend/sparse/__init__.py b/tensorrt_llm/_torch/attention_backend/sparse/__init__.py index f293f9547506..0b2994441bf0 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/__init__.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/__init__.py @@ -1,3 +1,13 @@ +# yapf: disable +from .minimax_m3 import (MiniMaxM3SparseAttention, + MiniMaxM3SparseAttentionMetadata, + MiniMaxM3SparseConfig, MiniMaxM3SparseIndexCache, + allocate_minimax_m3_static_buffers, + build_runtime_metadata_from_kv_manager, + get_minimax_m3_attention_backend_cls, + get_minimax_m3_kv_cache_manager_cls, + minimax_m3_sparse_decode, minimax_m3_sparse_prefill) +# yapf: enable from .utils import (get_flashinfer_sparse_attn_attention_backend, get_sparse_attn_kv_cache_manager, get_trtllm_sparse_attn_attention_backend, @@ -8,4 +18,14 @@ "get_vanilla_sparse_attn_attention_backend", "get_trtllm_sparse_attn_attention_backend", "get_flashinfer_sparse_attn_attention_backend", + "MiniMaxM3SparseAttention", + "MiniMaxM3SparseAttentionMetadata", + "MiniMaxM3SparseConfig", + "MiniMaxM3SparseIndexCache", + "allocate_minimax_m3_static_buffers", + "build_runtime_metadata_from_kv_manager", + "get_minimax_m3_attention_backend_cls", + "get_minimax_m3_kv_cache_manager_cls", + "minimax_m3_sparse_decode", + "minimax_m3_sparse_prefill", ] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/cache_manager.py b/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/cache_manager.py index 6eb9c366ff7c..2ff191fc8589 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/cache_manager.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/cache_manager.py @@ -14,13 +14,16 @@ # limitations under the License. from collections import defaultdict -from dataclasses import replace from typing import Dict, List, Optional, Tuple import torch from tensorrt_llm._torch.pyexecutor import llm_request -from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import GPU_LEVEL, KVCacheManagerV2 +from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import ( + GPU_LEVEL, + BlockReusePolicy, + KVCacheManagerV2, +) from tensorrt_llm._utils import ( TensorWrapper, convert_to_torch_tensor, @@ -36,11 +39,16 @@ from tensorrt_llm.runtime import ModelConfig from tensorrt_llm.runtime.kv_cache_manager_v2 import ( AttentionLayerConfig, + BatchDesc, BufferConfig, DataRole, + GpuCacheTierConfig, + HostCacheTierConfig, + KVCacheDesc, LayerId, PageIndexMode, ScratchDesc, + SwaScratchReuseConfig, ) from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheManagerConfig as KVCacheManagerConfigPy from tensorrt_llm.runtime.kv_cache_manager_v2._common import BAD_PAGE_INDEX @@ -246,6 +254,7 @@ def __init__( self._init_indexer_dtype(sparse_attn_config) + # _build_cache_config() needs them to build constraints self._max_input_len = max_input_len self._max_num_tokens = max_num_tokens @@ -262,7 +271,6 @@ def __init__( vocab_size=vocab_size, mapping=mapping, dtype=dtype, - max_num_tokens=max_num_tokens, **kwargs, ) self.is_vswa = True # DeepSeek-V4 must has VSWA @@ -306,10 +314,14 @@ def _format_kv_cache_pool_lifecycle_entry(self, layer_id: LayerId, role: DataRol return super()._format_kv_cache_pool_lifecycle_entry(layer_id, role) model_layer_idx, attn_type = layer_semantics + attr = self.impl._storage.get_buffer_attr(layer_id, role) + pool_group_id = self.impl._storage.get_pool_group_index(attr.life_cycle_id) + lifecycle = self.impl._life_cycles.get_life_cycle(attr.life_cycle_id) return ( f"deepseek_role={attn_type.name}, " f"compress_ratio={self._compress_ratios[model_layer_idx]}, " - f"{super()._format_kv_cache_pool_lifecycle_entry(layer_id, role)}" + f"pool_group_id={int(pool_group_id)}, " + f"lifecycle_id={int(attr.life_cycle_id)}, lifecycle={lifecycle}" ) def get_buffers(self, layer_idx: int, attn_type: DeepseekV4AttentionType) -> torch.Tensor: @@ -760,9 +772,16 @@ def _get_max_tokens_from_quota(self, quota: int) -> float: return float("inf") return self._max_num_tokens + (quota - context_limit_quota) / generation_size_per_token - def _build_cache_config(self, config: KVCacheManagerConfigPy) -> KVCacheManagerConfigPy: + def _build_cache_config( + self, + kv_cache_config: KvCacheConfig, + *, + tokens_per_block: int, + vocab_size: int | None, + cache_tiers: List[GpuCacheTierConfig | HostCacheTierConfig], + ) -> KVCacheManagerConfigPy: """ - Add DeepSeek-V4 layers to the cache config. + Create the cache manager config for DeepSeek-V4. """ layers: List[AttentionLayerConfig] = [] layer_attn_to_layer_id: Dict[Tuple[int, DeepseekV4AttentionType], LayerId] = {} @@ -849,9 +868,90 @@ def _add_layer( # number of layers in the KVCacheManagerPy self._num_manager_layers = len(layers) - return replace( - config, + # Build constraints and typical_step for better pool ratio. + max_batch_size = self.max_batch_size + max_seq_len = self.max_seq_len + max_num_tokens = self._max_num_tokens + max_draft_len = self._max_draft_len + typical_step = None + constraints = [] + if kv_cache_config.pool_ratio is None: + typical_seq_len = ( + kv_cache_config.avg_seq_len + if kv_cache_config.avg_seq_len is not None + else max_seq_len + ) + if typical_seq_len > max_seq_len: + raise ValueError( + f"kv_cache_config.avg_seq_len ({typical_seq_len}) must be less than or " + f"equal to max_seq_len ({max_seq_len})" + ) + + # For aggregated serving in large batch size: + # Use 1 context request + (max_batch_size - 1) generation requests as + # the typical step. An all-generation typical_step over-provisions the + # compressed-cache pool at the expense of the SWA pool, starving the + # SWA pool and artificially capping the achievable batch size. + ctx_capacity = max_num_tokens if max_num_tokens is not None else typical_seq_len + generation_history_length = max(0, typical_seq_len - max_draft_len - 1) + typical_step = BatchDesc( + kv_caches=[ + KVCacheDesc(capacity=ctx_capacity, history_length=0), + ] + + [ + KVCacheDesc( + capacity=typical_seq_len, + history_length=generation_history_length, + ) + ] + * (max_batch_size - 1), + ) + + # Constraint 1: cuda graph generation warmup — one decode request that has + # accumulated to the tail of max_seq_len. Using history_length=max_seq_len-1 + # (instead of 0) lets SWA / SSM pools collapse to their windowed working set, + # while full-cache pools still need max_seq_len/tokens_per_block blocks + # because they don't age. + constraints.append( + BatchDesc([KVCacheDesc(capacity=max_seq_len, history_length=max_seq_len - 1)]) + ) + + # Constraint 2: general / chunked-prefill warmup — one fresh context request + # at max_num_tokens (the per-iteration token budget). + if max_num_tokens is not None: + constraints.append( + BatchDesc( + [ + KVCacheDesc( + capacity=max_num_tokens + self.num_extra_kv_tokens, history_length=0 + ) + ] + ) + ) + + scratch_reuse_config = None + if self.enable_swa_scratch_reuse: + # Context requests will allocate num_extra_kv_tokens tokens for spec decoding. + # Cache manager should not take them into account when calculating scratch range. + # Therefore set max_rewind_len to num_extra_kv_tokens. + scratch_reuse_config = SwaScratchReuseConfig(max_rewind_len=self.num_extra_kv_tokens) + + return KVCacheManagerConfigPy( + tokens_per_block=tokens_per_block, + vocab_size=vocab_size, + cache_tiers=cache_tiers, + max_util_for_resume=kv_cache_config.max_util_for_resume, + enable_partial_reuse=kv_cache_config.enable_partial_reuse, + swa_scratch_reuse=scratch_reuse_config, + commit_min_snapshot=( + kv_cache_config.enable_block_reuse + and self.block_reuse_policy != BlockReusePolicy.ALL_REUSABLE + ), layers=layers, + typical_step=typical_step, + constraints=constraints, + enable_stats=self.enable_stats, + initial_pool_ratio=kv_cache_config.pool_ratio, ) def _init_indexer_dtype(self, sparse_attn_config: DeepSeekV4SparseAttentionConfig) -> None: @@ -1066,9 +1166,9 @@ def get_layer_bytes_per_token( local_layer_idx: int, data_role: DataRole, ) -> int: - # The generic layers in the base config are replaced by - # _build_cache_config, so their buffer sizes are only placeholders. - return 1 + raise NotImplementedError( + "DeepSeek-V4 doesn't support get_layer_bytes_per_token, use _get_attn_bytes_per_block" + ) def get_indexer_k_cache_buffers(self, layer_idx: int) -> torch.Tensor: """ @@ -1206,13 +1306,8 @@ def copy_batch_block_offsets( beam_width: int, num_contexts: int, num_seqs: int, - max_blocks: Optional[int] = None, ) -> None: - """For compatibility with AttentionOp, copy only the SWA block offsets. - - max_blocks is accepted for signature parity with KVCacheManager; the - copy below is already bounded by the precomputed SWA table width. - """ + """For compatibility with AttentionOp, copy only the SWA block offsets.""" assert beam_width == 1, "DSV4 only supports beam width 1 now" assert dst_tensor.is_cuda, "copy_batch_block_offsets expects a CUDA destination" dst_tensor.fill_(BAD_PAGE_INDEX) diff --git a/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/deepseek_v4.py b/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/deepseek_v4.py index 1f76e5a73e51..52f16c84ffe9 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/deepseek_v4.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/deepseek_v4/deepseek_v4.py @@ -32,7 +32,7 @@ from tensorrt_llm._torch.modules.multi_stream_utils import do_multi_stream from tensorrt_llm._torch.modules.rotary_embedding import RotaryEmbedding from tensorrt_llm._torch.utils import maybe_compile -from tensorrt_llm._utils import is_sm_100f, prefer_pinned +from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization.utils import fp8_utils from tensorrt_llm.runtime.kv_cache_manager_v2 import DataRole @@ -250,7 +250,6 @@ def make_deepseek_v4_sparse_metadata_params( ), enable_indexer_skip=sparse_attention_config.skip_indexer_for_short_seqs, enable_heuristic_topk=sparse_attention_config.enable_heuristic_topk, - use_cute_dsl_topk=sparse_attention_config.use_cute_dsl_topk, use_cute_dsl_paged_mqa_logits=(sparse_attention_config.use_cute_dsl_paged_mqa_logits), q_split_threshold=sparse_attention_config.q_split_threshold, compress_ratios=sparse_attention_config.compress_ratios, @@ -275,7 +274,6 @@ def __post_init__(self): super().__post_init__() self.num_total_compressed_tokens = {} self.max_ctx_compressed_tokens = {} - self._ctx_output_sizes: Optional[Dict[int, int]] = None sparse_metadata_params = self.sparse_metadata_params if not isinstance(sparse_metadata_params, DeepSeekV4MetadataParams): raise ValueError("DeepSeek-V4 sparse attention metadata params are not set") @@ -531,10 +529,6 @@ def prepare_for_indexer_k_cache(self): self.host_indexer_k_cache_block_offsets[: self.num_seqs], non_blocking=True, ) - # Columns beyond each sequence's allocated indexer blocks contain BAD_PAGE_INDEX (-1). - # CUDA-graph padded token slots may still compute scatter addresses from those columns - # before being ignored, so map them to block 0, matching the base DSA metadata path. - self.indexer_k_cache_block_offsets.clamp_(min=0) def prepare_for_block_tables(self): """Prepare block tables for sliding-window and compressed attention.""" @@ -738,18 +732,12 @@ def prepare(self): kv_lens_slice = kv_lens[:num_requests] cached_slice = cached_token_lens[:num_requests] - # Host-side per-ratio ctx compressed-token counts (Python ints), so - # _compute_ctx_compressed_position_ids never reads a device scalar - # (implicit D2H + stream sync) for its arange size / slice bound. - ctx_output_sizes: Optional[Dict[int, int]] = None if num_contexts > 0: # Prefill path: need per-request tensor ops for ctx scalar metadata. - ctx_output_sizes = {} for compress_ratio in self.compress_ratio_set: new_comp_kv_lens = kv_lens_slice // compress_ratio - cached_slice // compress_ratio cu_new = new_comp_kv_lens.cumsum(0) num_ctx_compressed_tokens = cu_new[num_contexts - 1].item() - ctx_output_sizes[compress_ratio] = num_ctx_compressed_tokens num_gen_compressed_tokens = num_generations * ( (num_gen_tokens_per_seq + compress_ratio - 1) // compress_ratio ) @@ -768,15 +756,12 @@ def prepare(self): ) self.max_ctx_compressed_tokens[compress_ratio] = 0 - # Cached for on_update_kv_lens(); see the reuse gate there. - self._ctx_output_sizes = ctx_output_sizes - # 2) CUDA-side: fill *_cuda buffers on device. kv_lens_cuda = ( self.cached_token_lens_cuda[:num_requests] + self._seq_lens_cuda[:num_requests] ) cached_tokens_cuda = self.cached_token_lens_cuda[:num_requests] - self.prepare_compressed_kv_metadata(kv_lens_cuda, cached_tokens_cuda, ctx_output_sizes) + self.prepare_compressed_kv_metadata(kv_lens_cuda, cached_tokens_cuda) self._compute_compressed_mask( self.new_comp_kv_lens_cuda, @@ -791,7 +776,6 @@ def prepare_compressed_kv_metadata( self, kv_lens: torch.Tensor, cached_tokens: torch.Tensor, - ctx_output_sizes: Optional[Dict[int, int]] = None, ): """Compute per-ratio compressed KV lens and position IDs on device. @@ -800,12 +784,6 @@ def prepare_compressed_kv_metadata( Args: kv_lens: Total KV lengths per request (device tensor, [batch_size]). cached_tokens: Cached token counts per request (device tensor, [batch_size]). - ctx_output_sizes: Optional per-ratio host-computed ctx - compressed-token counts (Python ints); avoids implicit - device-scalar reads (D2H + stream sync) in the ctx position-id - computation. prepare() always passes it; on_update_kv_lens() - reuses the cached copy unless the extend_ctx path may have - mutated ctx-row kv_lens on device. """ batch_size = kv_lens.shape[0] num_contexts = self.num_contexts @@ -829,7 +807,6 @@ def prepare_compressed_kv_metadata( self.compressed_position_ids_cuda, num_contexts, self._compress_ratios_sorted, - ctx_output_sizes, ) if self.num_gen_tokens_per_seq > 0 and num_generations > 0: @@ -866,13 +843,7 @@ def on_update_kv_lens(self): num_gen_tokens // self.num_generations if self.num_generations > 0 else 0 ) - # Reuse prepare()'s host-computed ctx sizes unless the extend_ctx path - # (num_chunked_ctx_requests > 0) may have mutated ctx-row kv_lens on - # device; every other path only changes gen rows. - ctx_output_sizes = ( - self._ctx_output_sizes if getattr(self, "num_chunked_ctx_requests", 0) == 0 else None - ) - self.prepare_compressed_kv_metadata(kv_lens, cached_tokens, ctx_output_sizes) + self.prepare_compressed_kv_metadata(kv_lens, cached_tokens) self._compute_compressed_mask( self.new_comp_kv_lens_cuda, @@ -1028,24 +999,14 @@ def _compute_ctx_compressed_position_ids( compressed_position_ids_bufs: Dict[int, torch.Tensor], num_contexts: int, compress_ratios: list, - ctx_output_sizes: Optional[Dict[int, int]] = None, ): - """Context-only compressed position IDs (eager, data-dependent shapes). - - ctx_output_sizes (host ints) keeps the arange size and slice bound off - the device; the 0-dim-CUDA fallback costs two implicit D2H syncs per - ratio. - """ + """Context-only compressed position IDs (eager, data-dependent shapes).""" device = past_kv_lens_bufs[compress_ratios[0]].device for compress_ratio in compress_ratios: past_kv = past_kv_lens_bufs[compress_ratio] cu_new_comp = cu_new_comp_kv_bufs[compress_ratio] - total_ctx_comp = ( - ctx_output_sizes[compress_ratio] - if ctx_output_sizes is not None - else cu_new_comp[num_contexts] - ) + total_ctx_comp = cu_new_comp[num_contexts] ctx_idx = torch.arange(total_ctx_comp, dtype=torch.int32, device=device) ctx_cu = cu_new_comp[: num_contexts + 1].to(torch.int32) ctx_req = torch.searchsorted(ctx_cu[1:], ctx_idx, right=True) @@ -1079,11 +1040,6 @@ def __init__( layer_idx, aux_stream, ) - # Preserve the checkpoint's 128x128 FP8 quantization while deriving a - # native-MXF8 (sf_vec=32) scale view for the TRT-LLM CuTe DSL fused - # indexer-Q projection. FP8BlockScalesLinearMethod materializes the - # derived, swizzled scale once after weight loading. - self.wq_b.use_indexer_q_cutedsl_fusion = True # Override base Indexer.weights_proj to bf16 (matches V4 checkpoint). self.weights_proj = Linear( self.hidden_size, @@ -1143,10 +1099,6 @@ def _qk_projection_and_rope(self, qr: torch.Tensor, position_ids: torch.Tensor): """ q = self.wq_b(qr) q = q.view(-1, self.n_heads, self.head_dim) - return self._apply_q_rope(q, position_ids) - - def _apply_q_rope(self, q: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor: - """Apply RoPE in-place to a projected indexer Q tensor.""" # Fused in-place RoPE on the rope portion of each head nope_dim = self.head_dim - self.rope_dim torch.ops.trtllm.mla_rope_inplace( @@ -1161,49 +1113,6 @@ def _apply_q_rope(self, q: torch.Tensor, position_ids: torch.Tensor) -> torch.Te ) return q - def _project_and_quantize_q( - self, qr: torch.Tensor, position_ids: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Project Q and produce the cache precision consumed by the indexer. - - The DSv4 MXFP4 configuration can fuse projection, interleaved RoPE, - and FP4 quantization. If the optional Hadamard transform is active, - retain the legacy path because that transform changes all 128 values - in a head and is not part of the CuTe DSL kernel. - """ - use_fused_project_mxfp4 = ( - self.indexer_cache_dtype == KVCacheDtype.MXFP4_BLOCKWISE - and not HAS_FAST_HADAMARD - and not self.rotary_emb.is_neox - and self.head_dim == 128 - and self.rope_dim == 64 - and self.wq_b.has_fp8_block_scales - and hasattr(self.wq_b, "indexer_q_weight_scale_cutedsl") - and hasattr( - torch.ops.trtllm, - "cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell", - ) - and qr.dtype == torch.bfloat16 - and is_sm_100f() - ) - if use_fused_project_mxfp4: - q_fp4, q_scale = torch.ops.trtllm.cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell( - qr, - self.wq_b.weight, - self.wq_b.indexer_q_weight_scale_cutedsl, - position_ids.view(-1), - self.rotary_emb.rotary_cos_sin.view(-1, self.rope_dim), - self.wq_b.indexer_q_alpha_cutedsl, - use_tvm_ffi=True, - ) - return q_fp4.view(-1, self.n_heads, self.head_dim // 2), q_scale.view( - -1, self.n_heads, 1 - ) - - q = self.wq_b(qr).view(-1, self.n_heads, self.head_dim) - q = self._apply_q_rope(q, position_ids) - return self._quantize_q(q) - def _quantize_q(self, q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: # Rotate + quantize (layout matches compressor K: [nope|pe]). After # rotate_activation (Hadamard) the nope/rope split becomes a linear @@ -1333,7 +1242,7 @@ def _run_overlapped_indexer_prepare( if pre_aux is None: self.indexer_start_event.record() - q_fp8, q_scale = self._project_and_quantize_q(qr, position_ids) + q = self._qk_projection_and_rope(qr, position_ids) with torch.cuda.stream(self.aux_stream): self.indexer_start_event.wait() @@ -1354,7 +1263,9 @@ def _run_overlapped_indexer_prepare( k_fp8.record_stream(cur_stream) if k_scale is not None: k_scale.record_stream(cur_stream) - q_fp8, q_scale = self._project_and_quantize_q(qr, position_ids) + q = self._qk_projection_and_rope(qr, position_ids) + + q_fp8, q_scale = self._quantize_q(q) self.weights_proj_event.wait() weights = self._apply_weight_scale(weights, q_scale) @@ -1371,7 +1282,9 @@ def _run_serial_indexer_prepare( ) -> Tuple[ torch.Tensor, torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], torch.Tensor ]: - q_fp8, q_scale = self._project_and_quantize_q(qr, position_ids) + q = self._qk_projection_and_rope(qr, position_ids) + + q_fp8, q_scale = self._quantize_q(q) weights = self.weights_proj(hidden_states) diff --git a/tensorrt_llm/_torch/attention_backend/sparse/dsa.py b/tensorrt_llm/_torch/attention_backend/sparse/dsa.py index 8466bbf59900..0a31adaecc08 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/dsa.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/dsa.py @@ -27,7 +27,7 @@ maybe_execute_in_parallel from tensorrt_llm._torch.modules.rotary_embedding import RotaryEmbedding from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager -from tensorrt_llm._torch.utils import Fp4QuantizedTensor, maybe_compile +from tensorrt_llm._torch.utils import maybe_compile from tensorrt_llm._utils import (get_size_in_bytes, get_sm_version, maybe_pin_memory, prefer_pinned) from tensorrt_llm.bindings import DataType @@ -90,7 +90,6 @@ class DSAMetadataParams(SparseMetadataParams): index_head_dim: int enable_indexer_skip: bool enable_heuristic_topk: bool - use_cute_dsl_topk: bool use_cute_dsl_paged_mqa_logits: bool q_split_threshold: int @@ -674,9 +673,6 @@ def __post_init__(self): self.indexer_head_dim = sparse_metadata_params.index_head_dim self.indexer_quant_block_size = 128 self.enable_indexer_skip = (sparse_metadata_params.enable_indexer_skip) - self.use_cute_dsl_topk = (sparse_metadata_params.use_cute_dsl_topk - and IS_CUTLASS_DSL_AVAILABLE) - self.kv_lens_row_reorder = None capture_graph = self.is_cuda_graph # Plain DSA has no compression and uses the default [1]. DeepSeek-V4's # metadata params carry the model-specific compression ratios. @@ -860,37 +856,8 @@ def on_update_kv_lens(self): _DG_SCHEDULE_BLOCK_KV, self.num_sms) self.scheduler_metadata_buffer_expanded.copy_( scheduler_metadata_buffer_expanded, non_blocking=True) - self._compute_kv_lens_row_reorder() self.prepare_dense_topk_indices(self.kv_lens_cuda, device=True) - def _compute_kv_lens_row_reorder(self): - """LJF (longest-job-first) row-reorder for the GVR DSL top-k path. - - Writes ``argsort(gen_kv_lens, descending)`` into the stable buffer when - the multi-wave threshold is met, otherwise leaves ``order_row`` None. - Called from ``on_update_kv_lens()`` (both base and DeepSeek-V4 via - super()) unconditionally every forward step so the GVR op sees a fresh - valid permutation and never a stale one from a prior step. Copies into - the stable buffer (not a fresh tensor) so the CUDA-Graph-captured op - reads a valid permutation on every replay. - """ - # Gate on row count (num_generations * next_n) rather than request count - # so the threshold aligns with the kernel-side tuning note that records - # the win region starting at num_rows >= 2 * num_sms. Using - # num_generations alone is only correct for next_n == 2; for next_n == 1 - # it engages inside the measured regression band, and for next_n == 4 it - # misses the win region between 2*num_sms and 4*num_sms rows. - next_n = 1 + self.max_draft_tokens - if (self.enable_heuristic_topk and self.use_cute_dsl_topk - and self.num_generations * next_n >= 2 * self.num_sms): - gen_kv_lens = self.kv_lens_cuda[self.num_contexts:self.num_seqs] - order = torch.argsort(gen_kv_lens, descending=True).to(torch.int32) - self.kv_lens_row_reorder_buffer[:self.num_generations].copy_(order) - self.kv_lens_row_reorder = \ - self.kv_lens_row_reorder_buffer[:self.num_generations] - else: - self.kv_lens_row_reorder = None - def update_for_spec_dec(self): super().update_for_spec_dec() # host @@ -1138,30 +1105,15 @@ def create_buffers_for_indexer(self, capture_graph=False): # Pre-allocated with stable address for CUDA Graph compatibility # (replaces cudaMallocAsync/cudaFreeAsync inside the kernel launcher). # Shape: [max_gen_tokens, topK] where max_gen_tokens = max_batch * (1 + max_draft). - # Only the C++ indexer_topk_decode path consumes it; the GVR DSL - # path does not, so skip the allocation when use_cute_dsl_topk. - if not self.use_cute_dsl_topk: - max_gen_tokens = self.max_num_sequences * ( - 1 + self.max_draft_tokens) - self.heuristic_scratch_values = self.get_empty( - self.cuda_graph_buffers, - (max_gen_tokens, self.num_sparse_topk), - cache_name="heuristic_scratch_values", - dtype=torch.float32, - capture_graph=capture_graph, - ) - # Stable-address buffer for the GVR DSL LJF row-reorder - # (order_row = argsort(gen_kv_lens, descending)). Must not be - # fresh-allocated per step: under CUDA Graph the captured op reads - # a frozen address, so prepare() copies into this buffer instead. - if self.use_cute_dsl_topk: - self.kv_lens_row_reorder_buffer = self.get_empty( - self.cuda_graph_buffers, - (self.max_num_sequences, ), - cache_name="kv_lens_row_reorder_buffer", - dtype=torch.int32, - capture_graph=capture_graph, - ) + max_gen_tokens = self.max_num_sequences * (1 + + self.max_draft_tokens) + self.heuristic_scratch_values = self.get_empty( + self.cuda_graph_buffers, + (max_gen_tokens, self.num_sparse_topk), + cache_name="heuristic_scratch_values", + dtype=torch.float32, + capture_graph=capture_graph, + ) # Persistent scratch for the Radix-split-work indexer path. Re-created # in update_spec_dec_param when max_draft_tokens changes so it stays @@ -1258,9 +1210,7 @@ def update_spec_dec_param( if self.max_num_sequences * (1 + self.max_draft_tokens) != init_shape: self.create_expanded_buffers(capture_graph=capture_graph) # Resize heuristic scratch buffer for new max_draft_tokens. - # Skip when use_cute_dsl_topk (GVR path never consumes it), matching - # the allocation guard in create_buffers_for_indexer. - if self.enable_heuristic_topk and not self.use_cute_dsl_topk: + if self.enable_heuristic_topk: max_gen_tokens = self.max_num_sequences * ( 1 + self.max_draft_tokens) self.heuristic_scratch_values = self.get_empty( @@ -1530,17 +1480,11 @@ def prepare_for_indexer_k_cache(self): 1) // tokens_per_block max_blocks_used = num_blocks_per_seq.max().item( ) if self.num_seqs > 0 else 1 - # pool_indices already has correct values; set padding to -1. - # Stage through a fresh pinned buffer: an async H2D from pageable - # memory would block the host behind the busy execution stream. - host_block_table = torch.empty((pool_indices.shape[0], max_blocks_used), - dtype=pool_indices.dtype, - pin_memory=prefer_pinned()) - host_block_table.copy_(pool_indices[:, :max_blocks_used]) - pad_cols = torch.arange(max_blocks_used, dtype=num_blocks_per_seq.dtype) - host_block_table.masked_fill_( - pad_cols.unsqueeze(0) - >= num_blocks_per_seq[:self.num_seqs].unsqueeze(1), -1) + # pool_indices already has correct values; set padding to -1 + host_block_table = pool_indices[:, :max_blocks_used].clone() + for i in range(self.num_seqs): + if num_blocks_per_seq[i] < max_blocks_used: + host_block_table[i, num_blocks_per_seq[i]:] = -1 # Copy to GPU self.block_table[:self.num_seqs, :max_blocks_used].copy_( host_block_table, non_blocking=True) @@ -1820,7 +1764,7 @@ def __init__(self, or self.use_cute_dsl_paged_mqa_logits) and layer_idx == 0: from tensorrt_llm._torch.custom_ops import cute_dsl_custom_ops - if self.use_cute_dsl_topk and not self._enable_heuristic_topk: + if self.use_cute_dsl_topk: # the dtype of topk input tensor, which is float32 now. # Note, need to update it if the dtype of topk input tensor is changed. cute_dsl_custom_ops.warmup_cute_dsl_indexer_topk( @@ -2806,34 +2750,17 @@ def sparse_attn_indexer( # handled inside the C++ kernel (preIdxOffset += 1). pre_idx = metadata.heuristic_prev_topk[ local_layer, :num_generations] - # heuristic_scratch is only consumed by the C++ - # indexer_topk_decode path; the GVR DSL op does not take it. - # Guard on the metadata flag so this stays consistent with - # the buffer allocation (also gated on the same flag). - if not metadata.use_cute_dsl_topk: - heuristic_scratch = \ - metadata.heuristic_scratch_values[ - :num_gen_tokens] - - if self.use_cute_dsl_topk and self._enable_heuristic_topk: - # GVR DSL: supports all compress_ratio and next_n values. - torch.ops.trtllm.cute_dsl_gvr_topk_decode( - logits_decode, - pre_idx, - gen_kv_lens_cuda, - topk_indices_buffer[num_ctx_tokens:num_ctx_tokens + - num_gen_tokens, :], - self.index_topk, - next_n=next_n, - compress_ratio=self.compress_ratio, - max_seq_len=indexer_max_seq_len, - order_row=metadata.kv_lens_row_reorder, - ) - # CuTE DSL radix top-k allocates O(num_gen_tokens * kv_len) - # global memory. Beyond 256 tokens the extra memory becomes - # significant, so we cap it at 256 and fall back to C++. - elif (self.use_cute_dsl_topk and num_gen_tokens <= 256 - and (self.compress_ratio == 1 or next_n == 1)): + heuristic_scratch = \ + metadata.heuristic_scratch_values[ + :num_gen_tokens] + + # CuTE DSL top-k allocates O(num_gen_tokens * kv_len) global + # memory. Beyond 256 tokens the extra memory becomes significant, + # so we cap it at 256 for now and fall back to the CUDA C++ + # indexer_topk_decode. This limit can be removed if GPU memory + # is not a bottleneck. + if (self.use_cute_dsl_topk and num_gen_tokens <= 256 + and (self.compress_ratio == 1 or next_n == 1)): torch.ops.trtllm.cute_dsl_indexer_topk_decode( logits_decode, context_lens if self.compress_ratio > 1 else gen_kv_lens_cuda, @@ -2953,18 +2880,7 @@ def pre_indexer_proj( """ assert self._fused_wk_wp_weight is not None, \ "cache_derived_state() must be called before forward()" - # When the boundary fusion pre-quantized the next layer's kv_a_proj - # NVFP4 input, hidden_states is an Fp4QuantizedTensor that also carries - # the BF16 post-RMSNorm value. Use that BF16 view here -- the indexer - # weight is BF16 and the matmul needs a float input. - if isinstance(hidden_states, Fp4QuantizedTensor): - assert hidden_states.unquantized_hidden_states is not None, ( - "pre_indexer_proj received Fp4QuantizedTensor without bf16 view; " - "the producer fusion must request return_norm_out=True") - hidden_states_bf = hidden_states.unquantized_hidden_states - else: - hidden_states_bf = hidden_states - hidden_float = _to_float(hidden_states_bf) + hidden_float = _to_float(hidden_states) with _tf32_matmul_enabled(): # F.linear computes input @ weight.T internally; no explicit .t() needed. # _fused_wk_wp_weight is [head_dim + n_heads, hidden_size] (nn.Linear convention). @@ -2975,7 +2891,7 @@ def pre_indexer_proj( indexer_k, weights = fused_out.split([self.head_dim, self.n_heads], dim=-1) # Cast indexer_k back to model dtype for downstream ops (k_norm, RoPE, FP8 quantize) - indexer_k = indexer_k.to(hidden_states_bf.dtype) + indexer_k = indexer_k.to(hidden_states.dtype) q_pe, q_nope, k_pe, k_nope = self._qk_projection_and_rope( qr, indexer_k, position_ids) diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/__init__.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/__init__.py index 522bf2d5b9ee..c248ffd91119 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/__init__.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/__init__.py @@ -4,37 +4,24 @@ Layered as: - * :mod:`.triton_metadata` -- ``MiniMaxM3TritonSparseAttentionMetadata`` - dataclass, CUDA-graph-stable buffer - allocator + builder, and the - :class:`AttentionMetadata` subclass - factory for the Triton reference path. - * :mod:`.cache_manager` -- standalone side index cache used by tests - and the :class:`KVCacheManagerV2` - subclass factory. Shared by both backends. - * :mod:`.common` -- backend-neutral config bundles, the paged - KV-slot writer, block-priority sentinels, and - the paged-cache slot mapping builder shared by - both backends. - * :mod:`.msa_utils` -- MSA-only (fmha_sm100) helpers: import guard, - kernel precondition constants, HND paged-cache - adapters, main-KV writer, page-table builder, - valid-block counting, and top-k selection. - * :mod:`.triton_kernels` -- OpenAI Triton kernels (per-block max - score, masked softmax for sparse GQA). - * :mod:`.triton_backend` -- the Triton reference algorithm (vectorized - paged-cache helpers, prefill / decode - entry points, the thin - :class:`MiniMaxM3TritonSparseAttention` - orchestrator) and its - :class:`AttentionBackend` subclass - factory. - * :mod:`.msa_backend` -- the MSA (fmha_sm100) backend, its flat - metadata, and the backend factory. - * :mod:`.msa_indexer` -- the MSA proxy scoring + top-k block - selection submodule. - * :mod:`.msa_availability`-- SM100 and fmha_sm100 gating for the MSA - path. + * :mod:`.kernels` -- OpenAI Triton kernels (per-block max + score, masked softmax for sparse GQA). + * :mod:`.metadata` -- ``MiniMaxM3SparseConfig`` / + ``MiniMaxM3SparseAttentionMetadata`` + dataclasses, CUDA-graph-stable buffer + allocator + builder, and the + :class:`AttentionMetadata` subclass + factory. + * :mod:`.cache_manager` -- standalone side index cache used by tests + and the :class:`KVCacheManagerV2` + subclass factory. + * :mod:`.backend` -- the algorithm itself (vectorized + paged-cache helpers, prefill / decode + entry points, the thin + :class:`MiniMaxM3SparseAttention` + orchestrator) and the + :class:`AttentionBackend` subclass + factory. This package's public surface re-exports the names callers historically imported from ``...sparse.minimax_m3`` so external @@ -42,20 +29,48 @@ working unchanged. """ -# The dense Triton oracle in the model imports these paged-cache helpers, so -# they stay importable from the package. They are package-private and are not -# part of __all__. Every other backend/metadata/config symbol is imported -# directly from its defining submodule by the code that needs it. -from .cache_manager import MiniMaxM3KVCacheManagerV2 -from .msa_backend import MiniMaxM3MsaSparseAttention -from .triton_backend import ( # noqa: F401 - MiniMaxM3SparseRuntimeBackend, +# Re-export the algorithm-internal helpers focused unit tests reach +# into so the package preserves the surface the monolithic module +# exposed. These are not part of ``__all__`` (still package-private) +# but stay importable as ``from ...minimax_m3 import _write_main_kv_slots``. +from .backend import ( # noqa: F401 + MiniMaxM3SparseAttention, + _compute_index_attn_chunk_q, + _compute_sparse_gqa_chunk_q, _gather_paged_batched, + _index_attention_and_select, + _write_main_kv_slots, _write_main_kv_slots_to_pool, + get_minimax_m3_attention_backend_cls, + minimax_m3_sparse_decode, + minimax_m3_sparse_prefill, +) +from .cache_manager import ( + MiniMaxM3KVCacheManagerV2, + MiniMaxM3SparseIndexCache, + get_minimax_m3_kv_cache_manager_cls, +) +from .metadata import ( + MiniMaxM3SparseAttentionMetadata, + MiniMaxM3SparseConfig, + allocate_minimax_m3_static_buffers, + build_runtime_metadata_from_kv_manager, + get_minimax_m3_attention_metadata_cls, + replace_metadata, ) __all__ = [ "MiniMaxM3KVCacheManagerV2", - "MiniMaxM3MsaSparseAttention", - "MiniMaxM3SparseRuntimeBackend", + "MiniMaxM3SparseAttention", + "MiniMaxM3SparseAttentionMetadata", + "MiniMaxM3SparseConfig", + "MiniMaxM3SparseIndexCache", + "allocate_minimax_m3_static_buffers", + "build_runtime_metadata_from_kv_manager", + "get_minimax_m3_attention_backend_cls", + "get_minimax_m3_attention_metadata_cls", + "get_minimax_m3_kv_cache_manager_cls", + "minimax_m3_sparse_decode", + "minimax_m3_sparse_prefill", + "replace_metadata", ] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_backend.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/backend.py similarity index 63% rename from tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_backend.py rename to tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/backend.py index 844fd979469c..090c6d7956fd 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_backend.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/backend.py @@ -46,7 +46,7 @@ ----------------- All scalar max lengths (``max_seqlen_q``, ``max_seqlen_k``) are -pre-computed CPU-side in :meth:`MiniMaxM3TritonSparseAttentionMetadata.prepare` +pre-computed CPU-side in :meth:`MiniMaxM3SparseAttentionMetadata.prepare` and stored as plain Python ints. The hot path uses only batched tensor ops with static shapes derived from those CPU-side scalars; no ``.item()`` or other GPU-CPU sync runs inside the forward @@ -56,29 +56,29 @@ from __future__ import annotations +import functools import os from dataclasses import dataclass from typing import TYPE_CHECKING, Optional, Tuple import torch -from ...interface import ( - AttentionBackend, - AttentionForwardArgs, - AttentionMetadata, - merge_attention_forward_args, -) -from .common import _INIT_SCORE, _LOCAL_SCORE, MiniMaxM3SparseConfig, write_kv_slots -from .triton_kernels import triton_block_max_score, triton_sparse_softmax -from .triton_metadata import ( - MiniMaxM3AttentionMetadata, - MiniMaxM3TritonSparseAttentionMetadata, +from .kernels import triton_block_max_score, triton_sparse_softmax +from .metadata import ( + MiniMaxM3SparseAttentionMetadata, + MiniMaxM3SparseConfig, ensure_metadata_on_device, + get_minimax_m3_attention_metadata_cls, ) if TYPE_CHECKING: from .cache_manager import MiniMaxM3SparseIndexCache - from .common import MiniMaxM3SparseParams + from .metadata import MiniMaxM3SparseParams + +# Sentinel block score for blocks that init / local priority forces into +# the top-k regardless of their numerical score. +_INIT_SCORE = 1e30 +_LOCAL_SCORE = 1e29 # --------------------------------------------------------------------------- @@ -143,6 +143,53 @@ def _gather_paged_batched( return flat.view(batch, max_k, *cache.shape[1:]) +def _assert_paged_write_in_bounds( + name: str, + cache: torch.Tensor, + page: torch.Tensor, + within: torch.Tensor, +) -> None: + """Optional CPU-side bounds check for paged-cache writes. + + The runtime computes per-token slot ids from + ``KVCacheManagerV2``'s block ids; if the runtime ever produces a + block id that does not fit the per-layer view's dim-0 the write + falls into another layer's coalesced memory and corrupts the + cache, or fires the CUDA ``IndexKernel.cu`` device-side assert + during fancy indexing. Both are far enough away from the root + cause to be hard to triage. + + When ``TRTLLM_MINIMAX_M3_DEBUG_BOUNDS`` is set this check runs a + CPU-side max/min comparison against the cache's dim-0 and dim-2 + bounds, surfacing the misindex with the exact tensor names and + values instead of a device-side assert spam. It is opt-in + because the comparison forces a CPU sync. + """ + if not os.environ.get("TRTLLM_MINIMAX_M3_DEBUG_BOUNDS"): + return + num_pages = int(cache.shape[0]) + tokens_per_block = int(cache.shape[1]) if cache.ndim == 4 else int(cache.shape[2]) + page_max = int(page.max().item()) if page.numel() else -1 + page_min = int(page.min().item()) if page.numel() else 0 + within_max = int(within.max().item()) if within.numel() else -1 + within_min = int(within.min().item()) if within.numel() else 0 + assert 0 <= page_min and page_max < num_pages, ( + f"{name}: page index out of bounds — page.min={page_min} " + f"page.max={page_max} but cache.shape[0]={num_pages} " + f"(shape={tuple(cache.shape)}). This usually means the " + f"runtime's get_block_ids_per_seq produced a block id wider " + f"than the per-layer paged view's dim-0; check that the M3 " + f"override path returns slot ids in [0, num_slots)." + ) + assert 0 <= within_min and within_max < tokens_per_block, ( + f"{name}: within-page offset out of bounds — within.min=" + f"{within_min} within.max={within_max} but tokens_per_block=" + f"{tokens_per_block} (shape={tuple(cache.shape)}). This " + f"usually means out_cache_loc was computed with a different " + f"tokens_per_block than the cache was allocated with." + ) + + def _write_main_kv_slots_to_pool( pool: torch.Tensor, kv_index: int, @@ -154,12 +201,40 @@ def _write_main_kv_slots_to_pool( ``pool`` is the 5-D main K/V pool returned by :meth:`KVCacheManagerV2.get_buffers` with the NHD layout ``[num_pages, kv_factor, tokens_per_block, num_kv_heads, head_dim]``. - ``values`` has shape ``[num_new_tokens, num_kv_heads, head_dim]`` and - ``out_cache_loc`` is the 1-D ``[num_new_tokens]`` int tensor of flat slot - ids to update. ``pool[:, kv_index]`` is a storage-sharing view, so the - shared :func:`common.write_kv_slots` propagates the write to the pool. + ``values`` has shape ``[num_new_tokens, num_kv_heads, head_dim]`` + and ``out_cache_loc`` is the 1-D ``[num_new_tokens]`` int tensor of + flat slot ids the caller wants to update. + + The write decomposes each flat slot id into + ``(page = s // tokens_per_block, within = s % tokens_per_block)`` + and uses multi-dim fancy-index assignment so the writes propagate + to the underlying pool storage. The previously used pattern + ``pool[:, kv_index].reshape(-1, num_kv_heads, head_dim) + .index_copy_(0, ...)`` instead wrote into a silent copy (see + :func:`_gather_paged_batched`), so the next forward call read + zeros for the prefilled positions. + + The optional CPU-side bounds assertion (enabled when the + ``TRTLLM_MINIMAX_M3_DEBUG_BOUNDS`` env var is set) catches + block_ids overflowing the pool's dim-0 before the device-side + ``IndexKernel.cu`` assert fires deep inside the kernel. The + assertion is a CPU sync, so the env var keeps it opt-in for + production runs that need a clean fast path. """ - write_kv_slots(pool[:, kv_index], out_cache_loc, values) + tokens_per_block = int(pool.shape[2]) + out_long = out_cache_loc.to(torch.long) + page = out_long // tokens_per_block + within = out_long % tokens_per_block + _assert_paged_write_in_bounds("pool", pool, page, within) + # KV-cache writes never need to participate in autograd. Wrap the + # fancy-index assignment in ``torch.no_grad()`` so callers that + # enter this path with an active grad context (e.g. unit tests + # exercising :class:`MiniMaxM3Attention` without ``inference_mode``) + # do not trip the "leaf Variable that requires grad is being used + # in an in-place operation" autograd guard on the view chain. + with torch.no_grad(): + # Multi-dim fancy assignment writes into the underlying pool buffer. + pool[page, kv_index, within] = values.to(pool.dtype) def _write_main_kv_slots( @@ -167,13 +242,39 @@ def _write_main_kv_slots( out_cache_loc: torch.Tensor, values: torch.Tensor, ) -> None: - """Write per-new-token K or V into a cache view via the shared writer. - - Delegates to :func:`common.write_kv_slots`, which handles both the 3-D - flat-slot layout used by focused unit tests and the 4-D paged view of - ``kv_pool[:, 0]`` / ``kv_pool[:, 1]``. + """Layout-aware writer for K (or V) caches used by the M3 backend. + + Supports two layouts, mirroring :func:`_gather_paged_batched`: + + * **3-D flat-slot** ``[num_slots, num_kv_heads, head_dim]``: used + by focused unit tests that allocate the cache as a contiguous + flat-slot tensor. ``index_copy_(0, ...)`` writes propagate + because the tensor IS the storage. + * **4-D multi-dim paged** ``[num_pages, tokens_per_block, + num_kv_heads, head_dim]``: used when the cache is a view of + ``kv_pool[:, 0]`` / ``kv_pool[:, 1]``. The view is + non-contiguous (its dim-0 stride is 2× the contiguous stride + because dim 1 separates K from V in the pool), so + ``index_copy_(0, ...)`` would silently fork a copy and the + write would be lost. Decompose the flat slot id into + ``(page, within)`` and use multi-dim fancy assignment so the + write propagates through the view to the underlying pool. """ - write_kv_slots(cache, out_cache_loc, values) + # KV-cache writes never need to participate in autograd. Wrap both + # branches in ``torch.no_grad()`` so callers that enter this path + # with an active grad context (e.g. unit tests exercising + # :class:`MiniMaxM3Attention` without ``inference_mode``) do not + # trip the autograd in-place guard on the cache view chain. + with torch.no_grad(): + if cache.ndim >= 4: + tokens_per_block = int(cache.shape[1]) + out_long = out_cache_loc.to(torch.long) + page = out_long // tokens_per_block + within = out_long % tokens_per_block + _assert_paged_write_in_bounds("cache", cache, page, within) + cache[page, within] = values.to(cache.dtype) + else: + cache.index_copy_(0, out_cache_loc.to(torch.long), values.to(cache.dtype)) def _scatter_topk_to_block_mask( @@ -598,7 +699,7 @@ def minimax_m3_sparse_decode( v_cache: torch.Tensor, idx_k_cache: torch.Tensor, idx_v_cache: Optional[torch.Tensor], - metadata: MiniMaxM3TritonSparseAttentionMetadata, + metadata: MiniMaxM3SparseAttentionMetadata, config: MiniMaxM3SparseConfig, *, disable_index_value: bool, @@ -693,7 +794,7 @@ def minimax_m3_sparse_prefill( idx_q: torch.Tensor, idx_k_cache: torch.Tensor, idx_v_cache: Optional[torch.Tensor], - metadata: MiniMaxM3TritonSparseAttentionMetadata, + metadata: MiniMaxM3SparseAttentionMetadata, config: MiniMaxM3SparseConfig, *, disable_index_value: bool, @@ -789,8 +890,18 @@ def minimax_m3_sparse_prefill( # --------------------------------------------------------------------------- +# Lazy import alias to avoid a circular import at module load — the side +# cache class lives in ``cache_manager`` which imports from this module +# is fine (no cycle), but keeping the indirection explicit makes the +# dependency direction in this file clearer. +def _import_index_cache_cls(): + from .cache_manager import MiniMaxM3SparseIndexCache + + return MiniMaxM3SparseIndexCache + + @dataclass -class MiniMaxM3TritonSparseAttention: +class MiniMaxM3SparseAttention: """Thin orchestrator for :func:`minimax_m3_sparse_prefill` and :func:`minimax_m3_sparse_decode`. @@ -798,7 +909,7 @@ class MiniMaxM3TritonSparseAttention: :class:`MiniMaxM3SparseIndexCache`. The caller is responsible for routing the projected Q, K, V, ``idx_q``, ``idx_k`` (and optional ``idx_v``) tensors plus the populated - :class:`MiniMaxM3TritonSparseAttentionMetadata`. + :class:`MiniMaxM3SparseAttentionMetadata`. """ config: MiniMaxM3SparseConfig @@ -835,7 +946,7 @@ def forward( idx_q: torch.Tensor, k_cache: torch.Tensor, v_cache: torch.Tensor, - metadata: MiniMaxM3TritonSparseAttentionMetadata, + metadata: MiniMaxM3SparseAttentionMetadata, *, disable_index_value: bool, sm_scale: Optional[float] = None, @@ -890,304 +1001,322 @@ def forward( # --------------------------------------------------------------------------- -class MiniMaxM3SparseRuntimeBackend(AttentionBackend[AttentionMetadata]): - """:class:`AttentionBackend` for MiniMax-M3 sparse layers. - - Constructed under the standard ``create_attention(...)`` dispatch - when ``SparseAttentionConfig(algorithm='minimax_m3', ...)`` is - configured. Drives the MiniMax-M3 sparse algorithm directly via - :func:`minimax_m3_sparse_prefill` and - :func:`minimax_m3_sparse_decode`. +@functools.lru_cache(maxsize=1) +def get_minimax_m3_attention_backend_cls(): + """Return :class:`MiniMaxM3SparseRuntimeBackend` (lazy import). - The standard :class:`AttentionForwardArgs` surface does not - carry ``idx_q`` / ``idx_k`` slots, so the model layer threads - the index branch through ``**kwargs`` of :meth:`forward`. When - ``forward`` is called without ``idx_q`` it raises - :class:`NotImplementedError` with a pointer at the model layer - — the backend's ``forward`` is **executable**, but it is not a - substitute for the MiniMax-specific projection / norm / RoPE - steps the model layer is responsible for. - - The backend exposes - :meth:`forward_sparse` for callers that want a name-explicit - entry point (the model layer calls it directly); :meth:`forward` - is the standard contract entry point and routes to - :meth:`forward_sparse` when ``idx_q`` is supplied. + Deferring the :class:`AttentionBackend` import keeps the algorithm + module usable from test paths that do not need the runtime backend. """ + from ...interface import ( + AttentionBackend, + AttentionForwardArgs, + AttentionMetadata, + merge_attention_forward_args, + ) - Metadata = MiniMaxM3AttentionMetadata - - def __init__( - self, - layer_idx: int, - num_heads: int, - head_dim: int, - num_kv_heads: Optional[int] = None, - quant_config=None, - sparse_params: Optional["MiniMaxM3SparseParams"] = None, - **kwargs, - ): - if sparse_params is None: - raise ValueError("sparse_params is required for MiniMaxM3SparseRuntimeBackend") - super().__init__( - layer_idx, - num_heads, - head_dim, - num_kv_heads=num_kv_heads, - quant_config=quant_config, - sparse_params=sparse_params, - **kwargs, - ) - self.m3_config = MiniMaxM3SparseConfig.from_sparse_params( - sparse_params, - num_q_heads=num_heads, - num_kv_heads=num_kv_heads or num_heads, - head_dim=head_dim, - ) - self.disable_index_value = bool(sparse_params.disable_index_value) + metadata_cls = get_minimax_m3_attention_metadata_cls() + + class MiniMaxM3SparseRuntimeBackend(AttentionBackend[AttentionMetadata]): + """:class:`AttentionBackend` for MiniMax-M3 sparse layers. + + Constructed under the standard ``create_attention(...)`` dispatch + when ``SparseAttentionConfig(algorithm='minimax_m3', ...)`` is + configured. Drives the MiniMax-M3 sparse algorithm directly via + :func:`minimax_m3_sparse_prefill` and + :func:`minimax_m3_sparse_decode`. + + The standard :class:`AttentionForwardArgs` surface does not + carry ``idx_q`` / ``idx_k`` slots, so the model layer threads + the index branch through ``**kwargs`` of :meth:`forward`. When + ``forward`` is called without ``idx_q`` it raises + :class:`NotImplementedError` with a pointer at the model layer + — the backend's ``forward`` is **executable**, but it is not a + substitute for the MiniMax-specific projection / norm / RoPE + steps the model layer is responsible for. + + The backend exposes + :meth:`forward_sparse` for callers that want a name-explicit + entry point (the model layer calls it directly); :meth:`forward` + is the standard contract entry point and routes to + :meth:`forward_sparse` when ``idx_q`` is supplied. + """ - @staticmethod - def support_fused_rope() -> bool: - # The MiniMax-M3 model layer applies RoPE explicitly because - # both the main and the index branches need partial RoPE, - # and the standard fused-RoPE attention op does not have a - # hook for the index branch. - return False + Metadata = metadata_cls - def forward_sparse( - self, - *, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - idx_q: torch.Tensor, - idx_k: torch.Tensor, - idx_v: Optional[torch.Tensor], - k_cache: torch.Tensor, - v_cache: torch.Tensor, - idx_k_cache: torch.Tensor, - idx_v_cache: Optional[torch.Tensor], - out_cache_loc: torch.Tensor, - m3_metadata: "MiniMaxM3TritonSparseAttentionMetadata", - sm_scale: Optional[float] = None, - idx_sm_scale: Optional[float] = None, - output: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Execute the MiniMax-M3 sparse path end-to-end. - - Inputs: - ``q``, ``k``, ``v`` : new-token projections, already - per-head norm + RoPE applied. - ``idx_q``, ``idx_k`` : index-branch projections, already - per-head norm + RoPE applied. - ``idx_v`` : index-V projection (only when - ``disable_index_value=False``). - ``k_cache``, ``v_cache``: flat-slot view of the paged - main K/V cache, - ``[num_slots, num_kv_heads, head_dim]``. - ``idx_k_cache`` : side index-K cache, - ``[num_slots, 1, sparse_index_dim]``. - ``idx_v_cache`` : side index-V cache (or ``None``). - ``out_cache_loc`` : ``[num_new_tokens]`` int slot - indices to write the new - token's K/V/idx_K to. - ``m3_metadata`` : populated - :class:`MiniMaxM3TritonSparseAttentionMetadata`. - ``output`` : optional preallocated final output, - ``[num_tokens, num_q_heads * head_dim]``. - - Returns ``[num_tokens, num_q_heads * head_dim]``. - """ - num_kv_heads = self.m3_config.num_kv_heads - head_dim = self.m3_config.head_dim - sparse_index_dim = self.m3_config.sparse_index_dim - num_idx_heads = self.m3_config.num_index_heads - - num_tokens = int(q.shape[0]) - q_view = q.view(num_tokens, self.num_heads, head_dim) - k_view = k.view(num_tokens, num_kv_heads, head_dim) - v_view = v.view(num_tokens, num_kv_heads, head_dim) - idx_q_view = idx_q.view(num_tokens, num_idx_heads, sparse_index_dim) - idx_k_view = idx_k.view(num_tokens, 1, sparse_index_dim) - - # Production paths build the M3 metadata on the cache - # device in ``MiniMaxM3AttentionMetadata.prepare`` (called - # outside any CUDA-graph capture window), and test paths - # construct it directly on the desired device. So all - # metadata tensors should already live on ``k_cache.device`` - # by this point. We keep a same-device pass for resilience - # against legacy test callers that produce metadata on a - # different device, but it must not introduce CPU->GPU - # copies inside the capture window. ``ensure_metadata_on_device`` - # is a no-op when each tensor is already on the target - # device; under capture that no-op path is the contract. - cache_device = k_cache.device - if any( - t is not None and t.device != cache_device - for t in ( - m3_metadata.req_to_token, - m3_metadata.slot_ids, - m3_metadata.seq_lens, - m3_metadata.prefix_lens, - m3_metadata.cu_seqlens_q, - m3_metadata.q_batch_row, - m3_metadata.q_positions, - ) + def __init__( + self, + layer_idx: int, + num_heads: int, + head_dim: int, + num_kv_heads: Optional[int] = None, + quant_config=None, + sparse_params: Optional["MiniMaxM3SparseParams"] = None, + **kwargs, ): - m3_metadata = ensure_metadata_on_device(m3_metadata, cache_device) - - # Write new K/V/idx_K to the configured slots. - # ``out_cache_loc`` comes from the pre-built attachment so - # it already lives on the cache device. The write goes - # through :func:`_write_main_kv_slots`, which is layout- - # aware: - # - # * 4-D multi-dim paged caches (the production V2 path: - # main K/V is ``kv_pool[:, 0]`` / ``kv_pool[:, 1]``, and - # ``idx_k_cache`` is the V2 4-D paged view ``[num_pages, - # tokens_per_block, 1, sparse_index_dim]``): decomposes - # each slot id into - # ``(page, within)`` and uses multi-dim fancy - # assignment so the write propagates to the underlying - # pool. A plain ``index_copy_(0, ...)`` would either - # silently fork a copy (non-contiguous main K/V view) - # or raise a shape mismatch (4-D index-K view), but - # the layout-aware helper sidesteps both failure - # modes. - # * 3-D flat-slot caches (focused unit tests that - # allocate plain ``torch.zeros((num_slots, num_heads, - # channel))`` tensors): falls back to - # ``index_copy_(0, ...)`` because the tensor IS the - # storage. - _write_main_kv_slots(k_cache, out_cache_loc, k_view) - _write_main_kv_slots(v_cache, out_cache_loc, v_view) - _write_main_kv_slots(idx_k_cache, out_cache_loc, idx_k_view) - if idx_v is not None and idx_v_cache is not None: - idx_v_view = idx_v.view(num_tokens, 1, sparse_index_dim) - _write_main_kv_slots(idx_v_cache, out_cache_loc, idx_v_view) - - if m3_metadata.is_prefill: - _, o = minimax_m3_sparse_prefill( - q_view, - k_cache, - v_cache, - idx_q_view, - idx_k_cache, - None if self.disable_index_value else idx_v_cache, - m3_metadata, - self.m3_config, - disable_index_value=self.disable_index_value, - sm_scale=sm_scale, - idx_sm_scale=idx_sm_scale, - output=output, + if sparse_params is None: + raise ValueError("sparse_params is required for MiniMaxM3SparseRuntimeBackend") + super().__init__( + layer_idx, + num_heads, + head_dim, + num_kv_heads=num_kv_heads, + quant_config=quant_config, + sparse_params=sparse_params, + **kwargs, ) - else: - _, o = minimax_m3_sparse_decode( - q_view, - idx_q_view, - k_cache, - v_cache, - idx_k_cache, - None if self.disable_index_value else idx_v_cache, - m3_metadata, - self.m3_config, - disable_index_value=self.disable_index_value, + self.m3_config = MiniMaxM3SparseConfig.from_sparse_params( + sparse_params, + num_q_heads=num_heads, + num_kv_heads=num_kv_heads or num_heads, + head_dim=head_dim, + ) + self.disable_index_value = bool(sparse_params.disable_index_value) + + @staticmethod + def support_fused_rope() -> bool: + # The MiniMax-M3 model layer applies RoPE explicitly because + # both the main and the index branches need partial RoPE, + # and the standard fused-RoPE attention op does not have a + # hook for the index branch. + return False + + def forward_sparse( + self, + *, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + idx_q: torch.Tensor, + idx_k: torch.Tensor, + idx_v: Optional[torch.Tensor], + k_cache: torch.Tensor, + v_cache: torch.Tensor, + idx_k_cache: torch.Tensor, + idx_v_cache: Optional[torch.Tensor], + out_cache_loc: torch.Tensor, + m3_metadata: "MiniMaxM3SparseAttentionMetadata", + sm_scale: Optional[float] = None, + idx_sm_scale: Optional[float] = None, + output: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Execute the MiniMax-M3 sparse path end-to-end. + + Inputs: + ``q``, ``k``, ``v`` : new-token projections, already + per-head norm + RoPE applied. + ``idx_q``, ``idx_k`` : index-branch projections, already + per-head norm + RoPE applied. + ``idx_v`` : index-V projection (only when + ``disable_index_value=False``). + ``k_cache``, ``v_cache``: flat-slot view of the paged + main K/V cache, + ``[num_slots, num_kv_heads, head_dim]``. + ``idx_k_cache`` : side index-K cache, + ``[num_slots, 1, sparse_index_dim]``. + ``idx_v_cache`` : side index-V cache (or ``None``). + ``out_cache_loc`` : ``[num_new_tokens]`` int slot + indices to write the new + token's K/V/idx_K to. + ``m3_metadata`` : populated + :class:`MiniMaxM3SparseAttentionMetadata`. + ``output`` : optional preallocated final output, + ``[num_tokens, num_q_heads * head_dim]``. + + Returns ``[num_tokens, num_q_heads * head_dim]``. + """ + num_kv_heads = self.m3_config.num_kv_heads + head_dim = self.m3_config.head_dim + sparse_index_dim = self.m3_config.sparse_index_dim + num_idx_heads = self.m3_config.num_index_heads + + num_tokens = int(q.shape[0]) + q_view = q.view(num_tokens, self.num_heads, head_dim) + k_view = k.view(num_tokens, num_kv_heads, head_dim) + v_view = v.view(num_tokens, num_kv_heads, head_dim) + idx_q_view = idx_q.view(num_tokens, num_idx_heads, sparse_index_dim) + idx_k_view = idx_k.view(num_tokens, 1, sparse_index_dim) + + # Production paths build the M3 metadata on the cache + # device in ``MiniMaxM3AttentionMetadata.prepare`` (called + # outside any CUDA-graph capture window), and test paths + # construct it directly on the desired device. So all + # metadata tensors should already live on ``k_cache.device`` + # by this point. We keep a same-device pass for resilience + # against legacy test callers that produce metadata on a + # different device, but it must not introduce CPU->GPU + # copies inside the capture window. ``ensure_metadata_on_device`` + # is a no-op when each tensor is already on the target + # device; under capture that no-op path is the contract. + cache_device = k_cache.device + if any( + t is not None and t.device != cache_device + for t in ( + m3_metadata.req_to_token, + m3_metadata.slot_ids, + m3_metadata.seq_lens, + m3_metadata.prefix_lens, + m3_metadata.cu_seqlens_q, + m3_metadata.q_batch_row, + m3_metadata.q_positions, + ) + ): + m3_metadata = ensure_metadata_on_device(m3_metadata, cache_device) + + # Write new K/V/idx_K to the configured slots. + # ``out_cache_loc`` comes from the pre-built attachment so + # it already lives on the cache device. The write goes + # through :func:`_write_main_kv_slots`, which is layout- + # aware: + # + # * 4-D multi-dim paged caches (the production V2 path: + # main K/V is ``kv_pool[:, 0]`` / ``kv_pool[:, 1]``, and + # ``idx_k_cache`` is the V2 4-D paged view ``[num_pages, + # tokens_per_block, 1, sparse_index_dim]``): decomposes + # each slot id into + # ``(page, within)`` and uses multi-dim fancy + # assignment so the write propagates to the underlying + # pool. A plain ``index_copy_(0, ...)`` would either + # silently fork a copy (non-contiguous main K/V view) + # or raise a shape mismatch (4-D index-K view), but + # the layout-aware helper sidesteps both failure + # modes. + # * 3-D flat-slot caches (focused unit tests that + # allocate plain ``torch.zeros((num_slots, num_heads, + # channel))`` tensors): falls back to + # ``index_copy_(0, ...)`` because the tensor IS the + # storage. + _write_main_kv_slots(k_cache, out_cache_loc, k_view) + _write_main_kv_slots(v_cache, out_cache_loc, v_view) + _write_main_kv_slots(idx_k_cache, out_cache_loc, idx_k_view) + if idx_v is not None and idx_v_cache is not None: + idx_v_view = idx_v.view(num_tokens, 1, sparse_index_dim) + _write_main_kv_slots(idx_v_cache, out_cache_loc, idx_v_view) + + if m3_metadata.is_prefill: + _, o = minimax_m3_sparse_prefill( + q_view, + k_cache, + v_cache, + idx_q_view, + idx_k_cache, + None if self.disable_index_value else idx_v_cache, + m3_metadata, + self.m3_config, + disable_index_value=self.disable_index_value, + sm_scale=sm_scale, + idx_sm_scale=idx_sm_scale, + output=output, + ) + else: + _, o = minimax_m3_sparse_decode( + q_view, + idx_q_view, + k_cache, + v_cache, + idx_k_cache, + None if self.disable_index_value else idx_v_cache, + m3_metadata, + self.m3_config, + disable_index_value=self.disable_index_value, + sm_scale=sm_scale, + idx_sm_scale=idx_sm_scale, + output=output, + ) + return o + + def forward( + self, + q: torch.Tensor, + k: Optional[torch.Tensor], + v: Optional[torch.Tensor], + metadata=None, + forward_args: Optional[AttentionForwardArgs] = None, + *, + output: Optional[torch.Tensor] = None, + idx_q: Optional[torch.Tensor] = None, + idx_k: Optional[torch.Tensor] = None, + idx_v: Optional[torch.Tensor] = None, + k_cache: Optional[torch.Tensor] = None, + v_cache: Optional[torch.Tensor] = None, + idx_k_cache: Optional[torch.Tensor] = None, + idx_v_cache: Optional[torch.Tensor] = None, + out_cache_loc: Optional[torch.Tensor] = None, + m3_metadata: Optional["MiniMaxM3SparseAttentionMetadata"] = None, + sm_scale: Optional[float] = None, + idx_sm_scale: Optional[float] = None, + **kwargs, + ) -> torch.Tensor: + """Standard ``AttentionBackend.forward`` entry point. + + The MiniMax-M3 sparse path needs the index branch projection + and the M3-shaped metadata; both arrive through keyword + arguments because the standard + :class:`AttentionForwardArgs` surface does not carry them. + + When ``idx_q`` is omitted, this method raises + :class:`NotImplementedError` to make the misuse loud — a + generic AttentionBackend dispatch site cannot drive this + backend without supplying the index branch. + """ + forward_args = merge_attention_forward_args(forward_args, kwargs) + if ( + output is not None + and forward_args.output is not None + and output is not forward_args.output + ): + raise ValueError("output was supplied both directly and through forward_args") + if output is None: + output = forward_args.output + if idx_q is None or idx_k is None or m3_metadata is None: + raise NotImplementedError( + f"MiniMaxM3SparseRuntimeBackend.forward (layer " + f"{self.layer_idx}) requires the M3 index branch and " + "metadata to be passed as keyword arguments " + "(`idx_q`, `idx_k`, `m3_metadata`, " + "`out_cache_loc`, `k_cache`, `v_cache`, " + "`idx_k_cache`). The standard AttentionForwardArgs " + "surface does not carry them; the model layer " + "(`MiniMaxM3Attention.forward`) supplies them when " + "calling this backend." + ) + if ( + k is None + or v is None + or k_cache is None + or v_cache is None + or idx_k_cache is None + or out_cache_loc is None + ): + raise ValueError( + "MiniMaxM3SparseRuntimeBackend.forward requires k, v, " + "k_cache, v_cache, idx_k_cache, and out_cache_loc to " + "be supplied alongside idx_q / idx_k / m3_metadata." + ) + return self.forward_sparse( + q=q, + k=k, + v=v, + idx_q=idx_q, + idx_k=idx_k, + idx_v=idx_v, + k_cache=k_cache, + v_cache=v_cache, + idx_k_cache=idx_k_cache, + idx_v_cache=idx_v_cache, + out_cache_loc=out_cache_loc, + m3_metadata=m3_metadata, sm_scale=sm_scale, idx_sm_scale=idx_sm_scale, output=output, ) - return o - def forward( - self, - q: torch.Tensor, - k: Optional[torch.Tensor], - v: Optional[torch.Tensor], - metadata=None, - forward_args: Optional[AttentionForwardArgs] = None, - *, - output: Optional[torch.Tensor] = None, - idx_q: Optional[torch.Tensor] = None, - idx_k: Optional[torch.Tensor] = None, - idx_v: Optional[torch.Tensor] = None, - k_cache: Optional[torch.Tensor] = None, - v_cache: Optional[torch.Tensor] = None, - idx_k_cache: Optional[torch.Tensor] = None, - idx_v_cache: Optional[torch.Tensor] = None, - out_cache_loc: Optional[torch.Tensor] = None, - m3_metadata: Optional["MiniMaxM3TritonSparseAttentionMetadata"] = None, - sm_scale: Optional[float] = None, - idx_sm_scale: Optional[float] = None, - **kwargs, - ) -> torch.Tensor: - """Standard ``AttentionBackend.forward`` entry point. - - The MiniMax-M3 sparse path needs the index branch projection - and the M3-shaped metadata; both arrive through keyword - arguments because the standard - :class:`AttentionForwardArgs` surface does not carry them. - - When ``idx_q`` is omitted, this method raises - :class:`NotImplementedError` to make the misuse loud — a - generic AttentionBackend dispatch site cannot drive this - backend without supplying the index branch. - """ - forward_args = merge_attention_forward_args(forward_args, kwargs) - if ( - output is not None - and forward_args.output is not None - and output is not forward_args.output - ): - raise ValueError("output was supplied both directly and through forward_args") - if output is None: - output = forward_args.output - if idx_q is None or idx_k is None or m3_metadata is None: - raise NotImplementedError( - f"MiniMaxM3SparseRuntimeBackend.forward (layer " - f"{self.layer_idx}) requires the M3 index branch and " - "metadata to be passed as keyword arguments " - "(`idx_q`, `idx_k`, `m3_metadata`, " - "`out_cache_loc`, `k_cache`, `v_cache`, " - "`idx_k_cache`). The standard AttentionForwardArgs " - "surface does not carry them; the model layer " - "(`MiniMaxM3Attention.forward`) supplies them when " - "calling this backend." - ) - if ( - k is None - or v is None - or k_cache is None - or v_cache is None - or idx_k_cache is None - or out_cache_loc is None - ): - raise ValueError( - "MiniMaxM3SparseRuntimeBackend.forward requires k, v, " - "k_cache, v_cache, idx_k_cache, and out_cache_loc to " - "be supplied alongside idx_q / idx_k / m3_metadata." - ) - return self.forward_sparse( - q=q, - k=k, - v=v, - idx_q=idx_q, - idx_k=idx_k, - idx_v=idx_v, - k_cache=k_cache, - v_cache=v_cache, - idx_k_cache=idx_k_cache, - idx_v_cache=idx_v_cache, - out_cache_loc=out_cache_loc, - m3_metadata=m3_metadata, - sm_scale=sm_scale, - idx_sm_scale=idx_sm_scale, - output=output, - ) + return MiniMaxM3SparseRuntimeBackend __all__ = [ - "MiniMaxM3SparseRuntimeBackend", - "MiniMaxM3TritonSparseAttention", + "MiniMaxM3SparseAttention", + "get_minimax_m3_attention_backend_cls", "minimax_m3_sparse_decode", "minimax_m3_sparse_prefill", ] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/cache_manager.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/cache_manager.py index 65c89b170c5c..b9badca6ea35 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/cache_manager.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/cache_manager.py @@ -1,17 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. """KV cache management for MiniMax-M3 sparse attention. Provides: @@ -25,17 +13,21 @@ from __future__ import annotations -from typing import List, Optional, Sequence +from typing import List, Optional import torch -from tensorrt_llm._torch.disaggregation.resource.page import MapperKind -from tensorrt_llm._utils import TensorWrapper, binding_to_torch_dtype, convert_to_torch_tensor +from tensorrt_llm._utils import ( + TensorWrapper, + binding_to_torch_dtype, + convert_to_torch_tensor, + prefer_pinned, +) from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType as CacheTypeCpp -from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig, PageIndexMode +from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig, LayerId from tensorrt_llm.runtime.kv_cache_manager_v2._common import BAD_PAGE_INDEX -from tensorrt_llm.runtime.kv_cache_manager_v2._config import DataRole +from tensorrt_llm.runtime.kv_cache_manager_v2._utils import typed_range from ....pyexecutor.kv_cache_manager_v2 import KVCacheManagerV2, Role @@ -176,7 +168,7 @@ def __init__( disable_index_value_layer_ids = list(sparse_layer_ids) # Must be set BEFORE super().__init__ — the base - # ``_build_base_config`` invokes ``_extra_buffers_per_layer`` + # ``_build_cache_config`` invokes ``_extra_buffers_per_layer`` # which reads these attributes. self.sparse_layer_ids = sorted(int(i) for i in sparse_layer_ids) self.disable_index_value_layer_ids = set(int(i) for i in disable_index_value_layer_ids) @@ -184,15 +176,6 @@ def __init__( super().__init__(*args, **kwargs) - index_v_layer_ids = set(self.sparse_layer_ids) - self.disable_index_value_layer_ids - if self.is_disagg and index_v_layer_ids: - raise ValueError( - "MiniMax M3 disaggregated serving requires disable_index_value=True " - "for every sparse layer because the optional test-only index-V cache " - "is not managed or transferred by KVCacheManagerV2; enabled layers=" - f"{sorted(index_v_layer_ids)}" - ) - # Optional plain-tensor index-V cache for non-disabled sparse # layers (test-only; production has disable_index_value=True # on every sparse layer). @@ -215,7 +198,7 @@ def _extra_buffers_per_layer(self, *, tokens_per_block): ``size`` is bytes per **block**: ``1 * sparse_index_dim * elem_bytes * tokens_per_block``. Keyed by **local** layer id — - the base ``_build_base_config`` iterates local ids, so keying + the base ``_build_cache_config`` iterates local ids, so keying by global ids would silently skip registration on non-trivial PP ranks. """ @@ -229,13 +212,6 @@ def _extra_buffers_per_layer(self, *, tokens_per_block): if layer_id in self.layer_offsets } - def get_disagg_role_mapper_kinds(self) -> dict[DataRole, MapperKind]: - """Declare MiniMax M3's token-major K/V and replicated index-K.""" - return { - Role.ALL: MapperKind.NHD, - Role.INDEX_KEY: MapperKind.REPLICATED, - } - def _compute_num_total_slots(self) -> int: """Total token slots across all blocks in the main K pool. @@ -284,12 +260,12 @@ def get_buffers(self, layer_idx: int, kv_layout: str = "NHD") -> Optional[torch. The base :meth:`KVCacheManagerV2.get_buffers` produces a ``[num_pages, kv_factor, ...]`` view with contiguous strides - that assume the slot holds exactly one layer's K+V. In M3's - pool the slot packs K+V for *all* layers of the group - (``scale >= 2 * num_layers_in_group``), so the base view's - dim-0 stride does not reach the next slot's K for this layer. - (When INDEX_KEY's per-block size coincides with K/V's, it is - coalesced into the same pool and contributes to ``scale`` too.) + that assume each slot contains exactly K+V. With INDEX_KEY + registered, sparse layers may have ``scale > 2`` per-slot + buffers (e.g. M3 TP=8 coalesces K, V, INDEX_K into one pool + where ``scale == 3 * num_sparse + 2 * num_dense``), and the + base view's dim-0 stride no longer reaches the next slot's K + for this layer. The override builds a ``[num_slots, scale, ...]`` view rooted at K's base, then slices ``[:, :2]`` to extract K+V. The slice @@ -365,30 +341,75 @@ def get_buffers(self, layer_idx: int, kv_layout: str = "NHD") -> Optional[torch. full_view = convert_to_torch_tensor(TensorWrapper(addr_key, torch_dtype, full_slot_shape)) return full_view[:, :2] - def _kv_pool_mapping_offset(self, layer_id, layer_group_id, key_base_addr) -> int: - """Pool-mapping offset from the layer's physical position in its pool. - - The base formula ``exact_div(addr_offset, key_bytes * kv_factor * - tokens_per_block)`` assumes each layer contributes exactly K+V to - its pool slot. When index-K coalesces into the K/V pool the layer - stride is non-uniform (sparse layers add an INDEX_KEY sub-page), - so no uniform-stride offset exists. The M3 forward path uses - :meth:`get_buffers` / :meth:`get_index_k_buffer` rather than this - mapping, so the offset just needs to be a consistent per-layer - position. Rank the group's layers by their K base address instead - of by ``layer_grouping`` iteration order: the ordering of - ``layer_grouping`` is not a V2 API contract, while the address - rank always reflects the physical slot layout (and keeps the - NVFP4 ``block_scale_offset == offset`` cross-check in the base - pool-mapping loop meaningful). + def _build_pool_mapping_tensors(self): + """Compute pool-mapping offsets from layer position in the pool group. + + The base method does ``exact_div(addr_offset, key_bytes * + kv_factor * tokens_per_block)``, which assumes each layer + contributes exactly K+V. When INDEX_KEY coincidentally shares + the same per-block size as K/V (M3 production at TP=8: all + three are 256 B/token), V2 coalesces all three into one pool + and the per-layer stride becomes ``3 * single_buffer_size`` — + the base ``exact_div`` then asserts. + + Compute ``offset`` directly from + ``self.impl.layer_grouping[group_id]`` so the formula stays + correct regardless of how many extra buffers coalesce with + K/V. The M3 forward path uses :meth:`get_buffers` / + :meth:`get_index_k_buffer` rather than this mapping, so the + offset just needs to be consistent (layer position in group). """ - layers_by_addr = sorted( - self.impl.layer_grouping[int(layer_group_id)], - key=lambda lid: self.impl.get_mem_pool_base_address( - lid, Role.KEY, PageIndexMode.SHARED - ), + kv_cache_pool_pointers = torch.tensor( + [ + [ + self.impl.get_mem_pool_base_address( + self.impl.layer_grouping[pool_id][0], Role.KEY + ), + 0, + ] + for pool_id in range(self.num_pools) + ], + dtype=torch.int64, + device="cpu", + pin_memory=prefer_pinned(), ) - return layers_by_addr.index(int(layer_id)) + + if self.dtype == DataType.NVFP4: + kv_cache_pool_pointers = torch.stack( + [ + kv_cache_pool_pointers, + torch.tensor( + [ + [ + self.impl.get_mem_pool_base_address( + self.impl.layer_grouping[pool_id][0], Role.KEY_BLOCK_SCALE + ), + 0, + ] + for pool_id in range(self.num_pools) + ], + dtype=torch.int64, + device="cpu", + pin_memory=prefer_pinned(), + ), + ], + dim=-1, + ) + + kv_cache_pool_mapping_list = [] + for layer_id in typed_range(LayerId(self.num_local_layers)): + layer_group_id = self.impl.get_layer_group_id(layer_id) + layers_in_group = list(self.impl.layer_grouping[int(layer_group_id)]) + offset = layers_in_group.index(int(layer_id)) + kv_cache_pool_mapping_list.append([int(layer_group_id), offset]) + + kv_cache_pool_mapping = torch.tensor( + kv_cache_pool_mapping_list, + dtype=torch.int32, + device="cpu", + pin_memory=prefer_pinned(), + ) + return kv_cache_pool_pointers, kv_cache_pool_mapping def _get_batch_cache_indices_by_pool_id( self, @@ -396,45 +417,34 @@ def _get_batch_cache_indices_by_pool_id( *, pool_id: int = 0, is_kv_aggregate: bool = True, - num_blocks_per_seq: Optional[Sequence[int]] = None, - index_scale: Optional[int] = None, ): - """Return page indices; padded entries remain ``BAD_PAGE_INDEX`` (-1). + """Return per-request slot ids in ``[0, num_slots)`` directly. The base method converts slot ids to V1-style block ids via ``base_idx * index_scales[pool_id] // kv_factor``, which is - only correct when each layer contributes exactly K+V. M3's slot - packs K+V for all layers of the group, so the scale breaks the - V1 conversion and produces out-of-bounds block ids during V2 - warmup. + only correct when each layer contributes exactly K+V. With + INDEX_KEY-coalesced sparse pools (M3 production), the scale + breaks the V1 conversion and produces out-of-bounds block ids + during V2 warmup. Bypass the conversion: the M3 forward path indexes paged views (built by :meth:`get_buffers` / :meth:`get_index_k_buffer`) directly by slot id. - ``BAD_PAGE_INDEX`` slots remain ``-1`` here because disaggregation's - :class:`KVRegionExtractorV1` filters ``region_ids >= 0``. - :meth:`get_block_ids_per_seq` maps them to zero for the attention - metadata's padded tensor. - - Args: - request_ids: Request IDs whose page-index rows are returned. - pool_id: V2 pool whose page indices are requested. - is_kv_aggregate: Kept for compatibility with the base virtual method. - num_blocks_per_seq: Optional per-request truncation limits. When - omitted, preserve the full padded width required by MiniMax - CUDA-graph metadata initialization. - index_scale: Kept for compatibility with the base virtual method; - M3 bypasses the V1 block-id conversion entirely, so any - caller-supplied scale is ignored alongside ``index_scales``. + ``BAD_PAGE_INDEX`` slots stay as 0 to match the legacy + padding contract. """ res = [] - for req_idx, req_id in enumerate(request_ids): - kv_cache = self.kv_cache_map[req_id] - base_page_indices = kv_cache.get_base_page_indices(pool_id) - if num_blocks_per_seq is not None: - num_blocks = min(kv_cache.num_blocks, num_blocks_per_seq[req_idx]) - base_page_indices = base_page_indices[:num_blocks] - res.append(list(base_page_indices)) + for req_id in request_ids: + idx_tensor = torch.as_tensor(self.kv_cache_map[req_id].get_base_page_indices(pool_id)) + res.append( + ( + torch.where( + idx_tensor != BAD_PAGE_INDEX, + idx_tensor, + torch.full_like(idx_tensor, BAD_PAGE_INDEX), + ) + ).tolist() + ) return res def get_block_ids_per_seq(self, request_ids): diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/common.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/common.py deleted file mode 100644 index 9f2863846df2..000000000000 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/common.py +++ /dev/null @@ -1,252 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Shared building blocks for the MiniMax-M3 sparse attention backends. - -Both the Triton reference and the MSA (fmha_sm100) path share these -backend-neutral pieces: the lowered parameter and per-rank kernel config -bundles, block-priority sentinels, KV-slot writers, and the paged-cache -slot mapping builder. MSA-only helpers live in :mod:`.msa_utils`. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, List, Literal, Optional, Tuple - -import torch - -from ..params import SparseMetadataParams, SparseParams - -if TYPE_CHECKING: - from tensorrt_llm.mapping import Mapping - -# Sentinel scores that force init and local blocks into the top-k regardless -# of their computed score. Init outranks local. -_INIT_SCORE = 1e30 -_LOCAL_SCORE = 1e29 - - -@dataclass(frozen=True) -class MiniMaxM3SparseParams(SparseParams): - """Lowered runtime parameters for the MiniMax-M3 sparse backend.""" - - algorithm: Literal["minimax_m3"] = field(init=False, default="minimax_m3") - num_index_heads: int = 4 - sparse_index_dim: int = 128 - block_size: int = 128 - topk: int = 16 - init_blocks: int = 0 - local_blocks: int = 1 - score_type: str = "max" - disable_index_value: bool = True - implementation: Literal["triton", "msa"] = "triton" - - @property - def indices_block_size(self) -> int: - """Block granularity of the selected sparse indices. - - Read by the shared TrtllmAttention forward when publishing the - sparse prediction. It equals the per-block scoring size. - """ - return self.block_size - - -@dataclass(frozen=True) -class MiniMaxM3SparseMetadataParams(SparseMetadataParams): - """Metadata-facing MiniMax-M3 sparse geometry.""" - - global_num_q_heads: int = 0 - global_num_kv_heads: int = 0 - num_index_heads: int = 4 - topk: int = 16 - - def sharded_head_counts(self, mapping: Optional["Mapping"] = None) -> Tuple[int, int]: - """Return per-rank (num_q_heads, num_kv_heads) for mapping. - - Matches the model's attention sharding: no split under attention data - parallelism, otherwise split by tp_size. - """ - if mapping is not None and not getattr(mapping, "enable_attention_dp", False): - tp_size = int(getattr(mapping, "tp_size", 1) or 1) - else: - tp_size = 1 - - def _shard(num_heads: int) -> int: - return (int(num_heads) + tp_size - 1) // tp_size - - return _shard(self.global_num_q_heads), _shard(self.global_num_kv_heads) - - -@dataclass(frozen=True) -class MiniMaxM3SparseConfig: - """Per-rank kernel parameter bundle for MiniMax-M3 sparse attention. - - This is **not** a user-facing config (use - :class:`tensorrt_llm.llmapi.llm_args.MiniMaxM3SparseAttentionConfig` - for that). It is the layer-invariant, post-TP-shard parameter bundle - that backend kernels and reference helpers consume. The user knobs - come from :class:`MiniMaxM3SparseParams`; ``num_q_heads`` / - ``num_kv_heads`` / ``head_dim`` come from the per-rank model - geometry and must be supplied by the caller (typically via - :meth:`from_sparse_params`). - """ - - num_q_heads: int - num_kv_heads: int - head_dim: int - num_index_heads: int - sparse_index_dim: int - block_size: int - topk: int - init_blocks: int = 0 - local_blocks: int = 1 - score_type: str = "max" - - def __post_init__(self) -> None: - if self.num_q_heads % self.num_kv_heads != 0: - raise ValueError( - f"num_q_heads ({self.num_q_heads}) must be divisible by " - f"num_kv_heads ({self.num_kv_heads})" - ) - if self.num_index_heads % self.num_kv_heads != 0: - raise ValueError( - f"num_index_heads ({self.num_index_heads}) must be divisible " - f"by num_kv_heads ({self.num_kv_heads})" - ) - if self.block_size <= 0: - raise ValueError(f"block_size must be > 0, got {self.block_size}") - if self.topk <= 0: - raise ValueError(f"topk must be > 0, got {self.topk}") - if self.init_blocks < 0: - raise ValueError(f"init_blocks must be >= 0, got {self.init_blocks}") - if self.local_blocks < 0: - raise ValueError(f"local_blocks must be >= 0, got {self.local_blocks}") - if self.score_type != "max": - # SGLang exposes only "max" today and that is what the MiniMax-M3 - # checkpoint config specifies. Reject anything else explicitly so - # a config drift surfaces immediately. - raise ValueError( - f"score_type={self.score_type!r} is not supported " - "(only 'max' matches the SGLang reference)" - ) - - @classmethod - def from_sparse_params( - cls, - sparse_params: "MiniMaxM3SparseParams", - *, - num_q_heads: int, - num_kv_heads: int, - head_dim: int, - ) -> "MiniMaxM3SparseConfig": - """Build a kernel param bundle from lowered ``MiniMaxM3SparseParams`` - and the per-rank model geometry. - """ - return cls( - num_q_heads=int(num_q_heads), - num_kv_heads=int(num_kv_heads), - head_dim=int(head_dim), - num_index_heads=int(sparse_params.num_index_heads), - sparse_index_dim=int(sparse_params.sparse_index_dim), - block_size=int(sparse_params.block_size), - topk=int(sparse_params.topk), - init_blocks=int(sparse_params.init_blocks), - local_blocks=int(sparse_params.local_blocks), - score_type=str(sparse_params.score_type), - ) - - -def write_kv_slots( - cache: torch.Tensor, - out_cache_loc: torch.Tensor, - values: torch.Tensor, - *, - layout: Literal["NHD", "HND"] = "NHD", -) -> None: - """Write per-token values into a K, V, or index-K cache at given slots. - - Handles a 3-D flat-slot cache and a 4-D paged view. `layout` sets the paged - axis order: "NHD" is [num_pages, tokens_per_block, num_heads, channel], - "HND" is [num_pages, num_heads, tokens_per_block, channel]. The paged view - is non-contiguous, so the slot id is split into (page, within) and written - by multi-dim assignment. `values` is always [num_tokens, num_heads, channel]. - """ - with torch.no_grad(): - if cache.ndim >= 4: - token_axis = 2 if layout == "HND" else 1 - tokens_per_block = int(cache.shape[token_axis]) - out_long = out_cache_loc.to(torch.long) - page = out_long // tokens_per_block - within = out_long % tokens_per_block - if layout == "HND": - # Advanced indices on dims 0 and 2 broadcast to [num_tokens] and - # move front, giving a [num_tokens, num_heads, channel] target. - cache[page, :, within, :] = values.to(cache.dtype) - else: - cache[page, within] = values.to(cache.dtype) - else: - cache.index_copy_(0, out_cache_loc.to(torch.long), values.to(cache.dtype)) - - -def build_paged_kv_slot_mapping( - *, - kv_cache_manager, - request_ids, - qo_lens_cpu: torch.Tensor, - qo_offset_cpu: torch.Tensor, - device: torch.device, -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Build the backend-neutral paged-cache slot mapping. - - Returns (req_to_token, slot_ids, out_cache_loc), derived only from the paged - KV cache manager and the per-request query geometry, with no dependency on - any backend-specific metadata. - - req_to_token is the [batch, max_kv_len] int32 map from (request, position) - to a global slot id, expanded from get_block_ids_per_seq with - tokens_per_block as block_id * tokens_per_block + offset_within_block. - slot_ids is the [batch] identity row index into req_to_token. out_cache_loc - lists the per-new-token slot ids in flattened query order: request b - contributes positions qo_offset[b] through qo_offset[b] + qo_lens[b] - 1. - That one formula covers prefill (qo_offset is the prefix length) and decode - (qo_offset is kv_len - 1 with qo_len 1). - - The req_to_token reads that build out_cache_loc sync the host, so call this - only from prepare(), never from the forward path. - """ - tokens_per_block = int(kv_cache_manager.tokens_per_block) - # block_ids_per_seq is a [batch, max_blocks_per_seq] tensor; row b holds the - # block ids assigned to request_ids[b] in order. - block_ids = kv_cache_manager.get_block_ids_per_seq(list(request_ids)) - batch = int(qo_lens_cpu.shape[0]) - max_blocks = int(block_ids.shape[1]) - max_kv_len = max_blocks * tokens_per_block - - # Expand block ids -> per-token slot ids. - block_ids_dev = block_ids.to(device).to(torch.int64) - within_block = torch.arange(tokens_per_block, device=device, dtype=torch.int64) - # Outer product per batch entry: [batch, max_blocks, tokens_per_block] - slot_grid = block_ids_dev.unsqueeze(-1) * tokens_per_block + within_block - req_to_token = slot_grid.reshape(batch, max_kv_len).to(torch.int32) - slot_ids = torch.arange(batch, device=device, dtype=torch.int32) - - # out_cache_loc: per-new-token slot ids, in flattened query-token order. - req_to_token_cpu = req_to_token.to("cpu") - qo_lens_list = qo_lens_cpu.to(torch.long).tolist() - qo_offset_list = qo_offset_cpu.to(torch.long).tolist() - out_cache_loc_list: List[int] = [] - for b in range(batch): - start = int(qo_offset_list[b]) - for offset in range(int(qo_lens_list[b])): - out_cache_loc_list.append(int(req_to_token_cpu[b, start + offset].item())) - out_cache_loc = torch.tensor(out_cache_loc_list, dtype=torch.int32, device=device) - return req_to_token, slot_ids, out_cache_loc - - -__all__ = [ - "MiniMaxM3SparseConfig", - "MiniMaxM3SparseMetadataParams", - "MiniMaxM3SparseParams", - "build_paged_kv_slot_mapping", - "write_kv_slots", -] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_kernels.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/kernels.py similarity index 100% rename from tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_kernels.py rename to tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/kernels.py diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_metadata.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/metadata.py similarity index 61% rename from tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_metadata.py rename to tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/metadata.py index be4b8ae24143..925ca6c0189b 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/triton_metadata.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/metadata.py @@ -1,31 +1,131 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""MiniMax-M3 Triton reference per-forward metadata. +"""MiniMax-M3 sparse attention configuration + per-forward metadata. Contains: - * :class:`MiniMaxM3TritonSparseAttentionMetadata` -- per-forward metadata - dataclass with a CUDA-graph-safe :meth:`prepare`. + * :class:`MiniMaxM3SparseConfig` -- post-TP-shard kernel + parameter bundle. + * :class:`MiniMaxM3SparseAttentionMetadata` -- per-forward metadata + dataclass with a + CUDA-graph-safe + :meth:`prepare`. * Helpers to migrate metadata across devices, build it from a real - :class:`KVCacheManagerV2`, and pre-allocate CUDA-graph-stable buffers. - * :class:`MiniMaxM3AttentionMetadata` -- the :class:`AttentionMetadata` - subclass the pyexecutor wires into the M3 sparse layer's forward path. + :class:`KVCacheManagerV2`, and pre-allocate CUDA-graph-stable + buffers. + * :func:`get_minimax_m3_attention_metadata_cls` -- lazy factory for + the :class:`AttentionMetadata` subclass the pyexecutor wires into + the M3 sparse layer's forward path. """ from __future__ import annotations import dataclasses +import functools from dataclasses import dataclass, field -from typing import List, Optional, Tuple +from typing import List, Literal, Optional, Tuple import torch -from ...interface import AttentionMetadata -from .common import build_paged_kv_slot_mapping +from ..params import SparseParams + + +@dataclass(frozen=True) +class MiniMaxM3SparseParams(SparseParams): + """Lowered runtime parameters for the MiniMax-M3 sparse backend.""" + + algorithm: Literal["minimax_m3"] = field(init=False, default="minimax_m3") + num_index_heads: int = 4 + sparse_index_dim: int = 128 + block_size: int = 128 + topk: int = 16 + init_blocks: int = 0 + local_blocks: int = 1 + score_type: str = "max" + disable_index_value: bool = True + + +@dataclass(frozen=True) +class MiniMaxM3SparseConfig: + """Per-rank kernel parameter bundle for MiniMax-M3 sparse attention. + + This is **not** a user-facing config (use + :class:`tensorrt_llm.llmapi.llm_args.MiniMaxM3SparseAttentionConfig` + for that). It is the layer-invariant, post-TP-shard parameter bundle + that backend kernels and reference helpers consume. The user knobs + come from :class:`MiniMaxM3SparseParams`; ``num_q_heads`` / + ``num_kv_heads`` / ``head_dim`` come from the per-rank model + geometry and must be supplied by the caller (typically via + :meth:`from_sparse_params`). + """ + + num_q_heads: int + num_kv_heads: int + head_dim: int + num_index_heads: int + sparse_index_dim: int + block_size: int + topk: int + init_blocks: int = 0 + local_blocks: int = 1 + score_type: str = "max" + + def __post_init__(self) -> None: + if self.num_q_heads % self.num_kv_heads != 0: + raise ValueError( + f"num_q_heads ({self.num_q_heads}) must be divisible by " + f"num_kv_heads ({self.num_kv_heads})" + ) + if self.num_index_heads % self.num_kv_heads != 0: + raise ValueError( + f"num_index_heads ({self.num_index_heads}) must be divisible " + f"by num_kv_heads ({self.num_kv_heads})" + ) + if self.block_size <= 0: + raise ValueError(f"block_size must be > 0, got {self.block_size}") + if self.topk <= 0: + raise ValueError(f"topk must be > 0, got {self.topk}") + if self.init_blocks < 0: + raise ValueError(f"init_blocks must be >= 0, got {self.init_blocks}") + if self.local_blocks < 0: + raise ValueError(f"local_blocks must be >= 0, got {self.local_blocks}") + if self.score_type != "max": + # SGLang exposes only "max" today and that is what the MiniMax-M3 + # checkpoint config specifies. Reject anything else explicitly so + # a config drift surfaces immediately. + raise ValueError( + f"score_type={self.score_type!r} is not supported " + "(only 'max' matches the SGLang reference)" + ) + + @classmethod + def from_sparse_params( + cls, + sparse_params: "MiniMaxM3SparseParams", + *, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + ) -> "MiniMaxM3SparseConfig": + """Build a kernel param bundle from lowered ``MiniMaxM3SparseParams`` + and the per-rank model geometry. + """ + return cls( + num_q_heads=int(num_q_heads), + num_kv_heads=int(num_kv_heads), + head_dim=int(head_dim), + num_index_heads=int(sparse_params.num_index_heads), + sparse_index_dim=int(sparse_params.sparse_index_dim), + block_size=int(sparse_params.block_size), + topk=int(sparse_params.topk), + init_blocks=int(sparse_params.init_blocks), + local_blocks=int(sparse_params.local_blocks), + score_type=str(sparse_params.score_type), + ) @dataclass -class MiniMaxM3TritonSparseAttentionMetadata: +class MiniMaxM3SparseAttentionMetadata: """Per-forward metadata for MiniMax-M3 sparse attention. Mirrors the shape of SGLang's @@ -169,12 +269,12 @@ def prepare(self) -> None: def ensure_metadata_on_device( - metadata: "MiniMaxM3TritonSparseAttentionMetadata", + metadata: "MiniMaxM3SparseAttentionMetadata", device: torch.device, -) -> "MiniMaxM3TritonSparseAttentionMetadata": +) -> "MiniMaxM3SparseAttentionMetadata": """Return ``metadata`` with every GPU-consumed tensor on ``device``. - Constructs a new :class:`MiniMaxM3TritonSparseAttentionMetadata` whose + Constructs a new :class:`MiniMaxM3SparseAttentionMetadata` whose tensor fields are migrated to ``device`` when they live elsewhere. The CPU-side mirror ``seq_lens_cpu`` is preserved because the algorithm reads it only when explicitly noted (e.g. for @@ -206,6 +306,18 @@ def _move(t: Optional[torch.Tensor]) -> Optional[torch.Tensor]: ) +def replace_metadata( + metadata: MiniMaxM3SparseAttentionMetadata, + **changes, +) -> MiniMaxM3SparseAttentionMetadata: + """Helper around :func:`dataclasses.replace` for ``metadata``. + + Provided so callers can build a decode metadata from a prefill + metadata without manually re-typing every field. + """ + return dataclasses.replace(metadata, **changes) + + def allocate_minimax_m3_static_buffers( *, max_num_sequences: int, @@ -293,82 +405,6 @@ def allocate_minimax_m3_static_buffers( } -def _build_runtime_metadata_fresh( - *, - kv_cache_manager, - request_ids, - seq_lens: torch.Tensor, - seq_lens_cpu: torch.Tensor, - is_prefill: bool, - prefix_lens: Optional[torch.Tensor], - extend_seq_lens_cpu: Optional[List[int]], - device: torch.device, -) -> Tuple[MiniMaxM3TritonSparseAttentionMetadata, torch.Tensor]: - """Fresh-allocation build of the Triton reference metadata. - - Delegates the backend-neutral req_to_token, slot_ids and out_cache_loc - derivation to common.build_paged_kv_slot_mapping and adds the Triton-only - fields: cu_seqlens_q, prefix_lens on device, and the scalar max_seqlen - values and per-query-token tensors that prepare() computes. Used when no - graph-stable static_buffers are supplied; the static_buffers path in - build_runtime_metadata_from_kv_manager keeps its own in-place buffer writes. - """ - batch = int(seq_lens.shape[0]) - seq_lens_dev = seq_lens.to(device) if seq_lens.device != device else seq_lens - - if is_prefill: - if extend_seq_lens_cpu is None: - raise ValueError("prefill metadata requires extend_seq_lens_cpu") - if prefix_lens is None: - raise ValueError("prefill metadata requires prefix_lens") - prefix_lens_dev = prefix_lens.to(device) if prefix_lens.device != device else prefix_lens - qo_lens_cpu = torch.tensor([int(x) for x in extend_seq_lens_cpu], dtype=torch.int32) - qo_offset_cpu = prefix_lens.detach().to(device="cpu", dtype=torch.int32) - req_to_token, slot_ids, out_cache_loc = build_paged_kv_slot_mapping( - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - qo_lens_cpu=qo_lens_cpu, - qo_offset_cpu=qo_offset_cpu, - device=device, - ) - cu_q: List[int] = [0] - for ext in extend_seq_lens_cpu: - cu_q.append(cu_q[-1] + int(ext)) - cu_seqlens_q = torch.tensor(cu_q, dtype=torch.int32, device=device) - meta = MiniMaxM3TritonSparseAttentionMetadata( - is_prefill=True, - req_to_token=req_to_token, - slot_ids=slot_ids, - seq_lens=seq_lens_dev, - seq_lens_cpu=seq_lens_cpu, - prefix_lens=prefix_lens_dev, - cu_seqlens_q=cu_seqlens_q, - extend_seq_lens_cpu=list(extend_seq_lens_cpu), - q_batch_row=None, - q_positions=None, - ) - else: - # Decode: the new token sits at position seq_lens[b] - 1. - qo_lens_cpu = torch.ones(batch, dtype=torch.int32) - qo_offset_cpu = seq_lens_cpu.detach().to(device="cpu", dtype=torch.int32) - 1 - req_to_token, slot_ids, out_cache_loc = build_paged_kv_slot_mapping( - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - qo_lens_cpu=qo_lens_cpu, - qo_offset_cpu=qo_offset_cpu, - device=device, - ) - meta = MiniMaxM3TritonSparseAttentionMetadata( - is_prefill=False, - req_to_token=req_to_token, - slot_ids=slot_ids, - seq_lens=seq_lens_dev, - seq_lens_cpu=seq_lens_cpu, - ) - meta.prepare() - return meta, out_cache_loc - - def build_runtime_metadata_from_kv_manager( *, kv_cache_manager, @@ -380,8 +416,8 @@ def build_runtime_metadata_from_kv_manager( extend_seq_lens_cpu: Optional[List[int]] = None, device: Optional[torch.device] = None, static_buffers: Optional[dict] = None, -) -> Tuple[MiniMaxM3TritonSparseAttentionMetadata, torch.Tensor]: - """Build a :class:`MiniMaxM3TritonSparseAttentionMetadata` from a real +) -> Tuple[MiniMaxM3SparseAttentionMetadata, torch.Tensor]: + """Build a :class:`MiniMaxM3SparseAttentionMetadata` from a real :class:`MiniMaxM3KVCacheManagerV2`. Returns the populated metadata plus an ``out_cache_loc`` tensor @@ -416,21 +452,6 @@ def build_runtime_metadata_from_kv_manager( the end-to-end runtime path without going through the full LLM forward. """ - if device is None: - device = seq_lens.device - - if static_buffers is None: - return _build_runtime_metadata_fresh( - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - seq_lens=seq_lens, - seq_lens_cpu=seq_lens_cpu, - is_prefill=is_prefill, - prefix_lens=prefix_lens, - extend_seq_lens_cpu=extend_seq_lens_cpu, - device=device, - ) - tokens_per_block = int(kv_cache_manager.tokens_per_block) # block_ids_per_seq is a [batch_size, max_blocks_per_seq] tensor; row b # holds the block ids assigned to request_ids[b] in order. @@ -438,6 +459,8 @@ def build_runtime_metadata_from_kv_manager( batch = int(seq_lens.shape[0]) max_blocks = int(block_ids.shape[1]) max_kv_len = max_blocks * tokens_per_block + if device is None: + device = seq_lens.device if static_buffers is not None: if static_buffers.get("device") != device: @@ -608,7 +631,7 @@ def build_runtime_metadata_from_kv_manager( cu_seqlens_q = torch.tensor(cu_q, dtype=torch.int32, device=device) q_batch_row = None q_positions = None - meta = MiniMaxM3TritonSparseAttentionMetadata( + meta = MiniMaxM3SparseAttentionMetadata( is_prefill=True, req_to_token=req_to_token, slot_ids=slot_ids, @@ -640,7 +663,7 @@ def build_runtime_metadata_from_kv_manager( out_cache_loc = out_cache_loc_buf[:batch] else: out_cache_loc = torch.tensor(out_cache_loc_list, dtype=torch.int32, device=device) - meta = MiniMaxM3TritonSparseAttentionMetadata( + meta = MiniMaxM3SparseAttentionMetadata( is_prefill=False, req_to_token=req_to_token, slot_ids=slot_ids, @@ -651,222 +674,242 @@ def build_runtime_metadata_from_kv_manager( return meta, out_cache_loc -class MiniMaxM3AttentionMetadata(AttentionMetadata): - """:class:`AttentionMetadata` that pre-builds MiniMax-M3 metadata. - - Overrides :meth:`prepare` so the M3-sparse - :class:`MiniMaxM3TritonSparseAttentionMetadata` and the per-new-token - ``out_cache_loc`` are built **once per scheduler step**, on the - cache device, before the model forward runs. The result is - stored as ``self.minimax_m3 = {"metadata": m3_meta, - "out_cache_loc": out_cache_loc}`` so the model layer's - ``_dense_forward`` and ``_sparse_forward`` can read it without - any device migration. +@functools.lru_cache(maxsize=1) +def get_minimax_m3_attention_metadata_cls(): + """Return :class:`MiniMaxM3AttentionMetadata` (lazy import). - Test paths that build their own metadata can short-circuit by - attaching ``attn_metadata.minimax_m3`` directly before calling - the forward; those paths do not go through :meth:`prepare`. + The class extends :class:`AttentionMetadata` so the pyexecutor's + metadata-creation/prepare hooks (model_engine.py) drive M3 metadata + construction outside the CUDA-graph capture window. Building the + M3-sparse ``req_to_token`` / ``slot_ids`` / ``out_cache_loc`` + tensors during ``prepare()`` lands them on the GPU **before** the + forward call; the forward path then reads from the pre-built + attachment and performs no CPU->GPU copies, which is required for + CUDA-graph capture safety (``cudaErrorStreamCaptureUnsupported`` + fires for CPU->GPU ``memcpyAsync`` calls inside a captured stream). """ - - minimax_m3: Optional[dict] = None - # Lazily allocated dict of persistent device buffers used to keep - # ``MiniMaxM3TritonSparseAttentionMetadata`` tensor addresses stable - # across CUDA-graph capture/replay. None until the first - # ``prepare()`` call decides to use them (``is_cuda_graph`` / - # graph-stable mode). - _m3_static_buffers: Optional[dict] = None - - def _maybe_get_m3_static_buffers( - self, cache_device: torch.device, kv_cache_manager - ) -> Optional[dict]: - """Return persistent M3 buffers when graph stability is - required. - - Allocates the persistent buffer dict the first time it is - needed and caches it on ``self._m3_static_buffers``. We - allocate the buffers under two conditions: - - * ``self.is_cuda_graph`` is True -- the captured graph - requires stable ``data_ptr()`` across replays; OR - * the previous prepare() already allocated buffers -- - we keep using them so the algorithm sees the same - addresses even between non-graph and graph-mode calls - (which can happen when the model engine alternates - between eager warmup and graph replay). - - Returns ``None`` when no static buffers should be used (e.g. - eager-only test paths that rely on per-call allocations). + from ...interface import AttentionMetadata + + class MiniMaxM3AttentionMetadata(AttentionMetadata): + """:class:`AttentionMetadata` that pre-builds MiniMax-M3 metadata. + + Overrides :meth:`prepare` so the M3-sparse + :class:`MiniMaxM3SparseAttentionMetadata` and the per-new-token + ``out_cache_loc`` are built **once per scheduler step**, on the + cache device, before the model forward runs. The result is + stored as ``self.minimax_m3 = {"metadata": m3_meta, + "out_cache_loc": out_cache_loc}`` so the model layer's + ``_dense_forward`` and ``_sparse_forward`` can read it without + any device migration. + + Test paths that build their own metadata can short-circuit by + attaching ``attn_metadata.minimax_m3`` directly before calling + the forward; those paths do not go through :meth:`prepare`. """ - need_static = ( - bool(getattr(self, "is_cuda_graph", False)) or self._m3_static_buffers is not None - ) - if not need_static: - return None - if self._m3_static_buffers is not None: - bufs = self._m3_static_buffers - if bufs.get("device") == cache_device: - return bufs - - # First-time use: return an empty placeholder dict. - # ``build_runtime_metadata_from_kv_manager`` performs the - # actual allocation lazily on the first call where the - # current scheduler step's geometry (max_kv_len from the - # manager's block-id table, total_q from extend_seq_lens, - # actual batch size after CUDA-graph padding) is known. - # That removes the need to predict the warmup geometry up - # front. The first allocation pins the buffer addresses - # for the rest of this metadata instance's lifetime, so all - # subsequent prepare() calls reuse the same ``data_ptr()``s - # and CUDA graph capture/replay stays valid. - placeholder: dict = { - "device": cache_device, - # Caller-provided hints used by the lazy allocator below - # when it sizes the persistent buffers on the first real - # prepare() call. - "max_num_sequences_hint": int( - getattr(self, "max_num_sequences", None) or self.max_num_requests - ), - "max_num_tokens_hint": int( - getattr(self, "max_num_tokens", None) - or (int(getattr(self, "max_num_sequences", None) or self.max_num_requests)) - ), - } - self._m3_static_buffers = placeholder - return placeholder - def prepare(self) -> None: - super().prepare() - - # Always rebuild the M3 metadata block on each prepare() - # call so it reflects the current scheduler step's seq_lens - # / request_ids / num_cached_tokens. Production - # ``model_engine`` invokes ``prepare()`` outside any CUDA - # graph capture window, so the (potentially expensive) build - # is safe to perform here. - # - # When CUDA graph is enabled the inner ``build_runtime_metadata_from_kv_manager`` - # call writes into the persistent ``_m3_static_buffers`` so - # the captured graph keeps reading from stable ``data_ptr()``s - # across replays. Without this the captured ``index_select`` - # over ``req_to_token``/``slot_ids`` reads from freed warmup - # memory and either produces wrong tokens or fires - # ``Indexing.cu:1515`` ``srcIndex < srcSelectDimSize``. - self.minimax_m3 = None - - # Production path: build the M3 metadata from the standard - # AttentionMetadata fields. Requires kv_cache_manager + the - # M3 sparse-cache contract. - kv_cache_manager = getattr(self, "kv_cache_manager", None) - if kv_cache_manager is None or not hasattr(kv_cache_manager, "get_index_k_buffer"): - # Not an M3 KV cache manager: nothing to build. The - # forward path will raise a clear error if the M3 - # backend ends up dispatched without the M3 cache. - return - request_ids = getattr(self, "request_ids", None) - seq_lens = self.seq_lens - if request_ids is None or seq_lens is None: - return - num_contexts = int(getattr(self, "num_contexts", 0) or 0) - batch_size = int(seq_lens.shape[0]) - if batch_size == 0: - return - - # The cache device hosts every paged buffer; this is the - # device the forward path consumes. - try: - layer_buf = kv_cache_manager.get_buffers(0) - cache_device = layer_buf.device - except Exception: - cache_device = torch.device(f"cuda:{torch.cuda.current_device()}") - - seq_lens_cpu = ( - getattr(self, "seq_lens_cpu", None) - if hasattr(self, "seq_lens_cpu") - else seq_lens.detach().to("cpu") - ) - if seq_lens_cpu is None: - seq_lens_cpu = seq_lens.detach().to("cpu") - - kv_cache_params = getattr(self, "kv_cache_params", None) - num_cached_per_seq = ( - kv_cache_params.num_cached_tokens_per_seq - if kv_cache_params is not None - else [0] * batch_size - ) - - # ``attn_metadata.seq_lens`` from the PyExecutor is the - # per-step new-token count. The M3 sparse-attention algorithm - # consumes a *cumulative* kv length: ``minimax_m3_sparse_*`` - # masks reads against ``metadata.seq_lens`` as the per-request - # K-side extent. Compute that cumulative kv length per request - # and feed it into the algorithm metadata builder. - kv_lens_cpu_list = [ - int(num_cached_per_seq[b]) + int(seq_lens_cpu[b].item()) for b in range(batch_size) - ] - kv_lens_cpu = torch.tensor(kv_lens_cpu_list, dtype=torch.int32) - kv_lens_dev = kv_lens_cpu.to(device=cache_device, non_blocking=True) - - static_buffers = self._maybe_get_m3_static_buffers(cache_device, kv_cache_manager) - - # Any batch containing a context (prefill or chunked extend) - # request takes the extend path. For prefill rows - # ``num_cached_per_seq`` is ``prefix_lens`` and the full new - # chunk is ``extend_seq_len``; for decode rows - # ``num_cached`` is ``kv_len - 1`` and ``extend_seq_len`` is - # 1, so the same builder produces the correct one-slot - # entry. Pure-decode batches (``num_contexts == 0``) still - # take the decode optimization for CUDA-graph warmup - # geometry. - # - # Mixed prefill+decode batches always take the extend path: - # the prefill kernel handles decode rows as 1-slot extends. - # The decode branch below is a pure-decode-only perf - # specialization. (iter-131 regression: previously a wrong - # predicate routed mixed batches into the decode branch and - # crashed in index_copy_.) - is_extend = num_contexts > 0 - if is_extend: - prefix_lens_list = [int(num_cached_per_seq[b]) for b in range(batch_size)] - extend_seq_lens_cpu = [ - kv_lens_cpu_list[b] - prefix_lens_list[b] for b in range(batch_size) - ] - prefix_lens = torch.tensor( - prefix_lens_list, - dtype=torch.int32, - device=cache_device, + minimax_m3: Optional[dict] = None + # Lazily allocated dict of persistent device buffers used to keep + # ``MiniMaxM3SparseAttentionMetadata`` tensor addresses stable + # across CUDA-graph capture/replay. None until the first + # ``prepare()`` call decides to use them (``is_cuda_graph`` / + # graph-stable mode). + _m3_static_buffers: Optional[dict] = None + + def _maybe_get_m3_static_buffers( + self, cache_device: torch.device, kv_cache_manager + ) -> Optional[dict]: + """Return persistent M3 buffers when graph stability is + required. + + Allocates the persistent buffer dict the first time it is + needed and caches it on ``self._m3_static_buffers``. We + allocate the buffers under two conditions: + + * ``self.is_cuda_graph`` is True -- the captured graph + requires stable ``data_ptr()`` across replays; OR + * the previous prepare() already allocated buffers -- + we keep using them so the algorithm sees the same + addresses even between non-graph and graph-mode calls + (which can happen when the model engine alternates + between eager warmup and graph replay). + + Returns ``None`` when no static buffers should be used (e.g. + eager-only test paths that rely on per-call allocations). + """ + need_static = ( + bool(getattr(self, "is_cuda_graph", False)) or self._m3_static_buffers is not None ) - m3_meta, out_cache_loc = build_runtime_metadata_from_kv_manager( - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - seq_lens=kv_lens_dev, - seq_lens_cpu=kv_lens_cpu, - is_prefill=True, - prefix_lens=prefix_lens, - extend_seq_lens_cpu=extend_seq_lens_cpu, - device=cache_device, - static_buffers=static_buffers, + if not need_static: + return None + if self._m3_static_buffers is not None: + bufs = self._m3_static_buffers + if bufs.get("device") == cache_device: + return bufs + + # First-time use: return an empty placeholder dict. + # ``build_runtime_metadata_from_kv_manager`` performs the + # actual allocation lazily on the first call where the + # current scheduler step's geometry (max_kv_len from the + # manager's block-id table, total_q from extend_seq_lens, + # actual batch size after CUDA-graph padding) is known. + # That removes the need to predict the warmup geometry up + # front. The first allocation pins the buffer addresses + # for the rest of this metadata instance's lifetime, so all + # subsequent prepare() calls reuse the same ``data_ptr()``s + # and CUDA graph capture/replay stays valid. + placeholder: dict = { + "device": cache_device, + # Caller-provided hints used by the lazy allocator below + # when it sizes the persistent buffers on the first real + # prepare() call. + "max_num_sequences_hint": int( + getattr(self, "max_num_sequences", None) or self.max_num_requests + ), + "max_num_tokens_hint": int( + getattr(self, "max_num_tokens", None) + or (int(getattr(self, "max_num_sequences", None) or self.max_num_requests)) + ), + } + self._m3_static_buffers = placeholder + return placeholder + + def prepare(self) -> None: + super().prepare() + + # Always rebuild the M3 metadata block on each prepare() + # call so it reflects the current scheduler step's seq_lens + # / request_ids / num_cached_tokens. Production + # ``model_engine`` invokes ``prepare()`` outside any CUDA + # graph capture window, so the (potentially expensive) build + # is safe to perform here. + # + # When CUDA graph is enabled the inner ``build_runtime_metadata_from_kv_manager`` + # call writes into the persistent ``_m3_static_buffers`` so + # the captured graph keeps reading from stable ``data_ptr()``s + # across replays. Without this the captured ``index_select`` + # over ``req_to_token``/``slot_ids`` reads from freed warmup + # memory and either produces wrong tokens or fires + # ``Indexing.cu:1515`` ``srcIndex < srcSelectDimSize``. + self.minimax_m3 = None + + # Production path: build the M3 metadata from the standard + # AttentionMetadata fields. Requires kv_cache_manager + the + # M3 sparse-cache contract. + kv_cache_manager = getattr(self, "kv_cache_manager", None) + if kv_cache_manager is None or not hasattr(kv_cache_manager, "get_index_k_buffer"): + # Not an M3 KV cache manager: nothing to build. The + # forward path will raise a clear error if the M3 + # backend ends up dispatched without the M3 cache. + return + request_ids = getattr(self, "request_ids", None) + seq_lens = self.seq_lens + if request_ids is None or seq_lens is None: + return + num_contexts = int(getattr(self, "num_contexts", 0) or 0) + batch_size = int(seq_lens.shape[0]) + if batch_size == 0: + return + + # The cache device hosts every paged buffer; this is the + # device the forward path consumes. + try: + layer_buf = kv_cache_manager.get_buffers(0) + cache_device = layer_buf.device + except Exception: + cache_device = torch.device(f"cuda:{torch.cuda.current_device()}") + + seq_lens_cpu = ( + getattr(self, "seq_lens_cpu", None) + if hasattr(self, "seq_lens_cpu") + else seq_lens.detach().to("cpu") ) - else: - m3_meta, out_cache_loc = build_runtime_metadata_from_kv_manager( - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - seq_lens=kv_lens_dev, - seq_lens_cpu=kv_lens_cpu, - is_prefill=False, - device=cache_device, - static_buffers=static_buffers, + if seq_lens_cpu is None: + seq_lens_cpu = seq_lens.detach().to("cpu") + + kv_cache_params = getattr(self, "kv_cache_params", None) + num_cached_per_seq = ( + kv_cache_params.num_cached_tokens_per_seq + if kv_cache_params is not None + else [0] * batch_size ) - self.minimax_m3 = { - "metadata": m3_meta, - "out_cache_loc": out_cache_loc, - } + # ``attn_metadata.seq_lens`` from the PyExecutor is the + # per-step new-token count. The M3 sparse-attention algorithm + # consumes a *cumulative* kv length: ``minimax_m3_sparse_*`` + # masks reads against ``metadata.seq_lens`` as the per-request + # K-side extent. Compute that cumulative kv length per request + # and feed it into the algorithm metadata builder. + kv_lens_cpu_list = [ + int(num_cached_per_seq[b]) + int(seq_lens_cpu[b].item()) for b in range(batch_size) + ] + kv_lens_cpu = torch.tensor(kv_lens_cpu_list, dtype=torch.int32) + kv_lens_dev = kv_lens_cpu.to(device=cache_device, non_blocking=True) + + static_buffers = self._maybe_get_m3_static_buffers(cache_device, kv_cache_manager) + + # Any batch containing a context (prefill or chunked extend) + # request takes the extend path. For prefill rows + # ``num_cached_per_seq`` is ``prefix_lens`` and the full new + # chunk is ``extend_seq_len``; for decode rows + # ``num_cached`` is ``kv_len - 1`` and ``extend_seq_len`` is + # 1, so the same builder produces the correct one-slot + # entry. Pure-decode batches (``num_contexts == 0``) still + # take the decode optimization for CUDA-graph warmup + # geometry. + # + # Mixed prefill+decode batches always take the extend path: + # the prefill kernel handles decode rows as 1-slot extends. + # The decode branch below is a pure-decode-only perf + # specialization. (iter-131 regression: previously a wrong + # predicate routed mixed batches into the decode branch and + # crashed in index_copy_.) + is_extend = num_contexts > 0 + if is_extend: + prefix_lens_list = [int(num_cached_per_seq[b]) for b in range(batch_size)] + extend_seq_lens_cpu = [ + kv_lens_cpu_list[b] - prefix_lens_list[b] for b in range(batch_size) + ] + prefix_lens = torch.tensor( + prefix_lens_list, + dtype=torch.int32, + device=cache_device, + ) + m3_meta, out_cache_loc = build_runtime_metadata_from_kv_manager( + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + seq_lens=kv_lens_dev, + seq_lens_cpu=kv_lens_cpu, + is_prefill=True, + prefix_lens=prefix_lens, + extend_seq_lens_cpu=extend_seq_lens_cpu, + device=cache_device, + static_buffers=static_buffers, + ) + else: + m3_meta, out_cache_loc = build_runtime_metadata_from_kv_manager( + kv_cache_manager=kv_cache_manager, + request_ids=request_ids, + seq_lens=kv_lens_dev, + seq_lens_cpu=kv_lens_cpu, + is_prefill=False, + device=cache_device, + static_buffers=static_buffers, + ) + + self.minimax_m3 = { + "metadata": m3_meta, + "out_cache_loc": out_cache_loc, + } + + return MiniMaxM3AttentionMetadata __all__ = [ - "MiniMaxM3AttentionMetadata", - "MiniMaxM3TritonSparseAttentionMetadata", + "MiniMaxM3SparseConfig", + "MiniMaxM3SparseAttentionMetadata", "allocate_minimax_m3_static_buffers", "build_runtime_metadata_from_kv_manager", "ensure_metadata_on_device", + "get_minimax_m3_attention_metadata_cls", + "replace_metadata", ] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_availability.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_availability.py deleted file mode 100644 index c4751cdec2e4..000000000000 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_availability.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Availability checks for the MiniMax-M3 MSA sparse attention kernels. - -The MSA kernels are provided by the fmha_sm100 package from the MSA git -submodule at 3rdparty/MSA and run only on the SM100 architecture family -(SM100 and SM103). These helpers gate backend selection so a request for the -MSA path fails early with a clear message on unsupported systems. -""" - -from __future__ import annotations - -from tensorrt_llm._utils import get_sm_version, is_sm_100f - -from .msa_utils import msa_package_available - -# fmha_sm100 runs on the SM100 architecture family (SM100 and SM103). Other -# architectures, including SM120, are not supported. -MSA_PACKAGE = "fmha_sm100" - - -def ensure_msa_available() -> None: - """Raise RuntimeError if the MSA sparse attention path cannot run here.""" - if not msa_package_available(): - raise RuntimeError( - f"MiniMax-M3 MSA sparse attention requires the {MSA_PACKAGE} kernels " - "from the MSA git submodule at 3rdparty/MSA. Initialize it with " - "'git submodule update --init --recursive'." - ) - if not is_sm_100f(): - sm_version = get_sm_version() - raise RuntimeError( - "MiniMax-M3 MSA sparse attention requires an SM100 or SM103 device, " - f"but the current device reports SM version {sm_version}." - ) - - -__all__ = [ - "MSA_PACKAGE", - "ensure_msa_available", -] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_backend.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_backend.py deleted file mode 100644 index 86cd67fa7272..000000000000 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_backend.py +++ /dev/null @@ -1,789 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""MSA-backed MiniMax-M3 sparse attention on the TrtllmAttention stack. - - * MiniMaxM3MsaSparseAttention subclasses TrtllmAttention and reuses its - inherited forward, overriding only the sparse hooks and owning an - MsaIndexer. - * The main sparse GQA runs through the registered MsaSparseGqaFmha. - * The indexer calls fmha_sm100 directly to produce the per-query selected - block indices, which the model layer threads through - forward_args.topk_indices. - * MiniMaxM3MsaSparseAttentionMetadata subclasses TrtllmAttentionMetadata and - stores its per-forward MSA tensors in CUDA-graph-stable buffers. - The buffers are allocated once in __post_init__ via - get_empty(capture_graph=...), and prepare() copies the per-step values - into them. The standard CUDAGraphRunner clones one metadata per graph - batch size (create_cuda_graph_metadata), so no per-batch-size cache is - needed here. - -The classes subclass TrtllmAttention and TrtllmAttentionMetadata, imported at -module scope. This is cycle-free because the fmha registry defers its -MsaSparseGqaFmha import (see fmha/registry.py), so trtllm's import chain does -not reach this module. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Optional, Tuple - -import torch - -from tensorrt_llm._torch.attention_backend.interface import AttentionForwardArgs -from tensorrt_llm._torch.attention_backend.trtllm import TrtllmAttention, TrtllmAttentionMetadata - -from .common import ( - MiniMaxM3SparseConfig, - MiniMaxM3SparseMetadataParams, - build_paged_kv_slot_mapping, - write_kv_slots, -) -from .msa_indexer import MsaIndexer -from .msa_utils import ( - MSA_REQUIRED_HEAD_DIM, - MSA_REQUIRED_TOPK, - build_kv_page_indices, - per_token_valid_blocks, - require_msa_module, -) - - -def _cache_device(meta) -> torch.device: - """Device hosting the paged KV buffers, else the current CUDA device.""" - kv_cache_manager = meta.kv_cache_manager - if kv_cache_manager is not None: - try: - return kv_cache_manager.get_buffers(0).device - except Exception: - pass - return torch.device(f"cuda:{torch.cuda.current_device()}") - - -def _worst_case_proxy_max_k_tiles( - fmha_sm100, - *, - num_index_heads: int, - kv_cache_manager, - max_batch: int, -) -> int: - """Return max_k_tiles for a proxy plan at the manager's max KV length.""" - page_size = int(kv_cache_manager.tokens_per_block) - max_kv_len = int(kv_cache_manager.max_blocks_per_seq) * page_size - qo_lens = torch.ones(max_batch, dtype=torch.int32) - kv_lens = torch.full((max_batch,), max_kv_len, dtype=torch.int32) - qo_offset = kv_lens - qo_lens - proxy_plan = fmha_sm100.fmha_sm100_plan( - qo_lens, - kv_lens, - num_index_heads, - num_kv_heads=1, - qo_offset=qo_offset, - page_size=page_size, - output_maxscore=True, - num_kv_splits=1, - causal=True, - ) - return int(proxy_plan[3]["max_k_tiles"]) - - -# Per-step fmha_sm100 plan tensors that must live in CUDA-graph-stable buffers. -# At num_kv_splits=1 the plan carries no split-KV workspaces, and -# cute_workspace_buffer is the vendor's cached scratch (kept by reference, not -# copied). -_MSA_PLAN_STABLE_KEYS = ( - "packed_work_range", - "packed_work_info", - "qo_segment_offsets", - "kv_segment_offsets", - "kv_page_indptr", - "qo_segment_lens", - "kv_segment_lens", - "qo_offset", -) -_MSA_PLAN_INT64_KEYS = ("packed_work_range", "packed_work_info") -# fmha_sm100 sizes packed_work_info at 131072 * max(num_kv_splits, 1); forcing -# num_kv_splits=1 pins this worklist width. -_MSA_PACKED_WORK_INFO_LEN = 131072 -_MSA_SPLIT_KV_KEYS = ( - "kv_tile_begin_indices", - "kv_tile_end_indices", - "kv_split_indices", - "num_kv_splits_per_row", - "workspace_o", - "workspace_lse", -) - - -class _MsaGraphSafePlan: - """CUDA-graph-stable mirror of one fmha_sm100 decode plan. - - Owns fixed device buffers for the per-step plan worklists. refresh() copies - a freshly built plan into them and returns a plan tuple pointing at the - stable buffers, so the captured fmha_sm100 run reads addresses that do not - change across replays. Mirrors FlashInfer's fixed indptr/indices buffers. - - Only valid at num_kv_splits=1: the plan then has no split-KV workspaces - (refresh() asserts this), and cute_workspace_buffer and the scalar fields - pass through unchanged. - """ - - def __init__(self, metadata, name: str, *, max_batch: int, num_ctas: int, capture_graph: bool): - buffers = metadata.cuda_graph_buffers - self._buf = {} - # Set by refresh(), read through the plan property. - self._plan: Optional[tuple] = None - # cute_workspace_buffer must keep a fixed address across steps for the - # captured graph to replay correctly. Pin it on first use and fail if - # it moves. - self._ws_ptr: Optional[int] = None - for key in _MSA_PLAN_STABLE_KEYS: - if key == "packed_work_range": - shape = (num_ctas,) - elif key == "packed_work_info": - shape = (_MSA_PACKED_WORK_INFO_LEN,) - elif key in ("qo_segment_offsets", "kv_segment_offsets", "kv_page_indptr"): - shape = (max_batch + 1,) - else: - shape = (max_batch,) - dtype = torch.int64 if key in _MSA_PLAN_INT64_KEYS else torch.int32 - self._buf[key] = metadata.get_empty( - buffers, - shape, - cache_name=f"{name}_{key}", - dtype=dtype, - capture_graph=capture_graph, - ) - - @property - def plan(self) -> Optional[tuple]: - """The current graph-safe plan tuple, or None if no decode plan is live.""" - return self._plan - - def reset(self) -> None: - """Drop the live plan tuple (e.g. for a prefill/mixed or captured step).""" - self._plan = None - - def refresh(self, plan_tuple) -> tuple: - has_mixed, split, batch, decode, prefill = plan_tuple - if has_mixed: - raise RuntimeError( - "MSA decode expects a single (non-mixed) fmha_sm100 plan; a decode " - "batch must be pure decode." - ) - for key in _MSA_SPLIT_KV_KEYS: - if decode.get(key) is not None: - raise RuntimeError( - f"MSA decode plan used split-KV workspace {key!r}; num_kv_splits=1 " - "is required for graph-safe decode." - ) - ws = decode.get("cute_workspace_buffer") - if ws is not None: - if self._ws_ptr is None: - self._ws_ptr = ws.data_ptr() - elif ws.data_ptr() != self._ws_ptr: - raise RuntimeError( - "cute_workspace_buffer moved across steps; the fmha_sm100 plan " - "is not CUDA-graph safe." - ) - rebuilt = dict(decode) - for key in _MSA_PLAN_STABLE_KEYS: - src = decode.get(key) - if src is None: - continue - n = int(src.shape[0]) - dst = self._buf[key] - if n > dst.shape[0]: - raise ValueError( - f"MSA plan buffer {key} ({dst.shape[0]}) is smaller than the plan tensor ({n})." - ) - dst[:n].copy_(src, non_blocking=True) - rebuilt[key] = dst[:n] - self._plan = (has_mixed, split, batch, rebuilt, prefill) - return self._plan - - -@dataclass(init=False) -class MiniMaxM3MsaSparseAttentionMetadata(TrtllmAttentionMetadata): - """TrtllmAttentionMetadata for MiniMax-M3 MSA sparse layers. - - Tensors read inside the captured forward are CUDA-graph-stable: the - cache slots (msa_out_cache_loc), page table (msa_kv_indices), and proxy - scratch (msa_max_score, msa_n_valid_blocks) are allocated once from the - manager's worst-case geometry. msa_out_cache_loc, msa_kv_indices, and - msa_n_valid_blocks are refreshed via copy_, while the fmha_sm100 proxy pass - writes msa_max_score directly (see msa_proxy_max_score_view). Decode-plan - worklists live on _MsaGraphSafePlan owners, surfaced via msa_decode_*_plan. - - Length inputs to fmha_sm100_plan (msa_qo_lens_cpu, msa_kv_lens_cpu, - msa_qo_offset_cpu) are host properties of the base seq_lens/kv_lens, - read only while building plans in prepare() (outside capture), so they - need no graph-stable storage. Plans are built in _build_decode_plans and - are absent for prefill/mixed batches, which run eagerly. - """ - - # Graph-stable buffers; consumers slice to the live count at the call - # site. Filled once the current step's cache write is prepared. - msa_out_cache_loc: Optional[torch.Tensor] = None - msa_kv_indices: Optional[torch.Tensor] = None - msa_max_score: Optional[torch.Tensor] = None - msa_n_valid_blocks: Optional[torch.Tensor] = None - - # _msa_buffers_ready gates the once-only device buffers; - # _msa_fields_ready marks that the current step's buffers are populated. - _msa_buffers_ready: bool = False - _msa_fields_ready: bool = False - # Sparse geometry the decode plans need. - _msa_params: Optional[MiniMaxM3SparseMetadataParams] = None - # Plan owners, created lazily when the decode plans are first built and - # reused across steps. Each owns its graph-safe plan buffers and the - # current refreshed plan tuple. - _msa_proxy_plan: Optional["_MsaGraphSafePlan"] = None - _msa_gqa_plan: Optional["_MsaGraphSafePlan"] = None - _msa_dense_plan: Optional["_MsaGraphSafePlan"] = None - - def __post_init__(self) -> None: - super().__post_init__() - params = self.sparse_metadata_params - self._msa_params = params if isinstance(params, MiniMaxM3SparseMetadataParams) else None - self._create_msa_buffers() - - @property - def msa_qo_lens_cpu(self) -> Optional[torch.Tensor]: - """Per-request query length (host int32), from the base seq_lens.""" - seq_lens = self.seq_lens - if seq_lens is None: - return None - out = seq_lens[: self.num_seqs] - return out if out.dtype == torch.int32 else out.to(torch.int32) - - @property - def msa_kv_lens_cpu(self) -> Optional[torch.Tensor]: - """Per-request KV length, cached plus new tokens (host int32).""" - kv_lens = getattr(self, "kv_lens", None) - if self.seq_lens is None or kv_lens is None: - return None - out = kv_lens[: self.num_seqs] - return out if out.dtype == torch.int32 else out.to(torch.int32) - - @property - def msa_qo_offset_cpu(self) -> Optional[torch.Tensor]: - """Per-request causal offset (kv_len - qo_len), the cached prefix length.""" - qo = self.msa_qo_lens_cpu - kv = self.msa_kv_lens_cpu - if qo is None or kv is None: - return None - return kv - qo - - @property - def msa_decode_proxy_plan(self) -> Optional[tuple]: - """Proxy (max-score) plan tuple, or None outside decode.""" - plan = self._msa_proxy_plan - return plan.plan if plan is not None else None - - @property - def msa_decode_gqa_plan(self) -> Optional[tuple]: - """Sparse GQA plan tuple, or None outside decode.""" - plan = self._msa_gqa_plan - return plan.plan if plan is not None else None - - @property - def msa_decode_dense_plan(self) -> Optional[tuple]: - """Dense GQA plan tuple, shared by dense layers 0 to 2.""" - plan = self._msa_dense_plan - return plan.plan if plan is not None else None - - def _create_msa_buffers(self) -> None: - """Allocate the CUDA-graph-stable MSA device buffers. - - Buffers come from the shared graph buffer pool so they are reserved - under capture. Sizing follows the worst-case graph geometry: - max_num_tokens for cache slots, max_num_sequences * max_blocks_per_seq - for the page table, and worst-case max_k_tiles for proxy scratch. - """ - kv_cache_manager = self.kv_cache_manager - self._msa_buffers_ready = False - if kv_cache_manager is None or not hasattr(kv_cache_manager, "get_index_k_buffer"): - return - capture_graph = self.is_cuda_graph - buffers = self.cuda_graph_buffers - max_num_sequences = int(self.max_num_sequences) - max_blocks_per_seq = int(kv_cache_manager.max_blocks_per_seq) - max_total_pages = max_num_sequences * max_blocks_per_seq - max_num_tokens = int(self.max_num_tokens) - - self.msa_out_cache_loc = self.get_empty( - buffers, - (max_num_tokens,), - cache_name="msa_out_cache_loc", - dtype=torch.int32, - capture_graph=capture_graph, - ) - self.msa_kv_indices = self.get_empty( - buffers, - (max_total_pages,), - cache_name="msa_kv_indices", - dtype=torch.int32, - capture_graph=capture_graph, - ) - # The proxy scratch needs the fmha_sm100 plan geometry. This metadata - # exists only for the MSA backend, whose selection already required the - # kernels, so a failed import here is a hard error rather than a reason - # to skip allocation. - params = self._msa_params - if params is not None: - fmha_sm100 = require_msa_module() - max_k_tiles = _worst_case_proxy_max_k_tiles( - fmha_sm100, - num_index_heads=params.num_index_heads, - kv_cache_manager=kv_cache_manager, - max_batch=max_num_sequences, - ) - self._alloc_msa_proxy_scratch( - num_index_heads=params.num_index_heads, - max_batch=max_num_sequences, - max_k_tiles=max_k_tiles, - capture_graph=capture_graph, - ) - self._msa_buffers_ready = True - - def _alloc_msa_proxy_scratch( - self, - *, - num_index_heads: int, - max_batch: int, - max_k_tiles: int, - capture_graph: bool, - ) -> None: - """Allocate the flat proxy max-score store and the valid-block scratch. - - The store is sized for the worst-case max_k_tiles so one allocation - serves every decode step. msa_proxy_max_score_view slices the per-step - shape out of it. - """ - buffers = self.cuda_graph_buffers - self.msa_max_score = self.get_empty( - buffers, - (num_index_heads * max_k_tiles * max_batch,), - cache_name="msa_max_score", - dtype=torch.float32, - capture_graph=capture_graph, - ) - self.msa_n_valid_blocks = self.get_empty( - buffers, - (max_batch,), - cache_name="msa_n_valid_blocks", - dtype=torch.int32, - capture_graph=capture_graph, - ) - - def _ensure_msa_decode_scratch_buffers( - self, - *, - num_index_heads: int, - max_batch: int, - capture_graph: bool, - required_max_k_tiles: int, - ) -> None: - """Ensure proxy scratch buffers exist and cover the current plan.""" - required_numel = num_index_heads * required_max_k_tiles * max_batch - if self.msa_max_score is not None: - if self.msa_max_score.numel() < required_numel: - raise ValueError( - f"msa_max_score backing store ({self.msa_max_score.numel()} " - f"elements) is smaller than the decode plan needs " - f"({required_numel} = {num_index_heads} heads * " - f"{required_max_k_tiles} k-tiles * {max_batch} batch)." - ) - return - - kv_cache_manager = self.kv_cache_manager - if kv_cache_manager is None: - return - - fmha_sm100 = require_msa_module() - max_k_tiles = _worst_case_proxy_max_k_tiles( - fmha_sm100, - num_index_heads=num_index_heads, - kv_cache_manager=kv_cache_manager, - max_batch=max_batch, - ) - if max_k_tiles < required_max_k_tiles: - raise ValueError( - f"Worst-case max_k_tiles ({max_k_tiles}) is less than the " - f"decode plan ({required_max_k_tiles})." - ) - self._alloc_msa_proxy_scratch( - num_index_heads=num_index_heads, - max_batch=max_batch, - max_k_tiles=max_k_tiles, - capture_graph=capture_graph, - ) - - def prepare(self) -> None: - super().prepare() - self._build_msa_fields() - self._build_decode_plans() - - def _build_decode_plans(self) -> None: - """Build the graph-safe decode plans and buffers for this step. - - Runs in prepare(), outside CUDA graph capture. The plans are - layer-invariant for MiniMax-M3, so they are built once per step from - the shared sparse geometry, mirrored into CUDA-graph-stable buffers, - and reused by every layer. Mirrors FlashInfer's plan() split. - Prefill/mixed batches leave the plans cleared and run eagerly. - """ - # Drop any plan tuples from the previous step; the msa_decode_*_plan - # properties then report None until they are rebuilt below. - for plan in (self._msa_proxy_plan, self._msa_gqa_plan, self._msa_dense_plan): - if plan is not None: - plan.reset() - if not self._msa_fields_ready: - return - # A decode batch is pure generation (no context requests). - if int(self.num_contexts or 0) > 0: - return - # Geometry is captured in __post_init__; skip when it is unavailable. - params = self._msa_params - if params is None: - return - num_index_heads = params.num_index_heads - num_q_heads, num_kv_heads = params.sharded_head_counts(self.mapping) - topk = params.topk - - fmha_sm100 = require_msa_module() - qo_lens_cpu = self.msa_qo_lens_cpu - kv_lens_cpu = self.msa_kv_lens_cpu - qo_offset_cpu = self.msa_qo_offset_cpu - if qo_lens_cpu is None or kv_lens_cpu is None or qo_offset_cpu is None: - return - batch = int(qo_lens_cpu.shape[0]) - device = _cache_device(self) - page_size = int(self.kv_cache_manager.tokens_per_block) - capture_graph = self.is_cuda_graph - max_batch = int(self.max_num_sequences) - - # Proxy plan: MQA (num_kv_heads=1) max-score pass over the index - # branch; output_maxscore feeds the indexer's top-k block selection. - proxy_plan = fmha_sm100.fmha_sm100_plan( - qo_lens_cpu, - kv_lens_cpu, - num_index_heads, - num_kv_heads=1, - qo_offset=qo_offset_cpu, - page_size=page_size, - output_maxscore=True, - num_kv_splits=1, - causal=True, - ) - # Sparse-layer plan: kv_block_num=topk limits attention to top-k blocks. - gqa_plan = fmha_sm100.fmha_sm100_plan( - qo_lens_cpu, - kv_lens_cpu, - num_q_heads, - num_kv_heads=num_kv_heads, - qo_offset=qo_offset_cpu, - page_size=page_size, - kv_block_num=topk, - num_kv_splits=1, - causal=True, - ) - # Dense-layer plan: no kv_block_num, so it attends the full page table. - dense_plan = fmha_sm100.fmha_sm100_plan( - qo_lens_cpu, - kv_lens_cpu, - num_q_heads, - num_kv_heads=num_kv_heads, - qo_offset=qo_offset_cpu, - page_size=page_size, - num_kv_splits=1, - causal=True, - ) - - required_max_k_tiles = int(proxy_plan[3]["max_k_tiles"]) - self._ensure_msa_decode_scratch_buffers( - num_index_heads=num_index_heads, - max_batch=max_batch, - capture_graph=capture_graph, - required_max_k_tiles=required_max_k_tiles, - ) - - # Allocate the graph-safe plan owners once per metadata; later steps - # only refresh their contents below. - if self._msa_proxy_plan is None: - num_ctas = torch.cuda.get_device_properties(device).multi_processor_count - self._msa_proxy_plan = _MsaGraphSafePlan( - self, - "msa_proxy_plan", - max_batch=max_batch, - num_ctas=num_ctas, - capture_graph=capture_graph, - ) - self._msa_gqa_plan = _MsaGraphSafePlan( - self, - "msa_gqa_plan", - max_batch=max_batch, - num_ctas=num_ctas, - capture_graph=capture_graph, - ) - self._msa_dense_plan = _MsaGraphSafePlan( - self, - "msa_dense_plan", - max_batch=max_batch, - num_ctas=num_ctas, - capture_graph=capture_graph, - ) - - # refresh() stores each plan tuple on its owner, surfaced by the - # msa_decode_*_plan properties. - self._msa_proxy_plan.refresh(proxy_plan) - self._msa_gqa_plan.refresh(gqa_plan) - self._msa_dense_plan.refresh(dense_plan) - - n_valid = per_token_valid_blocks( - qo_lens_cpu, kv_lens_cpu, qo_offset_cpu, causal=True, block_size=page_size - ) - self.msa_n_valid_blocks[:batch].copy_(n_valid.to(torch.int32), non_blocking=True) - - def _build_msa_fields(self) -> None: - """Populate the MSA cache-write buffers for this step. - - The page table and per-new-token cache slots are derived via the - build_paged_kv_slot_mapping helper, then copied into the persistent - buffers. The transient builder tensors are discarded. - """ - self._msa_fields_ready = False - if not self._msa_buffers_ready: - return - request_ids = self.request_ids - qo_lens_cpu = self.msa_qo_lens_cpu - kv_lens_cpu = self.msa_kv_lens_cpu - qo_offset_cpu = self.msa_qo_offset_cpu - if request_ids is None or qo_lens_cpu is None: - return - batch_size = int(qo_lens_cpu.shape[0]) - if batch_size == 0: - return - - kv_cache_manager = self.kv_cache_manager - cache_device = _cache_device(self) - page_size = int(kv_cache_manager.tokens_per_block) - - is_prefill = int(self.num_contexts or 0) > 0 - if not is_prefill and int(qo_lens_cpu.max().item()) > 1: - raise NotImplementedError( - "MiniMax-M3 MSA attention does not support speculative decoding " - "(multiple query tokens per decode step). Disable speculative " - "decoding or use the non-MSA MiniMax-M3 backend." - ) - - # Built in prepare() (outside capture), so these transients are - # fine: forwards read only the persistent buffers filled below. - # qo_offset is the prefix length, so one build covers prefill - # (num_cached) and decode (kv_len - 1 with qo_len 1). - req_to_token, slot_ids, out_cache_loc = build_paged_kv_slot_mapping( - kv_cache_manager=kv_cache_manager, - request_ids=request_ids, - qo_lens_cpu=qo_lens_cpu, - qo_offset_cpu=qo_offset_cpu, - device=cache_device, - ) - kv_indices = build_kv_page_indices(req_to_token, slot_ids, kv_lens_cpu, page_size) - - total_new_tokens = int(out_cache_loc.shape[0]) - total_pages = int(kv_indices.shape[0]) - if total_new_tokens > self.msa_out_cache_loc.shape[0]: - raise ValueError( - f"MSA out_cache_loc buffer ({self.msa_out_cache_loc.shape[0]}) is " - f"smaller than the step's new-token count ({total_new_tokens})." - ) - if total_pages > self.msa_kv_indices.shape[0]: - raise ValueError( - f"MSA kv_indices buffer ({self.msa_kv_indices.shape[0]}) is " - f"smaller than the step's page count ({total_pages})." - ) - - self.msa_out_cache_loc[:total_new_tokens].copy_(out_cache_loc, non_blocking=True) - self.msa_kv_indices[:total_pages].copy_(kv_indices, non_blocking=True) - self._msa_fields_ready = True - - def msa_idx_k_cache(self, layer_idx: int) -> torch.Tensor: - """Paged index-K view for the indexer; HND conversion is done there.""" - return self.kv_cache_manager.get_index_k_buffer(layer_idx) - - def msa_write_idx_k(self, layer_idx: int, idx_k: torch.Tensor) -> None: - """Write the new-token index-K into the side cache at out_cache_loc.""" - cache = self.msa_idx_k_cache(layer_idx) - sparse_index_dim = int(cache.shape[-1]) - num_tokens = int(idx_k.shape[0]) - write_kv_slots( - cache, - self.msa_out_cache_loc[:num_tokens], - idx_k.reshape(num_tokens, 1, sparse_index_dim), - ) - - def msa_proxy_max_score_view( - self, num_index_heads: int, plan_max_k_tiles: int, num_tokens: int - ) -> torch.Tensor: - """Return a contiguous [num_index_heads, plan_max_k_tiles, num_tokens] view. - - fmha_sm100 ignores the passed tensor's strides and writes a contiguous - [num_index_heads, plan_max_k_tiles, total_q] block sized by the current - decode plan, so it must receive a tensor contiguous in exactly that - shape. The view is taken from the flat store's prefix starting at offset - 0, so its data_ptr is stable for CUDA graph replay. Capture builds the - decode plan at the worst-case max_k_tiles, so replays only shrink it. - """ - store = self.msa_max_score - numel = num_index_heads * plan_max_k_tiles * num_tokens - if numel > store.numel(): - raise ValueError( - f"msa_max_score backing store ({store.numel()} elements) is " - f"smaller than the proxy view needs ({numel} = {num_index_heads} " - f"heads * {plan_max_k_tiles} k-tiles * {num_tokens} tokens)." - ) - return store[:numel].view(num_index_heads, plan_max_k_tiles, num_tokens) - - -class MiniMaxM3MsaSparseAttention(TrtllmAttention): - """MSA-backed MiniMax-M3 sparse attention.""" - - Metadata = MiniMaxM3MsaSparseAttentionMetadata - - def __init__( - self, - layer_idx: int, - num_heads: int, - head_dim: int, - num_kv_heads: Optional[int] = None, - quant_config=None, - *, - sparse_params, - **kwargs, - ): - TrtllmAttention.__init__( - self, - layer_idx, - num_heads, - head_dim, - num_kv_heads=num_kv_heads, - quant_config=quant_config, - sparse_params=sparse_params, - **kwargs, - ) - self.m3_config = MiniMaxM3SparseConfig.from_sparse_params( - sparse_params, - num_q_heads=num_heads, - num_kv_heads=num_kv_heads or num_heads, - head_dim=head_dim, - ) - self.disable_index_value = bool(sparse_params.disable_index_value) - self._validate_msa_preconditions() - self.indexer = MsaIndexer(self.m3_config) - - def _validate_msa_preconditions(self) -> None: - config = self.m3_config - if not self.disable_index_value: - raise NotImplementedError( - "MSA backend requires disable_index_value=True; the proxy pass " - "consumes only the max score and has no index-V path." - ) - if config.head_dim != MSA_REQUIRED_HEAD_DIM: - raise NotImplementedError( - f"MSA backend requires head_dim={MSA_REQUIRED_HEAD_DIM}, got {config.head_dim}." - ) - if config.sparse_index_dim != MSA_REQUIRED_HEAD_DIM: - raise NotImplementedError( - f"MSA backend requires sparse_index_dim={MSA_REQUIRED_HEAD_DIM}, " - f"got {config.sparse_index_dim}." - ) - if config.topk != MSA_REQUIRED_TOPK: - raise NotImplementedError( - f"MSA backend requires topk={MSA_REQUIRED_TOPK}, got {config.topk}." - ) - - @classmethod - def support_fused_rope(cls) -> bool: - # The MiniMax-M3 model layer applies partial RoPE to the main and - # index branches explicitly. - return False - - def run_indexer( - self, - idx_q: torch.Tensor, - idx_k: torch.Tensor, - metadata, - *, - idx_sm_scale: Optional[float] = None, - ) -> torch.Tensor: - """Write the index-K cache and return the selected block indices. - - The model layer runs this before forward and threads the result through - forward_args.topk_indices. Returns [total_q, num_kv_heads, topk]. - Decode uses the prebuilt graph-safe proxy plan; prefill plans - eagerly. - """ - config = self.m3_config - idx_sm_scale = idx_sm_scale if idx_sm_scale is not None else config.sparse_index_dim**-0.5 - num_tokens = int(idx_q.shape[0]) - idx_q_view = idx_q.view(num_tokens, config.num_index_heads, config.sparse_index_dim) - idx_k_view = idx_k.view(num_tokens, 1, config.sparse_index_dim) - - metadata.msa_write_idx_k(self.layer_idx, idx_k_view) - idx_k_cache = metadata.msa_idx_k_cache(self.layer_idx) - - # One selection path: decode passes the prebuilt graph-safe proxy - # plan plus the proxy scratch shaped to the live query count; prefill - # leaves them None and the proxy plan is built inline. - proxy_plan = metadata.msa_decode_proxy_plan - if proxy_plan is not None: - # proxy_plan is (has_mixed, split, batch, decode_dict, prefill); - # decode_dict carries max_k_tiles for the contiguous score view. - plan_max_k_tiles = int(proxy_plan[3]["max_k_tiles"]) - max_score = metadata.msa_proxy_max_score_view( - config.num_index_heads, plan_max_k_tiles, num_tokens - ) - n_valid_blocks = metadata.msa_n_valid_blocks[:num_tokens] - else: - max_score = None - n_valid_blocks = None - return self.indexer.select_blocks( - idx_q_view, - idx_k_cache, - idx_sm_scale=idx_sm_scale, - kv_indices=metadata.msa_kv_indices, - qo_lens_cpu=metadata.msa_qo_lens_cpu, - kv_lens_cpu=metadata.msa_kv_lens_cpu, - qo_offset_cpu=metadata.msa_qo_offset_cpu, - proxy_plan=proxy_plan, - max_score=max_score, - n_valid_blocks=n_valid_blocks, - ) - - def sparse_attn_predict( - self, - q: torch.Tensor, - k: Optional[torch.Tensor], - metadata, - forward_args: "AttentionForwardArgs", - ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: - # The model layer runs run_indexer and passes the selected block - # indices through forward_args.topk_indices. Publish them as the - # sparse attention indices MsaSparseGqaFmha reads. - return forward_args.topk_indices, None - - def sparse_kv_predict( - self, - q: torch.Tensor, - k: Optional[torch.Tensor], - metadata, - forward_args: "AttentionForwardArgs", - ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: - return None, None - - -__all__ = [ - "MiniMaxM3MsaSparseAttention", - "MiniMaxM3MsaSparseAttentionMetadata", -] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_indexer.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_indexer.py deleted file mode 100644 index a6a2e10e0dc1..000000000000 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_indexer.py +++ /dev/null @@ -1,201 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""MiniMax-M3 MSA sparse-attention indexer. - -Mirrors the DSA indexer pattern: a submodule owned by the sparse backend -that runs the predictor pass and returns the per-query selected KV block -indices the main attention consumes. It calls fmha_sm100 directly in -output_maxscore mode, reduces the per-index-head max score to KV-head -granularity, and selects the top-k blocks per query. - -Results are [total_q, num_kv_heads, topk] int32, ascending with -1 padding. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -import torch - -from .msa_utils import ( - MSA_REQUIRED_TOPK, - cache_view_to_msa_paged, - per_token_valid_blocks, - require_msa_module, - select_blocks_from_maxscore, -) - -if TYPE_CHECKING: - from .common import MiniMaxM3SparseConfig - - -def _proxy_max_score( - idx_q: torch.Tensor, - idx_k_paged: torch.Tensor, - *, - qo_lens_cpu: torch.Tensor, - kv_lens_cpu: torch.Tensor, - qo_offset_cpu: Optional[torch.Tensor], - kv_indices: torch.Tensor, - sm_scale: float, - causal: bool, -) -> torch.Tensor: - """Run the fmha_sm100 MQA proxy pass and return the per-block max score. - - Follows MSA's two-call pattern: fmha_sm100_plan builds the plan with - output_maxscore and num_kv_heads 1, then fmha_sm100 runs with output_o - disabled so only the per-block max score is produced. Returns - [num_index_heads, max_k_tiles, total_q] float32. - """ - fmha_sm100 = require_msa_module() - - if idx_q.dim() != 3: - raise ValueError( - "MsaIndexer expects idx_q [total_q, num_index_heads, head_dim]; " - f"got {tuple(idx_q.shape)}." - ) - if idx_k_paged.dim() != 4 or idx_k_paged.shape[1] != 1: - raise ValueError( - "MsaIndexer expects MQA paged index-K [num_pages, 1, page_size, head_dim]; " - f"got {tuple(idx_k_paged.shape)}." - ) - - page_size = int(idx_k_paged.shape[2]) - proxy_plan = fmha_sm100.fmha_sm100_plan( - qo_lens_cpu, - kv_lens_cpu, - idx_q.shape[1], - num_kv_heads=1, - qo_offset=qo_offset_cpu, - page_size=page_size, - output_maxscore=True, - causal=causal, - num_kv_splits=1, - ) - _, max_score = fmha_sm100.fmha_sm100( - idx_q, - idx_k_paged, - idx_k_paged, - proxy_plan, - kv_indices=kv_indices, - output_o=False, - output_maxscore=True, - sm_scale=sm_scale, - ) - return max_score - - -def _group_max_reduce( - max_score: torch.Tensor, - config: "MiniMaxM3SparseConfig", -) -> torch.Tensor: - """Reduce per-index-head max score to per-KV-head granularity by amax. - - Index heads are assumed to be grouped contiguously per KV head, so head h - maps to KV group h // group. - """ - group, rem = divmod(config.num_index_heads, config.num_kv_heads) - if rem != 0: - raise ValueError( - "num_index_heads must be divisible by num_kv_heads for group max " - f"reduce; got num_index_heads={config.num_index_heads}, " - f"num_kv_heads={config.num_kv_heads}." - ) - if group > 1: - return max_score.view( - config.num_kv_heads, group, max_score.shape[1], max_score.shape[2] - ).amax(dim=1) - return max_score - - -class MsaIndexer: - """Predictor submodule: proxy MQA scoring and top-k block selection. - - Owned by the MSA attention layer. Stateless in eager mode: it reads the - per-forward page table and lengths from the attention metadata and calls - the kernel directly. - """ - - def __init__(self, config: "MiniMaxM3SparseConfig"): - self.config = config - - def select_blocks( - self, - idx_q: torch.Tensor, - idx_k_cache: torch.Tensor, - *, - idx_sm_scale: float, - kv_indices: torch.Tensor, - qo_lens_cpu: Optional[torch.Tensor] = None, - kv_lens_cpu: Optional[torch.Tensor] = None, - qo_offset_cpu: Optional[torch.Tensor] = None, - proxy_plan: Optional[tuple] = None, - max_score: Optional[torch.Tensor] = None, - n_valid_blocks: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Return [total_q, num_kv_heads, topk] selected block indices. - - Plan/run split, mirroring the sparse GQA. When `proxy_plan` is None - (prefill and focused tests) the proxy plan is built inline and the - per-query valid-block count is derived here; when provided (CUDA-graph - decode) the proxy runs from the prebuilt plan into the preallocated - `max_score` buffer with a precomputed `n_valid_blocks`, so there is - no host sync inside the captured region. The same top-k selection serves - both, and generation is the one-query-token-per-request special case. - """ - config = self.config - idx_k_paged = cache_view_to_msa_paged(idx_k_cache) - - if proxy_plan is None: - max_score = _proxy_max_score( - idx_q, - idx_k_paged, - qo_lens_cpu=qo_lens_cpu, - kv_lens_cpu=kv_lens_cpu, - qo_offset_cpu=qo_offset_cpu, - kv_indices=kv_indices, - sm_scale=idx_sm_scale, - causal=True, - ) - else: - fmha_sm100 = require_msa_module() - _, max_score = fmha_sm100.fmha_sm100( - idx_q, - idx_k_paged, - idx_k_paged, - proxy_plan, - kv_indices=kv_indices, - output_o=False, - output_maxscore=True, - max_score=max_score, - sm_scale=idx_sm_scale, - ) - max_score_kv = _group_max_reduce(max_score, config) - - if n_valid_blocks is None: - n_valid_blocks = per_token_valid_blocks( - qo_lens_cpu, - kv_lens_cpu, - qo_offset_cpu, - causal=True, - block_size=int(idx_k_paged.shape[2]), - ) - # The empty-selection guard uses a host sync, so it only runs on the - # eager path; a decode batch always has valid blocks. - if n_valid_blocks.numel() == 0 or int(n_valid_blocks.max().item()) <= 0: - return torch.full( - (idx_q.shape[0], config.num_kv_heads, MSA_REQUIRED_TOPK), - -1, - dtype=torch.int32, - device=idx_q.device, - ) - return select_blocks_from_maxscore( - max_score_kv, - topk=MSA_REQUIRED_TOPK, - n_valid_blocks=n_valid_blocks, - init_blocks=config.init_blocks, - local_blocks=config.local_blocks, - ) - - -__all__ = ["MsaIndexer"] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_utils.py b/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_utils.py deleted file mode 100644 index 83380bc730c0..000000000000 --- a/tensorrt_llm/_torch/attention_backend/sparse/minimax_m3/msa_utils.py +++ /dev/null @@ -1,257 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""MSA (fmha_sm100) specific helpers for the MiniMax-M3 sparse backend.""" - -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path -from typing import Optional, Tuple - -import torch - -from .common import _INIT_SCORE, _LOCAL_SCORE, write_kv_slots - -# fmha_sm100 ships only head_dim 128 variants and the MiniMax-M3 checkpoint -# selects topk 16. Callers enforce these early so a misconfiguration fails -# with a clear message rather than a cryptic shape error inside the kernel. -MSA_REQUIRED_TOPK = 16 -MSA_REQUIRED_HEAD_DIM = 128 - -# Path of the fmha_sm100 package inside the MSA git submodule relative to the -# repository root (see 3rdparty/MSA/LICENSE and 3rdparty/MSA/NOTICE). -_MSA_PYTHON_RELPATH = Path("3rdparty") / "MSA" / "python" - - -def _find_msa_python_dir() -> Optional[Path]: - """Locate the fmha_sm100 package dir by walking up from this file. - - Returns None in installed layouts where the 3rdparty submodule is not - shipped. Walking up avoids hardcoding this module's depth below the - repository root. - """ - for parent in Path(__file__).resolve().parents: - candidate = parent / _MSA_PYTHON_RELPATH - if candidate.is_dir(): - return candidate - return None - - -def _ensure_msa_on_path() -> None: - """Prepend the MSA python package directory to sys.path if present.""" - msa_python = _find_msa_python_dir() - if msa_python is not None and str(msa_python) not in sys.path: - sys.path.insert(0, str(msa_python)) - - -def msa_package_available() -> bool: - """True if fmha_sm100 can be imported (submodule checkout or installed).""" - _ensure_msa_on_path() - return importlib.util.find_spec("fmha_sm100") is not None - - -def require_msa_module(): - """Import fmha_sm100 from the MSA submodule or raise a clear error. - - The import is deferred to first kernel use so the MSA backend can be - advertised in the config schema on systems where the kernels cannot load. - The 3rdparty/MSA/python directory is added to sys.path first, so a source - checkout with the submodule initialized resolves without a separate install. - A missing package is a hard error, never a silent fallback to another backend. - """ - _ensure_msa_on_path() - try: - import fmha_sm100 - except ImportError as exc: - raise RuntimeError( - "MiniMax-M3 MSA attention requires the fmha_sm100 kernels from the " - "MSA git submodule at 3rdparty/MSA. Initialize it with " - "'git submodule update --init --recursive', or install fmha_sm100." - ) from exc - return fmha_sm100 - - -def cache_view_to_msa_paged(cache_view: torch.Tensor) -> torch.Tensor: - """Convert a KV cache view to the fmha_sm100 HND paged layout. - - A 4-D paged view [num_pages, page_size, num_heads, head_dim] permutes to - [num_pages, num_heads, page_size, head_dim]. A 3-D flat-slot cache - [num_slots, num_heads, head_dim] is treated as one virtual page, giving - [1, num_heads, num_slots, head_dim]. - """ - if cache_view.dim() == 4: - return cache_view.permute(0, 2, 1, 3).contiguous() - if cache_view.dim() == 3: - return cache_view.permute(1, 0, 2).unsqueeze(0).contiguous() - raise ValueError( - f"Unsupported cache view rank {cache_view.dim()} for MSA paged conversion; " - "expected 3 (flat-slot) or 4 (paged)." - ) - - -def msa_paged_kv(kv_cache_manager, layer_idx: int) -> Tuple[torch.Tensor, torch.Tensor]: - """Return per-layer paged K and V in fmha_sm100 HND layout, zero-copy. - - The cache is stored head-major (see `write_msa_main_kv`), so the "HND" - buffer view is already the [num_slots, num_kv_heads, page_size, head_dim] - layout fmha_sm100 expects. The kernel reads the page and head strides at - runtime and needs only each page's [page_size, head_dim] block to be - contiguous, which this view satisfies, so no copy is required. - """ - buffers = kv_cache_manager.get_buffers(layer_idx, kv_layout="HND") - return buffers[:, 0], buffers[:, 1] - - -def write_msa_main_kv( - kv_cache_manager, - layer_idx: int, - out_cache_loc: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, -) -> None: - """Write new-token K and V into the paged main cache at out_cache_loc. - - fmha_sm100 reads the paged cache directly, so the new-token K and V must be - resident before the sparse GQA runs. The write uses the head-major HND view - so `msa_paged_kv` can return a zero-copy view. - """ - buffers = kv_cache_manager.get_buffers(layer_idx, kv_layout="HND") - k_view, v_view = buffers[:, 0], buffers[:, 1] - num_kv_heads = int(k_view.shape[1]) - head_dim = int(k_view.shape[3]) - num_tokens = int(k.shape[0]) - write_kv_slots( - k_view, out_cache_loc, k.reshape(num_tokens, num_kv_heads, head_dim), layout="HND" - ) - write_kv_slots( - v_view, out_cache_loc, v.reshape(num_tokens, num_kv_heads, head_dim), layout="HND" - ) - - -def build_kv_page_indices( - req_to_token: torch.Tensor, - slot_ids: torch.Tensor, - kv_lens_cpu: torch.Tensor, - page_size: int, -) -> torch.Tensor: - """Build the flattened per-request page table fmha_sm100 consumes. - - Returns int32 global page ids concatenated per request. A request's - pages come from the first slot of each page in its req_to_token row. - Page ids are global and non-contiguous in production, so they are not - clamped to a per-request bound. - """ - device = req_to_token.device - req_rows = req_to_token.index_select(0, slot_ids.to(torch.long)).to(torch.long) - batch = int(req_rows.shape[0]) - kv_lens_list = kv_lens_cpu.to(torch.long).tolist() - - page_lists = [] - for b in range(batch): - kv_len = int(kv_lens_list[b]) - if kv_len <= 0: - continue - num_pages = (kv_len + page_size - 1) // page_size - page_starts = torch.arange(num_pages, device=device, dtype=torch.long) * page_size - page_ids = req_rows[b].gather(0, page_starts) // page_size - page_lists.append(page_ids.to(torch.int32)) - - if page_lists: - return torch.cat(page_lists, dim=0) - return torch.empty(0, dtype=torch.int32, device=device) - - -def per_token_valid_blocks( - qo_lens_cpu: torch.Tensor, - kv_lens_cpu: torch.Tensor, - qo_offset_cpu: Optional[torch.Tensor], - *, - causal: bool, - block_size: int, -) -> torch.Tensor: - """Return the per-query number of valid KV blocks, on CPU. - - Expands per-request lengths and offsets to a per-token vector so block - selection can honour each query token's own causal extent. - """ - qo = qo_lens_cpu.to(torch.long) - kv = kv_lens_cpu.to(torch.long) - batch = int(qo.shape[0]) - total = int(qo.sum().item()) - if total == 0: - return torch.zeros(0, dtype=torch.long) - batch_row = torch.repeat_interleave(torch.arange(batch, dtype=torch.long), qo) - starts = torch.zeros(batch, dtype=torch.long) - if batch > 1: - starts[1:] = torch.cumsum(qo, 0)[:-1] - intra = torch.arange(total, dtype=torch.long) - starts[batch_row] - kv_per = kv[batch_row] - if causal: - if qo_offset_cpu is not None: - off = qo_offset_cpu.to(torch.long)[batch_row] - else: - off = (kv - qo)[batch_row] - eff = torch.minimum(off + intra + 1, kv_per) - else: - eff = kv_per - return (eff + block_size - 1) // block_size - - -def select_blocks_from_maxscore( - max_score_kv: torch.Tensor, - *, - topk: int, - n_valid_blocks: torch.Tensor, - init_blocks: int, - local_blocks: int, -) -> torch.Tensor: - """Select per-query top-k blocks from per-KV-head block scores. - - Applies init and local forced blocks and per-query valid-block masking - on the amax-reduced scores [num_kv_heads, n_blocks, total_q]. Returns - [total_q, num_kv_heads, topk] int32 ascending block ids with -1 tail - padding. - """ - num_kv_heads, n_blocks, total_q = max_score_kv.shape - device = max_score_kv.device - scores = max_score_kv.permute(2, 0, 1).to(torch.float32).clone() - block_ids = torch.arange(n_blocks, device=device, dtype=torch.long) - nvb = n_valid_blocks.to(device=device, dtype=torch.long) - - if init_blocks > 0: - init_mask = block_ids.view(1, 1, -1) < init_blocks - scores = torch.where(init_mask, torch.full_like(scores, _INIT_SCORE), scores) - if local_blocks > 0: - local_start = (nvb - local_blocks).clamp_min(0) - local_mask = (block_ids.view(1, -1) >= local_start.view(-1, 1)) & ( - block_ids.view(1, -1) < nvb.view(-1, 1) - ) - scores = torch.where(local_mask.unsqueeze(1), torch.full_like(scores, _LOCAL_SCORE), scores) - block_valid = block_ids.view(1, -1) < nvb.view(-1, 1) - scores = scores.masked_fill(~block_valid.unsqueeze(1), float("-inf")) - - k = min(topk, n_blocks) - vals, idx = scores.topk(k=k, dim=-1) - idx = torch.where(vals != float("-inf"), idx, torch.full_like(idx, -1)) - sort_key = torch.where(idx < 0, torch.full_like(idx, n_blocks), idx) - sort_key, _ = torch.sort(sort_key, dim=-1) - idx = torch.where(sort_key >= n_blocks, torch.full_like(sort_key, -1), sort_key) - if k < topk: - pad = torch.full((total_q, num_kv_heads, topk - k), -1, dtype=idx.dtype, device=device) - idx = torch.cat([idx, pad], dim=-1) - return idx.to(torch.int32) - - -__all__ = [ - "MSA_REQUIRED_HEAD_DIM", - "MSA_REQUIRED_TOPK", - "build_kv_page_indices", - "cache_view_to_msa_paged", - "msa_package_available", - "msa_paged_kv", - "per_token_valid_blocks", - "require_msa_module", - "select_blocks_from_maxscore", - "write_msa_main_kv", -] diff --git a/tensorrt_llm/_torch/attention_backend/sparse/utils.py b/tensorrt_llm/_torch/attention_backend/sparse/utils.py index 1762268bf242..f438d998939f 100644 --- a/tensorrt_llm/_torch/attention_backend/sparse/utils.py +++ b/tensorrt_llm/_torch/attention_backend/sparse/utils.py @@ -42,30 +42,14 @@ def get_sparse_attn_kv_cache_manager( ) -def _resolve_minimax_m3_backend_cls( - sparse_params: "SparseParams") -> Type["AttentionBackend"]: - """Select the MiniMax-M3 sparse backend from the lowered params. - - The Triton reference is the default. When implementation is 'msa' the MSA - (fmha_sm100) backend is used instead, gated on SM100 availability so an - unsupported system fails early rather than at kernel launch. - """ - from .minimax_m3 import MiniMaxM3SparseRuntimeBackend - if getattr(sparse_params, "implementation", "triton") == "msa": - from .minimax_m3 import MiniMaxM3MsaSparseAttention - from .minimax_m3.msa_availability import ensure_msa_available - ensure_msa_available() - return MiniMaxM3MsaSparseAttention - return MiniMaxM3SparseRuntimeBackend - - def get_vanilla_sparse_attn_attention_backend( sparse_params: "SparseParams") -> Type["AttentionBackend"]: + from .minimax_m3 import get_minimax_m3_attention_backend_cls from .rocket import RocketVanillaAttention if sparse_params.algorithm == "rocket": return RocketVanillaAttention elif sparse_params.algorithm == "minimax_m3": - return _resolve_minimax_m3_backend_cls(sparse_params) + return get_minimax_m3_attention_backend_cls() else: raise ValueError( f"Unsupported sparse attention algorithm in vanilla attention backend: {sparse_params.algorithm}" @@ -78,6 +62,7 @@ def get_trtllm_sparse_attn_attention_backend( from .deepseek_v4 import DeepseekV4TrtllmAttention from .dsa import DSATrtllmAttention + from .minimax_m3 import get_minimax_m3_attention_backend_cls from .rocket import RocketTrtllmAttention if sparse_params.algorithm == "rocket": return RocketTrtllmAttention @@ -93,7 +78,7 @@ def get_trtllm_sparse_attn_attention_backend( # `create_attention(...)` dispatch in `Attention.__init__` # returns an instantiable AttentionBackend under the trtllm # attention backend slot. - return _resolve_minimax_m3_backend_cls(sparse_params) + return get_minimax_m3_attention_backend_cls() else: raise ValueError( f"Unsupported sparse attention algorithm in trtllm attention backend: {sparse_params.algorithm}" @@ -102,8 +87,9 @@ def get_trtllm_sparse_attn_attention_backend( def get_flashinfer_sparse_attn_attention_backend( sparse_params: "SparseParams") -> Type["AttentionBackend"]: + from .minimax_m3 import get_minimax_m3_attention_backend_cls if sparse_params.algorithm == "minimax_m3": - return _resolve_minimax_m3_backend_cls(sparse_params) + return get_minimax_m3_attention_backend_cls() raise ValueError( f"Unsupported sparse attention algorithm in flashinfer attention backend: {sparse_params.algorithm}" ) diff --git a/tensorrt_llm/_torch/attention_backend/trtllm.py b/tensorrt_llm/_torch/attention_backend/trtllm.py index a3f4e89d8d85..931830253cba 100644 --- a/tensorrt_llm/_torch/attention_backend/trtllm.py +++ b/tensorrt_llm/_torch/attention_backend/trtllm.py @@ -31,7 +31,6 @@ from tensorrt_llm._utils import get_sm_version, maybe_pin_memory, prefer_pinned from tensorrt_llm.bindings.internal import thop from tensorrt_llm.functional import AttentionMaskType -from tensorrt_llm.math_utils import ceil_div from tensorrt_llm.models.modeling_utils import QuantConfig from ..utils import (compute_swizzled_sf_shape, get_global_attrs, @@ -117,7 +116,6 @@ def effective_beam_width(self) -> int: is_spec_dec_tree: bool = False # if spec-dec tree wouldn't be changed at all, the mask won't be computed every step. is_spec_dec_dynamic_tree: bool = False - force_prepare_spec_dec_tree_mask: bool = False # parameters required for spec-dec mode max_total_draft_tokens: Optional[int] = None @@ -588,47 +586,24 @@ def prepare(self) -> None: # kv block offsets assert self.request_ids is not None if self.kv_cache_manager is not None: - max_kv_len = int(self.kv_lens[:self.num_seqs].max()) - assert max_kv_len <= self.kv_cache_manager.max_seq_len, ( - f"The max KV cache length of input sequences ({max_kv_len}) " + self.kv_cache_manager.copy_batch_block_offsets( + self.kv_cache_block_offsets, self.request_ids, self.beam_width, + self.num_contexts, self.num_seqs) + + error_message = ( + f"The max KV cache length of input sequences ({self.kv_lens[:self.num_seqs].max()}) " f"exceeds the KV cache manager's maximum supported length " f"({self.kv_cache_manager.max_seq_len}).") - # On the non-speculative path the host kv_lens snapshot bounds - # every block-table access, so the staged/H2D width can be capped - # at the batch's maximum instead of max_seq_len's worth of - # columns. Speculative decoding must stage the full width: - # draft/tree sub-steps and the overlap scheduler advance - # kv_lens_cuda on device past the host snapshot, and their - # kernels dereference block columns a host-derived cap would - # leave unstaged (uninitialized in this buffer). - spec_active = (self.draft_kv_cache_manager is not None - or self.is_spec_decoding_enabled - or bool(self.kv_cache_params.num_extra_kv_tokens) or - (self.runtime_features is not None and - self.runtime_features.has_speculative_draft_tokens)) - max_blocks = None - if not spec_active and self.kv_cache_manager.tokens_per_block: - max_blocks = ceil_div(max_kv_len, - self.kv_cache_manager.tokens_per_block) - self.kv_cache_manager.copy_batch_block_offsets( - self.kv_cache_block_offsets, - self.request_ids, - self.beam_width, - self.num_contexts, - self.num_seqs, - max_blocks=max_blocks) + assert self.kv_lens[:self.num_seqs].max( + ) <= self.kv_cache_manager.max_seq_len, error_message # Also prepare draft KV cache block offsets if draft_kv_cache_manager exists if self.draft_kv_cache_manager is not None: # Use the wrapper method which works for both V1 and V2 self.draft_kv_cache_manager.copy_batch_block_offsets( - self.draft_kv_cache_block_offsets, - self.request_ids, - self.beam_width, - self.num_contexts, - self.num_seqs, - max_blocks=max_blocks) + self.draft_kv_cache_block_offsets, self.request_ids, + self.beam_width, self.num_contexts, self.num_seqs) # Don't pass self.kv_lens as kv_lens here because it includes extra # tokens. Use the actual KV length (without extra tokens) for diff --git a/tensorrt_llm/_torch/auto_deploy/__init__.py b/tensorrt_llm/_torch/auto_deploy/__init__.py index a8a22a945c05..0a2160c9356a 100644 --- a/tensorrt_llm/_torch/auto_deploy/__init__.py +++ b/tensorrt_llm/_torch/auto_deploy/__init__.py @@ -13,33 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The generated ``llmc`` symlink bootstraps the canonical package. Paragraf's -# redirect finder then ensures all legacy submodules share canonical identity. +# This source tree is copied verbatim to the standalone ``llmc`` package. +# Install the compatibility redirect before registration imports when running +# from that generated package; bundled AutoDeploy must remain unchanged. if __name__ == "llmc": - import paragraf as _paragraf # noqa: F401 -else: - # This source tree is copied verbatim to the standalone ``paragraf`` package. - # Install compatibility redirects before registration imports when running - # from that generated package; bundled AutoDeploy must remain unchanged. - if __name__ == "paragraf": - from . import trtllm_compat as _trtllm_compat + from . import trtllm_compat as _trtllm_compat - _trtllm_compat.install_autodeploy_redirect_from_env() - _trtllm_compat.install_llmc_alias() + _trtllm_compat.install_autodeploy_redirect_from_env() - # import submodules that require registration process - from . import compile, custom_ops, export, models # noqa: F401 - from ._compat import TRTLLM_AVAILABLE +# import submodules that require registration process +from . import compile, custom_ops, export, models # noqa: F401 +from ._compat import TRTLLM_AVAILABLE - if TRTLLM_AVAILABLE: - from . import shim # noqa: F401 +if TRTLLM_AVAILABLE: + from . import shim # noqa: F401 - # import AutoDeploy LLM and LlmArgs (require TRT-LLM base classes) - from .llm import * - from .llm_args import * + # import AutoDeploy LLM and LlmArgs (require TRT-LLM base classes) + from .llm import * + from .llm_args import * - try: - # This will overwrite the AutoModelForCausalLM.from_config to support modelopt quantization - import modelopt - except ImportError: - pass +try: + # This will overwrite the AutoModelForCausalLM.from_config to support modelopt quantization + import modelopt +except ImportError: + pass diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py index b44c5f040982..ba4a649f610c 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_qwen3_5_moe.py @@ -635,31 +635,25 @@ def __init__(self, config: Qwen3_5MoeTextConfig, intermediate_size: int): self.act_fn = ACT2FN[config.hidden_act] def forward(self, x: torch.Tensor) -> torch.Tensor: - # Tagged ``layer_type="mlp"`` so the shared expert is TP-sharded like the - # routed experts (colwise gate/up + rowwise down). Its rowwise down_proj - # therefore emits a per-rank partial that is summed with the routed - # partial and lifted to full by the single merge-point all_reduce in - # ``Qwen3_5MoeSparseMoeBlock.forward``. + # Intentionally left untagged: with no ``layer_type`` it defaults to "unknown", + # which the ``shard_layers`` inclusion whitelist excludes -> kept replicated. gate = torch.ops.auto_deploy.torch_linear_simple( x, self.gate_proj.weight, self.gate_proj.bias, tp_mode="colwise", - layer_type="mlp", ) up = torch.ops.auto_deploy.torch_linear_simple( x, self.up_proj.weight, self.up_proj.bias, tp_mode="colwise", - layer_type="mlp", ) return torch.ops.auto_deploy.torch_linear_simple( self.act_fn(gate) * up, self.down_proj.weight, self.down_proj.bias, tp_mode="rowwise", - layer_type="mlp", ) diff --git a/tensorrt_llm/_torch/auto_deploy/shim/demollm.py b/tensorrt_llm/_torch/auto_deploy/shim/demollm.py index 104ca0c7b266..9f12f444b880 100644 --- a/tensorrt_llm/_torch/auto_deploy/shim/demollm.py +++ b/tensorrt_llm/_torch/auto_deploy/shim/demollm.py @@ -24,7 +24,7 @@ from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import ( greedy_search_sampling_batch, - top_k_top_p_sampling_batch, + top_k_sampling_batch, ) from tensorrt_llm.executor import GenerationExecutor from tensorrt_llm.executor.request import GenerationRequest @@ -312,7 +312,7 @@ def _sample( logits_shape = logits.shape logits = logits.view(-1, logits_shape[-1]) # sampling_batch expects 2D logits if isinstance(sampling_params.top_k, int) and sampling_params.top_k > 1: - idx_next, probs = top_k_top_p_sampling_batch( + idx_next, probs = top_k_sampling_batch( logits, top_k=sampling_params.top_k, temperature=1.0 ) else: diff --git a/tensorrt_llm/_torch/auto_deploy/trtllm_compat.py b/tensorrt_llm/_torch/auto_deploy/trtllm_compat.py index 310d5e0ada92..b47eaa414d17 100644 --- a/tensorrt_llm/_torch/auto_deploy/trtllm_compat.py +++ b/tensorrt_llm/_torch/auto_deploy/trtllm_compat.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Compatibility redirects for the standalone Paragraf package.""" +"""Compatibility redirect from TensorRT-LLM's bundled AutoDeploy namespace to LLMC.""" from __future__ import annotations @@ -26,29 +26,24 @@ from collections.abc import Sequence from types import ModuleType -REDIRECT_ENV_VAR = "TRTLLM_REDIRECT_AD_TO_PARAGRAF" -LEGACY_REDIRECT_ENV_VAR = "TRTLLM_REDIRECT_AD_TO_LLMC" +REDIRECT_ENV_VAR = "TRTLLM_REDIRECT_AD_TO_LLMC" __all__ = [ - "LEGACY_REDIRECT_ENV_VAR", "REDIRECT_ENV_VAR", "install_autodeploy_redirect", "install_autodeploy_redirect_from_env", - "install_llmc_alias", ] _LEGACY_PACKAGE = "tensorrt_llm._torch.auto_deploy" -_TARGET_PACKAGE = "paragraf" -_LEGACY_STANDALONE_PACKAGE = "llmc" +_TARGET_PACKAGE = "llmc" _FALSE_VALUES = frozenset({"", "0", "false", "no", "off"}) _TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) -_AUTODEPLOY_FINDER_MARKER = "_is_paragraf_autodeploy_redirect" -_LLMC_FINDER_MARKER = "_is_paragraf_llmc_alias" +_FINDER_MARKER = "_is_llmc_autodeploy_redirect" _INSTALL_LOCK = threading.Lock() class _RedirectLoader(importlib.abc.Loader): - """Load an alias by returning its canonical Paragraf module object.""" + """Load an alias by returning its canonical LLMC module object.""" def __init__(self, target_name: str) -> None: self._target_name = target_name @@ -77,18 +72,15 @@ def exec_module(self, module: ModuleType) -> None: raise RuntimeError(f"Redirect loader for {self._target_name!r} was not initialized") # Import machinery temporarily applies the alias spec to the canonical - # module returned by create_module(). Restore its Paragraf identity so + # module returned by create_module(). Restore its LLMC identity so # introspection and pickling continue to use canonical module names. module.__dict__.update(self._canonical_attributes) -class _PrefixRedirectFinder(importlib.abc.MetaPathFinder): - """Map one import prefix to another while preserving canonical module identity.""" +class _AutoDeployRedirectFinder(importlib.abc.MetaPathFinder): + """Map the legacy AutoDeploy package prefix to the canonical LLMC prefix.""" - def __init__(self, source_package: str, target_package: str, marker: str) -> None: - self._source_package = source_package - self._target_package = target_package - setattr(self, marker, True) + _is_llmc_autodeploy_redirect = True def find_spec( self, @@ -97,14 +89,14 @@ def find_spec( target: ModuleType | None = None, ) -> importlib.machinery.ModuleSpec | None: del path, target - if fullname != self._source_package and not fullname.startswith(self._source_package + "."): + if fullname != _LEGACY_PACKAGE and not fullname.startswith(_LEGACY_PACKAGE + "."): return None - target_name = self._target_package + fullname[len(self._source_package) :] + target_name = _TARGET_PACKAGE + fullname[len(_LEGACY_PACKAGE) :] target_spec = importlib.util.find_spec(target_name) if target_spec is None: raise ModuleNotFoundError( - f"Cannot redirect {fullname!r}: target module {target_name!r} does not exist", + f"Cannot redirect {fullname!r}: LLMC module {target_name!r} does not exist", name=target_name, ) @@ -117,28 +109,20 @@ def find_spec( def _redirect_is_enabled() -> bool: - if REDIRECT_ENV_VAR in os.environ: - env_var = REDIRECT_ENV_VAR - elif LEGACY_REDIRECT_ENV_VAR in os.environ: - env_var = LEGACY_REDIRECT_ENV_VAR - else: + value = os.environ.get(REDIRECT_ENV_VAR) + if value is None: return False - value = os.environ[env_var] normalized = value.strip().lower() if normalized in _FALSE_VALUES: return False if normalized in _TRUE_VALUES: return True - raise ValueError(f"{env_var} must be a boolean value, got {value!r}") - - -def _finder_is_installed(marker: str) -> bool: - return any(getattr(finder, marker, False) for finder in sys.meta_path) + raise ValueError(f"{REDIRECT_ENV_VAR} must be a boolean value, got {value!r}") def install_autodeploy_redirect() -> None: - """Redirect legacy TensorRT-LLM AutoDeploy imports to canonical Paragraf modules. + """Redirect legacy TensorRT-LLM AutoDeploy imports to canonical LLMC modules. The redirect must be installed before any module in the legacy namespace is imported. Calling this function more than once is safe. @@ -147,8 +131,9 @@ def install_autodeploy_redirect() -> None: RuntimeError: If bundled AutoDeploy modules were imported before the redirect. """ with _INSTALL_LOCK: - if _finder_is_installed(_AUTODEPLOY_FINDER_MARKER): - return + for finder in sys.meta_path: + if getattr(finder, _FINDER_MARKER, False): + return loaded_legacy_modules = sorted( name @@ -161,45 +146,10 @@ def install_autodeploy_redirect() -> None: + ", ".join(loaded_legacy_modules) ) - sys.meta_path.insert( - 0, - _PrefixRedirectFinder( - _LEGACY_PACKAGE, - _TARGET_PACKAGE, - _AUTODEPLOY_FINDER_MARKER, - ), - ) - - -def install_llmc_alias() -> None: - """Make legacy ``llmc`` imports return canonical ``paragraf`` modules.""" - with _INSTALL_LOCK: - canonical_package = importlib.import_module(_TARGET_PACKAGE) - if _finder_is_installed(_LLMC_FINDER_MARKER): - sys.modules[_LEGACY_STANDALONE_PACKAGE] = canonical_package - return - - loaded_legacy_submodules = sorted( - name for name in sys.modules if name.startswith(_LEGACY_STANDALONE_PACKAGE + ".") - ) - if loaded_legacy_submodules: - raise RuntimeError( - "Cannot alias llmc to Paragraf after legacy submodules were loaded: " - + ", ".join(loaded_legacy_submodules) - ) - - sys.meta_path.insert( - 0, - _PrefixRedirectFinder( - _LEGACY_STANDALONE_PACKAGE, - _TARGET_PACKAGE, - _LLMC_FINDER_MARKER, - ), - ) - sys.modules[_LEGACY_STANDALONE_PACKAGE] = canonical_package + sys.meta_path.insert(0, _AutoDeployRedirectFinder()) def install_autodeploy_redirect_from_env() -> None: - """Install the redirect when either supported environment variable enables it.""" + """Install the AutoDeploy redirect when ``TRTLLM_REDIRECT_AD_TO_LLMC`` is enabled.""" if _redirect_is_enabled(): install_autodeploy_redirect() diff --git a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py index cace2f37e603..17b4012a83e4 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py @@ -810,7 +810,7 @@ def all_gather_ops() -> frozenset: selection) flow through as op arguments, not as separate op identities. The TRT-LLM-backed ops are silently skipped if their custom_ops module - failed to register (e.g. in the standalone ``paragraf`` package, where + failed to register (e.g. in the standalone ``llmc`` package, where ``trtllm_dist`` is not importable). """ return frozenset( @@ -828,7 +828,7 @@ def all_reduce_ops() -> frozenset: """All AllReduce custom op packets recognized by AutoDeploy. The TRT-LLM-backed op is silently skipped if its custom_ops module - failed to register (e.g. in the standalone ``paragraf`` package, where + failed to register (e.g. in the standalone ``llmc`` package, where ``trtllm_dist`` is not importable). """ return frozenset( diff --git a/tensorrt_llm/_torch/compilation/remove_copy_pass.py b/tensorrt_llm/_torch/compilation/remove_copy_pass.py index 0c8d3ede8b17..8e5eb7a81148 100644 --- a/tensorrt_llm/_torch/compilation/remove_copy_pass.py +++ b/tensorrt_llm/_torch/compilation/remove_copy_pass.py @@ -1,17 +1,3 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from operator import getitem import torch @@ -38,23 +24,12 @@ def remove_functionalize_inner(node: Node, mutates_args: dict, is_v2=False): kwargs = {k: v for k, v in node.kwargs.items() if not k.startswith("_")} if is_v2: - all_bases = node.kwargs["_all_bases"] - for arg in inplace_func._schema.arguments: - if arg.alias_info is None or not arg.alias_info.is_write: - continue - base_index_key = f"_{arg.name}_base_index" - base_index = node.kwargs[base_index_key] - kwargs[arg.name] = (None if base_index is None else - all_bases[base_index]) + for k, v in mutates_args.items(): + kwargs[v] = node.kwargs["_all_bases"][k - 1] for getitem_node in getitem_nodes: idx = getitem_node.args[1] - mutated_arg = mutates_args[idx] - replacement = kwargs[mutated_arg] - assert replacement is not None, ( - f"getitem user for optional output '{mutated_arg}' " - "has no base tensor -- graph is malformed") - getitem_node.replace_all_uses_with(replacement) + getitem_node.replace_all_uses_with(kwargs[mutates_args[idx]]) nodes_to_remove.append(getitem_node) with graph.inserting_before(node): @@ -76,11 +51,7 @@ def remove_functionalize_inner(node: Node, mutates_args: dict, is_v2=False): # We do not know the inplace op continue - remove_functionalize_inner( - node, - inplace_map[inplace_func], - is_v2=node.target == auto_functionalized_v2, - ) + remove_functionalize_inner(node, inplace_map[inplace_func]) for node in nodes_to_remove: graph.erase_node(node) diff --git a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py index cf7240b3be36..e82d81e219b2 100644 --- a/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cpp_custom_ops.py @@ -731,15 +731,6 @@ def _(input: torch.Tensor): dtype=torch.float8_e4m3fn), input.new_empty( (m, num_packed_sf_k), dtype=torch.int32) - @torch.library.register_fake("trtllm::fp8_quantize_1x128_cutedsl_ue8m0") - def _(input: torch.Tensor): - m, k = input.shape - padded_m = fp4_utils.pad_up(m, 128) - sf_cols = fp4_utils.pad_up(k // 32, 4) - return torch.empty_like(input, - dtype=torch.float8_e4m3fn), input.new_empty( - (padded_m * sf_cols, ), dtype=torch.uint8) - @torch.library.register_fake("trtllm::causal_conv1d_fwd") def _( x: torch.Tensor, @@ -1290,69 +1281,6 @@ def _( (m, n), dtype=input.dtype) if output_hp_norm else None return normed_output_fp4, output, sf_out, hp_output - def _swizzled_sf_size(m: int, n: int, sf_vec_size: int = 16) -> int: - # Mirrors tensorrt_llm::computeSwizzledLayoutSFSize: padUp(rows,128) * - # padUp(cols,4), with cols = n / sf_vec_size (SFMatrix columns). - cols = n // sf_vec_size - padded_row = ((m + 127) // 128) * 128 - padded_col = ((cols + 3) // 4) * 4 - return padded_row * padded_col - - @torch.library.register_fake("trtllm::fused_add_rmsnorm_fp4_quantize") - def _( - hidden_states: torch.Tensor, - residual: torch.Tensor, - norm_weight: torch.Tensor, - scale_factor: torch.Tensor, - eps: float, - return_norm_out: bool, - ) -> List[torch.Tensor]: - # quant_out: packed NVFP4 (E2M1x2) as uint8, last dim halved, leading - # dims preserved. scale_out: swizzled E4M3 scale bytes. residual_out: - # hidden+residual, a fresh tensor (the op does not mutate hidden_states). - # When return_norm_out, the BF16 post-RMSNorm value leads the tuple. - m = 1 - for d in hidden_states.shape[:-1]: - m *= d - k = hidden_states.shape[-1] - quant_shape = (*hidden_states.shape[:-1], k // 2) - quant_out = hidden_states.new_empty(quant_shape, dtype=torch.uint8) - sf_out = hidden_states.new_empty((_swizzled_sf_size(m, k), ), - dtype=torch.uint8) - # Fresh allocations (new_empty), matching the real op's empty_cuda; not - # empty_like, which would carry the input's layout in the trace. - residual_out = hidden_states.new_empty(tuple(hidden_states.shape), - dtype=hidden_states.dtype) - if return_norm_out: - norm_out = hidden_states.new_empty(tuple(hidden_states.shape), - dtype=hidden_states.dtype) - return [norm_out, quant_out, sf_out, residual_out] - return [quant_out, sf_out, residual_out] - - @torch.library.register_fake("trtllm::fused_rmsnorm_fp4_quantize") - def _( - hidden_states: torch.Tensor, - norm_weight: torch.Tensor, - scale_factor: torch.Tensor, - eps: float, - return_norm_out: bool, - ) -> List[torch.Tensor]: - # Residual-less variant. quant_out / sf_out as above; norm_out (packed, - # contiguous) leads the tuple when return_norm_out. - m = 1 - for d in hidden_states.shape[:-1]: - m *= d - k = hidden_states.shape[-1] - quant_shape = (*hidden_states.shape[:-1], k // 2) - quant_out = hidden_states.new_empty(quant_shape, dtype=torch.uint8) - sf_out = hidden_states.new_empty((_swizzled_sf_size(m, k), ), - dtype=torch.uint8) - if return_norm_out: - norm_out = hidden_states.new_empty(tuple(hidden_states.shape), - dtype=hidden_states.dtype) - return [norm_out, quant_out, sf_out] - return [quant_out, sf_out] - @torch.library.register_fake("trtllm::fused_gated_rmsnorm_quant") def _( x: torch.Tensor, @@ -1460,3 +1388,11 @@ def _(like: torch.Tensor, out_shape = shape if shape is not None else list(like.shape) dtype = out_dtype if out_dtype is not None else like.dtype return like.new_empty(out_shape, dtype=dtype), output_buffer_kind + + @torch.library.register_fake("trtllm::compute_probs_from_logits_op") + def _(logits: torch.Tensor, + temperatures: torch.Tensor, + top_k: Optional[torch.Tensor] = None, + top_p: Optional[torch.Tensor] = None, + skip_temperature: bool = False) -> torch.Tensor: + return logits.new_empty(list(logits.shape), dtype=torch.float32) diff --git a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py index 0db351140735..84d55a1f4c79 100644 --- a/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py @@ -23,9 +23,6 @@ get_last_power_of_2_num_tokens_buckets, is_gated_activation, last_positive_power_of_2, next_positive_power_of_2) -from .cutedsl_matmul_heuristics import (NVFP4_PRECISION, - nvmmh_enabled_for_nvfp4, nvmmh_fields, - nvmmh_max_tactics, rank_configs) try: from cuda.bindings import driver as cuda @@ -525,242 +522,7 @@ def get_valid_tactics( logger.debug( f"CuteDSL: Found {len(valid_tactics)} valid tactics for M={m}, N={n}, K={real_k}" ) - # Optionally rank/prune the sweep with nvMatmulHeuristics (opt-in via - # TRTLLM_CUTEDSL_NVMMH_ENABLE). Returns the full sweep unchanged when - # disabled, unconfigured, or on any failure. - return self._rank_prune_tactics(valid_tactics, m, n, real_k) - - def _swap_ab_candidates(self, m, n): - """Deterministic swap_ab choice (not swept), constrained by C-layout - alignment. - - swap_ab=True maps the kernel M to the GEMM N (and vice versa), which - is preferred when M is small (<=128) so the kernel works on the - larger dimension; large M does not swap. The chosen value must still - satisfy the output (C) 16-byte alignment: swap_ab=False needs - N%8==0, swap_ab=True needs M%8==0. Falls back to the feasible value - if the preferred one violates alignment; returns [] if neither works. - """ - m_aligned = m % 8 == 0 - n_aligned = n % 8 == 0 - prefer_swap = m <= 128 - if prefer_swap and m_aligned: - return [True] - if not prefer_swap and n_aligned: - return [False] - if n_aligned: - return [False] - if m_aligned: - return [True] - return [] - - @staticmethod - def _heuristic_to_tactic_tile(cta, cluster): - """Translate a nvMatmulHeuristics (cta, cluster) config to this - kernel's (mma_tiler_mn, cluster_shape_mn). - - nvMatmulHeuristics caps the per-CTA tile M at 128 on Blackwell and - encodes the 2-SM (2-CTA) MMA as cluster_m == 2 (in the queried - kernel frame), so its effective M tile is cluster_m * cta_m. This - kernel instead encodes the same 2-SM op as mma_tiler_m == 256 - (use_2cta_instrs). So a 2-CTA config (cluster_m == 2) doubles the M - tile while keeping the cluster; everything else passes through. - - The 2-SM op additionally requires the N tile to be aligned (the - ``mma_n_align_requirement_2cta`` in libheuristics; 16 for NVFP4/bf16 - output, 32 when the N tile exceeds the 256 UTCMMA max). If N is not - aligned it is not a valid 2-SM config, so leave it single-CTA. - """ - cta_m, cta_n = int(cta[0]), int(cta[1]) - cluster_m, cluster_n = int(cluster[0]), int(cluster[1]) - n_align = 32 if cta_n > 256 else 16 - if cluster_m == 2 and cta_n % n_align == 0: - # 2-SM MMA along the kernel M dimension. - return (2 * cta_m, cta_n), (cluster_m, cluster_n) - return (cta_m, cta_n), (cluster_m, cluster_n) - - @staticmethod - def _unpack_tactic(tactic): - """Unpack a tactic into the full knob set with back-compat defaults. - - The base tactic is (mma_tiler_mn, cluster_shape_mn, swap_ab, - use_prefetch). The heuristic path may append two tile-scheduler - knobs (swizzle_size, raster_along_m); older 4-tuples (and the full - sweep) default to the kernel's neutral values (no swizzle, M-major - raster). Non-tuple tactics use the default kernel tactic. - """ - if not isinstance(tactic, (tuple, list)): - return (128, 128), (1, 1), False, False, 1, True - mma_tiler_mn = tactic[0] - cluster_shape_mn = tactic[1] - swap_ab = tactic[2] - use_prefetch = tactic[3] - swizzle_size = int(tactic[4]) if len(tactic) > 4 else 1 - raster_along_m = bool(tactic[5]) if len(tactic) > 5 else True - return (mma_tiler_mn, cluster_shape_mn, swap_ab, use_prefetch, - swizzle_size, raster_along_m) - - def _tactic_is_supported(self, mma_tiler_mn, cluster_shape_mn, swap_ab, - use_prefetch, m, n, real_k) -> bool: - """Whether the kernel can run this (tile, cluster, swap, prefetch). - - Validity gate used by get_valid_tactics() (the full enumeration). - The nvMatmulHeuristics path does NOT call this -- it trusts the - model's mapped tiles and emits them directly. - """ - sf_vec_size = 16 - if swap_ab: - c_major, kernel_m, kernel_n = "m", n, m - else: - c_major, kernel_m, kernel_n = "n", m, n - - if not self.__class__.kernel_class.can_implement( - cutlass.Float4E2M1FN, # ab_dtype - cutlass.Float8E4M3FN, # sf_dtype - sf_vec_size, - cutlass.BFloat16, # c_dtype - mma_tiler_mn, - cluster_shape_mn, - kernel_m, - kernel_n, - real_k, - 1, # batch_size - "k", # a_major - "k", # b_major - c_major, - ): - return False - - # Prefetch pruning: only worthwhile for a CTA-wave ratio in (0.5, 1.0) - # or large K. - cta_nums = get_dense_gemm_approximate_cta_nums( - m, n, mma_tiler_mn, cluster_shape_mn) - cta_wave_ratio = cta_nums / torch.cuda.get_device_properties( - ).multi_processor_count - if use_prefetch and not any( - (0.5 < cta_wave_ratio < 1.0, real_k >= 8192)): - return False - return True - - def _rank_prune_tactics(self, tactics, m, n, real_k): - """Rank/prune the full-sweep tactics with nvMatmulHeuristics (opt-in). - - Called at the end of get_valid_tactics. A strict, re-validated subset - of ``tactics`` -- it never introduces a (tile, cluster, swap) the - kernel validator rejected. Gated by TRTLLM_CUTEDSL_NVMMH_ENABLE; - TRTLLM_CUTEDSL_NVMMH_FIELDS selects the model-driven knobs. When - "tile"/"cluster" is selected we keep only the top - TRTLLM_CUTEDSL_NVMMH_MAX_TACTICS ranked (mma_tiler, cluster, swap) - keys (all matching prefetch variants retained); when only scheduler - knobs (swizzle / cta_order) are selected we sweep every tile/cluster - and just annotate it with the model's swizzle / raster (a safe - 6-tuple extension -- they do not affect can_implement). Deterministic - swap_ab selection (small M swaps) is applied here, in the opt-in path - only. - - Purely additive: on any failure, an empty match, or when heuristics - are disabled / unconfigured, returns ``tactics`` unchanged so - profiling never loses a valid candidate. - """ - if not nvmmh_enabled_for_nvfp4() or not tactics: - return tactics - fields = nvmmh_fields() - if not fields: - return tactics - try: - max_tactics = nvmmh_max_tactics() - # Only prune the tile/cluster set when the model is asked to - # drive it; scheduler-only fields keep the full sweep. - prune_tile_cluster = bool(fields & {"tile", "cluster"}) - use_swizzle = "swizzle" in fields - use_cta_order = "cta_order" in fields - emit_extended = use_swizzle or use_cta_order - - # Deterministic swap orientation (opt-in only), intersected with - # the orientations actually present in the valid tactics. - valid_swaps = {t[2] for t in tactics} - swap_pref = [ - s - for s in self._swap_ab_candidates(m, n) if s in valid_swaps - ] or list(valid_swaps) - - # Build the model's ranked preference over (mma_tiler, cluster, - # swap) keys, plus the per-key scheduler knobs. Query enough - # configs to find matches within the valid candidate list. - query_count = max(max_tactics * 4, 16) - pref_rank = {} - pref_sched = {} - for swap_ab in swap_pref: - kernel_m, kernel_n = (n, m) if swap_ab else (m, n) - for cfg in rank_configs(kernel_m, kernel_n, real_k, - NVFP4_PRECISION, query_count): - # Map per-CTA tile + cluster to this kernel's mma_tiler / - # cluster (cluster_m==2 encodes an mma_tiler_m==256 2-SM - # op). Off-grid maps simply won't match the valid list. - mma_tiler_mn, cluster_shape_mn = \ - self._heuristic_to_tactic_tile(cfg.cta, cfg.cluster) - key = (mma_tiler_mn, cluster_shape_mn, swap_ab) - if key in pref_rank: - continue - pref_rank[key] = len(pref_rank) - swizzle_size = cfg.swizzle_factor if use_swizzle else 1 - # nvMatmulHeuristics cta_order==0 is row-major (N-major) - # raster -> raster_along_m=False; !=0 -> M-major -> True. - raster_along_m = ((cfg.cta_order != 0) - if use_cta_order else True) - pref_sched[key] = (max(1, int(swizzle_size)), - bool(raster_along_m)) - - # Restrict to the top-K ranked tile/cluster keys ONLY when the - # model drives tile/cluster. For scheduler-only fields keep every - # valid tile/cluster key and just annotate it below. - valid_keys = {(t[0], t[1], t[2]) for t in tactics} - if prune_tile_cluster: - matched_keys = valid_keys & pref_rank.keys() - if not matched_keys: - return tactics - kept_keys = set( - sorted(matched_keys, - key=lambda kk: pref_rank[kk])[:max_tactics]) - else: - kept_keys = valid_keys - - # Emit every supplied (already-valid) tactic whose key is kept, - # re-validating as a safety net and optionally annotating the - # model's swizzle / raster (neutral defaults for unranked keys). - selected = [] - for t in tactics: - key = (t[0], t[1], t[2]) - if key not in kept_keys: - continue - mma_tiler_mn, cluster_shape_mn, swap_ab, use_prefetch = t[: - 4] - if not self._tactic_is_supported( - mma_tiler_mn, cluster_shape_mn, swap_ab, - use_prefetch, m, n, real_k): - continue - if emit_extended: - swizzle_size, raster_along_m = pref_sched.get( - key, (1, True)) - selected.append( - (mma_tiler_mn, cluster_shape_mn, swap_ab, - use_prefetch, swizzle_size, raster_along_m)) - else: - selected.append((mma_tiler_mn, cluster_shape_mn, - swap_ab, use_prefetch)) - - logger.debug( - f"CuteDSL nvMatmulHeuristics: {len(tactics)} valid -> " - f"{len(selected)} tactics ({len(kept_keys)} keys) for " - f"M={m}, N={n}, K={real_k}; fields={sorted(fields)}") - return selected if selected else tactics - except Exception as e: # noqa: BLE001 - must never break tuning - logger.warning_once( - f"[nvMatmulHeuristics] NVFP4 tactic filtering failed: {e}. " - f"Falling back to full tactic list.", - key="nvmmh_nvfp4_filter_failure", - ) - return tactics + return valid_tactics def make_cute_dsl_global_pointer(self, tensor: torch.Tensor, dtype, assumed_align: int): @@ -797,8 +559,16 @@ def forward( """ sf_vec_size = 16 - (mma_tiler_mn, cluster_shape_mn, swap_ab, use_prefetch, - swizzle_size, raster_along_m) = self._unpack_tactic(tactic) + if isinstance(tactic, tuple): + mma_tiler_mn, cluster_shape_mn, swap_ab, use_prefetch = tactic + else: + # fallback to default tactic + mma_tiler_mn, cluster_shape_mn, swap_ab, use_prefetch = [ + (128, 128), + (1, 1), + False, + False, + ] a_tensor, b_tensor, a_sf_tensor, b_sf_tensor, alpha_tensor = inputs m, k, n = a_tensor.shape[0], a_tensor.shape[1], b_tensor.shape[0] @@ -861,8 +631,7 @@ def forward( stream = cuda.CUstream(torch_stream.cuda_stream) cache_key = (sf_vec_size, mma_tiler_mn, cluster_shape_mn, swap_ab, - use_prefetch, swizzle_size, raster_along_m, - self.use_tvm_ffi) + use_prefetch, self.use_tvm_ffi) if swap_ab: kernel_m = n kernel_n = m @@ -929,8 +698,6 @@ def forward( mma_tiler_mn, cluster_shape_mn, use_prefetch, - swizzle_size, - raster_along_m, ) # Compute max active clusters on current device hardware_info = cutlass.utils.HardwareInfo() @@ -957,7 +724,7 @@ def forward( max_active_clusters, stream, swap_ab, - options="--opt-level 2 --enable-tvm-ffi" + options=f"--opt-level 2 --enable-tvm-ffi" if self.use_tvm_ffi else "--opt-level 2", ) @@ -1414,7 +1181,7 @@ def forward( False, # swap_ab=False for SwiGLU ] compile_kwargs = dict( - options="--opt-level 2 --enable-tvm-ffi" + options=f"--opt-level 2 --enable-tvm-ffi" if self.use_tvm_ffi else "--opt-level 2", ) if has_bias: compile_kwargs["bias_ptr"] = bias_ptr @@ -1908,7 +1675,7 @@ def forward( compiled_gemm = cute.compile( *compile_args, - options="--opt-level 2 --enable-tvm-ffi" + options=f"--opt-level 2 --enable-tvm-ffi" if self.use_tvm_ffi else "--opt-level 2", ) @@ -3636,331 +3403,6 @@ def _fake_single_b( device=input_scale.device) return output, output_scale - _INDEXER_Q_CUTEDSL_TUNING_BUCKETS = ( - 4, - 8, - 16, - 32, - 64, - 128, - 256, - 512, - 1024, - 2048, - 4096, - 8192, - 16384, - ) - _INDEXER_Q_POSITION_IDS_INPUT_INDEX = 3 - - def _map_cutedsl_indexer_q_tuning_bucket(num_tokens: int) -> int: - if num_tokens <= 4: - return 4 - if num_tokens <= 8: - return 8 - if num_tokens <= 16: - return 16 - return max(32, last_positive_power_of_2(num_tokens)) - - def _prepare_cutedsl_indexer_q_tuning_inputs( - inputs: List[torch.Tensor]) -> List[torch.Tensor]: - # The autotuner resizes position_ids to the token bucket, leaving newly - # allocated values uninitialized. Use position zero for every tuning - # row so the fused RoPE lookup always stays inside cos_sin_cache. - position_ids = inputs[_INDEXER_Q_POSITION_IDS_INPUT_INDEX] - inputs[_INDEXER_Q_POSITION_IDS_INPUT_INDEX] = torch.zeros_like( - position_ids) - return inputs - - class CuteDSLIndexerQBlackwellRunner(TunableRunner): - """Native MXF8 GEMM with fused DSv4 indexer-Q RoPE/MXFP4 output.""" - - kernel_class = Sm100BlockScaledPersistentDenseGemmActFusionKernel - small_m_kernel_class = Sm100BlockScaledPersistentDenseGemmKernel - kernel_cache = dict() - tuning_config = TuningConfig( - dynamic_tensor_specs=(DynamicTensorSpec( - 0, - 0, - _INDEXER_Q_CUTEDSL_TUNING_BUCKETS, - _map_cutedsl_indexer_q_tuning_bucket, - ), ), - constraint_specs=(ConstraintSpec(3, 0, - lambda shapes: shapes[0][0]), ), - inputs_pre_hook=_prepare_cutedsl_indexer_q_tuning_inputs, - use_cold_l2_cache=True, - # CuTe kernels are JIT compiled into a process-local cache while - # the autotuner profiles tactics. Never persist only the selected - # tactic: a new process would otherwise skip that compilation and - # pay for cute.compile in its first inference forward. - exclude_from_cache=True, - # Every rank owns a process-local CuTe module cache. Profiling in - # parallel across ranks would leave the winning tactic uncompiled - # on ranks that benchmarked a different subset. - distributed_tuning_strategy=DistributedTuningStrategy.INDEPENDENT, - ) - - _small_m_tactics = ( - ("swap_ab", (128, 16), (1, 1), False, 4), - ("swap_ab", (128, 16), (1, 1), False, 8), - ) - _native_tactics = ( - ("native", (128, 128), (1, 1), False, 0), - ("native", (128, 128), (1, 2), False, 0), - ("native", (128, 128), (2, 1), False, 0), - ("native", (128, 128), (2, 1), True, 0), - ("native", (256, 128), (2, 1), False, 0), - ("native", (256, 128), (2, 1), True, 0), - ("native", (256, 128), (2, 2), True, 0), - ) - - def __init__(self, use_tvm_ffi: bool = True): - super().__init__() - self.use_tvm_ffi = use_tvm_ffi - - def unique_id(self): - return (self.use_tvm_ffi, ) - - def get_valid_tactics( - self, - inputs: List[torch.Tensor], - profile: OptimizationProfile, - **kwargs, - ) -> List[Tuple]: - if not is_sm_100f(): - return [] - m, k = inputs[0].shape - n = inputs[1].shape[0] - tactics = [] - if self._small_m_kernel_is_supported(m, n, k): - tactics.extend(self.__class__._small_m_tactics) - - tactics.extend([ - tactic for tactic in self.__class__._native_tactics - if self.__class__.kernel_class.can_implement( - cutlass.Float8E4M3FN, - cutlass.Float8E8M0FNU, - 32, - cutlass.Float4E2M1FN, - tactic[1], - tactic[2], - m, - n, - k, - 1, - "k", - "k", - "n", - ) - ]) - return tactics - - @staticmethod - def _small_m_kernel_is_supported(m: int, n: int, k: int) -> bool: - return 0 < m <= 16 and n % 128 == 0 and k % 128 == 0 - - @classmethod - def _fallback_tactic(cls, m: int, n: int, k: int) -> Tuple: - """Safe eager-mode fallback when TRT-LLM autotuning is disabled.""" - if cls._small_m_kernel_is_supported(m, n, k): - if m <= 4: - return ("swap_ab", (128, 16), (1, 1), False, 4) - if m <= 8: - return ("swap_ab", (128, 16), (1, 1), False, 8) - return ("native", (256, 128), (2, 1), False, 0) - - @staticmethod - def _ptr(tensor: torch.Tensor, dtype, align: int = 16): - return make_ptr( - dtype, - tensor.data_ptr(), - cute.AddressSpace.gmem, - assumed_align=align, - ) - - def forward( - self, - inputs: List[torch.Tensor], - tactic, - ) -> Tuple[torch.Tensor, torch.Tensor]: - input, weight, weight_scale, position_ids, cos_sin_cache, alpha = inputs - m, k = input.shape - n = weight.shape[0] - if tactic == -1: - tactic = self._fallback_tactic(m, n, k) - (kernel_kind, mma_tiler_mn, cluster_shape_mn, use_prefetch, - transform_warps) = tactic - if kernel_kind == "swap_ab": - if not 0 < m <= mma_tiler_mn[1]: - raise ValueError( - "The small-M indexer-Q kernel requires one non-empty " - f"token tile: M={m}, tile N={mma_tiler_mn[1]}") - if n % 128 != 0 or k % 128 != 0: - raise ValueError( - "The small-M indexer-Q kernel requires N and K to be " - f"divisible by 128, but got N={n}, K={k}") - - a, a_sf = torch.ops.trtllm.fp8_quantize_1x128_cutedsl_ue8m0(input) - packed = torch.empty((m, n // 2), - dtype=torch.uint8, - device=input.device) - output_scale = torch.empty((m, n // 32), - dtype=torch.uint8, - device=input.device) - - a_ptr = self._ptr(a, cutlass.Float8E4M3FN) - b_ptr = self._ptr(weight, cutlass.Float8E4M3FN) - a_sf_ptr = self._ptr(a_sf, cutlass.Float8E8M0FNU) - b_sf_ptr = self._ptr(weight_scale, cutlass.Float8E8M0FNU) - packed_ptr = self._ptr(packed, cutlass.Uint8) - output_scale_ptr = self._ptr(output_scale, cutlass.Float8E8M0FNU) - position_ids_ptr = self._ptr(position_ids, cutlass.Int32, 4) - cos_sin_ptr = self._ptr(cos_sin_cache, cutlass.Float32, 32) - alpha_cute = cute.runtime.from_dlpack(alpha) - - if self.use_tvm_ffi: - stream = cute.runtime.make_fake_stream( - use_tvm_ffi_env_stream=True) - else: - stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) - - cache_key = (kernel_kind, mma_tiler_mn, cluster_shape_mn, - use_prefetch, transform_warps, self.use_tvm_ffi) - if cache_key not in self.__class__.kernel_cache: - if kernel_kind == "swap_ab": - gemm = self.__class__.small_m_kernel_class( - 32, - mma_tiler_mn, - cluster_shape_mn, - use_prefetch=use_prefetch, - indexer_q_fusion=True, - indexer_transform_warps=transform_warps, - ) - compile_entry = gemm.wrapper_indexer_q_swap_ab - else: - gemm = self.__class__.kernel_class( - 32, - mma_tiler_mn, - cluster_shape_mn, - True, - use_prefetch, - activation_type=ActivationType.Identity, - indexer_q_fusion=True, - ) - compile_entry = gemm.wrapper_indexer_q - hardware_info = cutlass.utils.HardwareInfo() - max_active_clusters = hardware_info.get_max_active_clusters( - cluster_shape_mn[0] * cluster_shape_mn[1]) - compiled = cute.compile( - compile_entry, - m, - n, - k, - pad_up(m, 128) // 128, - pad_up(n, 128) // 128, - pad_up(k // 32, 4) // 4, - cos_sin_cache.shape[0], - 1, - a_ptr, - b_ptr, - a_sf_ptr, - b_sf_ptr, - packed_ptr, - output_scale_ptr, - position_ids_ptr, - cos_sin_ptr, - alpha_cute, - max_active_clusters, - stream, - options="--opt-level 2 --enable-tvm-ffi" - if self.use_tvm_ffi else "--opt-level 2", - ) - self.__class__.kernel_cache[cache_key] = compiled - else: - compiled = self.__class__.kernel_cache[cache_key] - - dynamic_args = [ - m, - n, - k, - pad_up(m, 128) // 128, - pad_up(n, 128) // 128, - pad_up(k // 32, 4) // 4, - cos_sin_cache.shape[0], - ] - if self.use_tvm_ffi: - compiled( - *dynamic_args, - a.data_ptr(), - weight.data_ptr(), - a_sf.data_ptr(), - weight_scale.data_ptr(), - packed.data_ptr(), - output_scale.data_ptr(), - position_ids.data_ptr(), - cos_sin_cache.data_ptr(), - alpha, - ) - else: - compiled( - *dynamic_args, - a_ptr, - b_ptr, - a_sf_ptr, - b_sf_ptr, - packed_ptr, - output_scale_ptr, - position_ids_ptr, - cos_sin_ptr, - alpha_cute, - stream, - ) - return packed.view(torch.int8), output_scale.view(torch.int32) - - @torch.library.custom_op( - "trtllm::cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell", - mutates_args=(), - device_types="cuda", - ) - def cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell( - input: torch.Tensor, - weight: torch.Tensor, - weight_scale: torch.Tensor, - position_ids: torch.Tensor, - cos_sin_cache: torch.Tensor, - alpha: torch.Tensor, - use_tvm_ffi: bool = True, - ) -> Tuple[torch.Tensor, torch.Tensor]: - runner = CuteDSLIndexerQBlackwellRunner(use_tvm_ffi) - inputs = [ - input, weight, weight_scale, position_ids, cos_sin_cache, alpha - ] - tuner = AutoTuner.get() - _, tactic = tuner.choose_one( - "trtllm::cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell", - [runner], - runner.__class__.tuning_config, - inputs, - ) - return runner(inputs, tactic=tactic) - - @torch.library.register_fake( - "trtllm::cute_dsl_fp8_indexer_q_gemm_rope_fp4_blackwell") - def _( - input: torch.Tensor, - weight: torch.Tensor, - weight_scale: torch.Tensor, - position_ids: torch.Tensor, - cos_sin_cache: torch.Tensor, - alpha: torch.Tensor, - use_tvm_ffi: bool = True, - ): - m, n = input.shape[0], weight.shape[0] - return ( - input.new_empty((m, n // 2), dtype=torch.int8), - input.new_empty((m, n // 128), dtype=torch.int32), - ) - class CuteDSLFp8BlackwellRunner(TunableRunner): kernel_class = Sm100BlockwiseGemmKernel kernel_cache = dict() @@ -4177,7 +3619,7 @@ def forward( c_cute_tensor, max_active_clusters=max_active_clusters, stream=stream, - options="--opt-level 2 --enable-tvm-ffi" + options=f"--opt-level 2 --enable-tvm-ffi" if self.use_tvm_ffi else "--opt-level 2", ) self.__class__.kernel_cache[cache_key] = compiled_gemm @@ -4491,7 +3933,7 @@ def forward( c_cute_tensor, max_active_clusters=max_active_clusters, stream=stream, - options="--opt-level 2 --enable-tvm-ffi" + options=f"--opt-level 2 --enable-tvm-ffi" if self.use_tvm_ffi else "--opt-level 2", ) self.__class__.kernel_cache[cache_key] = compiled_gemm @@ -5577,7 +5019,7 @@ def cute_dsl_topk_decode_blackwell( Args: input_values: Input logits tensor [batch_size * next_n, vocab_size] seq_lens: Sequence lengths for each batch [batch_size] - top_k: Number of top elements to select (max 16384) + top_k: Number of top elements to select (max 2048) next_n: Number of candidates per sequence (for speculative decoding) num_copy_bits: Number of bits for vectorized memory copy (128 or 256) load_balance: Enable persistent dynamic scheduling for load balancing @@ -5587,7 +5029,7 @@ def cute_dsl_topk_decode_blackwell( Note: This function requires Blackwell architecture (SM100+) and CuTE DSL support. - Maximum supported top_k is 16384. + Maximum supported top_k is 2048. """ # Validate SM version sm_version = get_sm_version() @@ -5597,10 +5039,10 @@ def cute_dsl_topk_decode_blackwell( "Use standard top-k implementation for older architectures.") # Validate inputs - if top_k <= 0 or top_k > 16384: + if top_k <= 0 or top_k > 2048: raise ValueError( - f"top_k must be in range [1, 16384], got {top_k}. " - "Maximum supported top_k is 16384 for Blackwell architecture.") + f"top_k must be in range [1, 2048], got {top_k}. " + "Maximum supported top_k is 2048 for Blackwell architecture.") if next_n <= 0: raise ValueError(f"next_n must be positive, got {next_n}") @@ -6312,7 +5754,7 @@ def cute_dsl_topk_decode_multi_cta_blackwell( Args: input_values: Input logits tensor [batch_size * next_n, vocab_size] seq_lens: Sequence lengths for each batch [batch_size] - top_k: Number of top elements to select (max 16384) + top_k: Number of top elements to select (max 2048) next_n: Number of candidates per sequence (for speculative decoding) num_copy_bits: Number of bits for vectorized memory copy (128 or 256) chunk_size_per_cta: Number of columns each CTA processes @@ -6332,10 +5774,10 @@ def cute_dsl_topk_decode_multi_cta_blackwell( "Use standard top-k implementation for older architectures.") # Validate inputs - if top_k <= 0 or top_k > 16384: + if top_k <= 0 or top_k > 2048: raise ValueError( - f"top_k must be in range [1, 16384], got {top_k}. " - "Maximum supported top_k is 16384 for Blackwell architecture.") + f"top_k must be in range [1, 2048], got {top_k}. " + "Maximum supported top_k is 2048 for Blackwell architecture.") if next_n <= 0: raise ValueError(f"next_n must be positive, got {next_n}") @@ -6443,7 +5885,7 @@ def cute_dsl_indexer_topk_decode( input_values: Input logits tensor [batch_size * next_n, vocab_size] seq_lens: Sequence lengths for each batch [batch_size] output_indices: Pre-allocated output buffer [batch_size * next_n, top_k] - top_k: Number of top elements to select (max 16384) + top_k: Number of top elements to select (max 2048) next_n: Number of candidates per sequence (for speculative decoding) num_copy_bits: Number of bits for vectorized memory copy (128 or 256) dynamic: Use dynamic multi-CTA scheduling (for 2-pass multi-CTA) @@ -6451,12 +5893,6 @@ def cute_dsl_indexer_topk_decode( single_pass_multi_cta_cluster: Force cluster-accelerated variant (only effective when single_pass_multi_cta=True) """ - # Validate inputs - if top_k <= 0 or top_k > 16384: - raise ValueError( - f"top_k must be in range [1, 16384], got {top_k}. " - "Maximum supported top_k is 16384 for Blackwell architecture.") - num_rows = input_values.shape[0] num_tokens = input_values.shape[1] @@ -7572,14 +7008,10 @@ def forward( SPLIT_KV = compute_block_kv * 2 # NUM_MATH_WG = 2 aligned_max_ctx = ( (max_context_len + SPLIT_KV - 1) // SPLIT_KV) * SPLIT_KV - # Use a persistent arena buffer instead of a per-forward torch.empty - # so the output address stays stable across CUDA-graph replays. - _reserve = torch.cuda.is_current_stream_capturing() - logits = get_memory_buffers().get_buffer( - [B * next_n, aligned_max_ctx], - output_dtype, - buffer_name="cute_dsl_mqa_logits", - reserve_buffer=_reserve, + logits = torch.empty( + (B * next_n, aligned_max_ctx), + device=q.device, + dtype=output_dtype, ) logits = logits[:, :max_context_len] @@ -8424,14 +7856,10 @@ def forward( SPLIT_KV = compute_block_kv * 2 # NUM_MATH_WG = 2 aligned_max_ctx = ( (max_context_len + SPLIT_KV - 1) // SPLIT_KV) * SPLIT_KV - # Use a persistent arena buffer instead of a per-forward torch.empty - # so the output address stays stable across CUDA-graph replays. - _reserve = torch.cuda.is_current_stream_capturing() - logits = get_memory_buffers().get_buffer( - [B * next_n, aligned_max_ctx], - output_dtype, - buffer_name="cute_dsl_mqa_logits", - reserve_buffer=_reserve, + logits = torch.empty( + (B * next_n, aligned_max_ctx), + device=q.device, + dtype=output_dtype, ) logits = logits[:, :max_context_len] diff --git a/tensorrt_llm/_torch/custom_ops/cutedsl_matmul_heuristics.py b/tensorrt_llm/_torch/custom_ops/cutedsl_matmul_heuristics.py deleted file mode 100644 index 0113a6c3a515..000000000000 --- a/tensorrt_llm/_torch/custom_ops/cutedsl_matmul_heuristics.py +++ /dev/null @@ -1,277 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Adapter around NVIDIA Matmul Heuristics (nvMatmulHeuristics) for the CuTe DSL -GEMM autotuner. - -The autotuner enumerates every valid CuTe DSL tactic and then compiles + -benchmarks all of them, which dominates autotuning wall-time. This module uses -nvMatmulHeuristics' analytical performance model to rank the candidate -(CTA-tile, cluster) configurations so the runner can keep only the top-K before -profiling. - -Everything here is best-effort and opt-in: if the library is missing, the -problem is unsupported, or the query fails, the public helpers return an empty -ranking so callers fall back to the full tactic list and never regress. -""" - -import os -import re -from functools import lru_cache -from typing import List, NamedTuple, Optional, Tuple - -from tensorrt_llm.logger import logger - -try: - import nvMatmulHeuristics as _nvmmh - - IS_NVMMH_AVAILABLE = True -except ImportError: - _nvmmh = None - IS_NVMMH_AVAILABLE = False - -# NVFP4 dense GEMM precision/layout for nvMatmulHeuristics, fixed to the -# CuteDSLNVFP4BlackwellRunner kernel: A/B are Float4E2M1FN (fp4), the output is -# BFloat16, and A/B are K-major (see the can_implement call in -# cute_dsl_custom_ops.py). "F4F4T" is the cuBLAS-style token (fp4 A, fp4 B, -# bf16 output; bf16->T) and TN_ROW_MAJOR is the K-major A/B layout enum name on -# NvMatmulHeuristicsMatmulLayout. The whole path is best-effort: if the token is -# ever rejected by the installed wheel, rank_configs degrades to an empty -# ranking and the caller falls back to the full tactic sweep. -NVFP4_PRECISION = "F4F4T" -NVFP4_LAYOUT = "TN_ROW_MAJOR" - -CtaCluster = Tuple[Tuple[int, int], Tuple[int, int]] - -# The knobs a caller may let nvMatmulHeuristics drive (instead of sweeping the -# runner's candidate list). A field named here is supplied by the model; a field -# not named is swept. "tile" and "cluster" are a coupled pair (the 2-SM encoding -# maps the joint (cta, cluster)), so selecting either drives both jointly. -# "swizzle" (swizzle_factor) and "cta_order" (rasterization order) are per-knob. -NVMMH_OPTIONAL_FIELDS = frozenset({"tile", "cluster", "swizzle", "cta_order"}) - -# Accepted spellings that normalize onto a canonical field name. -_NVMMH_FIELD_ALIASES = {"cta_tile": "tile", "mma_tiler": "tile"} - - -class HeuristicConfig(NamedTuple): - """One nvMatmulHeuristics candidate, reduced to the knobs we can map to a - CuTe DSL NVFP4 tactic. ``swizzle_factor``/``cta_order`` default to the - kernel's neutral values (no swizzle, M-major raster) when absent.""" - - cta: Tuple[int, int] - cluster: Tuple[int, int] - swizzle_factor: int = 1 - cta_order: int = 0 - - -def nvmmh_fields() -> set: - """Which knobs nvMatmulHeuristics drives (vs. the runner sweeping them). - - Read from TRTLLM_CUTEDSL_NVMMH_FIELDS (comma-separated). Recognized tokens - are ``NVMMH_OPTIONAL_FIELDS`` (plus the ``cta_tile``/``mma_tiler`` aliases - for "tile"); unknown tokens are warned about once and ignored. Because the - 2-SM encoding maps the joint (cta, cluster), "tile" and "cluster" are - coupled -- naming either drives both. Default: "tile,cluster" (today's - behavior: heuristic tile+cluster, swept swizzle/cta_order). - """ - raw = os.environ.get("TRTLLM_CUTEDSL_NVMMH_FIELDS", "tile,cluster") - fields = set() - for tok in (f.strip().lower() for f in raw.split(",")): - if not tok: - continue - canonical = _NVMMH_FIELD_ALIASES.get(tok, tok) - if canonical not in NVMMH_OPTIONAL_FIELDS: - logger.warning_once( - f"[nvMatmulHeuristics] Ignoring unknown field '{tok}' in " - f"TRTLLM_CUTEDSL_NVMMH_FIELDS. Recognized: " - f"{sorted(NVMMH_OPTIONAL_FIELDS)} (aliases: " - f"{sorted(_NVMMH_FIELD_ALIASES)}).", - key=f"nvmmh_unknown_field_{tok}", - ) - continue - fields.add(canonical) - # tile/cluster are a coupled pair: naming either drives both. - if fields & {"tile", "cluster"}: - fields.update({"tile", "cluster"}) - return fields - - -def nvmmh_enabled() -> bool: - """Master opt-in switch for heuristic tactic pruning.""" - return IS_NVMMH_AVAILABLE and os.environ.get("TRTLLM_CUTEDSL_NVMMH_ENABLE", "0") == "1" - - -def nvmmh_enabled_for_nvfp4() -> bool: - """Whether heuristic pruning applies to the NVFP4 dense GEMM. Gated solely - by the master switch (TRTLLM_CUTEDSL_NVMMH_ENABLE).""" - return nvmmh_enabled() - - -def nvmmh_max_tactics() -> int: - """Number of heuristic-ranked (tile, cluster) configs to keep per problem. - - Mirrors TorchInductor's nvgemm_max_profiling_configs. Both use_prefetch - variants of a kept config are profiled on top of this (prefetch is not - modeled by the heuristic), so the final tactic count is up to ~2x this. - """ - try: - return max(1, int(os.environ.get("TRTLLM_CUTEDSL_NVMMH_MAX_TACTICS", "5"))) - except ValueError: - return 5 - - -@lru_cache(maxsize=None) -def _get_interface(precision: str): - """Build and cache an interface for a given precision (None on failure).""" - if not IS_NVMMH_AVAILABLE: - return None - try: - return _nvmmh.NvMatmulHeuristicsInterface( - _nvmmh.NvMatmulHeuristicsTarget.CUTLASS3, - precision=precision, - flags=_nvmmh.NvMatmulHeuristicsFlags.PERF_MODEL_BASED_AUTO_TUNING, - ) - except Exception as e: # noqa: BLE001 - any failure must degrade gracefully - logger.warning_once( - f"[nvMatmulHeuristics] Failed to build interface for " - f"precision={precision}: {e}. Falling back to full tactic list.", - key="nvmmh_interface_init_failure", - ) - return None - - -def _get_layout(name: str): - try: - return getattr(_nvmmh.NvMatmulHeuristicsMatmulLayout, name) - except AttributeError: - logger.warning_once( - f"[nvMatmulHeuristics] Unknown layout '{name}'. Falling back to full tactic list.", - key="nvmmh_unknown_layout", - ) - return None - - -def _as_int_pair(values) -> Optional[Tuple[int, int]]: - try: - nums = [int(v) for v in values] - except (TypeError, ValueError): - return None - if len(nums) < 2: - return None - return (nums[0], nums[1]) - - -def _extract_int(config, kernel, dict_key: str, attr: str, str_re: str, default: int) -> int: - """Read one scalar field, tolerant of dict / object / printable-string.""" - if isinstance(config, dict) and dict_key in config: - try: - return int(config[dict_key]) - except (TypeError, ValueError): - pass - if hasattr(kernel, attr): - try: - return int(getattr(kernel, attr)) - except (TypeError, ValueError): - pass - m = re.search(str_re, kernel if isinstance(kernel, str) else str(kernel)) - if m: - try: - return int(m.group(1)) - except (TypeError, ValueError): - pass - return default - - -def _extract_config(config) -> Optional[HeuristicConfig]: - """Pull (cta, cluster, swizzle_factor, cta_order) out of one heuristic - config. Tolerant of the several shapes the config may take across versions: - a dict with flat fields, a dict whose ``kernel`` is an object with - attributes, or a dict whose ``kernel`` is a printable string. - """ - kernel = config.get("kernel") if isinstance(config, dict) else config - cta = cluster = None - - # Form 1: flat dict fields (DKG getEx-style wrapper). - if isinstance(config, dict) and "cta_tile_m" in config: - cta = _as_int_pair((config["cta_tile_m"], config["cta_tile_n"])) - cluster = _as_int_pair((config.get("cluster_m", 1), config.get("cluster_n", 1))) - - # Form 2: kernel object with attributes (public wheel's GemmConfig uses - # cluster_m/cluster_n; the DKG internal wrapper used cga_m/cga_n). - if (cta is None or cluster is None) and hasattr(kernel, "cta_tile_m"): - cta = _as_int_pair((kernel.cta_tile_m, kernel.cta_tile_n)) - cluster_m = getattr(kernel, "cluster_m", getattr(kernel, "cga_m", 1)) - cluster_n = getattr(kernel, "cluster_n", getattr(kernel, "cga_n", 1)) - cluster = _as_int_pair((cluster_m, cluster_n)) - - # Form 3: printable string like "... cta(128 16 128) ... cluster(2 1) ...". - if cta is None or cluster is None: - text = kernel if isinstance(kernel, str) else str(kernel) - cta_m = re.search(r"cta\(\s*([\d\s]+?)\)", text) - cluster_m = re.search(r"cluster\(\s*([\d\s]+?)\)", text) - if cta_m: - cta = _as_int_pair(cta_m.group(1).split()) - cluster = _as_int_pair(cluster_m.group(1).split()) if cluster_m else (1, 1) - - if cta is None or cluster is None: - return None - - # swizzle_factor -> "swizz(N)"; cta_order -> "ctaOrder(N)". Default to the - # kernel's neutral values (no swizzle, M-major raster) when the field is - # absent so an unselected knob never changes behavior. - swizzle = _extract_int( - config, kernel, "swizzle_factor", "swizzle_factor", r"swizz\(\s*(\d+)\)", 1 - ) - cta_order = _extract_int(config, kernel, "cta_order", "cta_order", r"ctaOrder\(\s*(\d+)\)", 0) - return HeuristicConfig(cta, cluster, max(1, swizzle), cta_order) - - -def rank_configs(m: int, n: int, k: int, precision: str, count: int) -> List[HeuristicConfig]: - """Return up to ``count`` HeuristicConfig entries ranked best-first. - - Returns an empty list if heuristics are unavailable or the query fails, so - callers can fall back to their full tactic list. - """ - interface = _get_interface(precision) - if interface is None: - return [] - layout = _get_layout(NVFP4_LAYOUT) - if layout is None: - return [] - try: - # hw=None targets the current GPU. - try: - interface.loadInternalDiscoverySet(layout, None) - except Exception as e: # noqa: BLE001 - discovery data is optional - logger.debug(f"[nvMatmulHeuristics] loadInternalDiscoverySet skipped: {e}") - configs = interface.get_with_mnk(int(m), int(n), int(k), layout, int(count), None) - except Exception as e: # noqa: BLE001 - any failure must degrade gracefully - logger.warning_once( - f"[nvMatmulHeuristics] Query failed for " - f"precision={precision}, mnk=({m},{n},{k}): {e}. " - f"Falling back to full tactic list.", - key="nvmmh_query_failure", - ) - return [] - - if not configs: - return [] - - # Sort by estimated runtime when present; the API may already be ranked. - def _runtime(c): - return c.get("runtime", float("inf")) if isinstance(c, dict) else float("inf") - - ordered = sorted(configs, key=_runtime) - ranked: List[HeuristicConfig] = [] - for c in ordered: - cfg = _extract_config(c) - if cfg is not None and cfg not in ranked: - ranked.append(cfg) - return ranked - - -def rank_tile_cluster_configs( - m: int, n: int, k: int, precision: str, count: int -) -> List[CtaCluster]: - """Back-compat helper: (cta, cluster) pairs only, ranked best-first.""" - return [(c.cta, c.cluster) for c in rank_configs(m, n, k, precision, count)] diff --git a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py index 714c77be63ab..1d24db4fcc64 100644 --- a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py @@ -1112,10 +1112,6 @@ def get_valid_tactics(self, inputs: List[torch.Tensor], # SM version OK, check if CuteDSL supports the current shape cutedsl_runner = CuteDSLNVFP4BlackwellRunner( self.output_dtype) - # get_valid_tactics ranks/prunes with nvMatmulHeuristics - # internally when TRTLLM_CUTEDSL_NVMMH_ENABLE=1 (no-op - # otherwise), so the returned list already reflects any - # opt-in pruning before it enters the unified tactic list. cutedsl_tactics = cutedsl_runner.get_valid_tactics( inputs, profile) diff --git a/tensorrt_llm/_torch/custom_ops/trtllm_gen_custom_ops.py b/tensorrt_llm/_torch/custom_ops/trtllm_gen_custom_ops.py index 083892888fbb..a8bbb4cb2f09 100644 --- a/tensorrt_llm/_torch/custom_ops/trtllm_gen_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/trtllm_gen_custom_ops.py @@ -128,19 +128,6 @@ def prepare_dummy_topk_and_hook( # experts and only ~1/ep_size of slots hit local. is_local = use_dp - def pad_with_remote_dummy_experts(topk_ids_local: torch.Tensor, - device: torch.device) -> torch.Tensor: - """Fill remaining slots with deterministic out-of-shard ids, mirroring - production rows when local_num_experts < top_k: a row's top_k picks - are distinct global experts, at most local_num_experts of them local. - """ - num_tokens, k_local = topk_ids_local.shape - remote = (local_expert_offset + local_num_experts + torch.arange( - top_k - k_local, device=device, dtype=torch.int32)) % num_experts - return torch.cat( - [topk_ids_local, - remote.unsqueeze(0).expand(num_tokens, -1)], dim=1) - def make_balanced_dummy_topk( num_tokens: int, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]: @@ -167,26 +154,24 @@ def make_balanced_dummy_topk( n_target = local_num_experts if is_local else num_experts topk_ids[t, k] = (t + k * stride) % n_target (+ local_expert_offset if is_local) - Stride is picked so each row's filled entries stay distinct; when - n_target < top_k (local regime only) the row is padded with - out-of-shard ids (see pad_with_remote_dummy_experts). + Stride is picked so each row's `top_k` entries stay distinct; + the assertion guards against degenerate (n_target < top_k) configs + that would produce in-row duplicates. """ n_target = local_num_experts if is_local else num_experts - assert is_local or n_target >= top_k, ( - f"make_balanced_dummy_topk requires num_experts>={top_k} in " - f"the global regime; got num_experts={num_experts}") - k_fill = min(top_k, n_target) - stride = max(1, min(local_num_experts, n_target // k_fill)) - if stride * k_fill > n_target: - stride = max(1, n_target // k_fill) - base = torch.arange(k_fill, device=device, dtype=torch.int32) * stride + assert n_target >= top_k, ( + f"make_balanced_dummy_topk requires n_target>={top_k}; " + f"got n_target={n_target}, is_local={is_local}, " + f"num_experts={num_experts}, local_num_experts={local_num_experts}") + stride = max(1, min(local_num_experts, n_target // top_k)) + if stride * top_k > n_target: + stride = max(1, n_target // top_k) + base = torch.arange(top_k, device=device, dtype=torch.int32) * stride token_idx = torch.arange(num_tokens, device=device, dtype=torch.int32).unsqueeze(1) topk_ids = (base + token_idx) % n_target if is_local: topk_ids = topk_ids + local_expert_offset - if k_fill < top_k: - topk_ids = pad_with_remote_dummy_experts(topk_ids, device) topk_weights = torch.ones(num_tokens, top_k, dtype=torch.bfloat16, @@ -276,7 +261,9 @@ def make_routing_dummy_topk( would observe. """ if is_local: - k_fill = min(top_k, local_num_experts) + assert local_num_experts >= top_k, ( + f"random_local requires local_num_experts >= top_k; " + f"got local_num_experts={local_num_experts}, top_k={top_k}") if (logits is None or logits.shape[0] != num_tokens or logits.shape[-1] != local_num_experts): logits = torch.randn(num_tokens, @@ -285,12 +272,9 @@ def make_routing_dummy_topk( device=device) # Plain topk over local logits — bypasses the model's # routing_method on purpose (see docstring caveat re: - # grouped routings like DeepSeekV3 / Llama4). If the shard has - # fewer than top_k experts, take all and pad (see helper). - topk_ids = torch.topk(logits.float(), k_fill, dim=-1).indices.to( + # grouped routings like DeepSeekV3 / Llama4). + topk_ids = torch.topk(logits.float(), top_k, dim=-1).indices.to( torch.int32) + local_expert_offset - if k_fill < top_k: - topk_ids = pad_with_remote_dummy_experts(topk_ids, device) topk_weights = torch.ones(num_tokens, top_k, dtype=torch.bfloat16, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_act_fusion.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_act_fusion.py index 88215a759fc2..b96186049f67 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_act_fusion.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_act_fusion.py @@ -125,7 +125,6 @@ def __init__( vectorized_f32: bool, use_prefetch: bool = False, activation_type: ActivationType = ActivationType.Swiglu, - indexer_q_fusion: bool = False, ): """Initializes the configuration for a Blackwell dense GEMM kernel with fused activation. @@ -183,28 +182,19 @@ def __init__( self.vectorized_f32 = vectorized_f32 self.activation_type = activation_type - self.indexer_q_fusion = indexer_q_fusion self.is_gated = is_gated_activation(activation_type) # Precompute per-activation flags as plain Python bools so the epilogue # dispatch can use cutlass.const_expr(...) on them (like is_gated). An # inline enum comparison inside @cute.jit is not folded as a constant. self._act_is_swiglu = activation_type == ActivationType.Swiglu self._act_is_gelu = activation_type == ActivationType.Gelu - self._act_is_identity = activation_type == ActivationType.Identity # Host-side guard (raise is not allowed inside the @cute.kernel epilogue): # only SwiGLU and GELU(tanh) are wired in the fused epilogue. - if not (self._act_is_swiglu or self._act_is_gelu or self._act_is_identity): + if not (self._act_is_swiglu or self._act_is_gelu): raise NotImplementedError( f"Fused epilogue activation {activation_type} not implemented " - f"(only Swiglu, Gelu, and the indexer-Q identity epilogue are wired)." + f"(only Swiglu and Gelu are wired)." ) - if self.indexer_q_fusion: - if activation_type != ActivationType.Identity: - raise ValueError("indexer_q_fusion requires ActivationType.Identity") - if sf_vec_size != 32 or mma_tiler_mn[1] != 128: - raise ValueError( - "indexer_q_fusion requires MXF8 sf_vec_size=32 and a 128-column MMA tile" - ) def _setup_attributes(self): """Set up configurations that are dependent on GEMM inputs @@ -308,7 +298,7 @@ def _setup_attributes(self): self.is_b_mcast = self.num_mcast_ctas_b > 1 self.is_sfb_mcast = self.num_mcast_ctas_sfb > 1 - # SwiGLU: hardcoded epilogue tile matching grouped swiglu variant. + # SwiGLU: hardcoded epilogue tile matching grouped swiglu variant self.epi_tile = (128, 64) self.epi_tile_cnt = ( self.cta_tile_shape_mnk_c[0] // self.epi_tile[0], @@ -399,9 +389,6 @@ def __call__( sfc_tensor: Optional[cute.Tensor] = None, norm_const_tensor: Optional[cute.Tensor] = None, bias_tensor: Optional[cute.Tensor] = None, - indexer_scale_tensor: Optional[cute.Tensor] = None, - position_ids_tensor: Optional[cute.Tensor] = None, - cos_sin_cache_tensor: Optional[cute.Tensor] = None, ): """Execute the GEMM operation with SwiGLU fusion in steps: - Setup static attributes before smem/grid/tma computation @@ -457,20 +444,6 @@ def __call__( # Setup sfc tensor by filling C tensor to scale factor atom layout self.generate_sfc = sfc_tensor is not None and norm_const_tensor is not None - if cutlass.const_expr( - self.indexer_q_fusion - and ( - not self.generate_sfc - or self.c_dtype != cutlass.Float4E2M1FN - or indexer_scale_tensor is None - or position_ids_tensor is None - or cos_sin_cache_tensor is None - ) - ): - raise ValueError( - "indexer_q_fusion requires FP4 C, SFC generation, contiguous " - "output scales, position IDs, and a cos/sin cache" - ) if cutlass.const_expr(self.generate_sfc): sfc_layout = blockscaled_utils.tile_atom_to_shape_SF(c_tensor.shape, self.sf_vec_size) sfc_tensor = cute.make_tensor(sfc_tensor.iterator, sfc_layout) @@ -648,9 +621,6 @@ class SharedStorage: sfc_tensor, norm_const_tensor, bias_tensor, - indexer_scale_tensor, - position_ids_tensor, - cos_sin_cache_tensor, self.cluster_layout_vmnk, self.cluster_layout_sfb_vmnk, self.a_smem_layout_staged, @@ -692,9 +662,6 @@ def kernel( mSFC_mnl: Optional[cute.Tensor], norm_const_tensor: Optional[cute.Tensor], mBias_mnl: Optional[cute.Tensor], - mIndexerScale: Optional[cute.Tensor], - mPositionIds: Optional[cute.Tensor], - mCosSinCache: Optional[cute.Tensor], cluster_layout_vmnk: cute.Layout, cluster_layout_sfb_vmnk: cute.Layout, a_smem_layout_staged: cute.ComposedLayout, @@ -1398,7 +1365,6 @@ def kernel( tTR_rAcc_up, tTR_rAcc_gate, tTR_gBias_base, - tTR_cC, ) = self.epilog_tmem_copy_and_partition( epi_tidx, tCtAcc_base, tCgC, epi_tile, use_2cta_instrs, tCgBias ) @@ -1537,7 +1503,6 @@ def kernel( # up * silu(gate) # subtile_cnt = cute.size(tTR_tAcc.shape, mode=[3]) - packed_indexer_scale = cutlass.Uint32(0) for subtile_idx in cutlass.range(0, subtile_cnt, 2 if self.is_gated else 1): if cutlass.const_expr(self.is_gated): @@ -1603,60 +1568,6 @@ def kernel( self._apply_gelu_epilogue( acc_vec_up, alpha_val, tCompute, bias_vec=bias_vec ) - elif cutlass.const_expr(self._act_is_identity): - # Preserve the existing Q-projection contract: the GEMM - # accumulator is rounded to BF16 before RoPE. The - # indexer-Q block below performs that boundary rounding; - # this copy keeps the accumulator in FP32 until then. - tCompute.store(acc_vec_up) - - if cutlass.const_expr(self.indexer_q_fusion): - # The legacy path rounds GEMM output to BF16 before - # RoPE, then rounds the rotated values to BF16 again. - tCompute.store(tCompute.load().to(cutlass.BFloat16).to(cutlass.Float32)) - subtile_n_for_rope = real_subtile_idx % self.epi_tile_cnt[1] - subtiles_per_head = 128 // self.epi_tile[1] - rotary_subtile_begin = 64 // self.epi_tile[1] - subtile_in_head = subtile_n_for_rope % subtiles_per_head - if subtile_in_head >= rotary_subtile_begin: - subtile_m_for_rope = real_subtile_idx // self.epi_tile_cnt[1] - rope_base_m = ( - cur_tile_coord[0] * self.cta_tile_shape_mnk_c[0] - + subtile_m_for_rope * self.epi_tile[0] - ) - rope_m_idx = rope_base_m + tTR_cC[0][0] - if rope_m_idx < mPositionIds.shape[0]: - position = mPositionIds[rope_m_idx] - rope_row = mCosSinCache[position, None] - copy_atom_f32x8 = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), - cutlass.Float32, - num_bits_per_copy=256, - ) - cos_values = cute.make_rmem_tensor((8,), cutlass.Float32) - sin_values = cute.make_rmem_tensor((8,), cutlass.Float32) - for chunk_idx in cutlass.range_constexpr(4): - cos_tile = cute.local_tile(rope_row, (8,), (chunk_idx,)) - sin_tile = cute.local_tile(rope_row, (8,), (chunk_idx + 4,)) - cute.copy( - copy_atom_f32x8, - cute.coalesce(cos_tile), - cute.coalesce(cos_values), - ) - cute.copy( - copy_atom_f32x8, - cute.coalesce(sin_tile), - cute.coalesce(sin_values), - ) - for pair_in_chunk in cutlass.range_constexpr(8): - pair_idx = chunk_idx * 8 + pair_in_chunk - cosine = cos_values[pair_in_chunk] - sine = sin_values[pair_in_chunk] - x = tCompute[pair_idx * 2] - y = tCompute[pair_idx * 2 + 1] - tCompute[pair_idx * 2] = cosine * x - sine * y - tCompute[pair_idx * 2 + 1] = cosine * y + sine * x - tCompute.store(tCompute.load().to(cutlass.BFloat16).to(cutlass.Float32)) if cutlass.const_expr(self.generate_sfc): # @@ -1728,52 +1639,19 @@ def kernel( * norm_const ) - # CuTe's packed f32x2 -> UE8M0 conversion currently - # lowers a two-element result through vector<0xi32>. - # MXF8 with sf_vec_size=32 has exactly two output scale - # values per epilogue thread, so keep these conversions - # scalar until that lowering is fixed upstream. - if cutlass.const_expr(self.sf_vec_size == 32): - for vi in cutlass.range_constexpr(cute.size(tCrSFC)): - tCrSFC[vi] = tCrSFC_pvscale[vi].to(self.sf_dtype) - else: - tCrSFC.store(tCrSFC_pvscale.load().to(self.sf_dtype)) + # TODO: need to add f32x2 -> f8x2 conversion + tCrSFC.store(tCrSFC_pvscale.load().to(self.sf_dtype)) # # Store SFC to global memory # - if cutlass.const_expr(self.indexer_q_fusion): - subtile_m = real_subtile_idx // self.epi_tile_cnt[1] - subtile_n = real_subtile_idx % self.epi_tile_cnt[1] - scale_m = ( - cur_tile_coord[0] * self.cta_tile_shape_mnk_c[0] - + subtile_m * self.epi_tile[0] - + tTR_cC[0][0] - ) - if scale_m < mIndexerScale.shape[0]: - scale_bytes = cute.recast_tensor(tCrSFC, cutlass.Uint8) - for vi in cutlass.range_constexpr(cute.size(tCrSFC)): - scale_byte_idx = subtile_n * 2 + vi - packed_indexer_scale = packed_indexer_scale | ( - cutlass.Uint32(scale_bytes[vi]) - << cutlass.Uint32(scale_byte_idx * 8) - ) - else: - cute.autovec_copy(tCrSFC, tCgSFC) + cute.autovec_copy(tCrSFC, tCgSFC) # # Compute quantized output values and convert to C type # - # Same two-element UE8M0 lowering issue applies in the - # reverse direction. - if cutlass.const_expr(self.sf_vec_size == 32): - tCrSFC_qpvscale_up = cute.make_rmem_tensor( - tCrSFC.shape, cutlass.Float32 - ) - for vi in cutlass.range_constexpr(cute.size(tCrSFC)): - tCrSFC_qpvscale_up[vi] = cutlass.Float32(tCrSFC[vi]) - else: - tCrSFC_qpvscale_up = tCrSFC.load().to(cutlass.Float32) + # TODO: need to add f8x2 -> f32x2 conversion + tCrSFC_qpvscale_up = tCrSFC.load().to(cutlass.Float32) fp32_max = cutlass.Float32(3.40282346638528859812e38) if cutlass.const_expr(self.vectorized_f32): for vi in cutlass.range_constexpr(0, cute.size(tCrSFC), 2): @@ -1808,30 +1686,6 @@ def kernel( acc_vec = tiled_copy_r2s.retile(tCompute).load() tRS_rC.store(acc_vec.to(self.c_dtype)) - if cutlass.const_expr(self.indexer_q_fusion): - packed_words = cute.recast_tensor(tTR_rC, cutlass.Uint32) - for word_idx in cutlass.range_constexpr(cute.size(packed_words)): - packed = packed_words[word_idx] - midpoint_adjust = cutlass.Uint32(0) - for nibble_idx in cutlass.range_constexpr(8): - scaled_value = tCompute[word_idx * 8 + nibble_idx] - abs_value = cute.arch.fmax(scaled_value, -scaled_value) - nibble_adjust = cutlass.Uint32(1) << (nibble_idx * 4) - if abs_value == cutlass.Float32(0.75): - midpoint_adjust = midpoint_adjust | nibble_adjust - if abs_value == cutlass.Float32(1.75): - midpoint_adjust = midpoint_adjust | nibble_adjust - if abs_value == cutlass.Float32(3.5): - midpoint_adjust = midpoint_adjust | nibble_adjust - packed = packed - midpoint_adjust - magnitude = packed & cutlass.Uint32(0x77777777) - nonzero_sign = ( - (magnitude | (magnitude << 1) | (magnitude << 2)) - & cutlass.Uint32(0x44444444) - ) << 1 - packed_words[word_idx] = packed & ( - cutlass.Uint32(0x77777777) | nonzero_sign - ) else: # # Convert to C type (non-SFC path) @@ -1878,10 +1732,6 @@ def kernel( barrier_id=self.epilog_sync_bar_id, number_of_threads=epilog_threads, ) - if cutlass.const_expr(self.indexer_q_fusion and self.generate_sfc): - scale_m = cur_tile_coord[0] * self.cta_tile_shape_mnk_c[0] + tTR_cC[0][0] - if scale_m < mIndexerScale.shape[0]: - mIndexerScale[scale_m, cur_tile_coord[1], 0] = packed_indexer_scale # # Async arrive accumulator buffer empty @@ -2094,14 +1944,7 @@ def epilog_tmem_copy_and_partition( epi_tile: cute.Tile, use_2cta_instrs: Union[cutlass.Boolean, bool], tCgBias: Optional[cute.Tensor] = None, - ) -> Tuple[ - cute.TiledCopy, - cute.Tensor, - cute.Tensor, - cute.Tensor, - Optional[cute.Tensor], - cute.Tensor, - ]: + ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor, cute.Tensor, Optional[cute.Tensor]]: """ Make tiledCopy for tensor memory load, then use it to partition tensor memory (source) and register array (destination). @@ -2164,8 +2007,6 @@ def epilog_tmem_copy_and_partition( tTR_rAcc_gate = cute.make_rmem_tensor( tTR_gC[(None, None, None, 0, 0, 0, 0, 0)].shape, self.acc_dtype ) - cC = cute.make_identity_tensor(epi_tile) - tTR_cC = thr_copy_t2r.partition_D(cC) # Partition the per-N bias EXACTLY like C (broadcast over M via stride 0). # Same flat_divide + partition_D path -> tTR_gBias has the same layout as @@ -2177,14 +2018,7 @@ def epilog_tmem_copy_and_partition( ) tTR_gBias = thr_copy_t2r.partition_D(gBias_mnl_epi) - return ( - tiled_copy_t2r, - tTR_tAcc, - tTR_rAcc_up, - tTR_rAcc_gate, - tTR_gBias, - tTR_cC, - ) + return tiled_copy_t2r, tTR_tAcc, tTR_rAcc_up, tTR_rAcc_gate, tTR_gBias def epilog_smem_copy_and_partition( self, @@ -2772,98 +2606,6 @@ def wrapper( bias_tensor=bias_tensor, ) - @cute.jit - def wrapper_indexer_q( - self, - m: cutlass.Int64, - n: cutlass.Int64, - k: cutlass.Int64, - sf_m: cutlass.Int64, - sf_n: cutlass.Int64, - sf_k: cutlass.Int64, - cos_sin_rows: cutlass.Int64, - l: cutlass.Constexpr, # noqa: E741 - a_ptr: cute.Pointer, - b_ptr: cute.Pointer, - a_sf_ptr: cute.Pointer, - b_sf_ptr: cute.Pointer, - packed_ptr: cute.Pointer, - scale_ptr: cute.Pointer, - position_ids_ptr: cute.Pointer, - cos_sin_ptr: cute.Pointer, - alpha_tensor: cute.Tensor, - max_active_clusters: cutlass.Constexpr, - current_stream: cuda.CUstream, - ): - """MXF8 GEMM with the DeepSeek-V4 indexer-Q RoPE/MXFP4 epilogue. - - ``packed_ptr`` and ``scale_ptr`` use the indexer's existing contiguous - layouts: uint8 [M, N/2] and UE8M0 [M, N/32]. The output tensor is a - logical FP4 view so the repository's native FP4 shared-memory layout - and TMA store path can be reused. - """ - a_tensor = cute.make_tensor( - a_ptr, - layout=cute.make_ordered_layout((m, k, l), order=(1, 0, 2)), - ) - b_tensor = cute.make_tensor( - b_ptr, - layout=cute.make_ordered_layout((n, k, l), order=(1, 0, 2)), - ) - c_tensor = cute.make_tensor( - cute.recast_ptr(packed_ptr, dtype=cutlass.Float4E2M1FN), - layout=cute.make_ordered_layout((m, n, l), order=(1, 0, 2)), - ) - scale_tensor = cute.make_tensor( - cute.recast_ptr(scale_ptr, dtype=cutlass.Uint32), - layout=cute.make_ordered_layout((m, n // 128, l), order=(1, 0, 2)), - ) - # generate_sfc needs a scale-factor-shaped tensor to derive its register - # partition. Indexer-Q stores the values through scale_tensor above, - # preserving the consumer's contiguous per-head four-byte layout. - sfc_tensor = cute.make_tensor( - scale_ptr, - layout=cute.make_ordered_layout((32, 4, sf_m, 4, sf_n, l), order=(2, 1, 4, 0, 3, 5)), - ) - position_ids_tensor = cute.make_tensor( - position_ids_ptr, - layout=cute.make_layout((m,)), - ) - cos_sin_cache_tensor = cute.make_tensor( - cos_sin_ptr, - layout=cute.make_ordered_layout((cos_sin_rows, 64), order=(1, 0)), - ) - sfa_tensor = cute.make_tensor( - a_sf_ptr, - layout=cute.make_ordered_layout( - (32, 4, sf_m, 4, sf_k, l), - order=(2, 1, 4, 0, 3, 5), - ), - ) - sfb_tensor = cute.make_tensor( - b_sf_ptr, - layout=cute.make_ordered_layout( - (32, 4, sf_n, 4, sf_k, l), - order=(2, 1, 4, 0, 3, 5), - ), - ) - - self( - a_tensor, - b_tensor, - sfa_tensor, - sfb_tensor, - c_tensor, - alpha_tensor, - max_active_clusters, - current_stream, - sfc_tensor=sfc_tensor, - norm_const_tensor=alpha_tensor, - indexer_scale_tensor=scale_tensor, - position_ids_tensor=position_ids_tensor, - cos_sin_cache_tensor=cos_sin_cache_tensor, - ) - @cute.jit def wrapper_fp4out( self, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py index 0194cbc8359e..3e77045eb36a 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/dense_blockscaled_gemm_persistent.py @@ -52,48 +52,13 @@ import cutlass.utils as utils import cutlass.utils.blackwell_helpers as sm100_utils import cutlass.utils.blockscaled_layout as blockscaled_utils -from cutlass._mlir.dialects import llvm from cutlass.cute.nvgpu import cpasync, tcgen05 -from cutlass.cutlass_dsl import dsl_user_op from .custom_pipeline import PipelineTmaUmma, PipelineUmmaAsync from .utils import (TRTLLM_ENABLE_PDL, griddepcontrol_launch_dependents, griddepcontrol_wait, is_power_of_2) -@dsl_user_op -def _indexer_q_pack_fp4x4(value0: cutlass.Float32, - value1: cutlass.Float32, - value2: cutlass.Float32, - value3: cutlass.Float32, - *, - loc=None, - ip=None) -> cutlass.Uint16: - """Pack four FP32 values with two native E2M1x2 conversions.""" - return cutlass.Uint16( - llvm.inline_asm( - cutlass.Uint16.mlir_type, - [ - value0.ir_value(loc=loc, ip=ip), - value1.ir_value(loc=loc, ip=ip), - value2.ir_value(loc=loc, ip=ip), - value3.ir_value(loc=loc, ip=ip), - ], - """{ - .reg .b8 byte0, byte1; - cvt.rn.satfinite.e2m1x2.f32 byte0, $2, $1; - cvt.rn.satfinite.e2m1x2.f32 byte1, $4, $3; - mov.b16 $0, {byte0, byte1}; - }""", - "=h,f,f,f,f", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - loc=loc, - ip=ip, - )) - - class Sm100BlockScaledPersistentDenseGemmKernel: """Implements batched matrix multiplication (C = A x SFA x B x SFB) with support for various data types and Blackwell GPU architectural features, including persistent tile scheduling and warp specialization. @@ -133,10 +98,6 @@ def __init__( mma_tiler_mn: Tuple[int, int], cluster_shape_mn: Tuple[int, int], use_prefetch: bool = False, - swizzle_size: int = 1, - raster_along_m: bool = True, - indexer_q_fusion: bool = False, - indexer_transform_warps: int = 4, ): """Initializes the configuration for a Blackwell dense GEMM kernel. @@ -150,16 +111,10 @@ def __init__( 2. Cluster Shape: - cluster_shape_mn: The (ClusterM, ClusterN) shape of the CTA cluster. - 3. Persistent tile scheduler: - - swizzle_size: cluster-swizzle size for L2-locality (1 = no swizzle). - - raster_along_m: cluster rasterization order (M-major when True). - Args: sf_vec_size (int): Scalefactor vector size. mma_tiler_mn (Tuple[int, int]): Shape of the Matrix Multiply-Accumulate (MMA) tile (M, N). cluster_shape_mn (Tuple[int, int]): Cluster dimensions (M, N) for parallel processing. - swizzle_size (int): Tile-scheduler swizzle size in clusters (default 1). - raster_along_m (bool): Tile-scheduler rasterization order (default M-major). """ self.acc_dtype = cutlass.Float32 @@ -169,18 +124,6 @@ def __init__( # K dimension is deferred in _setup_attributes self.mma_tiler = (*mma_tiler_mn, 1) self.use_prefetch = use_prefetch - self.swizzle_size = swizzle_size - self.raster_along_m = raster_along_m - self.indexer_q_fusion = indexer_q_fusion - self.indexer_transform_warps = indexer_transform_warps - if self.indexer_q_fusion and (sf_vec_size != 32 or mma_tiler_mn[0] - != 128 or mma_tiler_mn[1] != 16 - or cluster_shape_mn != (1, 1) - or indexer_transform_warps not in (4, 8)): - raise ValueError( - "The swapped indexer-Q epilogue currently requires " - "MXF8, a 128x16 MMA tile, a 1x1 cluster, and 4 or 8 " - "row-transform warps") self.cta_group = (tcgen05.CtaGroup.TWO if self.use_2cta_instrs else tcgen05.CtaGroup.ONE) @@ -194,12 +137,8 @@ def __init__( ) self.mma_warp_id = 4 self.tma_warp_id = 5 - self.indexer_extra_warp_id = tuple( - range(self.tma_warp_id + 1, self.tma_warp_id + 1 + - indexer_transform_warps - 4)) if self.indexer_q_fusion else () self.threads_per_cta = 32 * len( - (self.mma_warp_id, self.tma_warp_id, *self.epilog_warp_id, - *self.indexer_extra_warp_id)) + (self.mma_warp_id, self.tma_warp_id, *self.epilog_warp_id)) # Set barrier id for cta sync, epilogue sync and tmem ptr sync self.cta_sync_bar_id = 0 self.epilog_sync_bar_id = 1 @@ -308,14 +247,6 @@ def _setup_attributes(self): ) self.epi_tile_n = cute.size(self.epi_tile[1]) - expected_indexer_epi_n = min(self.mma_tiler[1], 32) - if (self.indexer_q_fusion - and (cute.size(self.epi_tile[0]) != 128 - or self.epi_tile_n != expected_indexer_epi_n)): - raise ValueError( - f"Unexpected swapped indexer-Q epilogue tile {self.epi_tile}; " - f"expected a 128-feature by {expected_indexer_epi_n}-token " - "BF16 stage") # Setup A/B/C stage count in shared memory and ACC stage count in tensor memory self.num_acc_stage, self.num_ab_stage, self.num_c_stage = self._compute_stages( @@ -367,10 +298,6 @@ def _setup_attributes(self): ) self.overlapping_accum = self.num_acc_stage == 1 - if self.indexer_q_fusion and self.overlapping_accum: - raise ValueError( - "The swapped indexer-Q transform-warp schedule requires at " - "least two accumulator stages") sf_atom_mn = 32 self.num_sfa_tmem_cols = (self.cta_tile_shape_mnk[0] // sf_atom_mn) * mma_inst_tile_k @@ -399,10 +326,6 @@ def __call__( max_active_clusters: cutlass.Constexpr, stream: cuda.CUstream, epilogue_op: cutlass.Constexpr = lambda x: x, - packed_tensor: Optional[cute.Tensor] = None, - indexer_scale_tensor: Optional[cute.Tensor] = None, - position_ids_tensor: Optional[cute.Tensor] = None, - cos_sin_cache_tensor: Optional[cute.Tensor] = None, ): """Execute the GEMM operation in steps: - Setup static attributes before smem/grid/tma computation @@ -435,17 +358,6 @@ def __call__( b_tensor).mma_major_mode() self.c_layout = utils.LayoutEnum.from_tensor(c_tensor) - if cutlass.const_expr( - self.indexer_q_fusion and - (self.c_dtype != cutlass.BFloat16 - or self.c_layout != utils.LayoutEnum.COL_MAJOR - or packed_tensor is None or indexer_scale_tensor is None - or position_ids_tensor is None or cos_sin_cache_tensor is None)): - raise ValueError( - "The swapped indexer-Q epilogue requires a column-major BF16 " - "GEMM boundary plus packed output, scale, position, and RoPE tensors" - ) - # Check if input data types are compatible with MMA instruction if cutlass.const_expr(self.a_dtype != self.b_dtype): raise TypeError( @@ -581,8 +493,6 @@ def __call__( self.cta_tile_shape_mnk, self.cluster_shape_mn, max_active_clusters, - self.swizzle_size, - self.raster_along_m, ) self.buffer_align_bytes = 1024 @@ -651,10 +561,6 @@ class SharedStorage: tma_tensor_sfb, tma_atom_c, tma_tensor_c, - packed_tensor, - indexer_scale_tensor, - position_ids_tensor, - cos_sin_cache_tensor, self.cluster_layout_vmnk, self.cluster_layout_sfb_vmnk, self.a_smem_layout_staged, @@ -693,10 +599,6 @@ def kernel( mSFB_nkl: cute.Tensor, tma_atom_c: Optional[cute.CopyAtom], mC_mnl: cute.Tensor, - mPacked_nml: Optional[cute.Tensor], - mIndexerScale_nml: Optional[cute.Tensor], - mPositionIds: Optional[cute.Tensor], - mCosSinCache: Optional[cute.Tensor], cluster_layout_vmnk: cute.Layout, cluster_layout_sfb_vmnk: cute.Layout, a_smem_layout_staged: cute.ComposedLayout, @@ -1516,9 +1418,7 @@ def kernel( "async.shared", space="cta", ) - epilog_threads = 32 * (self.indexer_transform_warps - if self.indexer_q_fusion else len( - self.epilog_warp_id)) + epilog_threads = 32 * len(self.epilog_warp_id) cute.arch.barrier( barrier_id=self.epilog_sync_bar_id, number_of_threads=epilog_threads, @@ -1527,31 +1427,15 @@ def kernel( # # TMA store C to global memory # - if cutlass.const_expr(self.indexer_q_fusion): - self._indexer_q_transform_rows( - sC, - c_buffer, - warp_idx, - self.indexer_transform_warps, - cur_tile_coord[0], - cur_tile_coord[1], - cur_tile_coord[2], - real_subtile_idx, - mPacked_nml, - mIndexerScale_nml, - mPositionIds, - mCosSinCache, + if warp_idx == self.epilog_warp_id[0]: + cute.copy( + tma_atom_c, + bSG_sC[(None, c_buffer)], + bSG_gC[(None, real_subtile_idx)], ) - else: - if warp_idx == self.epilog_warp_id[0]: - cute.copy( - tma_atom_c, - bSG_sC[(None, c_buffer)], - bSG_gC[(None, real_subtile_idx)], - ) - # Fence and barrier to make sure shared memory store is visible to TMA store - c_pipeline.producer_commit() - c_pipeline.producer_acquire() + # Fence and barrier to make sure shared memory store is visible to TMA store + c_pipeline.producer_commit() + c_pipeline.producer_acquire() cute.arch.barrier( barrier_id=self.epilog_sync_bar_id, number_of_threads=epilog_threads, @@ -1591,184 +1475,10 @@ def kernel( # # Wait for C store complete # - if cutlass.const_expr(not self.indexer_q_fusion): - c_pipeline.producer_tail() - - # Optional row-transform-only warps. The four canonical epilogue - # warps remain the sole TMEM drain owners; these warps join only the - # two shared-stage barriers and process disjoint complete token rows. - if cutlass.const_expr(self.indexer_q_fusion - and self.indexer_transform_warps > 4): - if (warp_idx >= self.indexer_extra_warp_id[0] - and warp_idx <= self.indexer_extra_warp_id[-1]): - tile_sched = utils.StaticPersistentTileScheduler.create( - tile_sched_params, cute.arch.block_idx(), - cute.arch.grid_dim()) - work_tile = tile_sched.initial_work_tile_info() - subtile_cnt = (self.cta_tile_shape_mnk[1] // self.epi_tile_n) - transform_threads = 32 * self.indexer_transform_warps - transform_warp_idx = (len(self.epilog_warp_id) + warp_idx - - self.indexer_extra_warp_id[0]) - - while work_tile.is_valid_tile: - cur_tile_coord = work_tile.tile_idx - num_prev_subtiles = (tile_sched.num_tiles_executed * - subtile_cnt) - for subtile_idx in cutlass.range(subtile_cnt): - c_buffer = ((num_prev_subtiles + subtile_idx) % - self.num_c_stage) - cute.arch.barrier( - barrier_id=self.epilog_sync_bar_id, - number_of_threads=transform_threads, - ) - self._indexer_q_transform_rows( - sC, - c_buffer, - transform_warp_idx, - self.indexer_transform_warps, - cur_tile_coord[0], - cur_tile_coord[1], - cur_tile_coord[2], - subtile_idx, - mPacked_nml, - mIndexerScale_nml, - mPositionIds, - mCosSinCache, - ) - cute.arch.barrier( - barrier_id=self.epilog_sync_bar_id, - number_of_threads=transform_threads, - ) - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() + c_pipeline.producer_tail() griddepcontrol_launch_dependents() - @cute.jit - def _indexer_q_transform_rows( - self, - sC: cute.Tensor, - c_buffer: cutlass.Int32, - row_start: cutlass.Int32, - row_stride: cutlass.Constexpr, - head_idx: cutlass.Int32, - token_tile_idx: cutlass.Int32, - batch_idx: cutlass.Int32, - real_subtile_idx: cutlass.Int32, - mPacked_nml: cute.Tensor, - mIndexerScale_nml: cute.Tensor, - mPositionIds: cute.Tensor, - mCosSinCache: cute.Tensor, - ): - """Transform complete BF16 token rows from the shared epilogue tile.""" - lane_idx = cute.arch.lane_idx() - for row in cutlass.range(row_start, - self.epi_tile_n, - row_stride, - unroll_full=True): - token_idx = (token_tile_idx * self.cta_tile_shape_mnk[1] + - real_subtile_idx * self.epi_tile_n + row) - if token_idx < mPositionIds.shape[0]: - values = cute.make_rmem_tensor((4, ), cutlass.Float32) - for value_idx in cutlass.range_constexpr(4): - feature_idx = lane_idx * 4 + value_idx - values[value_idx] = cutlass.Float32(sC[(feature_idx, row, - c_buffer)]) - - if lane_idx >= 16: - position = mPositionIds[token_idx] - pair_base = (lane_idx * 4 - 64) // 2 - for value_idx in cutlass.range_constexpr(0, 4, 2): - cosine = mCosSinCache[position, - pair_base + value_idx // 2] - sine = mCosSinCache[position, - pair_base + value_idx // 2 + 32] - x = values[value_idx] - y = values[value_idx + 1] - values[value_idx] = (cosine * x - sine * y).to( - cutlass.BFloat16).to(cutlass.Float32) - values[value_idx + 1] = (cosine * y + sine * x).to( - cutlass.BFloat16).to(cutlass.Float32) - - amax = cute.arch.fmax( - cute.arch.fmax(values[0], -values[0]), - cute.arch.fmax(values[1], -values[1]), - ) - amax = cute.arch.fmax( - amax, - cute.arch.fmax( - cute.arch.fmax(values[2], -values[2]), - cute.arch.fmax(values[3], -values[3]), - ), - ) - amax = cute.arch.fmax( - amax, cute.arch.shuffle_sync_bfly(amax, offset=1)) - amax = cute.arch.fmax( - amax, cute.arch.shuffle_sync_bfly(amax, offset=2)) - amax = cute.arch.fmax( - amax, cute.arch.shuffle_sync_bfly(amax, offset=4)) - if amax < cutlass.Float32(1.0e-12): - amax = cutlass.Float32(1.0e-12) - - scale_reg = cute.make_rmem_tensor((1, ), cutlass.Float8E8M0FNU) - scale_reg[0] = (amax * cutlass.Float32(1.0 / 6.0)).to( - cutlass.Float8E8M0FNU) - scale_byte = cute.recast_tensor(scale_reg, cutlass.Uint8)[0] - exponent = cutlass.Uint32(scale_byte) - inverse_bits = cutlass.Uint32(0) - if exponent == cutlass.Uint32(254): - inverse_bits = cutlass.Uint32(0x00400000) - else: - inverse_bits = (cutlass.Uint32(254) - exponent) << 23 - inverse_scale = cutlass.Float32( - llvm.bitcast(cutlass.Float32.mlir_type, - inverse_bits.ir_value())) - for value_idx in cutlass.range_constexpr(4): - values[value_idx] = values[value_idx] * inverse_scale - - packed = _indexer_q_pack_fp4x4( - values[0], - values[1], - values[2], - values[3], - ) - midpoint_adjust = cutlass.Uint16(0) - for value_idx in cutlass.range_constexpr(4): - abs_value = cute.arch.fmax(values[value_idx], - -values[value_idx]) - nibble_adjust = cutlass.Uint16(1 << (value_idx * 4)) - if abs_value == cutlass.Float32(0.75): - midpoint_adjust = midpoint_adjust | nibble_adjust - if abs_value == cutlass.Float32(1.75): - midpoint_adjust = midpoint_adjust | nibble_adjust - if abs_value == cutlass.Float32(3.5): - midpoint_adjust = midpoint_adjust | nibble_adjust - packed = packed - midpoint_adjust - magnitude = packed & cutlass.Uint16(0x7777) - nonzero_sign = ((magnitude | (magnitude << 1) - | (magnitude << 2)) - & cutlass.Uint16(0x4444)) << 1 - packed = cutlass.Uint16(packed & (cutlass.Uint16(0x7777) - | nonzero_sign)) - mPacked_nml[head_idx * 32 + lane_idx, token_idx, - batch_idx] = packed - - exponent0 = cutlass.Uint32( - cute.arch.shuffle_sync(exponent, cutlass.Int32(0))) - exponent1 = cutlass.Uint32( - cute.arch.shuffle_sync(exponent, cutlass.Int32(8))) - exponent2 = cutlass.Uint32( - cute.arch.shuffle_sync(exponent, cutlass.Int32(16))) - exponent3 = cutlass.Uint32( - cute.arch.shuffle_sync(exponent, cutlass.Int32(24))) - if lane_idx == 0: - mIndexerScale_nml[ - head_idx, - token_idx, - batch_idx, - ] = (exponent0 | (exponent1 << 8) | (exponent2 << 16) - | (exponent3 << 24)) - def mainloop_s2t_copy_and_partition( self, sSF: cute.Tensor, @@ -2058,8 +1768,6 @@ def _compute_grid( cta_tile_shape_mnk: Tuple[int, int, int], cluster_shape_mn: Tuple[int, int], max_active_clusters: cutlass.Constexpr, - swizzle_size: cutlass.Constexpr = 1, - raster_along_m: cutlass.Constexpr = True, ) -> Tuple[utils.PersistentTileSchedulerParams, Tuple[int, int, int]]: """Computes the grid size for the kernel launch. @@ -2073,10 +1781,6 @@ def _compute_grid( cluster_shape_mn (Tuple[int, int]): The shape of a CTA cluster. max_active_clusters (cutlass.Constexpr): The maximum number of active clusters supported by the hardware. - swizzle_size (cutlass.Constexpr): Tile-scheduler swizzle size in - clusters (1 = no swizzle). - raster_along_m (cutlass.Constexpr): Tile-scheduler rasterization - order (M-major when True). Returns: A tuple containing the tile scheduler parameters and the computed @@ -2088,10 +1792,7 @@ def _compute_grid( cluster_shape_mnl = (*cluster_shape_mn, 1) tile_sched_params = utils.PersistentTileSchedulerParams( - num_ctas_mnl, - cluster_shape_mnl, - swizzle_size=swizzle_size, - raster_along_m=raster_along_m) + num_ctas_mnl, cluster_shape_mnl) grid = utils.StaticPersistentTileScheduler.get_grid_shape( tile_sched_params, max_active_clusters) @@ -2218,7 +1919,7 @@ def is_valid_tensor_alignment( m: cutlass.Int64, n: cutlass.Int64, k: cutlass.Int64, - l: cutlass.Int64, # noqa: E741 - CUTLASS names the batch mode L. + l: cutlass.Int64, ab_dtype: Type[cutlass.Numeric], c_dtype: Type[cutlass.Numeric], a_major: str, @@ -2270,7 +1971,7 @@ def can_implement( m: cutlass.Int64, n: cutlass.Int64, k: cutlass.Int64, - l: cutlass.Int64, # noqa: E741 - CUTLASS names the batch mode L. + l: cutlass.Int64, a_major: str, b_major: str, c_major: str, @@ -2328,7 +2029,7 @@ def wrapper( sf_m: cutlass.Int64, sf_n: cutlass.Int64, sf_k: cutlass.Int64, - l: cutlass.Constexpr, # noqa: E741 - Preserve the generic wrapper API. + l: cutlass.Constexpr, a_ptr: cute.Pointer, b_ptr: cute.Pointer, a_sf_ptr: cute.Pointer, @@ -2404,102 +2105,6 @@ def wrapper( self(a_tensor, b_tensor, sfa_tensor, sfb_tensor, c_tensor, alpha_tensor, max_active_clusters, current_stream, epilogue_op) - @cute.jit - def wrapper_indexer_q_swap_ab( - self, - m: cutlass.Int64, - n: cutlass.Int64, - k: cutlass.Int64, - sf_m: cutlass.Int64, - sf_n: cutlass.Int64, - sf_k: cutlass.Int64, - cos_sin_rows: cutlass.Int64, - batch_count: cutlass.Constexpr, - a_ptr: cute.Pointer, - b_ptr: cute.Pointer, - a_sf_ptr: cute.Pointer, - b_sf_ptr: cute.Pointer, - packed_ptr: cute.Pointer, - scale_ptr: cute.Pointer, - position_ids_ptr: cute.Pointer, - cos_sin_ptr: cute.Pointer, - alpha_tensor: cute.Tensor, - max_active_clusters: cutlass.Constexpr, - current_stream: cuda.CUstream, - ): - """Run indexer Q with features on MMA-M and tokens on MMA-N. - - The GEMM boundary is a logical column-major BF16 ``[N, M]`` view. - It is never written to global memory; the custom epilogue drains it to - the existing BF16 shared-memory stage, then writes the production - packed-FP4 and four-UE8M0-per-head outputs directly. - """ - # Swap A/B so the fixed 8192 output features occupy hardware MMA-M and - # the small dynamic token count occupies hardware MMA-N. - weight_tensor = cute.make_tensor( - b_ptr, - layout=cute.make_ordered_layout((n, k, batch_count), - order=(1, 0, 2)), - ) - input_tensor = cute.make_tensor( - a_ptr, - layout=cute.make_ordered_layout((m, k, batch_count), - order=(1, 0, 2)), - ) - weight_sf_tensor = cute.make_tensor( - b_sf_ptr, - layout=cute.make_ordered_layout( - (32, 4, sf_n, 4, sf_k, batch_count), - order=(2, 1, 4, 0, 3, 5), - ), - ) - input_sf_tensor = cute.make_tensor( - a_sf_ptr, - layout=cute.make_ordered_layout( - (32, 4, sf_m, 4, sf_k, batch_count), - order=(2, 1, 4, 0, 3, 5), - ), - ) - - # Only the shape/layout participate in scheduling and TMEM partitioning. - # The custom epilogue does not issue a TMA store through this tensor. - c_boundary_tensor = cute.make_tensor( - cute.recast_ptr(packed_ptr, dtype=cutlass.BFloat16), - layout=cute.make_ordered_layout((n, m, batch_count), - order=(0, 1, 2)), - ) - packed_tensor = cute.make_tensor( - cute.recast_ptr(packed_ptr, dtype=cutlass.Uint16), - layout=cute.make_ordered_layout((n // 4, m, batch_count), - order=(0, 1, 2)), - ) - scale_tensor = cute.make_tensor( - cute.recast_ptr(scale_ptr, dtype=cutlass.Uint32), - layout=cute.make_ordered_layout((n // 128, m, batch_count), - order=(0, 1, 2)), - ) - position_ids_tensor = cute.make_tensor(position_ids_ptr, - cute.make_layout((m, ))) - cos_sin_cache_tensor = cute.make_tensor( - cos_sin_ptr, - layout=cute.make_ordered_layout((cos_sin_rows, 64), order=(1, 0)), - ) - - self( - weight_tensor, - input_tensor, - weight_sf_tensor, - input_sf_tensor, - c_boundary_tensor, - alpha_tensor, - max_active_clusters, - current_stream, - packed_tensor=packed_tensor, - indexer_scale_tensor=scale_tensor, - position_ids_tensor=position_ids_tensor, - cos_sin_cache_tensor=cos_sin_cache_tensor, - ) - @cute.jit def cvt_sf_MKL_to_M32x4xrm_K4xrk_L( diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_decode_varlen.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_decode_varlen.py index ab16d033dba2..66b79c6986f8 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_decode_varlen.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_decode_varlen.py @@ -56,6 +56,7 @@ --top_k 2048 --do_ref_check --return_val --do_benchmark Constraints for this example: +* The problem size of top_k <= 2048. * The input tensor has data contiguous on the n dimension (row-major). * The supported input data types are Float32, Float16, or BFloat16. """ @@ -345,7 +346,7 @@ def filtered_topk_kernel( g_num_input = None s_indices = smem.allocate_tensor( element_type=self.index_type, - layout=cute.make_ordered_layout((self.top_k,), order=(0)), + layout=cute.make_ordered_layout((self.filtered_topk_max_k,), order=(0)), byte_alignment=128, ) s_input_idx = smem.allocate_tensor( @@ -1392,6 +1393,9 @@ def run_topk_decode( parser.add_argument("--use_cold_l2", action="store_true", default=True, help="Use cold L2") args = parser.parse_args() + if args.top_k % 2 != 0: + parser.error("top_k must be a multiple of 2 (got top_k={})".format(args.top_k)) + run_topk_decode( dtype=args.dtype, batch_size=args.batch_size, diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_varlen_util.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_varlen_util.py index 6725526407c2..be796763e8e2 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_varlen_util.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/filtered_top_k_varlen_util.py @@ -58,17 +58,8 @@ def __init__( self.num_ctas_per_row = num_ctas_per_row self.merge_blocks = merge_blocks - # top_k sizes the shared-memory index staging (s_indices), so bound it - # here. Direct callers and the run_topk_decode CLI bypass the decode - # wrappers, and an oversized top_k would otherwise surface as an opaque - # smem launch failure. 16384 matches the wrapper guards in - # cute_dsl_custom_ops.py. - if top_k <= 0 or top_k > 16384: - raise ValueError( - f"top_k must be in range [1, 16384], got {top_k}. " - "Maximum supported top_k is 16384 for Blackwell architecture." - ) - + # Note: now we only support top_k <= 2048, we could change the code here to support larger top_k. + self.filtered_topk_max_k = 2048 # 8 bits for radix-based filter. self.radix = 256 @@ -972,14 +963,13 @@ def filtered_topk_kernel_per_row( cute.arch.barrier() # Phase 3: Output phase - output_vector_width = 2 if self.top_k % 2 == 0 else 1 vecsize_out = cutlass.const_expr( min( self.top_k, cute.ceil_div(self.top_k, self.num_threads_per_cta), self.num_copy_bits // self.dtype.width, # TODO: only tested for float32. need to check for other dtypes. - output_vector_width, + 2, ) ) assert self.top_k % vecsize_out == 0 diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py index c879c57c36cf..72849ee9adc6 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/top_k/gvr_topk_decode.py @@ -231,18 +231,7 @@ def __init__( enable_smem_cache: bool = False, smem_cache_elems: int = 32768, seqlen_sorted: bool = False, - p4_warp_redundant: bool = True, - p2_warp_redundant: bool = True, ): - # Redundant-warp sync reduction: every warp replays the block - # reduce + decision from the same staged SMEM partials in the - # same fp32 order, so results are bit-identical across warps and - # the publish barrier + leader serialization disappear. - # p4_warp_redundant: P4 k-th bin search + snap loop (1 barrier/iter). - # p2_warp_redundant: P2 secant cadence (cluster_size == 1 only). - # Both default ON; OFF restores the leader-based paths (A/B). - self.p4_warp_redundant = p4_warp_redundant - self.p2_warp_redundant = p2_warp_redundant # cluster_size: number of CTAs cooperating per row. 1 = single-CTA # path; 2/4 = thread-block cluster with DSMEM aggregation. Capped at # 16 by B200's per-GPC SM count. @@ -503,38 +492,6 @@ def warp_reduce_max_f32(self, val): # PTX redux.sync.fmax.f32 (sm_100). return cute.arch.warp_redux_sync(val, "fmax") - # ------------------------------------------------------------------ - # Raw-address SMEM scalar access through a pre-hoisted window base. - # - # Tensor-indexed SMEM access (smem_keys[i]) makes the compiler - # re-derive the cluster SMEM window per access (S2R SR_CgaCtaId + - # LEA<<24) — ncu shows this as the top single-instruction stall in - # the P3 stream-write and P4 snap loops. Hoisting the base once via - # iterator.toint() (one S2R per call site) turns every subsequent - # access into plain integer addressing — the same pattern the P2 - # scan loops already use for smem_input, whose SASS regions show no - # S2R at all. - # ------------------------------------------------------------------ - @cute.jit - def _smem_ref(self, dtype: cutlass.Constexpr, base_addr, idx): - elem_bytes = cutlass.const_expr(dtype.width // 8) - p = cute.make_ptr( - dtype, - base_addr + cutlass.Int64(idx) * cutlass.Int64(elem_bytes), - cute.AddressSpace.smem, - assumed_align=4, - ) - return cute.make_tensor(p, cute.make_layout((1,))) - - @cute.jit - def _smem_ld(self, dtype: cutlass.Constexpr, base_addr, idx): - return self._smem_ref(dtype, base_addr, idx)[0] - - @cute.jit - def _smem_st(self, dtype: cutlass.Constexpr, base_addr, idx, val): - t = self._smem_ref(dtype, base_addr, idx) - t[0] = val - # ------------------------------------------------------------------ # Phase 1: preIdx Min/Max/Mean -> initial threshold # ------------------------------------------------------------------ @@ -726,31 +683,17 @@ def block_count_ge( lane, do_cluster_sync, # bool: False = skip DSMEM aggregation (cs=1 / short-row degrade) smem_input=None, # optional SMEM-cached slice (smem_input[i] == input_row[slice_start+i]) - redundant=False, # trace-time: every-warp reduce, return the total - wcnt_off=None, # int32 staging bank offset into smem_wcnt (parity) ): """Count input[i] >= threshold across this CTA's row slice, then DSMEM-aggregate across the cluster. - ``redundant=True`` (p2_warp_redundant, cluster_size == 1 only): - after the staging barrier EVERY warp reduces the warp counts - lane-parallel and the block total RETURNS in a register — - bit-identical across warps — instead of a leader writing - s_iscalars[0] for a barrier-published broadcast. ``wcnt_off`` - parity-banks the smem_wcnt staging so a warp that has moved on - to the next Phase-2 round cannot clobber a slot a slower warp is - still reading (the per-round staging barrier bounds the drift to - one round). - Vectorized scan: each thread loads vec_w elements per iter (128 or 256 bits) over ``input_row[slice_start : slice_end)``; scalar tail handles the remainder. Cluster aggregation (cluster_size > 1): every CTA stages its - slice-local count into ``s_cluster_partial[call & 1]`` (parity - double-buffer; slot 2 is the tid0-private call counter), syncs the - cluster, then DSMEM-reads every peer's slot and sums into - ``s_iscalars[0]``. + slice-local count into ``s_cluster_partial[0]``, syncs the cluster, + then DSMEM-reads every peer's slot and sums into ``s_iscalars[0]``. After this every CTA's ``s_iscalars[0]`` holds the same cluster-wide cand_count, so Phase 2's secant update stays a leader-only scalar op on a value all CTAs agree on. @@ -875,22 +818,10 @@ def block_count_ge( # Warp reduce + lane-0 write wc = self.warp_reduce_sum_i32(c) - stage_base = cutlass.Int32(0) - if cutlass.const_expr(wcnt_off is not None): - stage_base = wcnt_off if lane == 0: - smem_wcnt[stage_base + warp_id] = wc + smem_wcnt[warp_id] = wc cute.arch.barrier() - if cutlass.const_expr(redundant): - # Every warp reduces the staged counts itself; no leader, no - # publish barrier, no s_iscalars[0] round-trip. - v_r = cutlass.Int32(0) - if lane < cutlass.Int32(self.num_warps): - v_r = smem_wcnt[stage_base + lane] - total_r = self.warp_reduce_sum_i32(v_r) - return total_r - # Block aggregate (sum reduce over num_warps slots). No trailing # barrier: caller is expected to insert its own __syncthreads after # its post-processing of cand_count. @@ -930,18 +861,8 @@ def block_count_ge( if cutlass.const_expr(cluster_size > 1): if do_cluster_sync: cute.arch.barrier() # publish s_iscalars[0] to all threads of this CTA - # Parity double-buffer: with a single slot, a straggler's - # post-wait DSMEM read races the peer's next-call overwrite - # (PTX-model data race). Writing call k into slot k&1 orders - # the call-(k+2) overwrite after my call-k reads via the - # call-(k+1) rendezvous. Slot 2 = tid0-private call counter - # (zeroed per row); do_cluster_sync is row-uniform, so CTAs - # step the counter in lockstep and parity stays aligned. - par = cutlass.Int32(0) if tidx == cutlass.Int32(0): - par = s_cluster_partial[2] - s_cluster_partial[par & cutlass.Int32(1)] = s_iscalars[0] - s_cluster_partial[2] = par + cutlass.Int32(1) + s_cluster_partial[0] = s_iscalars[0] # Non-relaxed arrive: pairs with the peer cluster_wait acquire # to release s_cluster_partial writes so the DSMEM ld below # observes them. cluster_arrive_relaxed would skip the release @@ -951,15 +872,13 @@ def block_count_ge( cute.arch.cluster_wait() if tidx == cutlass.Int32(0): total = cutlass.Int32(0) - local_ptr = s_cluster_partial.iterator + (par & cutlass.Int32(1)) + local_ptr = s_cluster_partial.iterator + cutlass.Int32(0) for peer in cutlass.range_constexpr(cluster_size): peer_addr = mapa_shared_cluster(local_ptr, cutlass.Int32(peer)) total = total + ld_shared_cluster_i32(peer_addr) s_iscalars[0] = total cute.arch.barrier() # broadcast cluster total within this CTA - return cutlass.Int32(0) - # ------------------------------------------------------------------ # Phase 2: Secant-interpolation threshold search # Refines threshold to bring cand_count into [kK, kCC] using secant @@ -977,7 +896,7 @@ def phase2_secant_search( smem_wcnt, s_thr, # [threshold, val_lo, val_hi] s_iscalars, # [cand_count, done, cnt_lo, cnt_hi, out_count] - s_cluster_partial, # [3] int32 cluster scratch (parity slots + counter) + s_cluster_partial, # [1] int32 cluster scratch tidx, warp_id, lane, @@ -994,120 +913,6 @@ def phase2_secant_search( kCC = cutlass.const_expr(self.kC) kFTarget = cutlass.const_expr(self.kFTarget) - if cutlass.const_expr(self.p2_warp_redundant and self.cluster_size == 1): - # ---- Redundant-warp cadence: ONE barrier per round ---- - # The whole secant state (threshold, bracket, counts, done) - # lives in registers; every warp reduces the staged warp - # counts itself (block_count_ge redundant mode) and replays - # the identical classify + secant update, so the per-round - # publish barriers and every s_thr/s_iscalars SMEM round-trip - # (with its per-access cluster-window S2R recompute) - # disappear. Canonical exit state is written once for P3. - nwp2 = cutlass.const_expr(self.num_warps) - thr_r = s_thr[0] - vlo_r = s_thr[1] - vhi_r = s_thr[2] - clo_r = s_iscalars[2] - chi_r = s_iscalars[3] - done_r = cutlass.Int32(0) - par_r = cutlass.Int32(0) - cnt_r = self.block_count_ge( - input_row, - slice_start, - slice_end, - thr_r, - smem_ptcnt, - smem_wcnt, - s_iscalars, - s_cluster_partial, - tidx, - warp_id, - lane, - cutlass.Boolean(False), # do_cluster_sync (cs==1 gate) - smem_input=smem_input, - redundant=True, - wcnt_off=par_r * cutlass.Int32(nwp2), - ) - if cnt_r >= cutlass.Int32(kK) and cnt_r <= cutlass.Int32(kCC): - done_r = cutlass.Int32(1) - elif cnt_r > cutlass.Int32(kCC): - vlo_r = thr_r - clo_r = cnt_r - else: - vhi_r = thr_r - chi_r = cnt_r - it = cutlass.Int32(0) - while it < cutlass.Int32(self.MAX_REFINE_ITERS) and done_r == cutlass.Int32(0): - rng = vhi_r - vlo_r - nv = cutlass.Float32(0.0) - if clo_r > chi_r and rng > cutlass.Float32(1e-10): - f = cutlass.Float32(clo_r - cutlass.Int32(kFTarget)) / cutlass.Float32( - clo_r - chi_r - ) - f = cute.arch.fmax(cutlass.Float32(0.05), f) - f = _fmin_f32_inline(f, cutlass.Float32(0.95)) - if it == cutlass.Int32(0): - f = _fmin_f32_inline(f, cutlass.Float32(0.5)) - nv = vlo_r + rng * f - else: - nv = (vlo_r + vhi_r) * cutlass.Float32(0.5) - if nv <= vlo_r: - nv = vlo_r + rng * cutlass.Float32(0.05) - if nv >= vhi_r: - nv = vhi_r - rng * cutlass.Float32(0.05) - if nv == vlo_r or nv == vhi_r: - nv = (vlo_r + vhi_r) * cutlass.Float32(0.5) - if nv == vlo_r or nv == vhi_r: - thr_r = vlo_r - done_r = cutlass.Int32(2) - if done_r == cutlass.Int32(0): - thr_r = nv - par_r = par_r ^ cutlass.Int32(1) - cnt_r = self.block_count_ge( - input_row, - slice_start, - slice_end, - thr_r, - smem_ptcnt, - smem_wcnt, - s_iscalars, - s_cluster_partial, - tidx, - warp_id, - lane, - cutlass.Boolean(False), # do_cluster_sync (cs==1 gate) - smem_input=smem_input, - redundant=True, - wcnt_off=par_r * cutlass.Int32(nwp2), - ) - if cnt_r >= cutlass.Int32(kK) and cnt_r <= cutlass.Int32(kCC): - done_r = cutlass.Int32(1) - elif cnt_r > cutlass.Int32(kCC): - vlo_r = thr_r - clo_r = cnt_r - else: - vhi_r = thr_r - chi_r = cnt_r - it = it + cutlass.Int32(1) - if done_r == cutlass.Int32(0): - if clo_r <= cutlass.Int32(kCC * 2): - thr_r = vlo_r - else: - thr_r = vhi_r - done_r = cutlass.Int32(2) - # Canonical exit state for Phase 3/4 (byte-compatible with the - # leader path), published once. - if tidx == 0: - s_thr[0] = thr_r - s_thr[1] = vlo_r - s_thr[2] = vhi_r - s_iscalars[0] = cnt_r - s_iscalars[1] = done_r - s_iscalars[2] = clo_r - s_iscalars[3] = chi_r - cute.arch.barrier() - return - # ---- Initial count with the Phase-1 mean as threshold ---- # TODO: smem_ptcnt is not always needed? only for the last block_count_ge. # Do we have methods to reduce its write? @@ -1396,11 +1201,6 @@ def phase3_collect_candidates( copy_atom = self._make_load_copy_atom() row_addr = input_row.iterator.toint() step_elem = cutlass.const_expr(num_threads * vec_w) - # Hoisted SMEM window bases (one S2R here vs one per emitted - # candidate below — this loop is the kernel's biggest instruction - # region at production shapes). - keys_base = smem_keys.iterator.toint() - vals_base = smem_vals.iterator.toint() slice_len = slice_end - slice_start # When reading from the cached slice, scan indices are slice-LOCAL; @@ -1458,10 +1258,8 @@ def phase3_collect_candidates( else: vj = cutlass.Float32(rng_frag[j]) if vj >= thr_final and wc < cutlass.Int32(kCC): - self._smem_st(cutlass.Float32, keys_base, wc, vj) - self._smem_st( - cutlass.Int32, vals_base, wc, global_base + cutlass.Int32(j) - ) + smem_keys[wc] = vj + smem_vals[wc] = global_base + cutlass.Int32(j) wc = wc + cutlass.Int32(1) # Advance ic past all consumed vec_w-aligned positions. ic = ic + big_iters * cutlass.Int32(step_elem) @@ -1493,8 +1291,8 @@ def phase3_collect_candidates( else: vj = cutlass.Float32(tail_frag[j]) if vj >= thr_final and wc < cutlass.Int32(kCC): - self._smem_st(cutlass.Float32, keys_base, wc, vj) - self._smem_st(cutlass.Int32, vals_base, wc, global_base_t + cutlass.Int32(j)) + smem_keys[wc] = vj + smem_vals[wc] = global_base_t + cutlass.Int32(j) wc = wc + cutlass.Int32(1) ic = ic + step @@ -1510,8 +1308,8 @@ def phase3_collect_candidates( v = self._load_fp32(input_row, it) pos_global = it if v >= thr_final and wc < cutlass.Int32(kCC): - self._smem_st(cutlass.Float32, keys_base, wc, v) - self._smem_st(cutlass.Int32, vals_base, wc, pos_global) + smem_keys[wc] = v + smem_vals[wc] = pos_global wc = wc + cutlass.Int32(1) it = it + cutlass.Int32(num_threads) cute.arch.barrier() @@ -1522,7 +1320,7 @@ def phase3_collect_candidates( @cute.jit def block_fused_snap_iter( self, - keys_base, # hoisted SMEM window base of smem_keys (iterator.toint()) + smem_keys, smem_wcnt, smem_hist, # reused as scratch for s_up/s_down warp aggregates s_thr, @@ -1547,7 +1345,7 @@ def block_fused_snap_iter( isi = tidx while isi < count: - v = self._smem_ld(cutlass.Float32, keys_base, isi) + v = smem_keys[isi] if v >= thr: lge = lge + cutlass.Int32(1) if v > thr: @@ -1636,257 +1434,6 @@ def block_fused_snap_iter( s_thr[0] = total_down cute.arch.barrier() - # ------------------------------------------------------------------ - # P4 helpers: histogram build + parallel k-th bin search. Factored - # out so the level-2 refinement can rerun both over a narrowed window. - # ------------------------------------------------------------------ - @cute.jit - def _hist_build(self, keys_base, smem_hist, cand_count, lo, inv, tidx): - """Zero smem_hist[0:kBins], then histogram keys[0:cand_count] with - bin = clamp(int((v - lo) * inv), 0, kBins-1). Out-of-window values - clamp into the edge bins, which keeps cumulative counts from the - top exact for the k-th search (everything above the window lands - in the top bin). Barrier after the zero pass and after the build.""" - kBins = cutlass.const_expr(self.kNumBins) - num_threads = cutlass.const_expr(self.num_threads) - i6 = tidx - while i6 < cutlass.Int32(kBins): - smem_hist[i6] = cutlass.Int32(0) - i6 = i6 + cutlass.Int32(num_threads) - cute.arch.barrier() - i7 = tidx - while i7 < cand_count: - vk = self._smem_ld(cutlass.Float32, keys_base, i7) - bin_f = (vk - lo) * inv - # Clamp in the FLOAT domain before the int cast: fptosi is - # undefined for out-of-range/NaN inputs at the IR level (PTX - # cvt.rzi saturates, but LLVM may optimize on the poison). - # fmax first canonicalizes NaN to 0; the pair keeps the - # edge-bin clamping semantics bit-identical for in-range - # values. - bin_f = cute.arch.fmax(bin_f, cutlass.Float32(0.0)) - bin_f = _fmin_f32_inline(bin_f, cutlass.Float32(kBins - 1)) - bin_i = cutlass.Int32(bin_f) - atomicAdd(smem_hist.iterator + bin_i, cutlass.Int32(1)) - i7 = i7 + cutlass.Int32(num_threads) - cute.arch.barrier() - - @cute.jit - def _kth_bin_search( - self, smem_hist, smem_wcnt, s_thr, s_iscalars, lo, binw, tidx, warp_id, lane - ): - """Parallel k-th bin search (3-step, high→low). Writes - s_thr[0] = lower edge of the selected bin (lo + bidx*binw) and - s_iscalars[4] = selected bin's count (gates the level-2 histogram - refinement). Clobbers s_iscalars[2]/[3] as staging (both are - rewritten by the snap loop before anyone else reads them). - Trailing barrier.""" - kK = cutlass.const_expr(self.top_k) - kBins = cutlass.const_expr(self.kNumBins) - bins_per_warp = cutlass.const_expr(kBins // self.num_warps) - - # Step 1: each warp sums BINS_PER_WARP bins (high→low slice). - # Lane-parallel when the slice divides evenly across the warp: - # each lane sums bins_per_warp/32 bins + one warp reduce, instead - # of every lane redundantly walking a bins_per_warp-deep serial - # LDS+IADD dependency chain (~7% of stall samples at N=8K). - warp_bin_sum = cutlass.Int32(0) - if cutlass.const_expr(bins_per_warp % self.WARP_SIZE == 0): - for jm in cutlass.range_constexpr(bins_per_warp // self.WARP_SIZE): - bidx_s = ( - cutlass.Int32(kBins - 1) - - warp_id * cutlass.Int32(bins_per_warp) - - (lane + cutlass.Int32(jm * self.WARP_SIZE)) - ) - warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] - warp_bin_sum = self.warp_reduce_sum_i32(warp_bin_sum) - else: - for jb in cutlass.range_constexpr(bins_per_warp): - bidx_s = ( - cutlass.Int32(kBins - 1) - - warp_id * cutlass.Int32(bins_per_warp) - - cutlass.Int32(jb) - ) - warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] - if lane == 0: - smem_wcnt[warp_id] = warp_bin_sum - cute.arch.barrier() - - # Step 2: tid==0 finds target warp; stores prefix-count + warp index - # into s_iscalars[2] (=cnt_lo: prefix before target warp) - # and s_iscalars[3] (=cnt_hi: target warp index) - if tidx == 0: - cum = cutlass.Int32(0) - tw = cutlass.Int32(self.num_warps - 1) - found = cutlass.Int32(0) - for w2 in cutlass.range_constexpr(self.num_warps): - cum = cum + smem_wcnt[w2] - if cum >= cutlass.Int32(kK) and found == cutlass.Int32(0): - tw = cutlass.Int32(w2) - found = cutlass.Int32(1) - # Recompute prefix BEFORE target warp - cum2 = cutlass.Int32(0) - for w3 in cutlass.range_constexpr(self.num_warps): - if cutlass.Int32(w3) < tw: - cum2 = cum2 + smem_wcnt[w3] - s_iscalars[2] = cum2 # prefix - s_iscalars[3] = tw # target warp index - cute.arch.barrier() - - # Step 3: target warp's lane 0 scans BINS_PER_WARP bins → - # threshold. Single-thread serial; the unrolled - # range_constexpr beats a runtime `for+break` (tried it: -544 - # SASS insts but -7pp fp32 / -14pp bf16, since the - # branch/counter overhead in a single thread dominates the - # static math). - target_warp = s_iscalars[3] - if warp_id == target_warp and lane == cutlass.Int32(0): - base_cum = s_iscalars[2] - thr_local = lo - sel_cnt = cutlass.Int32(0) - set_done = cutlass.Int32(0) - for jb2 in cutlass.range_constexpr(bins_per_warp): - bidx2 = ( - cutlass.Int32(kBins - 1) - - target_warp * cutlass.Int32(bins_per_warp) - - cutlass.Int32(jb2) - ) - cnt_here = smem_hist[bidx2] - base_cum = base_cum + cnt_here - if base_cum >= cutlass.Int32(kK) and set_done == cutlass.Int32(0): - thr_local = lo + cutlass.Float32(bidx2) * binw - sel_cnt = cnt_here - set_done = cutlass.Int32(1) - s_thr[0] = thr_local - s_iscalars[4] = sel_cnt - cute.arch.barrier() - - # ------------------------------------------------------------------ - # _kth_bin_search_rw — redundant-warp variant (p4_warp_redundant). - # Step 1 stages per-warp bin-slice sums exactly like _kth_bin_search - # (the ONE barrier). Then EVERY warp redundantly (a) walks the - # num_warps slot sums with broadcast SMEM reads + predicated adds to - # locate the target warp, and (b) lane-parallel walks the target - # slice — each lane owns a contiguous descending sub-range, a - # shuffle-up prefix + the unique sub-range crossing test find the - # k-th bin in O(bins_per_warp/32) LDS instead of a 64-deep serial - # LDS+IADD chain in one thread. Same inputs in the same order on - # every warp -> bit-identical results, so there is no leader, no - # publish barrier, and no s_thr/s_iscalars staging; the selected - # (threshold, bin count) return in registers. - # ------------------------------------------------------------------ - @cute.jit - def _kth_bin_search_rw(self, smem_hist, smem_wcnt, lo, binw, tidx, warp_id, lane): - kK = cutlass.const_expr(self.top_k) - kBins = cutlass.const_expr(self.kNumBins) - bins_per_warp = cutlass.const_expr(kBins // self.num_warps) - - # Step 1: identical staging to _kth_bin_search. - warp_bin_sum = cutlass.Int32(0) - if cutlass.const_expr(bins_per_warp % self.WARP_SIZE == 0): - for jm in cutlass.range_constexpr(bins_per_warp // self.WARP_SIZE): - bidx_s = ( - cutlass.Int32(kBins - 1) - - warp_id * cutlass.Int32(bins_per_warp) - - (lane + cutlass.Int32(jm * self.WARP_SIZE)) - ) - warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] - warp_bin_sum = self.warp_reduce_sum_i32(warp_bin_sum) - else: - for jb in cutlass.range_constexpr(bins_per_warp): - bidx_s = ( - cutlass.Int32(kBins - 1) - - warp_id * cutlass.Int32(bins_per_warp) - - cutlass.Int32(jb) - ) - warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] - if lane == 0: - smem_wcnt[warp_id] = warp_bin_sum - cute.arch.barrier() - - # Step 2 (every warp, lane-parallel): lane w holds slot w; an - # inclusive idx-shuffle scan + ballot locate the target warp. - # (shuffle_sync with a computed source lane is the working shfl - # idiom; shuffle_sync_up ignores its offset — probed.) - v_s = cutlass.Int32(0) - if lane < cutlass.Int32(self.num_warps): - v_s = smem_wcnt[lane] - run2 = v_s - for d2 in cutlass.range_constexpr(5): - off2 = cutlass.const_expr(1 << d2) - src2 = lane - cutlass.Int32(off2) - if src2 < cutlass.Int32(0): - src2 = cutlass.Int32(0) - up2 = cute.arch.shuffle_sync(run2, src2) - if lane >= cutlass.Int32(off2): - run2 = run2 + up2 - m2 = cute.arch.vote_ballot_sync(run2 >= cutlass.Int32(kK)) - tw = cutlass.Int32(self.num_warps - 1) - if m2 != cutlass.Uint32(0): - low2 = m2 & (cutlass.Uint32(0) - m2) - tw = cutlass.Int32(cute.arch.popc(low2 - cutlass.Uint32(1))) - incl_tw = cute.arch.shuffle_sync(run2, tw) - slot_tw = cute.arch.shuffle_sync(v_s, tw) - prefix = incl_tw - slot_tw - - # Step 3 (every warp, lane-parallel): lane l owns the contiguous - # descending positions [l*ppl, (l+1)*ppl) of the target slice. - ppl = cutlass.const_expr((bins_per_warp + self.WARP_SIZE - 1) // self.WARP_SIZE) - cnt_frag = cute.make_fragment((ppl,), cutlass.Int32) - my_sum = cutlass.Int32(0) - for j3 in cutlass.range_constexpr(ppl): - pos = lane * cutlass.Int32(ppl) + cutlass.Int32(j3) - cnt_j = cutlass.Int32(0) - if pos < cutlass.Int32(bins_per_warp): - bidx3 = cutlass.Int32(kBins - 1) - tw * cutlass.Int32(bins_per_warp) - pos - cnt_j = smem_hist[bidx3] - cnt_frag[j3] = cnt_j - my_sum = my_sum + cnt_j - # Exclusive cross-lane prefix of the lane partial sums via the - # idx-shuffle scan (5 log-steps; shuffle_sync_up ignores its - # offset — probed — so the scan uses computed source lanes). - run3 = my_sum - for d3 in cutlass.range_constexpr(5): - off3 = cutlass.const_expr(1 << d3) - src3 = lane - cutlass.Int32(off3) - if src3 < cutlass.Int32(0): - src3 = cutlass.Int32(0) - up3 = cute.arch.shuffle_sync(run3, src3) - if lane >= cutlass.Int32(off3): - run3 = run3 + up3 - base3 = prefix + (run3 - my_sum) - - # Unique crossing: the lane where the running count passes kK. - thr_loc = lo - sel_loc = cutlass.Int32(0) - hit = cutlass.Int32(0) - r3 = base3 - for j4 in cutlass.range_constexpr(ppl): - pos4 = lane * cutlass.Int32(ppl) + cutlass.Int32(j4) - cnt4 = cnt_frag[j4] - if ( - pos4 < cutlass.Int32(bins_per_warp) - and r3 < cutlass.Int32(kK) - and r3 + cnt4 >= cutlass.Int32(kK) - and hit == cutlass.Int32(0) - ): - bidx4 = cutlass.Int32(kBins - 1) - tw * cutlass.Int32(bins_per_warp) - pos4 - thr_loc = lo + cutlass.Float32(bidx4) * binw - sel_loc = cnt4 - hit = cutlass.Int32(1) - r3 = r3 + cnt4 - # Broadcast from the (at most one) hitting lane; no hit keeps - # (lo, 0) — same fallback as _kth_bin_search's set_done guard. - mask3 = cute.arch.vote_ballot_sync(hit != cutlass.Int32(0)) - thr_out = lo - sel_out = cutlass.Int32(0) - if mask3 != cutlass.Uint32(0): - low = mask3 & (cutlass.Uint32(0) - mask3) - src = cutlass.Int32(cute.arch.popc(low - cutlass.Uint32(1))) - thr_out = cute.arch.shuffle_sync(thr_loc, src) - sel_out = cute.arch.shuffle_sync(sel_loc, src) - return thr_out, sel_out - # ------------------------------------------------------------------ # Phase 4: Histogram-based k-th selection + two-pass writeback # ------------------------------------------------------------------ @@ -1914,191 +1461,151 @@ def phase4_histogram_snap( kK = cutlass.const_expr(self.top_k) kBins = cutlass.const_expr(self.kNumBins) num_threads = cutlass.const_expr(self.num_threads) - # Hoisted SMEM window bases: every keys/vals element access below - # goes through raw integer addressing (see _smem_ref rationale). - keys_base = smem_keys.iterator.toint() - vals_base = smem_vals.iterator.toint() - # Scalars base for the snap-loop convergence check (read by ALL - # threads once per snap iteration — a measured per-iteration - # LDS hotspot). - isc_base = s_iscalars.iterator.toint() + num_warps = cutlass.const_expr(self.num_warps) + bins_per_warp = cutlass.const_expr(kBins // self.num_warps) # ----- Branch A: cand_count == kK (fast path) ----- if cand_count == cutlass.Int32(kK): i4 = tidx while i4 < cutlass.Int32(kK): if cutlass.const_expr(self.return_output_values): - output_values_row[i4] = self.dtype( - self._smem_ld(cutlass.Float32, keys_base, i4) - ) - output_indices_row[i4] = self._smem_ld(cutlass.Int32, vals_base, i4) + output_values_row[i4] = self.dtype(smem_keys[i4]) + output_indices_row[i4] = smem_vals[i4] i4 = i4 + cutlass.Int32(num_threads) elif cand_count > cutlass.Int32(kK): # ----- Branch B: cand_count > kK → histogram snap ----- - # ---- Histogram window ---- - # Fast path: reuse the P2 exit bracket [vlo, vhi) instead of - # scanning candidates for min/max. P3 collected v >= s_thr[0] - # and P2's exit sets s_thr[0] = vlo (= s_thr[1]), so vlo - # lower-bounds every candidate; the bracket invariant - # cnt(>= vhi) < kK puts the k-th value inside [vlo, vhi). - # Out-of-window candidates (row max etc.) clamp into the edge - # bins — cumulative counts from the top stay exact — and the - # bracket is P2's acceptance band, far narrower than - # [cand_min, cand_max], so level-1 bin resolution IMPROVES. - # Any path that leaves the bracket stale (degenerate-bracket - # fallback, probe variants) fails the guard and takes the - # original min/max scan; a plausible-but-wrong bracket can - # only cost extra snap/refinement steps, never exactness. - # Uniform branch: SMEM scalars read after the P3-exit barrier. - w_lo = s_thr[1] - w_hi = s_thr[2] - bmin_r = cutlass.Float32(0.0) - bmax_r = cutlass.Float32(1e-6) - if s_thr[0] == w_lo and w_hi > w_lo and w_hi < cutlass.Float32(self.FLT_MAX): - bmin_r = w_lo - bmax_r = w_hi - else: - # Block min/max over keys[0:cand_count] - local_cmin = cutlass.Float32(self.FLT_MAX) - local_cmax = cutlass.Float32(self.NEG_FLT_MAX) - i5 = tidx - while i5 < cand_count: - v = self._smem_ld(cutlass.Float32, keys_base, i5) - local_cmin = _fmin_f32_inline(local_cmin, v) - local_cmax = cute.arch.fmax(local_cmax, v) - i5 = i5 + cutlass.Int32(num_threads) - cmin = self.warp_reduce_min_f32(local_cmin) - cmax = self.warp_reduce_max_f32(local_cmax) - # Stage warp results into smem_wcnt[w] (cmin) and smem_hist[w] (cmax) - # as bit-cast int32. cmax stored at smem_hist[0..NW-1]. - if lane == 0: - smem_wcnt[warp_id] = float_as_uint32(cmin) - smem_hist[warp_id] = float_as_uint32(cmax) - cute.arch.barrier() + # Block min/max over keys[0:cand_count] + local_cmin = cutlass.Float32(self.FLT_MAX) + local_cmax = cutlass.Float32(self.NEG_FLT_MAX) + i5 = tidx + while i5 < cand_count: + v = smem_keys[i5] + local_cmin = _fmin_f32_inline(local_cmin, v) + local_cmax = cute.arch.fmax(local_cmax, v) + i5 = i5 + cutlass.Int32(num_threads) + cmin = self.warp_reduce_min_f32(local_cmin) + cmax = self.warp_reduce_max_f32(local_cmax) + # Stage warp results into smem_wcnt[w] (cmin) and smem_hist[w] (cmax) + # as bit-cast int32. cmax stored at smem_hist[0..NW-1]. + if lane == 0: + smem_wcnt[warp_id] = float_as_uint32(cmin) + smem_hist[warp_id] = float_as_uint32(cmax) + cute.arch.barrier() - # Every thread independently recomputes block_min/block_max - # from the warp-staged smem slots (CUDA heuristic_topk.cuh:891-898 - # pattern). No tid==0 → s_thr broadcast → saves a block barrier. - bmin_r = cutlass.Float32(self.FLT_MAX) - bmax_r = cutlass.Float32(self.NEG_FLT_MAX) - # Unrolled num_warps times (16 or 32 — fixed at compile time). - for w in cutlass.range_constexpr(self.num_warps): - vmin_bits = smem_wcnt[w] - vmax_bits = smem_hist[w] - vmin = cutlass.Float32( - llvm.bitcast(cutlass.Float32.mlir_type, vmin_bits.ir_value()) - ) - vmax = cutlass.Float32( - llvm.bitcast(cutlass.Float32.mlir_type, vmax_bits.ir_value()) - ) - bmin_r = _fmin_f32_inline(bmin_r, vmin) - bmax_r = cute.arch.fmax(bmax_r, vmax) - if bmax_r <= bmin_r: - bmax_r = bmin_r + cutlass.Float32(1e-6) - # Barrier required: smem_hist[0..NW-1] above doubles as cmax - # scratch and below as the histogram. Without this sync the - # zeroing pass below can clobber a cmax slot a later warp is - # still reading → wrong bmax_r → all candidates squashed into - # bin 0 (hit-rate-dependent race). - cute.arch.barrier() + # Every thread independently recomputes block_min/block_max + # from the warp-staged smem slots (CUDA heuristic_topk.cuh:891-898 + # pattern). No tid==0 → s_thr broadcast → saves a block barrier. + bmin_r = cutlass.Float32(self.FLT_MAX) + bmax_r = cutlass.Float32(self.NEG_FLT_MAX) + # Unrolled num_warps times (16 or 32 — fixed at compile time). + for w in cutlass.range_constexpr(self.num_warps): + vmin_bits = smem_wcnt[w] + vmax_bits = smem_hist[w] + vmin = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vmin_bits.ir_value()) + ) + vmax = cutlass.Float32( + llvm.bitcast(cutlass.Float32.mlir_type, vmax_bits.ir_value()) + ) + bmin_r = _fmin_f32_inline(bmin_r, vmin) + bmax_r = cute.arch.fmax(bmax_r, vmax) + if bmax_r <= bmin_r: + bmax_r = bmin_r + cutlass.Float32(1e-6) + # Barrier required: smem_hist[0..NW-1] above doubles as cmax + # scratch and below as the histogram. Without this sync the + # zeroing pass below can clobber a cmax slot a later warp is + # still reading → wrong bmax_r → all candidates squashed into + # bin 0 (hit-rate-dependent race). + cute.arch.barrier() + + # Zero histogram (must zero ALL slots since smem_hist[0..NW-1] was + # used as cmax scratch above). + i6 = tidx + while i6 < cutlass.Int32(kBins): + smem_hist[i6] = cutlass.Int32(0) + i6 = i6 + cutlass.Int32(num_threads) + cute.arch.barrier() range1 = bmax_r - bmin_r - # Overflow hardening (pre-existing): - # a candidate span > FLT_MAX (needs |v| ~ 1.7e38; fuzz-only for - # real logits) overflows range1 to +inf → inv1 = +0 → every - # candidate lands in bin 0 → thr = lo + 0*inf = NaN → all snap - # comparisons false, the walk never moves, and the whole row - # writes as padding. Clamp to FLT_MAX: the start threshold - # stays ORDERED (±inf is fine — snap's monotone walk rescues - # any ordered start; only NaN breaks it). - if range1 > cutlass.Float32(self.FLT_MAX): - range1 = cutlass.Float32(self.FLT_MAX) # inv1 = (kBins - 1 + 0.99) / range1 (range1 > 0 guaranteed by 1e-6 patch) inv1 = (cutlass.Float32(kBins - 1) + cutlass.Float32(0.99)) / range1 - binw1 = range1 / cutlass.Float32(kBins) - - # Predeclared register state for the redundant-warp path - # (threshold / counts / staging parity live in registers; the - # leader path below keeps them in s_thr/s_iscalars instead). - thr_reg = bmin_r - selc_reg = cutlass.Int32(0) - thr_s = bmin_r - cge_r = cutlass.Int32(0) - cgt_r = cutlass.Int32(0) - win_par = cutlass.Int32(0) - - # Level-1: histogram over [bmin, bmax] + k-th bin search. - self._hist_build(keys_base, smem_hist, cand_count, bmin_r, inv1, tidx) - if cutlass.const_expr(self.p4_warp_redundant): - thr_reg, selc_reg = self._kth_bin_search_rw( - smem_hist, smem_wcnt, bmin_r, binw1, tidx, warp_id, lane - ) - else: - self._kth_bin_search( - smem_hist, smem_wcnt, s_thr, s_iscalars, bmin_r, binw1, tidx, warp_id, lane + + # Build histogram by atomicAdd. + i7 = tidx + while i7 < cand_count: + vk = smem_keys[i7] + bin_f = (vk - bmin_r) * inv1 + bin_i = cutlass.Int32(bin_f) + if bin_i < cutlass.Int32(0): + bin_i = cutlass.Int32(0) + if bin_i > cutlass.Int32(kBins - 1): + bin_i = cutlass.Int32(kBins - 1) + atomicAdd(smem_hist.iterator + bin_i, cutlass.Int32(1)) + i7 = i7 + cutlass.Int32(num_threads) + cute.arch.barrier() + + # ---- Parallel k-th bin search (3-step) ---- + # Step 1: each warp sums BINS_PER_WARP bins (high→low slice) + warp_bin_sum = cutlass.Int32(0) + for jb in cutlass.range_constexpr(bins_per_warp): + bidx_s = ( + cutlass.Int32(kBins - 1) + - warp_id * cutlass.Int32(bins_per_warp) + - cutlass.Int32(jb) ) + warp_bin_sum = warp_bin_sum + smem_hist[bidx_s] + if lane == 0: + smem_wcnt[warp_id] = warp_bin_sum + cute.arch.barrier() - # ---- Level-2 histogram refinement ---- - # The snap loop below steps ONE distinct value per iteration - # (~0.45us each: full candidate re-scan + 2 barriers), and real - # logits concentrate count mass right at the k-th boundary, so - # the selected level-1 bin often holds tens of values → snap - # stragglers of 10+ us set the wall clock at N<=32K. When the - # selected bin is dense, re-histogram just that bin (bin width - # shrinks kBins x) for ~1us of extra scan, leaving the snap - # loop 0-2 steps. The snap loop converges monotonically from - # any starting threshold, so this only moves the start point — - # exactness is untouched (a level-2 edge-rounding error at - # worst costs one extra snap step). Uniform branch: everyone - # reads the same post-barrier SMEM scalar. - # Level 2 fires when a snap walk would cost more than one - # rebuild (~2 snap steps break even); level 3 only when level 2 - # failed to split the bin (>8: heavy ties or a sub-ulp-wide - # window — both rare on real logits, where ties at the k-th - # are ~1 and the acceptance band spans >>1 ulp). - binw_cur = binw1 - for _lvl in cutlass.range_constexpr(2): - if cutlass.const_expr(self.p4_warp_redundant): - sel_cnt_l = selc_reg - else: - sel_cnt_l = s_iscalars[4] - gate_l = cutlass.const_expr(2 if _lvl == 0 else 8) - if sel_cnt_l > cutlass.Int32(gate_l): - if cutlass.const_expr(self.p4_warp_redundant): - thr_el = thr_reg - # _kth_bin_search_rw has no trailing barrier; the - # zero pass of the rebuild below must not clobber - # smem_hist under a warp still in its step 3. - cute.arch.barrier() - else: - thr_el = s_thr[0] - # 2% slop each side absorbs the inv-vs-binw rounding - # difference in the previous level's edge estimate. - lo_l = thr_el - cutlass.Float32(0.02) * binw_cur - range_l = cutlass.Float32(1.04) * binw_cur - inv_l = (cutlass.Float32(kBins - 1) + cutlass.Float32(0.99)) / range_l - binw_next = range_l / cutlass.Float32(kBins) - self._hist_build(keys_base, smem_hist, cand_count, lo_l, inv_l, tidx) - if cutlass.const_expr(self.p4_warp_redundant): - thr_l2, selc_l2 = self._kth_bin_search_rw( - smem_hist, smem_wcnt, lo_l, binw_next, tidx, warp_id, lane - ) - thr_reg = thr_l2 - selc_reg = selc_l2 - else: - self._kth_bin_search( - smem_hist, - smem_wcnt, - s_thr, - s_iscalars, - lo_l, - binw_next, - tidx, - warp_id, - lane, + # Step 2: tid==0 finds target warp; stores prefix-count + warp index + # into s_iscalars[2] (=cnt_lo: prefix before target warp) + # and s_iscalars[3] (=cnt_hi: target warp index) + if tidx == 0: + cum = cutlass.Int32(0) + tw = cutlass.Int32(num_warps - 1) + found = cutlass.Int32(0) + for w2 in cutlass.range_constexpr(self.num_warps): + cum = cum + smem_wcnt[w2] + if cum >= cutlass.Int32(kK) and found == cutlass.Int32(0): + tw = cutlass.Int32(w2) + found = cutlass.Int32(1) + # Recompute prefix BEFORE target warp + cum2 = cutlass.Int32(0) + for w3 in cutlass.range_constexpr(self.num_warps): + if cutlass.Int32(w3) < tw: + cum2 = cum2 + smem_wcnt[w3] + s_iscalars[2] = cum2 # prefix + s_iscalars[3] = tw # target warp index + cute.arch.barrier() + + # Step 3: target warp's lane 0 scans BINS_PER_WARP bins → + # threshold. Single-thread serial; the unrolled + # range_constexpr beats a runtime `for+break` (tried it: -544 + # SASS insts but -7pp fp32 / -14pp bf16, since the + # branch/counter overhead in a single thread dominates the + # static math). + target_warp = s_iscalars[3] + if warp_id == target_warp and lane == cutlass.Int32(0): + base_cum = s_iscalars[2] + thr_local = bmin_r + bmin_local = bmin_r + set_done = cutlass.Int32(0) + for jb2 in cutlass.range_constexpr(bins_per_warp): + bidx2 = ( + cutlass.Int32(kBins - 1) + - target_warp * cutlass.Int32(bins_per_warp) + - cutlass.Int32(jb2) + ) + base_cum = base_cum + smem_hist[bidx2] + if base_cum >= cutlass.Int32(kK) and set_done == cutlass.Int32(0): + thr_local = bmin_local + cutlass.Float32(bidx2) * range1 / cutlass.Float32( + kBins ) - binw_cur = binw_next + set_done = cutlass.Int32(1) + s_thr[0] = thr_local + cute.arch.barrier() # ---- Snap convergence loop ---- # Upper bound = cand_count (matches CUDA heuristic_topk.cuh:985). @@ -2110,267 +1617,100 @@ def phase4_histogram_snap( # Runtime break via a guard flag — no `break` in cute.range. si = cutlass.Int32(0) done_snap = cutlass.Int32(0) - if cutlass.const_expr(self.p4_warp_redundant): - # Redundant-warp snap: threshold + convergence state live - # in registers (every warp reduces the staged partials - # itself, bit-identically), so each iteration needs ONE - # barrier (staging visibility) instead of two. Staging is - # parity double-buffered in smem_hist[par*3NW ..] so a - # warp one iteration ahead writes the other bank while a - # slow warp still reads the old one; the staging barrier - # bounds the drift to a single iteration. - cute.arch.barrier() # rw-search step-3 readers vs staging - nwc = cutlass.const_expr(self.num_warps) - thr_s = thr_reg - par4 = cutlass.Int32(0) - while si < snap_limit and done_snap == cutlass.Int32(0): - lge4 = cutlass.Int32(0) - lgt4 = cutlass.Int32(0) - up4 = cutlass.Float32(self.FLT_MAX) - dn4 = cutlass.Float32(self.NEG_FLT_MAX) - isi4 = tidx - while isi4 < cand_count: - v4 = self._smem_ld(cutlass.Float32, keys_base, isi4) - if v4 >= thr_s: - lge4 = lge4 + cutlass.Int32(1) - if v4 > thr_s: - lgt4 = lgt4 + cutlass.Int32(1) - up4 = _fmin_f32_inline(up4, v4) - if v4 < thr_s: - dn4 = cute.arch.fmax(dn4, v4) - isi4 = isi4 + cutlass.Int32(num_threads) - packed4 = (lge4 << cutlass.Int32(16)) | lgt4 - packed4 = self.warp_reduce_sum_i32(packed4) - up4 = self.warp_reduce_min_f32(up4) - dn4 = self.warp_reduce_max_f32(dn4) - off4 = par4 * cutlass.Int32(3 * nwc) - if lane == 0: - smem_hist[off4 + warp_id] = packed4 - smem_hist[off4 + cutlass.Int32(nwc) + warp_id] = float_as_uint32(up4) - smem_hist[off4 + cutlass.Int32(2 * nwc) + warp_id] = float_as_uint32(dn4) - cute.arch.barrier() - v_tp = cutlass.Int32(0) - v_up = cutlass.Float32(self.FLT_MAX) - v_dn = cutlass.Float32(self.NEG_FLT_MAX) - if lane < cutlass.Int32(nwc): - v_tp = smem_hist[off4 + lane] - vu_b = smem_hist[off4 + cutlass.Int32(nwc) + lane] - vd_b = smem_hist[off4 + cutlass.Int32(2 * nwc) + lane] - v_up = cutlass.Float32( - llvm.bitcast(cutlass.Float32.mlir_type, vu_b.ir_value()) - ) - v_dn = cutlass.Float32( - llvm.bitcast(cutlass.Float32.mlir_type, vd_b.ir_value()) - ) - tp4 = self.warp_reduce_sum_i32(v_tp) - tup4 = self.warp_reduce_min_f32(v_up) - tdn4 = self.warp_reduce_max_f32(v_dn) - cge_r = tp4 >> cutlass.Int32(16) - cgt_r = tp4 & cutlass.Int32(0xFFFF) - win_par = par4 - if cgt_r >= cutlass.Int32(kK): - if tup4 < cutlass.Float32(self.FLT_MAX): - thr_s = tup4 - elif cge_r < cutlass.Int32(kK): - if tdn4 > cutlass.Float32(self.NEG_FLT_MAX): - thr_s = tdn4 - if cgt_r < cutlass.Int32(kK) and cge_r >= cutlass.Int32(kK): - done_snap = cutlass.Int32(1) - par4 = par4 ^ cutlass.Int32(1) - si = si + cutlass.Int32(1) - else: - while si < snap_limit and done_snap == cutlass.Int32(0): - self.block_fused_snap_iter( - keys_base, - smem_wcnt, - smem_hist, - s_thr, - s_iscalars, - cand_count, - tidx, - warp_id, - lane, - ) - # After block_fused_snap_iter, s_iscalars[2]=cge, s_iscalars[3]=cgt. - cgt_c = self._smem_ld(cutlass.Int32, isc_base, cutlass.Int32(3)) - cge_c = self._smem_ld(cutlass.Int32, isc_base, cutlass.Int32(2)) - if cgt_c < cutlass.Int32(kK) and cge_c >= cutlass.Int32(kK): - done_snap = cutlass.Int32(1) - si = si + cutlass.Int32(1) - - # ---- Writeback (ballot + popc) ---- - # Converged snap (the overwhelmingly common case): SINGLE pass. - # The converged iteration's cgt (s_iscalars[3]) is the exact - # strictly-greater count at sel_thr (block_fused_snap_iter does - # not move the threshold when cgt < kK <= cge), so gt entries - # can pack into [0, cgt) via counter s_iscalars[4] while - # tie(==) entries start at offset cgt via counter s_iscalars[5] - # — same [gt | eq | pad] output partition as the two-pass - # original, one candidate sweep and one barrier fewer. The - # non-converged fallback keeps the original two-pass (its cgt - # would be stale: the last iter may have moved the threshold - # after counting). - if cutlass.const_expr(self.p4_warp_redundant): - sel_thr = thr_s - else: - sel_thr = s_thr[0] + while si < snap_limit and done_snap == cutlass.Int32(0): + self.block_fused_snap_iter( + smem_keys, + smem_wcnt, + smem_hist, + s_thr, + s_iscalars, + cand_count, + tidx, + warp_id, + lane, + ) + # After block_fused_snap_iter, s_iscalars[2]=cge, s_iscalars[3]=cgt. + if s_iscalars[3] < cutlass.Int32(kK) and s_iscalars[2] >= cutlass.Int32(kK): + done_snap = cutlass.Int32(1) + si = si + cutlass.Int32(1) + + # ---- Two-pass writeback (ballot + popc) ---- + # Per-iter: ballot collects emit flags into a 32-bit mask; + # popc gives the within-warp count; lane 0 atomicAdds the + # output base; shuffle broadcasts it. One barrier between + # passes, none within a pass. + sel_thr = s_thr[0] if tidx == 0: - s_iscalars[4] = cutlass.Int32(0) # gt out_count - # s_iscalars[5] (cluster-local scratch, consumed before P4) - # is reused as the eq counter for the single-pass path. - s_iscalars[5] = cutlass.Int32(0) + s_iscalars[4] = cutlass.Int32(0) # out_count cute.arch.barrier() - if done_snap == cutlass.Int32(1): - # Zero-atomic single pass. The converged snap iteration - # staged each warp's packed(ge<<16|gt) counts AT sel_thr in - # smem_wcnt[w] (nothing touches smem_wcnt between the snap - # exit and here), and the snap scan's tidx-strided - # partition covers exactly the same element set per warp - # as this warp-chunk scan. So every warp derives its - # deterministic output bases from a prefix over - # smem_wcnt — the ~2*cand/32 serialized SMEM atomics of - # the claim-based scheme (a top stall region in ncu at - # N=8K) disappear. Output order within the [gt | eq] - # segments changes (deterministic instead of claim order), - # which the contract allows. - if cutlass.const_expr(self.p4_warp_redundant): - cgt_base = cgt_r - else: - cgt_base = s_iscalars[3] - gt_run = cutlass.Int32(0) - eq_run = cutlass.Int32(0) - for wpre in cutlass.range_constexpr(self.num_warps): - if cutlass.const_expr(self.p4_warp_redundant): - # Converged iteration's packed counts live in the - # winning parity bank of smem_hist, not smem_wcnt. - pk_w = smem_hist[win_par * cutlass.Int32(3 * self.num_warps) + wpre] - else: - pk_w = smem_wcnt[wpre] - if cutlass.Int32(wpre) < warp_id: - wge_w = pk_w >> cutlass.Int32(16) - wgt_w = pk_w & cutlass.Int32(0xFFFF) - gt_run = gt_run + wgt_w - eq_run = eq_run + (wge_w - wgt_w) - eq_run = cgt_base + eq_run - base_w = warp_id * cutlass.Int32(self.WARP_SIZE) - while base_w < cand_count: - ix1 = base_w + lane - emit_gt = cutlass.Int32(0) - emit_eq = cutlass.Int32(0) - v_p1 = cutlass.Float32(self.NEG_FLT_MAX) - if ix1 < cand_count: - v_p1 = self._smem_ld(cutlass.Float32, keys_base, ix1) - if v_p1 > sel_thr: - emit_gt = cutlass.Int32(1) - if v_p1 == sel_thr: - emit_eq = cutlass.Int32(1) - mask_gt = cute.arch.vote_ballot_sync(emit_gt != cutlass.Int32(0)) - lane_mask = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32(1) - if mask_gt != cutlass.Uint32(0): - moff_gt = cutlass.Int32(cute.arch.popc(mask_gt & lane_mask)) - wpos_p1 = gt_run + moff_gt - if emit_gt != cutlass.Int32(0) and wpos_p1 < cutlass.Int32(kK): - if cutlass.const_expr(self.return_output_values): - output_values_row[wpos_p1] = self.dtype(v_p1) - output_indices_row[wpos_p1] = self._smem_ld( - cutlass.Int32, vals_base, ix1 - ) - gt_run = gt_run + cutlass.Int32(cute.arch.popc(mask_gt)) - mask_eq = cute.arch.vote_ballot_sync(emit_eq != cutlass.Int32(0)) - if mask_eq != cutlass.Uint32(0): - moff_eq = cutlass.Int32(cute.arch.popc(mask_eq & lane_mask)) - wpos_p2 = eq_run + moff_eq - if emit_eq != cutlass.Int32(0) and wpos_p2 < cutlass.Int32(kK): - if cutlass.const_expr(self.return_output_values): - output_values_row[wpos_p2] = self.dtype(v_p1) - output_indices_row[wpos_p2] = self._smem_ld( - cutlass.Int32, vals_base, ix1 - ) - eq_run = eq_run + cutlass.Int32(cute.arch.popc(mask_eq)) - base_w = base_w + cutlass.Int32(num_threads) - cute.arch.barrier() - else: - # Pass 1: v > sel_thr, strided over (warp_id * WARP_SIZE, ...). - base_w = warp_id * cutlass.Int32(self.WARP_SIZE) - while base_w < cand_count: - ix1 = base_w + lane - emit_gt = cutlass.Int32(0) - v_p1 = cutlass.Float32(self.NEG_FLT_MAX) - if ix1 < cand_count: - v_p1 = self._smem_ld(cutlass.Float32, keys_base, ix1) - if v_p1 > sel_thr: - emit_gt = cutlass.Int32(1) - mask_gt = cute.arch.vote_ballot_sync(emit_gt != cutlass.Int32(0)) - if mask_gt != cutlass.Uint32(0): - cnt_gt = cutlass.Int32(cute.arch.popc(mask_gt)) - lane_mask_gt = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32( - 1 + # Pass 1: v > sel_thr, strided over (warp_id * WARP_SIZE, ...). + # The `if mask_gt != 0` guard skips popc + atomicAdd + shuffle + # when no lane in the warp emits — the SMEM atomicAdd alone is + # ~10-30 cycles. + base_w = warp_id * cutlass.Int32(self.WARP_SIZE) + while base_w < cand_count: + ix1 = base_w + lane + emit_gt = cutlass.Int32(0) + v_p1 = cutlass.Float32(self.NEG_FLT_MAX) + if ix1 < cand_count: + v_p1 = smem_keys[ix1] + if v_p1 > sel_thr: + emit_gt = cutlass.Int32(1) + mask_gt = cute.arch.vote_ballot_sync(emit_gt != cutlass.Int32(0)) + if mask_gt != cutlass.Uint32(0): + cnt_gt = cutlass.Int32(cute.arch.popc(mask_gt)) + lane_mask_gt = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32(1) + moff_gt = cutlass.Int32(cute.arch.popc(mask_gt & lane_mask_gt)) + bp_gt = cutlass.Int32(0) + if lane == cutlass.Int32(0): + bp_gt = atomicAdd( + s_iscalars.iterator + cutlass.Int32(4), + cnt_gt, ) - moff_gt = cutlass.Int32(cute.arch.popc(mask_gt & lane_mask_gt)) - bp_gt = cutlass.Int32(0) - if lane == cutlass.Int32(0): - bp_gt = atomicAdd( - s_iscalars.iterator + cutlass.Int32(4), - cnt_gt, - ) - bp_gt = cute.arch.shuffle_sync(bp_gt, cutlass.Int32(0)) - wpos_p1 = bp_gt + moff_gt - if emit_gt != cutlass.Int32(0) and wpos_p1 < cutlass.Int32(kK): - if cutlass.const_expr(self.return_output_values): - output_values_row[wpos_p1] = self.dtype(v_p1) - output_indices_row[wpos_p1] = self._smem_ld( - cutlass.Int32, vals_base, ix1 - ) - base_w = base_w + cutlass.Int32(num_threads) - cute.arch.barrier() + bp_gt = cute.arch.shuffle_sync(bp_gt, cutlass.Int32(0)) + wpos_p1 = bp_gt + moff_gt + if emit_gt != cutlass.Int32(0) and wpos_p1 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[wpos_p1] = self.dtype(v_p1) + output_indices_row[wpos_p1] = smem_vals[ix1] + base_w = base_w + cutlass.Int32(num_threads) + cute.arch.barrier() - # Pass 2: v == sel_thr (same pattern + guard as Pass 1). - base_w2 = warp_id * cutlass.Int32(self.WARP_SIZE) - while base_w2 < cand_count: - ix2 = base_w2 + lane - emit_eq = cutlass.Int32(0) - v_p2 = cutlass.Float32(self.NEG_FLT_MAX) - if ix2 < cand_count: - v_p2 = self._smem_ld(cutlass.Float32, keys_base, ix2) - if v_p2 == sel_thr: - emit_eq = cutlass.Int32(1) - mask_eq = cute.arch.vote_ballot_sync(emit_eq != cutlass.Int32(0)) - if mask_eq != cutlass.Uint32(0): - cnt_eq = cutlass.Int32(cute.arch.popc(mask_eq)) - lane_mask_eq = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32( - 1 + # Pass 2: v == sel_thr (same pattern + guard as Pass 1). Empty + # iterations are much more common here since only tie-at-K + # values qualify. + base_w2 = warp_id * cutlass.Int32(self.WARP_SIZE) + while base_w2 < cand_count: + ix2 = base_w2 + lane + emit_eq = cutlass.Int32(0) + v_p2 = cutlass.Float32(self.NEG_FLT_MAX) + if ix2 < cand_count: + v_p2 = smem_keys[ix2] + if v_p2 == sel_thr: + emit_eq = cutlass.Int32(1) + mask_eq = cute.arch.vote_ballot_sync(emit_eq != cutlass.Int32(0)) + if mask_eq != cutlass.Uint32(0): + cnt_eq = cutlass.Int32(cute.arch.popc(mask_eq)) + lane_mask_eq = (cutlass.Uint32(1) << cutlass.Uint32(lane)) - cutlass.Uint32(1) + moff_eq = cutlass.Int32(cute.arch.popc(mask_eq & lane_mask_eq)) + bp_eq = cutlass.Int32(0) + if lane == cutlass.Int32(0): + bp_eq = atomicAdd( + s_iscalars.iterator + cutlass.Int32(4), + cnt_eq, ) - moff_eq = cutlass.Int32(cute.arch.popc(mask_eq & lane_mask_eq)) - bp_eq = cutlass.Int32(0) - if lane == cutlass.Int32(0): - bp_eq = atomicAdd( - s_iscalars.iterator + cutlass.Int32(4), - cnt_eq, - ) - bp_eq = cute.arch.shuffle_sync(bp_eq, cutlass.Int32(0)) - wpos_p2 = bp_eq + moff_eq - if emit_eq != cutlass.Int32(0) and wpos_p2 < cutlass.Int32(kK): - if cutlass.const_expr(self.return_output_values): - output_values_row[wpos_p2] = self.dtype(v_p2) - output_indices_row[wpos_p2] = self._smem_ld( - cutlass.Int32, vals_base, ix2 - ) - base_w2 = base_w2 + cutlass.Int32(num_threads) - cute.arch.barrier() + bp_eq = cute.arch.shuffle_sync(bp_eq, cutlass.Int32(0)) + wpos_p2 = bp_eq + moff_eq + if emit_eq != cutlass.Int32(0) and wpos_p2 < cutlass.Int32(kK): + if cutlass.const_expr(self.return_output_values): + output_values_row[wpos_p2] = self.dtype(v_p2) + output_indices_row[wpos_p2] = smem_vals[ix2] + base_w2 = base_w2 + cutlass.Int32(num_threads) + cute.arch.barrier() - # Pad remainder with -self.FLT_MAX / -1. Single-pass filled = - # cge (= cgt + total ties at sel_thr, from the converged snap - # iteration; the zero-atomic path leaves counters untouched); - # two-pass filled = counter [4] (gt + eq accumulated). - filled_par = cutlass.Int32(0) - if done_snap == cutlass.Int32(1): - if cutlass.const_expr(self.p4_warp_redundant): - filled_par = cge_r - else: - filled_par = s_iscalars[2] - else: - filled_par = s_iscalars[4] + # Pad remainder with -self.FLT_MAX / -1 + filled_par = s_iscalars[4] if filled_par > cutlass.Int32(kK): filled_par = cutlass.Int32(kK) ipad = filled_par + tidx @@ -2379,17 +1719,14 @@ def phase4_histogram_snap( output_values_row[ipad] = self.dtype(self.NEG_FLT_MAX) output_indices_row[ipad] = cutlass.Int32(-1) ipad = ipad + cutlass.Int32(num_threads) - else: # ----- Branch C: cand_count < kK ----- # Emit cand_count + pad i10 = tidx while i10 < cand_count: if cutlass.const_expr(self.return_output_values): - output_values_row[i10] = self.dtype( - self._smem_ld(cutlass.Float32, keys_base, i10) - ) - output_indices_row[i10] = self._smem_ld(cutlass.Int32, vals_base, i10) + output_values_row[i10] = self.dtype(smem_keys[i10]) + output_indices_row[i10] = smem_vals[i10] i10 = i10 + cutlass.Int32(num_threads) i11 = cand_count + tidx while i11 < cutlass.Int32(kK): @@ -2569,14 +1906,9 @@ def run_one_row( byte_alignment=128, ) # warp_counts[NUM_WARPS] int32 (P3 prefix-sum scratch) - # p2_warp_redundant parity-banks the Phase-2 staging (a warp one - # round ahead writes the other half) — costs num_warps*4 bytes. smem_wcnt = smem.allocate_tensor( element_type=cutlass.Int32, - layout=cute.make_ordered_layout( - (2 * num_warps if cutlass.const_expr(self.p2_warp_redundant) else num_warps,), - order=(0,), - ), + layout=cute.make_ordered_layout((num_warps,), order=(0,)), byte_alignment=128, ) # Phase-1 warp aggregates (fp32 + int32; ~256 bytes total) @@ -2619,26 +1951,17 @@ def run_one_row( layout=cute.make_ordered_layout((6,), order=(0,)), byte_alignment=16, ) - # Per-CTA DSMEM scratch for the cluster all-reduce of cand_count: - # slots 0/1 = parity double-buffered count exchange (call k writes - # slot k&1 — closes the straggler-read-vs-next-write DSMEM race), - # slot 2 = tid0-private call counter. mapa.shared::cluster relies - # on every CTA holding this block at the SAME SMEM offset, so it's - # allocated once here. Only needed at cs>1; skipped at cs=1 (all - # uses are gated by const_expr(cs>1) and None propagates - # harmlessly through the unused parameters). + # Per-CTA DSMEM scratch for the cluster all-reduce of cand_count. + # mapa.shared::cluster relies on every CTA holding this slot at the + # SAME SMEM offset, so it's allocated once here. Only needed at + # cs>1; skipped at cs=1 (all uses are gated by const_expr(cs>1) and + # None propagates harmlessly through the unused parameters). if cutlass.const_expr(cluster_size > 1): s_cluster_partial = smem.allocate_tensor( element_type=cutlass.Int32, - layout=cute.make_ordered_layout((3,), order=(0,)), + layout=cute.make_ordered_layout((1,), order=(0,)), byte_alignment=16, ) - # Zero the call counter before any block_count_ge call. tid0- - # private (same thread reads/increments it), so program order - # suffices — but parity must start at 0 on EVERY CTA of the - # cluster for lockstep alignment. - if tidx == cutlass.Int32(0): - s_cluster_partial[2] = cutlass.Int32(0) else: s_cluster_partial = None diff --git a/tensorrt_llm/_torch/disaggregation/native/bounce/config.py b/tensorrt_llm/_torch/disaggregation/native/bounce/config.py index cfdcd4e45557..bcc09f9935ec 100644 --- a/tensorrt_llm/_torch/disaggregation/native/bounce/config.py +++ b/tensorrt_llm/_torch/disaggregation/native/bounce/config.py @@ -15,37 +15,11 @@ """Bounce configuration and pluggable sizing policy. A config enables bounce; leaving it unset keeps the per-block path. The size knob doubles as the on and off switch.""" -import os from dataclasses import dataclass, field from typing import Optional -from tensorrt_llm import logger - _MIB = 1024 * 1024 -# Test/advanced override for the block-count gate below (users only tune the bounce size). Read on -# the generation side, so set it there; unset uses the default. -_MIN_BLOCKS_ENV = "TRTLLM_KV_CACHE_BOUNCE_MIN_BLOCKS" - - -def _env_min_blocks(default: int) -> int: - """Read the gate from the env, defensively: unset or malformed falls back to the default (never - crashing), and the value is clamped to at least 1.""" - raw = os.environ.get(_MIN_BLOCKS_ENV) - if raw is None: - return default - try: - value = int(raw) - except ValueError: - logger.warning(f"{_MIN_BLOCKS_ENV}={raw!r} is not an integer; using default {default}") - return default - if value < 1: - logger.warning( - f"{_MIN_BLOCKS_ENV}={value} < 1; clamping to 1 (bounce always clears the gate)" - ) - return 1 - return value - def _round_up(a: int, b: int) -> int: return (a + b - 1) // b * b @@ -111,12 +85,9 @@ class Config: min_blocks: int = 96 -def config_from_size(size_mb: int, min_blocks: Optional[int] = None) -> Optional[Config]: - """Build a bounce config from a per-region size in MiB, or None to leave bounce off (size <= 0). - Size is both the capacity and the on/off switch. min_blocks is the gate below which a transfer - stays on the per-block path; when unset it comes from the env, else the default.""" +def config_from_size(size_mb: int) -> Optional[Config]: + """Build a bounce config from a per-region size in MiB, or None to leave bounce off when the size + is not positive. The size is both the capacity and the on and off switch.""" if size_mb is None or size_mb <= 0: return None - if min_blocks is None: - min_blocks = _env_min_blocks(Config.min_blocks) - return Config(sizing=FixedSizing(capacity_mb=size_mb), min_blocks=min_blocks) + return Config(sizing=FixedSizing(capacity_mb=size_mb)) diff --git a/tensorrt_llm/_torch/disaggregation/native/bounce/core.py b/tensorrt_llm/_torch/disaggregation/native/bounce/core.py index 1878fa90fd3a..94b63da1b8a4 100644 --- a/tensorrt_llm/_torch/disaggregation/native/bounce/core.py +++ b/tensorrt_llm/_torch/disaggregation/native/bounce/core.py @@ -207,10 +207,6 @@ def is_bounced(self, rid_slice) -> bool: def release_idle_reservation(self, rid_slice) -> None: """Immediately free a reservation cancelled before any address went out.""" - @abstractmethod - def orphan_reservation(self, rid_slice) -> None: - """Give up on an in-flight reservation (cancel/timeout/lost result); quarantine, don't leak.""" - @abstractmethod def record_result( self, rid_slice, peer_rank, dst_ptrs=None, sizes=None, src_base=None, on_done=None diff --git a/tensorrt_llm/_torch/disaggregation/native/bounce/impl.py b/tensorrt_llm/_torch/disaggregation/native/bounce/impl.py index a9597b7cbb2a..8fefc2c761e4 100644 --- a/tensorrt_llm/_torch/disaggregation/native/bounce/impl.py +++ b/tensorrt_llm/_torch/disaggregation/native/bounce/impl.py @@ -146,21 +146,22 @@ def _start_scatter_worker(self, name: str) -> None: def _new_stream(self): return CUASSERT(cudart.cudaStreamCreate())[0] - def _gather_blocking(self, src_addr: int, write_meta, total: int) -> None: - """Gather the scattered fragments into the send region and block until done. The whole gather - runs under the stream lock so a second sender thread can't overwrite the shared staging buffer - mid-copy and corrupt this region; only the fast gather serializes, the writes stay parallel.""" + def _launch_gather(self, src_addr: int, write_meta, total: int): + """Launch the gather of the scattered fragments into the send region and return an event to + wait on.""" plan = Plan(write_meta.src_ptrs, write_meta.dst_ptrs, write_meta.sizes, total) with self._send_stream_lock: gather_contiguous( src_addr, plan.src_ptrs, plan.sizes, plan.offsets, stream=self._send_stream ) event = CUASSERT(cudart.cudaEventCreate())[0] - try: - CUASSERT(cudart.cudaEventRecord(event, self._send_stream)) - CUASSERT(cudart.cudaEventSynchronize(event)) - finally: - CUASSERT(cudart.cudaEventDestroy(event)) + CUASSERT(cudart.cudaEventRecord(event, self._send_stream)) + return event + + def _wait_gather(self, event) -> None: + if event is not None: + CUASSERT(cudart.cudaEventSynchronize(event)) + CUASSERT(cudart.cudaEventDestroy(event)) def _make_write(self, src_addr: int, write_meta, total: int): # one coalesced descriptor spanning the whole region @@ -188,20 +189,20 @@ def _reserve_and_gather(self, write_meta, *, timeout): ) return None slot_id, src_addr = res - try: - self._gather_blocking(src_addr, write_meta, total) - except Exception: - self._send_alloc.release(slot_id) # free the slot if the gather raises - raise - return slot_id, src_addr, total + return slot_id, src_addr, total, self._launch_gather(src_addr, write_meta, total) def build_request(self, write_meta): - """Gather into a send slot and build the coalesced write, or None on backpressure. The gather - blocks (and frees the slot on failure) inside _reserve_and_gather.""" + """Gather into a send slot and build the coalesced write, or None on backpressure. Frees the + slot if the gather raises.""" gathered = self._reserve_and_gather(write_meta, timeout=_RESERVE_TIMEOUT_S) if gathered is None: # backpressure: fall back return None - slot_id, src_addr, total = gathered + slot_id, src_addr, total, event = gathered + try: + self._wait_gather(event) + except Exception: + self._send_alloc.release(slot_id) + raise return self._make_write(src_addr, write_meta, total), slot_id def release_send(self, slot_id) -> None: @@ -238,21 +239,6 @@ def reserve( f"({total % num_writers}B remainder); head-mismatch explosion NOT mitigated", warn_key="kv-bounce-uneven-fanin", ) - if num_writers > 1: - # Fan-in gives each writer an equal share of the region, which only matches where it - # writes when all writers send the same bytes. Equal layer count guarantees that only - # when the per-block sizes match, so require that here, else fall back. - present_slot_bytes = { - self._block_bytes_per_group[g] - for g, block_ids in enumerate(recv_req.block_ids_per_layer_groups) - if int(block_ids.size) > 0 - } - if len(present_slot_bytes) > 1: - return self._skip_bounce( - f"fan-in across {num_writers} senders with non-uniform layer-group slot bytes " - f"{sorted(present_slot_bytes)}; the equal split would overrun a sub-region", - warn_key="kv-bounce-heterogeneous-fanin", - ) if total > self._recv_alloc.capacity: # too big to ever fit, unlike transient backpressure return self._skip_bounce( f"transfer {total // _MIB}MiB exceeds the {self._recv_alloc.capacity // _MIB}MiB bounce " @@ -275,14 +261,6 @@ def reserve( num_writers=num_writers, ) self._reserved_map[ctx.rid_slice] = ctx # inactive until the first writer reports - # Positive marker: all fall-back guards above passed, so this transfer provably takes the - # coalesced-bounce WRITE path. Logged once (per process) so an e2e test can assert that - # bounce actually engaged instead of silently falling back to the per-fragment path. - logger.info_once( - f"[kv-bounce] coalesced {nblocks} blocks / {total // _MIB}MiB into one region " - f"across {num_writers} writer(s)", - key="kv-bounce-coalesced", - ) return True def writer_base(self, rid_slice: RidSlice, writer_index: int) -> Optional[int]: @@ -303,12 +281,6 @@ def release_idle_reservation(self, rid_slice: RidSlice) -> None: if ctx is not None: self._recv_alloc.release(ctx.slot_id) - def orphan_reservation(self, rid_slice: RidSlice) -> None: - """Give up on a reservation whose write may still be in flight (cancel/timeout/lost result). - The write can't be aborted, so quarantine the region (reclaimed later) rather than releasing - or leaking it. Idempotent; a no-op once the transfer has settled.""" - self._apply(rid_slice, lambda ctx: ctx.mark_orphaned()) - def _apply(self, rid_slice: RidSlice, mutate: Callable[[TransferContext], None]) -> None: """Mutate the state under the lock, then do what it asks (scatter or settle) with the lock released, never holding it across a CUDA sync, a queue put, or a callback. No-op if the @@ -449,9 +421,6 @@ def is_bounced(self, rid_slice) -> bool: def release_idle_reservation(self, rid_slice) -> None: pass - def orphan_reservation(self, rid_slice) -> None: - pass - def record_result( self, rid_slice, peer_rank, dst_ptrs=None, sizes=None, src_base=None, on_done=None ): diff --git a/tensorrt_llm/_torch/disaggregation/native/mixers/attention/peer.py b/tensorrt_llm/_torch/disaggregation/native/mixers/attention/peer.py index 405810661fba..12d7cadfee4f 100644 --- a/tensorrt_llm/_torch/disaggregation/native/mixers/attention/peer.py +++ b/tensorrt_llm/_torch/disaggregation/native/mixers/attention/peer.py @@ -1,20 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections.abc import Sequence - import numpy as np from tensorrt_llm import logger @@ -29,124 +12,107 @@ from tensorrt_llm._utils import nvtx_range -class IntactMapper(RegionMapperBase): +class IdentityMapper(RegionMapperBase): """ - ---- mapper_intact ---- - - Copy the selected layers' class regions between slots. Consumes slot-base - pointers from the extractor and expands them with slot-relative per-layer - byte offsets taken from the logical view's buffer entries, so it handles - non-uniform layer strides (a slot may interleave other role classes - between this class's layers). + ---- mapper_identity ---- - src slot: [ base ]--+off(L2)--> [L2 region] --+off(L3)--> [L3 region] ... - dst slot: [ base ]--+off'(L2)-> [L2 region] --+off'(L3)-> [L3 region] ... + Pass-through mapping. Do not change pointers or sizes. - Layers whose regions are contiguous on BOTH sides are merged into one - fragment, so a fully contiguous class (e.g. K/V in a dedicated pool) - degrades to a single whole-region copy per block. + src_ptrs: [ S0 ] [ S1 ] [ S2 ] ... + | | | + v v v + dst_ptrs: [ D0 ] [ D1 ] [ D2 ] ... """ - def __init__( - self, - src_layer_offsets: Sequence[int] | np.ndarray, - dst_layer_offsets: Sequence[int] | np.ndarray, - self_bytes_per_layer: int, - peer_bytes_per_layer: int, - *, - mapper_name: str = "Intact", - ) -> None: - if self_bytes_per_layer != peer_bytes_per_layer: - raise ValueError( - f"{mapper_name} cache region size mismatch: " - f"local={self_bytes_per_layer}, peer={peer_bytes_per_layer}" - ) - src = np.asarray(src_layer_offsets, dtype=np.int64) - dst = np.asarray(dst_layer_offsets, dtype=np.int64) - if src.size == 0 or src.size != dst.size: - raise ValueError( - f"{mapper_name} layer offsets must be non-empty and equal-length: " - f"src={src.size}, dst={dst.size}" - ) - self._runs = self._merge_contiguous(src, dst, self_bytes_per_layer) - - @staticmethod - def _merge_contiguous( - src: np.ndarray, dst: np.ndarray, bytes_per_layer: int - ) -> list[tuple[int, int, int]]: - runs: list[tuple[int, int, int]] = [] - run_start = 0 - for i in range(1, src.size + 1): - if ( - i == src.size - or src[i] != src[i - 1] + bytes_per_layer - or dst[i] != dst[i - 1] + bytes_per_layer - ): - run_layers = i - run_start - runs.append( - (int(src[run_start]), int(dst[run_start]), run_layers * bytes_per_layer) - ) - run_start = i - return runs - - @nvtx_range("IntactMapper.map") - def map(self, src_regions: SpecRegion, dst_regions: SpecRegion): + @nvtx_range("IdentityMapper.map") + def map(self, src_regions: SpecRegion, dst_regions: SpecRegion) -> SpecRegionPair: src_group = src_regions.memory dst_group = dst_regions.memory - if src_group.ptrs.size != dst_group.ptrs.size: - raise ValueError( - f"Number of regions of src({src_group.ptrs.size}) and " - f"dst({dst_group.ptrs.size}) must match" - ) - pairs = [ - SpecRegionPair( - src=SpecRegion( - memory=MemRegionGroup( - ptrs=src_group.ptrs + src_off, bytes_per_region=run_bytes - ), - spec=src_regions.spec, - ), - dst=SpecRegion( - memory=MemRegionGroup( - ptrs=dst_group.ptrs + dst_off, bytes_per_region=run_bytes - ), - spec=dst_regions.spec, - ), - ) - for src_off, dst_off, run_bytes in self._runs - ] - return pairs[0] if len(pairs) == 1 else pairs - + assert src_group.ptrs.size == dst_group.ptrs.size, ( + f"Number of regions of src({src_group.ptrs.size}) and dst({dst_group.ptrs.size}) must match" + ) + return SpecRegionPair( + src=SpecRegion(memory=src_group, spec=src_regions.spec), + dst=SpecRegion(memory=dst_group, spec=dst_regions.spec), + ) -class ReplicatedMapper(IntactMapper): - """Copy TP-replicated per-layer regions without KV-head remapping. - Every TP rank holds identical bytes per layer (MiniMax M3 index-key, DSA - indexer K), so no head slicing applies. Layer selection under partial PP - overlap happens through the per-layer offsets, and fan-in routing (one - owning sender per destination) is decided upstream by - ``PeerRegistrar.should_send_pool``. +class HeadMatchMapper(RegionMapperBase): + """ + ---- mapper_head_match ---- + + Move/copy entire contiguous block(s) (multi-layer fragment) as a single chunk. + Align by whole fragment size (frag_size) and apply a constant source/destination block offset. + + src_ptrs: [ S0 ] [ S1 ] ... + | | + + src_off + src_off + | | + [ S0 + src_off ] [ S1 + src_off ] -> (each points to a frag of size frag_size) + copy whole frag + | | + v v + [ D0 + dst_off ] [ D1 + dst_off ] -> (destination frags) + + Contiguous-layer assumption: + This mapper assumes that ``transfer_layers`` consecutive layers + starting at ``src_layer_off`` (and ``dst_layer_off``) are laid out + contiguously within each slot. This holds because + ``buffer_attributes()`` in the storage config assigns buffer + offsets sequentially from 0 for each layer_group (life cycle), + and each PoolDescriptor only contains layers belonging to a single + layer_group. Even when multiple layer_groups share the same + physical storage pool_group, each layer_group independently + occupies the full slot (offsets start from 0), so the contiguous + layout is preserved. """ def __init__( self, - src_layer_offsets: Sequence[int] | np.ndarray, - dst_layer_offsets: Sequence[int] | np.ndarray, - self_bytes_per_layer: int, - peer_bytes_per_layer: int, - ) -> None: - super().__init__( - src_layer_offsets, - dst_layer_offsets, - self_bytes_per_layer, - peer_bytes_per_layer, - mapper_name="Replicated", + transfer_layers: int, + src_layer_off: int, + dst_layer_off: int, + self_ri: RankInfo, + peer_ri: RankInfo, + slot_size_per_layer: int, + ): + if not isinstance(slot_size_per_layer, int): + raise TypeError( + f"slot_size_per_layer must be int, got {type(slot_size_per_layer).__name__} " + f"(value={slot_size_per_layer}). Use // instead of / for integer division." + ) + self._kv_factor = self_ri.attention.kv_factor + self._frag_size = self._block_size(transfer_layers, slot_size_per_layer=slot_size_per_layer) + self._src_block_off = self._block_size( + src_layer_off, slot_size_per_layer=slot_size_per_layer + ) + self._dst_block_off = self._block_size( + dst_layer_off, slot_size_per_layer=slot_size_per_layer ) + @nvtx_range("HeadMatchMapper.map") + def map(self, src_regions: SpecRegion, dst_regions: SpecRegion) -> SpecRegionPair: + src_group = src_regions.memory + dst_group = dst_regions.memory + assert src_group.ptrs.size == dst_group.ptrs.size, ( + f"Number of regions of src({src_group.ptrs.size}) and dst({dst_group.ptrs.size}) must match" + ) + new_src_ptrs = src_group.ptrs + self._src_block_off + new_dst_ptrs = dst_group.ptrs + self._dst_block_off + new_src = MemRegionGroup(ptrs=new_src_ptrs, bytes_per_region=self._frag_size) + new_dst = MemRegionGroup(ptrs=new_dst_ptrs, bytes_per_region=self._frag_size) + return SpecRegionPair( + src=SpecRegion(memory=new_src, spec=src_regions.spec), + dst=SpecRegion(memory=new_dst, spec=dst_regions.spec), + ) + + def _block_size(self, layer_num: int, slot_size_per_layer: int) -> int: + return layer_num * slot_size_per_layer -class HNDHeadMismatchMapper(RegionMapperBase): + +class HeadMismatchMapper(RegionMapperBase): """ - ---- mapper_hnd_head_mismatch ---- + ---- mapper_head_mismatch ---- Fine-grained mapping when head counts or TP/DP partitioning differ. Split layers into per-head (or contiguous-heads) fragments and map them individually. @@ -168,60 +134,26 @@ class HNDHeadMismatchMapper(RegionMapperBase): def __init__( self, - *, - src_layer_offsets: "Sequence[int] | np.ndarray", - dst_layer_offsets: "Sequence[int] | np.ndarray", + transfer_layers: int, + src_layer_off: int, + peer_layer_off: int, self_ri: RankInfo, peer_ri: RankInfo, - self_bytes_per_layer: int, - peer_bytes_per_layer: int, - self_buffers_per_layer: int, - peer_buffers_per_layer: int, ): self._ri = self_ri self._peer_ri = peer_ri + kv_factor = self_ri.attention.kv_factor self_tp_per_dp = self_ri.tp_size_per_dp_group peer_tp_per_dp = peer_ri.tp_size_per_dp_group self_tp_rank = self_ri.tp_rank peer_tp_rank = peer_ri.tp_rank - if self_buffers_per_layer != peer_buffers_per_layer: - raise ValueError( - "HND buffer count per layer mismatch: " - f"local={self_buffers_per_layer}, peer={peer_buffers_per_layer}" - ) - src_offsets = np.asarray(src_layer_offsets, dtype=np.int64) - dst_offsets = np.asarray(dst_layer_offsets, dtype=np.int64) - if src_offsets.size == 0 or src_offsets.size != dst_offsets.size: - raise ValueError( - "HND layer offsets must be non-empty and equal-length: " - f"src={src_offsets.size}, dst={dst_offsets.size}" - ) - - # Byte geometry is derived from the per-layer region size registered - # by storage (always whole bytes) rather than element_bytes x dims - # arithmetic, so sub-byte dtypes (e.g. NVFP4) stay exact-integer. - src_buffer_bytes = self._bytes_per_buffer( - bytes_per_layer=self_bytes_per_layer, - buffers_per_layer=self_buffers_per_layer, - side="local", - ) - dst_buffer_bytes = self._bytes_per_buffer( - bytes_per_layer=peer_bytes_per_layer, - buffers_per_layer=peer_buffers_per_layer, - side="peer", - ) - bytes_per_head = self._bytes_per_head( - src_buffer_bytes, self._ri.attention.kv_heads_per_rank, side="local" + bytes_per_head = ( + self._ri.attention.tokens_per_block + * self._ri.attention.dims_per_head + * self._ri.attention.element_bytes ) - peer_bytes_per_head = self._bytes_per_head( - dst_buffer_bytes, peer_ri.attention.kv_heads_per_rank, side="peer" - ) - if bytes_per_head != peer_bytes_per_head: - raise ValueError( - f"HND bytes per head mismatch: local={bytes_per_head}, peer={peer_bytes_per_head}" - ) self._bytes_cont_heads = ( min(self._ri.attention.kv_heads_per_rank, peer_ri.attention.kv_heads_per_rank) * bytes_per_head @@ -236,41 +168,55 @@ def __init__( peer_kv_heads=peer_ri.attention.kv_heads_per_rank, bytes_per_head=bytes_per_head, ) + self._peer_layer_off = peer_layer_off + + # --- Pre-compute flat 1D offset arrays --- + # + # Each KV cache block (slot) is laid out as: + # + # block_base ──► [layer_0 kv_0] [layer_0 kv_1] [layer_1 kv_0] [layer_1 kv_1] ... + # ◄─ layer_kv ─► ◄─ layer_kv ─► + # ◄────── layer_num (= layer_kv * kv_factor) ──────► + # + # To address fragment (layer=j, kv=k) within a block at base_ptr: + # + # frag_ptr = base_ptr + # + layer_num * (layer_off + j) # skip to the right layer + # + layer_kv * k # skip to key or value + # + head_off # head offset for TP mismatch + # + # The original code computed this as a 3D broadcast in map(): + # bases[:, None, None] + layer_offsets[None, :, None] + # + kv_offsets[None, None, :] + head_off + # producing shape (n_blocks, transfer_layers, kv_factor) then .ravel(). + # + # Optimization: since the (layer, kv) offsets are independent of the + # per-call block base pointers, we pre-compute them here as a flat 1D + # array of length (transfer_layers * kv_factor). At map() time we only + # need np.add.outer(bases, flat_offsets).ravel(), which produces the + # same result in the same C-order traversal (blocks outer, offsets inner) + # but with fewer intermediate allocations. + layer_indices = np.arange(transfer_layers, dtype=np.int64) + kv_indices = np.arange(kv_factor, dtype=np.int64) + + src_layer_kv_num = self._get_layer_kv_num(self._ri) + src_layer_num = src_layer_kv_num * kv_factor + # Shape (transfer_layers, kv_factor) → ravel to 1D + self._src_flat_offsets = ( + src_layer_num * (src_layer_off + layer_indices)[:, None] + + src_layer_kv_num * kv_indices[None, :] + + self._src_head_off + ).ravel() - # Pre-compute flat 1D offset arrays: one fragment per (layer, buffer) - # where buffers are the layer's K/V (or scale) buffers laid out - # back-to-back within the layer's region. Layer starts come from the - # view's buffer entries, so interleaved role classes (non-uniform - # layer strides) are handled the same way as everywhere else. At - # map() time a single np.add.outer(bases, flat_offsets) expands the - # per-block base pointers. - self._src_flat_offsets = self._build_flat_offsets( - layer_offsets=src_offsets, - buffers_per_layer=self_buffers_per_layer, - buffer_bytes=src_buffer_bytes, - head_offset=self._src_head_off, - ) - self._dst_flat_offsets = self._build_flat_offsets( - layer_offsets=dst_offsets, - buffers_per_layer=peer_buffers_per_layer, - buffer_bytes=dst_buffer_bytes, - head_offset=self._dst_head_off, - ) - - @staticmethod - def _build_flat_offsets( - *, - layer_offsets: np.ndarray, - buffers_per_layer: int, - buffer_bytes: int, - head_offset: int, - ) -> np.ndarray: - buffer_indices = np.arange(buffers_per_layer, dtype=np.int64) - return ( - layer_offsets[:, None] + buffer_bytes * buffer_indices[None, :] + head_offset + dst_layer_kv_num = self._get_layer_kv_num(self._peer_ri) + dst_layer_num = dst_layer_kv_num * kv_factor + self._dst_flat_offsets = ( + dst_layer_num * (peer_layer_off + layer_indices)[:, None] + + dst_layer_kv_num * kv_indices[None, :] + + self._dst_head_off ).ravel() - @nvtx_range("HNDHeadMismatchMapper.map") + @nvtx_range("HeadMismatchMapper.map") def map(self, src_regions: SpecRegion, dst_regions: SpecRegion) -> SpecRegionPair: src_group = src_regions.memory dst_group = dst_regions.memory @@ -315,167 +261,57 @@ def _compute_head_offsets( return 0, dst_head_idx * bytes_per_head @staticmethod - def _bytes_per_buffer(*, bytes_per_layer: int, buffers_per_layer: int, side: str) -> int: - """Bytes of one buffer (K or V) within a layer's region.""" - if buffers_per_layer <= 0 or bytes_per_layer % buffers_per_layer != 0: - raise ValueError( - f"HND layer geometry is not evenly divisible ({side}): " - f"bytes_per_layer={bytes_per_layer}, buffers_per_layer={buffers_per_layer}" - ) - return bytes_per_layer // buffers_per_layer - - @staticmethod - def _bytes_per_head(layer_kv_bytes: int, heads: int, *, side: str) -> int: - """Bytes of one head's rows within a K/V buffer. - - Byte-granular head slicing is only valid when a head lands on a byte - boundary; sub-byte dtypes (e.g. NVFP4) satisfy this whenever the - per-head element count covers whole bytes, which this divisibility - check enforces without any fractional arithmetic. - """ - if heads <= 0 or layer_kv_bytes % heads != 0: - raise ValueError( - f"HND head slicing is not byte-aligned ({side}): " - f"layer_kv_bytes={layer_kv_bytes}, kv_heads={heads}" - ) - return layer_kv_bytes // heads + def _get_layer_kv_num(ri: RankInfo) -> int: + return ( + ri.attention.kv_heads_per_rank + * ri.attention.tokens_per_block + * ri.attention.dims_per_head + * ri.attention.element_bytes + ) -class NHDHeadMismatchMapper(HNDHeadMismatchMapper): - """Map heterogeneous KV heads stored token-major as ``[N, H, D]``. +class IndexerKCacheHeadMatchMapper(RegionMapperBase): + """ + Mapper for indexer K cache when head counts match. - ``HNDHeadMismatchMapper`` selects one contiguous head range per K/V buffer, - which is correct for HND storage. In NHD storage, the selected head range - is contiguous only within one token, so this mapper emits one fragment per - ``(layer, K/V, token)``. Only offset precomputation differs from the - parent; the inherited :meth:`map` consumes ``_src_flat_offsets``, - ``_dst_flat_offsets``, and ``_bytes_cont_heads``. + Moves contiguous block(s) as a single chunk, aligned by block_size_per_layer, + with constant source/destination block offsets. """ def __init__( self, - *, - src_layer_offsets: Sequence[int] | np.ndarray, - dst_layer_offsets: Sequence[int] | np.ndarray, + transfer_layers: int, + src_layer_off: int, + dst_layer_off: int, self_ri: RankInfo, peer_ri: RankInfo, - self_bytes_per_layer: int, - peer_bytes_per_layer: int, - self_buffers_per_layer: int, - peer_buffers_per_layer: int, - ) -> None: - # Deliberately do not call HNDHeadMismatchMapper.__init__: its offsets - # assume HND-contiguous heads. Initialize the three attributes consumed - # by the inherited map() with NHD token-granular geometry instead. - self_tpb = self_ri.attention.tokens_per_block - peer_tpb = peer_ri.attention.tokens_per_block - if self_tpb != peer_tpb: - raise ValueError( - "NHDHeadMismatchMapper requires equal tokens_per_block; " - f"local={self_tpb}, peer={peer_tpb}" - ) - - self_heads = self_ri.attention.kv_heads_per_rank - peer_heads = peer_ri.attention.kv_heads_per_rank - if self_buffers_per_layer != peer_buffers_per_layer: - raise ValueError( - "NHD buffer count per layer mismatch: " - f"local={self_buffers_per_layer}, peer={peer_buffers_per_layer}" - ) - src_offsets = np.asarray(src_layer_offsets, dtype=np.int64) - dst_offsets = np.asarray(dst_layer_offsets, dtype=np.int64) - if src_offsets.size == 0 or src_offsets.size != dst_offsets.size: - raise ValueError( - "NHD layer offsets must be non-empty and equal-length: " - f"src={src_offsets.size}, dst={dst_offsets.size}" + block_size_per_layer: int, + ): + if not isinstance(block_size_per_layer, int): + raise TypeError( + f"block_size_per_layer must be int, got {type(block_size_per_layer).__name__} " + f"(value={block_size_per_layer}). Use // instead of / for integer division." ) + self._frag_size = block_size_per_layer * transfer_layers + self._src_block_off = block_size_per_layer * src_layer_off + self._dst_block_off = block_size_per_layer * dst_layer_off - self_bytes_per_token_head = self._bytes_per_token_head( - bytes_per_layer=self_bytes_per_layer, - buffers_per_layer=self_buffers_per_layer, - tokens_per_block=self_tpb, - heads=self_heads, - ) - peer_bytes_per_token_head = self._bytes_per_token_head( - bytes_per_layer=peer_bytes_per_layer, - buffers_per_layer=peer_buffers_per_layer, - tokens_per_block=peer_tpb, - heads=peer_heads, + @nvtx_range("IndexerKCacheHeadMatchMapper.map") + def map(self, src_regions: SpecRegion, dst_regions: SpecRegion) -> SpecRegionPair: + src_group = src_regions.memory + dst_group = dst_regions.memory + assert src_group.ptrs.size == dst_group.ptrs.size, ( + f"Number of regions of src({src_group.ptrs.size}) and dst({dst_group.ptrs.size}) must match" ) - if self_bytes_per_token_head != peer_bytes_per_token_head: - raise ValueError( - "NHD bytes per token/head mismatch: " - f"local={self_bytes_per_token_head}, peer={peer_bytes_per_token_head}" - ) - self._bytes_cont_heads = min(self_heads, peer_heads) * self_bytes_per_token_head - - src_head_off, dst_head_off = HNDHeadMismatchMapper._compute_head_offsets( - self_ri.tp_size_per_dp_group, - peer_ri.tp_size_per_dp_group, - self_ri.tp_rank, - peer_ri.tp_rank, - self_kv_heads=self_heads, - peer_kv_heads=peer_heads, - bytes_per_head=self_bytes_per_token_head, + new_src_ptrs = src_group.ptrs + self._src_block_off + new_dst_ptrs = dst_group.ptrs + self._dst_block_off + new_src = MemRegionGroup(ptrs=new_src_ptrs, bytes_per_region=self._frag_size) + new_dst = MemRegionGroup(ptrs=new_dst_ptrs, bytes_per_region=self._frag_size) + return SpecRegionPair( + src=SpecRegion(memory=new_src, spec=src_regions.spec), + dst=SpecRegion(memory=new_dst, spec=dst_regions.spec), ) - self._src_flat_offsets = self._build_flat_offsets( - layer_offsets=src_offsets, - buffers_per_layer=self_buffers_per_layer, - tokens_per_block=self_tpb, - heads=self_heads, - bytes_per_token_head=self_bytes_per_token_head, - head_offset=src_head_off, - ) - self._dst_flat_offsets = self._build_flat_offsets( - layer_offsets=dst_offsets, - buffers_per_layer=peer_buffers_per_layer, - tokens_per_block=peer_tpb, - heads=peer_heads, - bytes_per_token_head=peer_bytes_per_token_head, - head_offset=dst_head_off, - ) - - @staticmethod - def _bytes_per_token_head( - *, - bytes_per_layer: int, - buffers_per_layer: int, - tokens_per_block: int, - heads: int, - ) -> int: - denominator = buffers_per_layer * tokens_per_block * heads - if denominator <= 0 or bytes_per_layer % denominator != 0: - raise ValueError( - "NHD region geometry is not evenly divisible: " - f"bytes_per_layer={bytes_per_layer}, " - f"buffers_per_layer={buffers_per_layer}, " - f"tokens_per_block={tokens_per_block}, kv_heads={heads}, " - f"denominator={denominator}" - ) - return bytes_per_layer // denominator - - @staticmethod - def _build_flat_offsets( - *, - layer_offsets: np.ndarray, - buffers_per_layer: int, - tokens_per_block: int, - heads: int, - bytes_per_token_head: int, - head_offset: int, - ) -> np.ndarray: - buffer_indices = np.arange(buffers_per_layer, dtype=np.int64) - token_indices = np.arange(tokens_per_block, dtype=np.int64) - token_bytes = heads * bytes_per_token_head - buffer_bytes = tokens_per_block * token_bytes - return ( - layer_offsets[:, None, None] - + buffer_bytes * buffer_indices[None, :, None] - + token_bytes * token_indices[None, None, :] - + head_offset - ).ravel() - class AttentionPolicy: def __init__(self, self_rank_info: RankInfo): @@ -499,29 +335,9 @@ def _mismatch(self, field: str, local, peer) -> bool: local != peer, f"{field} mismatch", field=field, local=local, peer=peer ) - @staticmethod - def _uses_exact_tpb_mapper(ri: RankInfo) -> bool: - """NHD / replicated pools address bytes inside a block, so their - geometry only lines up when both sides use the same tokens_per_block.""" - if ri.page_table is None: - return False - return any( - pool_view.mapper_kind in (MapperKind.NHD, MapperKind.REPLICATED) - for layer_group in ri.page_table.layer_groups - for pool_view in getattr(layer_group, "pool_views", ()) - ) - - def _tpb_check(self, local: int, peer: int, peer_ri: RankInfo) -> bool: + def _tpb_check(self, local: int, peer: int) -> bool: if local == peer: return False - if self._uses_exact_tpb_mapper(self._ri) or self._uses_exact_tpb_mapper(peer_ri): - logger.warning( - "AttentionPolicy: incompatible: tokens_per_block mismatch for " - "NHD/replicated pools; local=%d peer=%d", - local, - peer, - ) - return True larger, smaller = max(local, peer), min(local, peer) if larger % smaller != 0: logger.warning( @@ -551,15 +367,7 @@ def check_peer_compatible(self, peer_ri: RankInfo) -> bool: peer=peer_ri.cp_size, ) or self._mismatch("element_bytes", a.element_bytes, b.element_bytes) - or self._fail_if( - not self.head_match(peer_ri)[0] - and not float(a.tokens_per_block * a.dims_per_head * a.element_bytes).is_integer(), - "sub-byte head slicing is not byte-aligned", - tokens_per_block=a.tokens_per_block, - dims_per_head=a.dims_per_head, - element_bytes=a.element_bytes, - ) - or self._tpb_check(a.tokens_per_block, b.tokens_per_block, peer_ri) + or self._tpb_check(a.tokens_per_block, b.tokens_per_block) or self._mismatch("dims_per_head", a.dims_per_head, b.dims_per_head) or self._fail_if( a.is_mla and (a.kv_heads_per_rank != 1 or b.kv_heads_per_rank != 1), @@ -597,66 +405,53 @@ def build_kv_mapper( *, peer_ri: RankInfo, mapper_kind: MapperKind, - self_layer_offsets: "Sequence[int] | np.ndarray", - peer_layer_offsets: "Sequence[int] | np.ndarray", - self_bytes_per_layer: int, - peer_bytes_per_layer: int, - self_buffers_per_layer: int = 1, - peer_buffers_per_layer: int = 1, + transfer_layers: int, + self_layer_offset: int, + peer_layer_offset: int, + self_pool_num_layers: int, + peer_pool_num_layers: int, + self_pool_slot_bytes: int, + peer_pool_slot_bytes: int, ) -> RegionMapperBase: - """Pick the mapper for one view pair. - - Every view is entries-driven: layer selection always uses explicit - slot-relative per-layer byte offsets, never positional arithmetic - (other role classes may interleave between this class's layers). - The kind only decides the two irreducible semantic differences: - - - REPLICATED skips head matching entirely (bytes are identical on - every TP rank; fan-in ownership is decided upstream). - - Under head mismatch, HND (INDEXED) slices one contiguous head - range per K/V buffer, while NHD must slice inside every token. - - Head-matched views of any kind collapse into IntactMapper, - whose run merging degrades to a single whole-region copy per block - when the selected layers are contiguous on both sides. - """ - if mapper_kind == MapperKind.REPLICATED: - return ReplicatedMapper( - self_layer_offsets, - peer_layer_offsets, - self_bytes_per_layer, - peer_bytes_per_layer, - ) - head_match, _ = self.head_match(peer_ri) + + if head_match and transfer_layers == self_pool_num_layers == peer_pool_num_layers: + return IdentityMapper() + if head_match: - return IntactMapper( - self_layer_offsets, - peer_layer_offsets, - self_bytes_per_layer, - peer_bytes_per_layer, - mapper_name=mapper_kind.name, - ) + if mapper_kind == MapperKind.FLAT: + block_size_per_layer = self_pool_slot_bytes // self_pool_num_layers + return IndexerKCacheHeadMatchMapper( + transfer_layers=transfer_layers, + src_layer_off=self_layer_offset, + dst_layer_off=peer_layer_offset, + self_ri=self._ri, + peer_ri=peer_ri, + block_size_per_layer=block_size_per_layer, + ) - if mapper_kind == MapperKind.NHD: - return NHDHeadMismatchMapper( - src_layer_offsets=self_layer_offsets, - dst_layer_offsets=peer_layer_offsets, + slot_size_per_layer = self_pool_slot_bytes // self_pool_num_layers + peer_size_per_layer = peer_pool_slot_bytes // peer_pool_num_layers + assert slot_size_per_layer == peer_size_per_layer, ( + f"slot_size_per_layer mismatch between self ({slot_size_per_layer}) " + f"and peer ({peer_size_per_layer}) for HeadMatchMapper" + ) + return HeadMatchMapper( + transfer_layers=transfer_layers, + src_layer_off=self_layer_offset, + dst_layer_off=peer_layer_offset, self_ri=self._ri, peer_ri=peer_ri, - self_bytes_per_layer=self_bytes_per_layer, - peer_bytes_per_layer=peer_bytes_per_layer, - self_buffers_per_layer=self_buffers_per_layer, - peer_buffers_per_layer=peer_buffers_per_layer, + slot_size_per_layer=slot_size_per_layer, ) - return HNDHeadMismatchMapper( - src_layer_offsets=self_layer_offsets, - dst_layer_offsets=peer_layer_offsets, + if mapper_kind == MapperKind.FLAT: + raise ValueError("IndexerKCacheHeadMatchMapper is not supported for head mismatch case") + + return HeadMismatchMapper( + transfer_layers=transfer_layers, + src_layer_off=self_layer_offset, + peer_layer_off=peer_layer_offset, self_ri=self._ri, peer_ri=peer_ri, - self_bytes_per_layer=self_bytes_per_layer, - peer_bytes_per_layer=peer_bytes_per_layer, - self_buffers_per_layer=self_buffers_per_layer, - peer_buffers_per_layer=peer_buffers_per_layer, ) diff --git a/tensorrt_llm/_torch/disaggregation/native/mixers/attention/spec.py b/tensorrt_llm/_torch/disaggregation/native/mixers/attention/spec.py index 931eef38dc16..4c3db7cc8638 100644 --- a/tensorrt_llm/_torch/disaggregation/native/mixers/attention/spec.py +++ b/tensorrt_llm/_torch/disaggregation/native/mixers/attention/spec.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from dataclasses import asdict, dataclass @@ -22,7 +7,7 @@ class AttentionInfo: kv_heads_per_rank: int tokens_per_block: int dims_per_head: int - element_bytes: int | float + element_bytes: int enable_attention_dp: bool is_mla: bool diff --git a/tensorrt_llm/_torch/disaggregation/native/peer.py b/tensorrt_llm/_torch/disaggregation/native/peer.py index ebea1dfba5ee..1902c9d14f4f 100644 --- a/tensorrt_llm/_torch/disaggregation/native/peer.py +++ b/tensorrt_llm/_torch/disaggregation/native/peer.py @@ -1,38 +1,19 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import Counter from dataclasses import dataclass, field from typing import Dict, List, Tuple -import numpy as np - from tensorrt_llm import logger from tensorrt_llm._torch.disaggregation.base.region import RegionMapperBase from tensorrt_llm._torch.disaggregation.native.mixers.attention.peer import AttentionPolicy from tensorrt_llm._torch.disaggregation.native.rank_info import RankInfo from tensorrt_llm._torch.disaggregation.resource.kv_extractor import KVRegionExtractorV1 -from tensorrt_llm._torch.disaggregation.resource.page import ( - AttentionLayerGroup, - MapperKind, - PoolView, -) +from tensorrt_llm._torch.disaggregation.resource.page import AttentionLayerGroup, MapperKind from tensorrt_llm._torch.disaggregation.resource.utils import ( - get_layer_byte_ranges, + get_global_layer_ids, + get_layer_group_num_layers, get_layer_to_layer_group, + get_physical_pool, get_pool_view_global_layer_ids, + get_pool_view_num_layers, ) # Type alias for (lg_idx, pool_idx) pair @@ -76,33 +57,6 @@ def register(self, peer_name: str, peer_rank: int, peer_ri: RankInfo): extractor = KVRegionExtractorV1(peer_ri.page_table) self._peer_ext_cache[key] = extractor - head_match, _ = self._attention_policy.head_match(peer_ri) - if not head_match: - self_page_table = self._self_ext_cache.page_table - nhd_fragments_per_token = sum( - len( - self_page_table.layer_groups[layer_group_id].pool_views[pool_idx].buffer_entries - ) - for layer_group_id, pool_idx in self.get_pool_mapping(peer_ri) - if self_page_table.layer_groups[layer_group_id].pool_views[pool_idx].mapper_kind - == MapperKind.NHD - ) - if nhd_fragments_per_token: - local_heads = self._ri.attention.kv_heads_per_rank - peer_heads = peer_ri.attention.kv_heads_per_rank - logger.warning_once( - "NHD head-mismatched disaggregated KV transfer has no " - "contiguous staging path and will emit approximately " - f"{nhd_fragments_per_token} NIXL descriptors per transferred " - "token per peer, excluding block-level replicated pools " - f"(local_kv_heads={local_heads}, peer_kv_heads={peer_heads}). " - "Long-context TEP/DEP transfers may have high latency.", - key=( - "native-nhd-head-mismatch-" - f"{local_heads}-{peer_heads}-{nhd_fragments_per_token}" - ), - ) - def peer_extractor(self, peer_name: str, peer_rank: int) -> KVRegionExtractorV1: return self._peer_ext_cache[self._unique_key(peer_name, peer_rank)] @@ -164,13 +118,6 @@ def get_pool_mapping(self, peer_ri: RankInfo) -> Dict[LGPoolKey, LGPoolKey]: Layer-overlap is required: a peer pool with the same pool_role but zero layer overlap with self is *not* a match — the two pools cover disjoint layers and have nothing to transfer. - - A self layer group never matches multiple peer layer groups, so the - result is one peer pool per self pool. Layer groups partition each - rank's layers by attention/life-cycle class, which both sides derive - from the same model config; PP only changes which layers overlap (the - fan-out across peer PP ranks is handled by calling this method once - per peer rank). Step 1 raises if this invariant is ever violated. """ key = self._unique_key(peer_ri.instance_name, peer_ri.instance_rank) if key in self._lg_pool_mapping_cache: @@ -193,40 +140,27 @@ def get_pool_mapping(self, peer_ri: RankInfo) -> Dict[LGPoolKey, LGPoolKey]: if not isinstance(self_lg, AttentionLayerGroup): continue for self_pi, self_pv in enumerate(self_lg.pool_views): - # Every view carries buffer_entries, so a view's exact layer - # set always comes from its entries (a view may cover a - # subset of the LG when V2 splits an LG into multiple pools - # by buffer-size class, or when a role class exists only on - # some layers, e.g. sparse-layer index-K). - pv_global_ids = get_pool_view_global_layer_ids(self_pv, self_lg) + # The only place mapper_kind affects pool matching: + # INDEXED → pool may cover a subset of the LG; read + # buffer_entries to find the exact layer set. + # FLAT → pool covers the entire LG by convention; + # use the LG's layer ids directly. + self_is_flat = self_pv.mapper_kind == MapperKind.FLAT + pv_global_ids = ( + get_global_layer_ids(self_lg) + if self_is_flat + else get_pool_view_global_layer_ids(self_pv, self_lg) + ) if not pv_global_ids: continue - # Step 1: find the peer layer_group via overlapping global_layer_ids. - # A self layer group (hence each of its pool views) never matches - # multiple peer layer groups: layer groups partition a rank's - # layers by attention/life-cycle class, both sides derive that - # class from the same model config, and global ids are - # PP-invariant. So PP only changes WHICH layers overlap — layers - # the peer doesn't hold are a legal skip (PP slices, one-sided - # MTP layers) — never how many peer LGs they land in; the PP - # fan-out is handled by per-peer-rank calls of this method. A - # multi-LG hit therefore means the two peers group layers - # differently (unsupported topology), and we fail loudly instead - # of silently transferring only the first LG's overlap. - peer_lg_indices = { - peer_layer_to_group[g] for g in pv_global_ids if g in peer_layer_to_group - } - if not peer_lg_indices: + # Step 1: find peer layer_group via any overlapping global_layer_id. + peer_lg_idx = next( + (peer_layer_to_group[g] for g in pv_global_ids if g in peer_layer_to_group), + None, + ) + if peer_lg_idx is None: continue - if len(peer_lg_indices) > 1: - raise ValueError( - "PeerRegistrar.get_pool_mapping: pool view " - f"(lg={self_lg_idx}, pool={self_pi}) spans multiple peer " - f"layer groups {sorted(peer_lg_indices)}; mismatched layer " - "grouping between peers is not supported" - ) - peer_lg_idx = next(iter(peer_lg_indices)) peer_lg = peer_pt.layer_groups[peer_lg_idx] # Step 2: pick the first peer pool with the same pool_role @@ -247,7 +181,11 @@ def get_pool_mapping(self, peer_ri: RankInfo) -> Dict[LGPoolKey, LGPoolKey]: for peer_pi, peer_pv in enumerate(peer_lg.pool_views): if peer_pv.pool_role != self_pv.pool_role: continue - peer_global_ids = get_pool_view_global_layer_ids(peer_pv, peer_lg) + peer_global_ids = ( + get_global_layer_ids(peer_lg) + if peer_pv.mapper_kind == MapperKind.FLAT + else get_pool_view_global_layer_ids(peer_pv, peer_lg) + ) if not set(peer_global_ids) & self_layer_set: continue if peer_pv.mapper_kind != self_pv.mapper_kind: @@ -302,89 +240,53 @@ def get_kv_map( f"(local={self_pv.mapper_kind.name}, peer={peer_pv.mapper_kind.name})" ) - # Every view is entries-driven: resolve the overlap layers to - # slot-relative byte offsets on each side from the views' buffer - # entries. Layer selection is explicit, so mappers never assume a - # uniform layer stride (other role classes may interleave), and no - # convention about global-id/byte-offset ordering is needed. - self_global_ids = get_pool_view_global_layer_ids(self_pv, self_lg) - peer_global_ids = get_pool_view_global_layer_ids(peer_pv, peer_lg) - # Iterate the overlap in self's physical slot order (not sorted by - # global id) so that layers whose regions are contiguous on both - # sides stay adjacent in the offset arrays and the mappers can merge - # them into one fragment even when global-id order diverges from the - # physical layout. Order only affects run merging (a perf property): - # each layer's byte offset is looked up explicitly below, so any - # iteration order transfers correct bytes. - overlap = set(self_global_ids) & set(peer_global_ids) - overlapping_layers = [gid for gid in self_global_ids if gid in overlap] - - self_starts, self_bytes_per_layer = get_layer_byte_ranges(self_pv) - peer_starts, peer_bytes_per_layer = get_layer_byte_ranges(peer_pv) - self_g2l = {ll.global_layer_id: ll.local_layer_id for ll in self_lg.local_layers} - peer_g2l = {ll.global_layer_id: ll.local_layer_id for ll in peer_lg.local_layers} - self_layer_offsets = np.array( - [self_starts[self_g2l[gid]] for gid in overlapping_layers], dtype=np.int64 - ) - peer_layer_offsets = np.array( - [peer_starts[peer_g2l[gid]] for gid in overlapping_layers], dtype=np.int64 - ) - # Per-layer buffer count (K and V are separate buffers within a - # layer's region); head-mismatch mappers slice heads inside each. - self_buffers_per_layer = self._get_buffers_per_layer( - self_pv, - layer_group_id=self_lg_idx, - pool_idx=self_pi, - ) - peer_buffers_per_layer = self._get_buffers_per_layer( - peer_pv, - layer_group_id=peer_lg_idx, - pool_idx=peer_pi, - ) + # FLAT pools carry no per-buffer layer info, so layer ids and + # layer count come from the layer_group itself. + # + # Sort by global_layer_id so that ``.index(first_overlap_layer)`` + # below returns the layer's slot position. This relies on the + # convention that managers (V1 / V2 / DSv4) assign global_layer_id + # monotonically with the layer's byte offset in the slot. + if self_pv.mapper_kind == MapperKind.FLAT: + self_global_ids = sorted(get_global_layer_ids(self_lg)) + peer_global_ids = sorted(get_global_layer_ids(peer_lg)) + self_num_layers = get_layer_group_num_layers(self_lg) + peer_num_layers = get_layer_group_num_layers(peer_lg) + else: + self_global_ids = sorted(get_pool_view_global_layer_ids(self_pv, self_lg)) + peer_global_ids = sorted(get_pool_view_global_layer_ids(peer_pv, peer_lg)) + self_num_layers = get_pool_view_num_layers(self_pv) + peer_num_layers = get_pool_view_num_layers(peer_pv) + + overlapping_layers = sorted(set(self_global_ids) & set(peer_global_ids)) + transfer_layers = len(overlapping_layers) + + if transfer_layers > 0: + first_overlap_layer = overlapping_layers[0] + self_layer_offset = self_global_ids.index(first_overlap_layer) + peer_layer_offset = peer_global_ids.index(first_overlap_layer) + else: + self_layer_offset = 0 + peer_layer_offset = 0 + + self_phys = get_physical_pool(self_pt, self_lg_idx, self_pv.pool_idx) + peer_phys = get_physical_pool(peer_pt, peer_lg_idx, peer_pv.pool_idx) mapper = self._attention_policy.build_kv_mapper( peer_ri=peer_ri, mapper_kind=self_pv.mapper_kind, - self_layer_offsets=self_layer_offsets, - peer_layer_offsets=peer_layer_offsets, - self_bytes_per_layer=self_bytes_per_layer, - peer_bytes_per_layer=peer_bytes_per_layer, - self_buffers_per_layer=self_buffers_per_layer, - peer_buffers_per_layer=peer_buffers_per_layer, + transfer_layers=transfer_layers, + self_layer_offset=self_layer_offset, + peer_layer_offset=peer_layer_offset, + self_pool_num_layers=self_num_layers, + peer_pool_num_layers=peer_num_layers, + self_pool_slot_bytes=self_phys.slot_bytes, + peer_pool_slot_bytes=peer_phys.slot_bytes, ) self._kv_map_cache[cache_key] = mapper return mapper - @staticmethod - def _get_buffers_per_layer( - pool_view: PoolView, - *, - layer_group_id: int, - pool_idx: int, - ) -> int: - """Per-layer buffer count of a view (e.g. K+V -> 2, key-only -> 1). - - Views are bucketed per (layer group, pool, mapper kind) at page-table - build time, so every layer in a view carries the same role set and - hence the same entry count — a skewed distribution should never occur. - Still verify it per layer rather than via total-count divisibility: - e.g. 1 + 3 entries over two layers passes ``total % layers == 0`` yet - would make head-slicing mappers split every layer at wrong offsets. - """ - entries = pool_view.buffer_entries - if len(entries) == 0: - return 1 - counts = Counter(int(e["local_layer_id"]) for e in entries) - distinct = set(counts.values()) - if len(distinct) != 1: - raise ValueError( - "PoolView buffer entries are not evenly distributed across layers: " - f"layer_group={layer_group_id}, pool={pool_idx}, " - f"per-layer entry counts={sorted(counts.items())}" - ) - return distinct.pop() - @staticmethod def _find_overlap(self_val, peer_val, self_rank, peer_rank=None): if self_val <= peer_val: @@ -466,44 +368,6 @@ def should_send_kv(self, peer_overlap: PeerOverlap, peer_rank_info: RankInfo) -> self_tp_rank_in_dp_group % dup_head_factor ) - def _owns_tp_fan_in(self, peer_rank_info: RankInfo) -> bool: - """Elect one owner when replicated bytes fan in across TP ranks. - - A peer with fewer TP shards receives identical replicated data from - several local ranks. Rotate the elected owner by the destination's - DP rank (mirroring ``should_send_kv``'s head-duplication pairing) so - that with a multi-DP-group generation side the extra replicated - traffic spreads across local ranks instead of always landing on the - first rank of each fan-in group. - """ - ratio = max( - 1, - self._ri.tp_size_per_dp_group // peer_rank_info.tp_size_per_dp_group, - ) - self_tp_rank = self._ri.tp_rank % self._ri.tp_size_per_dp_group - return self_tp_rank % ratio == peer_rank_info.dp_rank % ratio - - def should_send_pool( - self, - peer_overlap: PeerOverlap, - peer_rank_info: RankInfo, - layer_group_id: int, - pool_idx: int, - ) -> bool: - """Return whether this rank owns the transfer of one view pair. - - ``pool_idx`` indexes the layer group's ``pool_views`` list (one view - per role class; several views may share a physical pool). Each view - is kind-homogeneous, so ownership is a single per-view decision: - replicated views use one sender per fan-in group, sharded views - retain head-duplication routing. - """ - layer_group = self._self_ext_cache.page_table.layer_groups[layer_group_id] - pool_view = layer_group.pool_views[pool_idx] - if pool_view.mapper_kind == MapperKind.REPLICATED: - return self._owns_tp_fan_in(peer_rank_info) - return self.should_send_kv(peer_overlap, peer_rank_info) - def should_send_aux(self, peer_rank_info: RankInfo) -> bool: # to ensure the transfer aux is not duplicated diff --git a/tensorrt_llm/_torch/disaggregation/native/perf_logger.py b/tensorrt_llm/_torch/disaggregation/native/perf_logger.py index 5528b8f1eb08..3c491df3495b 100644 --- a/tensorrt_llm/_torch/disaggregation/native/perf_logger.py +++ b/tensorrt_llm/_torch/disaggregation/native/perf_logger.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -130,13 +130,11 @@ def get_task_latency(self, peer_rank: int) -> float: class PerfLogManager: """Singleton manager for KV transfer performance logging. - Logic (checked in priority order): - 1. TRTLLM_KVCACHE_TIME_OUTPUT_PATH set (C++ standard): enabled, CSV to - ``{path}/{instance_name}_{instance_rank}.csv`` - 2. TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1 with TLLM_KV_TRANSFER_PERF_LOG_FILE: - enabled, CSV to ``{base}_{instance_name}_{instance_rank}.csv`` - 3. TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1 alone: enabled, logger.info to stdout - 4. None of the above: disabled + Logic: + - TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO not set: no output + - TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO set, TLLM_KV_TRANSFER_PERF_LOG_FILE not set: + logger.info to stdout + - Both set: CSV output to {TLLM_KV_TRANSFER_PERF_LOG_FILE}_{instance_name}_{instance_rank}.csv """ _instance = None @@ -156,18 +154,8 @@ def __init__(self): self._initialized = True self._file_loggers = {} # (instance_name, instance_rank) -> logger self._file_lock = threading.Lock() - - # Primary: C++ standard env var (directory path) - cpp_output_path = os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH") - if cpp_output_path: - self._perf_enabled = True - self._log_file_base = cpp_output_path - self._use_cpp_naming = True - else: - # Fallback: existing Python env vars - self._perf_enabled = os.getenv("TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO", "0") == "1" - self._log_file_base = os.getenv("TLLM_KV_TRANSFER_PERF_LOG_FILE") - self._use_cpp_naming = False + self._perf_enabled = os.getenv("TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO", "0") == "1" + self._log_file_base = os.getenv("TLLM_KV_TRANSFER_PERF_LOG_FILE") @property def enabled(self) -> bool: @@ -187,12 +175,8 @@ def _get_or_create_file_logger(self, instance_name: str, instance_rank: int): if key in self._file_loggers: return self._file_loggers[key] - if self._use_cpp_naming: - # C++ pattern: {dir}/{instance_name}_{instance_rank}.csv - log_file = os.path.join(self._log_file_base, f"{instance_name}_{instance_rank}.csv") - else: - # Legacy Python pattern: {base}_{instance_name}_{instance_rank}.csv - log_file = f"{self._log_file_base}_{instance_name}_{instance_rank}.csv" + # Create file path: {base}_{instance_name}_{instance_rank}.csv + log_file = f"{self._log_file_base}_{instance_name}_{instance_rank}.csv" try: # Create directory if needed @@ -331,73 +315,6 @@ def log_recv_task_perf( ) self.log(instance_name, instance_rank, csv_line, info_msg) - def log_gen_transfer_summary( - self, - unique_rid: int, - instance_name: str, - instance_rank: int, - gen_side_transfer_time_ms: float, - kv_cache_size: int, - ) -> None: - """Log a gen-side transfer summary row to a separate CSV. - - Written after timing sync across ranks so values are globally - consistent. Only active when ``TRTLLM_KVCACHE_TIME_OUTPUT_PATH`` - is set. - - Args: - unique_rid: Unique request id. - instance_name: Instance name for file naming. - instance_rank: Instance rank for file naming. - gen_side_transfer_time_ms: Synced gen-side transfer time in ms. - kv_cache_size: Total KV cache size across ranks (bytes). - """ - cpp_output_path = os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH") - if not cpp_output_path: - return - - _GEN_SUMMARY_HEADER = "timestamp,RequestID,gen_side_transfer_time(ms),kv_cache_size" - key = ("gen_summary", instance_name, instance_rank) - - if key not in self._file_loggers: - with self._file_lock: - if key not in self._file_loggers: - log_file = os.path.join( - cpp_output_path, - f"{instance_name}_{instance_rank}_gen_transfer_summary.csv", - ) - try: - log_dir = os.path.dirname(log_file) - if log_dir and not os.path.exists(log_dir): - os.makedirs(log_dir, exist_ok=True) - - write_header = not os.path.exists(log_file) - file_logger = logging.getLogger( - f"kv_gen_summary_{instance_name}_{instance_rank}" - ) - file_logger.setLevel(logging.INFO) - file_logger.propagate = False - file_handler = logging.FileHandler(log_file, mode="a", encoding="utf-8") - formatter = logging.Formatter( - fmt="%(asctime)s.%(msecs)03d,%(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - file_handler.setFormatter(formatter) - file_logger.addHandler(file_handler) - if write_header: - file_handler.stream.write(_GEN_SUMMARY_HEADER + "\n") - file_handler.stream.flush() - self._file_loggers[key] = file_logger - except OSError as e: - sys.stderr.write( - f"[KV Transfer] Warning: Failed to create gen summary log file {log_file}: {e}\n" - ) - return - - file_logger = self._file_loggers.get(key) - if file_logger: - file_logger.info(f"{unique_rid},{gen_side_transfer_time_ms:.3f},{kv_cache_size}") - # Singleton instance perf_log_manager = PerfLogManager() diff --git a/tensorrt_llm/_torch/disaggregation/native/rank_info.py b/tensorrt_llm/_torch/disaggregation/native/rank_info.py index 4c0dba242199..12a614ca2dad 100644 --- a/tensorrt_llm/_torch/disaggregation/native/rank_info.py +++ b/tensorrt_llm/_torch/disaggregation/native/rank_info.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from dataclasses import asdict, dataclass from typing import List, Optional @@ -74,15 +59,6 @@ def from_kv_cache_manager( m = kv_cache_manager.mapping kvm = kv_cache_manager enable_attention_dp = m.enable_attention_dp - kv_heads_per_rank = next((h for h in kvm.num_kv_heads_per_layer if h > 0), 0) - # Eight is the smallest element count guaranteed to occupy whole bytes - # for every supported sub-byte cache dtype (including NVFP4). - bytes_for_eight_elements = get_size_in_bytes(8, kvm.dtype) - element_bytes = ( - bytes_for_eight_elements // 8 - if bytes_for_eight_elements % 8 == 0 - else bytes_for_eight_elements / 8 - ) return cls( instance_name=instance_name, instance_rank=m.rank, @@ -101,10 +77,10 @@ def from_kv_cache_manager( self_endpoint="", transfer_engine_info=bytes(), attention=AttentionInfo( - kv_heads_per_rank=kv_heads_per_rank, + kv_heads_per_rank=kvm.num_kv_heads_per_layer[0], tokens_per_block=kvm.tokens_per_block, dims_per_head=kvm.head_dim, - element_bytes=element_bytes, + element_bytes=get_size_in_bytes(1, kvm.dtype), enable_attention_dp=enable_attention_dp, is_mla=kvm.kv_factor == 1, ), diff --git a/tensorrt_llm/_torch/disaggregation/native/transfer.py b/tensorrt_llm/_torch/disaggregation/native/transfer.py index 353444eeca6f..fe5105bc8652 100644 --- a/tensorrt_llm/_torch/disaggregation/native/transfer.py +++ b/tensorrt_llm/_torch/disaggregation/native/transfer.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from __future__ import annotations import os @@ -63,7 +48,6 @@ from tensorrt_llm._torch.disaggregation.native.utils import get_local_ip from tensorrt_llm._torch.disaggregation.nixl.agent import NixlTransferAgent from tensorrt_llm._torch.disaggregation.resource.kv_extractor import KVRegionExtractorV1 -from tensorrt_llm._torch.disaggregation.resource.page import MapperKind from tensorrt_llm._torch.disaggregation.resource.utils import get_unique_pool_memory_descs from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequest from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager @@ -176,16 +160,16 @@ class AgentResult(Enum): FAILED = "FAILED" -# KV_AGENT_RESULT prefix in one struct frame (was ascii frames serialized/parsed under the -# GIL per slice per writer): instance_rank, unique_rid, slice_id, is_last, status, -# transfer_size. The optional bounce tail follows at message[2:]. -_KV_RESULT_PREFIX = struct.Struct(" 0: - try: - request, send_slot_id = build_send_request( - self._bounce, - write_meta, - lambda: Sender._make_agent_request(write_meta, device_id=self._device_id), - ) - except Exception as e: - # Don't let a gather fault escape: without a result the receiver would hang and its - # region leak. Tell the receiver it failed and fail the local task instead. - logger.error( - f"_deliver_kv_to_agent: failed to build the KV send request for " - f"{write_meta.unique_rid} slice={write_meta.slice_id}: {e}" - ) - task.fail(RuntimeError(f"build_send_request failed: {e}")) - self._get_or_connect_dealer(write_meta.peer_endpoint).send( - _make_kv_result_msg( - self._instance_rank, - write_meta.unique_rid, - write_meta.slice_id, - True, # is_last_slice — ensures receiver resolves its task future - AgentResult.FAILED, - ) - ) - return + request, send_slot_id = build_send_request( + self._bounce, + write_meta, + lambda: Sender._make_agent_request(write_meta, device_id=self._device_id), + ) if timer: timer.record_transfer_start(write_meta.peer_rank) try: @@ -625,14 +589,12 @@ def _deliver_kv_to_agent(self, write_meta: WriteMeta): if send_slot_id is not None and agent_result == AgentResult.SUCCESS else None ) - transfer_size = timer.get_transfer_size(write_meta.peer_rank) if timer else 0 result_msg = _make_kv_result_msg( self._instance_rank, write_meta.unique_rid, write_meta.slice_id, write_meta.is_last_slice, agent_result, - transfer_size=transfer_size, tail=tail, ) self._get_or_connect_thread_dealer(write_meta.peer_endpoint).send(result_msg) @@ -658,8 +620,6 @@ def _deliver_kv_to_agent(self, write_meta: WriteMeta): ) else: task.complete() - if all(t.status == TaskStatus.TRANSFERRED for t in session.kv_tasks): - session.transfer_end_time = tensorrt_llm.bindings.global_steady_clock_now() logger.debug( f"deliver_kv_to_agent completed: unique_rid={write_meta.unique_rid}, " @@ -787,86 +747,89 @@ def _build_kv_write_meta(self, task: KVSendTask, req_info: RecvReqInfo) -> Write peer_extractor = self._registrar.peer_extractor( peer_ri.instance_name, peer_ri.instance_rank ) - pool_mapping = self._registrar.get_pool_mapping(peer_ri) - dst_block_ids_per_groups = req_info.block_ids_per_layer_groups - src_block_ids_per_groups = task._slice.block_ids_per_layer_groups - - # Aggregate fragments from all matching pools using numpy concatenation. - # Send ownership is per pool: replicated pools elect one fan-in - # owner, sharded pools keep head-duplication routing. - for (self_lg, self_pi), (peer_lg, peer_pi) in pool_mapping.items(): - if not self._registrar.should_send_pool(targets, peer_ri, self_lg, self_pi): - continue - src_block_ids = src_block_ids_per_groups[self_lg] - dst_block_ids = dst_block_ids_per_groups[peer_lg] - - # Both sides trim block lists to ceil(prompt_len / tpb) in - # _create_kv_slice, so dst must never exceed src. A smaller dst - # (generation prefix-cache reuse) is handled via dst_start below. - block_diff = dst_block_ids.size - src_block_ids.size - if block_diff > 0: - raise ValueError( - f"src/dst block count mismatch: {src_block_ids.size} vs " - f"{dst_block_ids.size} (dst must not exceed src)" + if self._registrar.should_send_kv(targets, peer_ri): + pool_mapping = self._registrar.get_pool_mapping(peer_ri) + dst_block_ids_per_groups = req_info.block_ids_per_layer_groups + src_block_ids_per_groups = task._slice.block_ids_per_layer_groups + + # Aggregate fragments from all matching pools using numpy concatenation + for (self_lg, self_pi), (peer_lg, peer_pi) in pool_mapping.items(): + src_block_ids = src_block_ids_per_groups[self_lg] + dst_block_ids = dst_block_ids_per_groups[peer_lg] + + # Speculative decoding: generation may have one extra draft-token block. + block_diff = dst_block_ids.size - src_block_ids.size + if block_diff == 1: + logger.debug( + f"Trimming 1 extra dst block for draft tokens: " + f"src={src_block_ids.size}, dst={dst_block_ids.size}" + ) + dst_block_ids = dst_block_ids[:-1] + elif block_diff > 1: + raise ValueError( + f"src/dst block count mismatch: {src_block_ids.size} vs " + f"{dst_block_ids.size} (expected diff <= 1)" + ) + tpb = extractor.page_table.tokens_per_block + token_range = task._slice.token_range + lg_info = extractor.page_table.layer_groups[self_lg] + window_size = getattr(lg_info, "sliding_window_size", None) + + # Block lists are the suffix of [..., slice_end); cached prefix + # is implicit in their size. token_start = (total_blocks - n) * tpb. + slice_end = token_range.end if token_range is not None else 0 + total_blocks = (slice_end + tpb - 1) // tpb + src_beam0_blocks = Sender._beam0_block_count( + src_block_ids, total_blocks, task._beam_width ) - tpb = extractor.page_table.tokens_per_block - token_range = task._slice.token_range - lg_info = extractor.page_table.layer_groups[self_lg] - window_size = getattr(lg_info, "sliding_window_size", None) - - # Block lists are the suffix of [..., slice_end); cached prefix - # is implicit in their size. token_start = (total_blocks - n) * tpb. - slice_end = token_range.end if token_range is not None else 0 - total_blocks = (slice_end + tpb - 1) // tpb - src_beam0_blocks = Sender._beam0_block_count( - src_block_ids, total_blocks, task._beam_width - ) - dst_beam0_blocks = Sender._beam0_block_count( - dst_block_ids, total_blocks, task._beam_width - ) - assert src_beam0_blocks <= total_blocks, ( - f"src beam-0 block list ({src_beam0_blocks}) exceeds total slice " - f"blocks ({total_blocks}); slice_end={slice_end}, tpb={tpb}" - ) - assert dst_beam0_blocks <= total_blocks, ( - f"dst beam-0 block list ({dst_beam0_blocks}) exceeds total slice " - f"blocks ({total_blocks}); slice_end={slice_end}, tpb={tpb}" - ) - src_start = (total_blocks - src_beam0_blocks) * tpb - dst_start = (total_blocks - dst_beam0_blocks) * tpb - if req_info.dst_start_token is not None: - dst_start = max(dst_start, req_info.dst_start_token) - if window_size is not None: - # SWA stale_end uses the request prompt_len (not slice_end — - # they differ for non-final slices). prompt_len must be plumbed - # via the session; falling back to slice_end is wrong on - # non-final slices. - assert task._prompt_len is not None, ( - "SWA layer requires session.prompt_len; " - "set TxSession(prompt_len=request.prompt_len)." + dst_beam0_blocks = Sender._beam0_block_count( + dst_block_ids, total_blocks, task._beam_width + ) + assert src_beam0_blocks <= total_blocks, ( + f"src beam-0 block list ({src_beam0_blocks}) exceeds total slice " + f"blocks ({total_blocks}); slice_end={slice_end}, tpb={tpb}" + ) + assert dst_beam0_blocks <= total_blocks, ( + f"dst beam-0 block list ({dst_beam0_blocks}) exceeds total slice " + f"blocks ({total_blocks}); slice_end={slice_end}, tpb={tpb}" + ) + src_start = (total_blocks - src_beam0_blocks) * tpb + dst_start = (total_blocks - dst_beam0_blocks) * tpb + if req_info.dst_start_token is not None: + dst_start = max(dst_start, req_info.dst_start_token) + if window_size is not None: + # SWA stale_end uses the request prompt_len (not slice_end — + # they differ for non-final slices). prompt_len must be plumbed + # via the session; falling back to slice_end is wrong on + # non-final slices. + assert task._prompt_len is not None, ( + "SWA layer requires session.prompt_len; " + "set TxSession(prompt_len=request.prompt_len)." + ) + stale_end = max(0, (task._prompt_len + 1 - window_size) // tpb) + src_start = max(stale_end * tpb, src_start) + dst_start = max(stale_end * tpb, dst_start) + src_block_ids, dst_block_ids = Sender._align_kv_blocks( + src_block_ids, + dst_block_ids, + src_token_start=src_start, + dst_token_start=dst_start, + tokens_per_block=tpb, ) - stale_end = max(0, (task._prompt_len + 1 - window_size) // tpb) - src_start = max(stale_end * tpb, src_start) - dst_start = max(stale_end * tpb, dst_start) - src_block_ids, dst_block_ids = Sender._align_kv_blocks( - src_block_ids, - dst_block_ids, - src_token_start=src_start, - dst_token_start=dst_start, - tokens_per_block=tpb, - ) - src_region = extractor.extract(src_block_ids, layer_group_id=self_lg, pool_idx=self_pi) - dst_region = peer_extractor.extract( - dst_block_ids, layer_group_id=peer_lg, pool_idx=peer_pi - ) - mapper = self._registrar.get_kv_map(peer_ri, (self_lg, self_pi), (peer_lg, peer_pi)) - region_pair = mapper.map(src_region, dst_region) - region_pairs = region_pair if isinstance(region_pair, list) else [region_pair] - for rp in region_pairs: - src_frag_parts.append(rp.src.memory.ptrs) - dst_frag_parts.append(rp.dst.memory.ptrs) - size_specs.append((rp.src.memory.ptrs.size, rp.src.memory.bytes_per_region)) + src_region = extractor.extract( + src_block_ids, layer_group_id=self_lg, pool_idx=self_pi + ) + dst_region = peer_extractor.extract( + dst_block_ids, layer_group_id=peer_lg, pool_idx=peer_pi + ) + mapper = self._registrar.get_kv_map(peer_ri, (self_lg, self_pi), (peer_lg, peer_pi)) + region_pair = mapper.map(src_region, dst_region) + region_pairs = region_pair if isinstance(region_pair, list) else [region_pair] + for rp in region_pairs: + src_frag_parts.append(rp.src.memory.ptrs) + dst_frag_parts.append(rp.dst.memory.ptrs) + size_specs.append((rp.src.memory.ptrs.size, rp.src.memory.bytes_per_region)) if src_frag_parts: src_frags = np.concatenate(src_frag_parts) @@ -1211,8 +1174,6 @@ def __init__( self._exception: Optional[Exception] = None self._closed = False self._terminal_status: Optional[SessionStatus] = None - self.transfer_start_time = None - self.transfer_end_time = None # Must be last: makes session visible to listener thread, # so all attributes above must be initialized first. self._sender.setup_session(self) @@ -1245,8 +1206,6 @@ def status(self) -> SessionStatus: return SessionStatus.READY if self.receiver_ready else SessionStatus.INIT def send(self, slice: KVSlice) -> None: - if self.transfer_start_time is None: - self.transfer_start_time = tensorrt_llm.bindings.global_steady_clock_now() with self.lock: params = self._base_args.params slice_id = len(self.kv_tasks) @@ -1520,23 +1479,13 @@ def _get_session(self, unique_rid: Optional[int]) -> Optional["RxSession"]: def _build_recv_req_info(self, task: KVRecvTask) -> RecvReqInfo: self_ri = self._registrar.self_rank_info + assert task._params.ctx_request_id is not None, ( + f"ctx_request_id is None for task unique_rid={task._unique_rid}" + ) assert task._unique_rid is not None, "KVRecvTask unique_rid is None" - # Some requests arrive with ctx_request_id None while disagg_request_id - # is set; disagg_request_id is the receive-session key, so fall back to - # it instead of failing here (nvbugs/6482576). - sender_req_id = task._params.ctx_request_id - if sender_req_id is None: - sender_req_id = task._params.disagg_request_id - if sender_req_id is None: - # Not an assert: must survive python -O so a None id never reaches - # RecvReqInfo.sender_req_id / the wire. - raise ValueError( - "both ctx_request_id and disagg_request_id are None for task " - f"unique_rid={task._unique_rid}" - ) # Receiver's cached prefix is implicit in block_ids size; sender derives dst_start. return RecvReqInfo( - sender_req_id=sender_req_id, + sender_req_id=task._params.ctx_request_id, instance_name=self_ri.instance_name, instance_rank=self_ri.instance_rank, block_ids_per_layer_groups=task._kv_slice.block_ids_per_layer_groups, @@ -1558,22 +1507,15 @@ def _fanin_bounce_safe(overlap, peer_ri) -> bool: (peer_ri.layer_num_per_pp all-equal) or per-writer sizes differ. If that full per-stage list isn't available (shorter than the fan-in degree), be conservative and fall back. Otherwise fall back to the per-fragment path (correct, just not coalesced). - Equal layer count means equal bytes only when the per-block sizes match; reserve() rejects - the mismatched case, so this only needs the count to split evenly.""" + NOTE: equal layer COUNT == equal BYTES only when per-layer KV is uniform; for mixed layer + groups (e.g. differing slot_bytes) reserve()'s `total % num_writers` divisibility guard is the + backstop, and a fully size-aware split (per-writer offsets) would be the general fix.""" if overlap.duplicate_head_factor != 1: return False if overlap.overlap_pp_size > 1: lpp = getattr(peer_ri, "layer_num_per_pp", None) if not lpp or len(lpp) < overlap.overlap_pp_size or len(set(lpp)) != 1: return False - # Replicated pools (e.g. MiniMax M3 index-key) are sent by one elected - # fan-in owner only, so with multiple writers their contributions - # differ in size and the equal split is invalid. - if len(overlap.ranks) > 1 and peer_ri.page_table is not None: - for layer_group in peer_ri.page_table.layer_groups: - for pool_view in getattr(layer_group, "pool_views", ()): - if pool_view.mapper_kind == MapperKind.REPLICATED: - return False return True def dispatch_task(self, task: KVRecvTask): @@ -1744,7 +1686,7 @@ def _process_kv_agent_result(self, _send_id: bytes, message: list[bytes]): f"_process_kv_agent_result: unexpected msg_type={message[0]!r}, expected KV_AGENT_RESULT" ) return - peer_rank, unique_rid, sender_slice_id, is_last_slice, status_code, transfer_size = ( + peer_rank, unique_rid, sender_slice_id, is_last_slice, status_code = ( _KV_RESULT_PREFIX.unpack(message[1]) ) from .bounce import decode_result_tail @@ -1764,7 +1706,6 @@ def _process_kv_agent_result(self, _send_id: bytes, message: list[bytes]): dst_ptrs=dst_ptrs, sizes=sizes, src_base=src_base, - transfer_size=transfer_size, ) def _process_aux_agent_result(self, _send_id: bytes, message: list[bytes]): @@ -1822,9 +1763,6 @@ def __init__( self._exception: Optional[Exception] = None self._closed = False self._terminal_status: Optional[SessionStatus] = None - self.transfer_start_time = None - self.transfer_end_time = None - self.kv_cache_size_bytes: int = 0 self._kv_tasks: list[KVRecvTask] = [] self._aux_count = 0 self._aux_status: TaskStatus = TaskStatus.INIT @@ -1865,8 +1803,6 @@ def mark_transferring(self, slice_id: int): self._kv_tasks[slice_id].status = TaskStatus.TRANSFERRING def receive(self, slice: KVSlice) -> None: - if self.transfer_start_time is None: - self.transfer_start_time = tensorrt_llm.bindings.global_steady_clock_now() params = self._base_args.params slice_id = len(self._kv_tasks) task = KVRecvTask( @@ -1888,10 +1824,8 @@ def process_kv_agent_result( dst_ptrs=None, sizes=None, src_base=None, - transfer_size: int = 0, ): with self.lock: - self.kv_cache_size_bytes += transfer_size assert sender_slice_id < len(self._kv_tasks), ( f"Receiver got slice_id={sender_slice_id} from sender but only has " f"{len(self._kv_tasks)} receive task(s) for request {self.request_id}. " @@ -1946,13 +1880,6 @@ def on_done( f"slice={sender_slice_id}: {e}" ) task.complete() - # Transfer end for perf/time-sync: only meaningful once every slice has - # landed. Plain attribute write (atomic under the GIL); on_done must stay - # lock-free, and consumers only read it after wait_complete succeeds. - if all(t.status == TaskStatus.TRANSFERRED for t in self._kv_tasks): - self.transfer_end_time = ( - tensorrt_llm.bindings.global_steady_clock_now() - ) logger.debug( f"KV transfer complete for request {request_id} " f"slice={sender_slice_id}" @@ -2067,16 +1994,13 @@ def cancel(self) -> None: self._terminal_status = SessionStatus.CANCELLED exc = RuntimeError(f"RxSession {self.disagg_request_id} cancelled") for task in self._kv_tasks: - rid_slice = (self.disagg_request_id, task.slice_id) if task.status == TaskStatus.INIT: # INIT = reserved but no write in flight, so freeing its bounce reservation here - # is safe. - self._receiver._bounce.release_idle_reservation(rid_slice) + # is safe (TRANSFERRING tasks keep running and self-release on SUCCESS/FAILED). + self._receiver._bounce.release_idle_reservation( + (self.disagg_request_id, task.slice_id) + ) task.fail(exc) - elif task.status == TaskStatus.TRANSFERRING: - # A write may still be mid-flight, so quarantine the region rather than freeing - # it; this keeps a cancelled transfer from leaking. No-op when bounce is off. - self._receiver._bounce.orphan_reservation(rid_slice) # Send outside the lock to avoid holding it during I/O. self._receiver.send_cancel_to_senders(self.disagg_request_id, self._sender_endpoints) @@ -2134,10 +2058,6 @@ def close(self): self.aux_slot = None # Unregister from Receiver; keep fields alive for in-flight listener messages. if self._receiver is not None: - # Reclaim any bounce region still live at teardown (closed mid-transfer) so it isn't - # leaked; a no-op for finished or non-bounce transfers. - for task in self._kv_tasks: - self._receiver._bounce.orphan_reservation((self.disagg_request_id, task.slice_id)) self._receiver.clear_session(self.disagg_request_id) def __enter__(self): diff --git a/tensorrt_llm/_torch/disaggregation/resource/cache_reuse.py b/tensorrt_llm/_torch/disaggregation/resource/cache_reuse.py index 142161093f12..3fbbe070c44f 100644 --- a/tensorrt_llm/_torch/disaggregation/resource/cache_reuse.py +++ b/tensorrt_llm/_torch/disaggregation/resource/cache_reuse.py @@ -64,15 +64,7 @@ def get_block_ids( group_idx: int, lg: AttentionLayerGroup, ) -> np.ndarray: - """Per-layer-group block identifiers for *req* (dtype ``int64``). - - Returned values are **primary memory-pool slot indices**, not raw block IDs: - ``KVRegionExtractorV1.extract`` and downstream transfer code do - ``base_ptr + slot_idx * slot_bytes`` and require the value to be a current - primary-pool offset. With host offload enabled, a block's logical ID can - diverge from its primary slot index after offload/onboard, so each backend - must translate before returning. - """ + """All block IDs for *req* in layer group *lg* (dtype ``int64``).""" @abstractmethod def commit_blocks_for_reuse(self, req: LlmRequest) -> None: @@ -105,23 +97,12 @@ def _global_cached_token_count(self, req: LlmRequest) -> int: def get_block_ids(self, req, group_idx, lg): # noqa: ARG002 first_layer = get_global_layer_ids(lg)[0] beam_width = req.py_beam_width - raw_ids = self._mgr.get_batch_cache_indices( - [req.py_request_id], layer_idx=first_layer, beam_width=beam_width - )[0] - if not raw_ids: - return np.array([], dtype=np.int64) - # block_id != primary-pool slot index once host offload kicks in; translate - # so the cache transceiver's pointer arithmetic is correct. The manager aborts - # if any referenced block is currently offloaded — disagg transfer cannot read - # from the secondary pool, and a held block can never be offloaded. - window_size = lg.sliding_window_size - # V1 layer groups carry the manager's window key (full-attention layers get the - # max window), so this is always set; see kv_extractor.build_page_table. - assert window_size is not None - pool_indices = self._mgr.get_memory_pool_block_indices( - list(raw_ids), window_size=window_size + return np.asarray( + self._mgr.get_batch_cache_indices( + [req.py_request_id], layer_idx=first_layer, beam_width=beam_width + )[0], + dtype=np.int64, ) - return np.asarray(pool_indices, dtype=np.int64) def commit_blocks_for_reuse(self, req: LlmRequest) -> None: if not self.enable_block_reuse: @@ -153,11 +134,6 @@ def _global_cached_token_count(self, req: LlmRequest) -> int: return (kv_cache.num_committed_tokens // tpb) * tpb def get_block_ids(self, req, group_idx, lg): # noqa: ARG002 - # V2 already returns per-cache-level pool slot indices (not logical block - # IDs), and active sequences GPU-lock their pages (_UniqPageLock enforces - # cache_level==GPU), so the slot_ids yielded here are already the right - # offsets for primary-pool pointer arithmetic. No translation is needed, - # unlike V1 (see _CacheReuseAdapterV1.get_block_ids). return np.fromiter( self._mgr.kv_cache_map[req.py_request_id].get_aggregated_page_indices( group_idx, valid_only=True diff --git a/tensorrt_llm/_torch/disaggregation/resource/kv_extractor.py b/tensorrt_llm/_torch/disaggregation/resource/kv_extractor.py index 4aa76c5b9605..c124d678de5d 100644 --- a/tensorrt_llm/_torch/disaggregation/resource/kv_extractor.py +++ b/tensorrt_llm/_torch/disaggregation/resource/kv_extractor.py @@ -1,19 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import defaultdict from typing import Dict, List import numpy as np @@ -36,28 +20,12 @@ PhysicalPoolGroup, PoolView, ) -from tensorrt_llm._torch.disaggregation.resource.utils import ( - compute_layer_byte_ranges, - get_physical_pool, -) -from tensorrt_llm._torch.pyexecutor.kv_cache_manager_v2 import Role +from tensorrt_llm._torch.disaggregation.resource.utils import get_physical_pool from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import MambaHybridCacheManager from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager from tensorrt_llm._utils import get_size_in_bytes, nvtx_range from tensorrt_llm.bindings import DataType -# Mapper kinds a V2 manager may declare via get_disagg_role_mapper_kinds(). -# A physical pool may mix kinds (V2 storage coalesces buffers purely by -# size within a life cycle); the page-table builder emits one PoolView per -# (pool, kind) so each view stays kind-homogeneous. -_V2_ROLE_MAPPER_KINDS = frozenset( - { - MapperKind.INDEXED, - MapperKind.REPLICATED, - MapperKind.NHD, - } -) - class KVRegionExtractorV1(RegionExtractorBase): """ @@ -92,9 +60,8 @@ def extract( described by region_ids. For KV cache: each ptr = base_address + slot_id * slot_bytes, pointing - to the start of a full slot. Sub-slot selection (layers, role classes, - heads) is the mappers' responsibility; logical views carry that - geometry in their buffer entries. + to the start of a full slot. The slot contains buffer entries for all + layers in this layer_group laid out contiguously from offset 0. Args: layer_group_id: The layer group index (= life cycle index). @@ -226,15 +193,11 @@ def build_page_table(kv_cache_manager: KVCacheManager) -> KVCachePageTable: buffer_entries=np.array(entries, dtype=BUFFER_ENTRY_DTYPE), pool_role=frozenset(kv_role_names), mapper_kind=MapperKind.INDEXED, - bytes_per_layer=stride, ) physical_pools = [kv_physical] pool_views = [kv_view] - # Indexer K cache support. The DSA indexer K cache is identical on - # every TP rank (single index head), so its view is REPLICATED with - # one synthesized buffer entry per local layer: the slot packs the - # layers equal-sized in local-layer order. + # Indexer K cache support if getattr(kv_cache_manager, "enable_indexer_k_cache", False): indexer_pool = kv_cache_manager.impl.get_indexer_k_cache_pool() # indexer_pool shape: (numBlocks, numLayers, kvFactor, blockSize), dtype=UINT8 @@ -248,19 +211,11 @@ def build_page_table(kv_cache_manager: KVCacheManager) -> KVCachePageTable: slot_bytes=indexer_slot_bytes, num_slots=num_blocks, ) - indexer_bytes_per_layer = indexer_slot_bytes // len(local_layer_ids) indexer_view = PoolView( pool_idx=1, - buffer_entries=np.array( - [ - (lid, i * indexer_bytes_per_layer, indexer_bytes_per_layer) - for i, lid in enumerate(local_layer_ids) - ], - dtype=BUFFER_ENTRY_DTYPE, - ), + buffer_entries=np.array([], dtype=BUFFER_ENTRY_DTYPE), pool_role=frozenset({"indexer_k"}), - mapper_kind=MapperKind.REPLICATED, - bytes_per_layer=indexer_bytes_per_layer, + mapper_kind=MapperKind.FLAT, ) physical_pools.append(indexer_physical) pool_views.append(indexer_view) @@ -333,191 +288,136 @@ def _compute_global_layer_ids(manager, lg_idx: int) -> List[int]: def _build_page_table_v2(manager) -> KVCachePageTable: """Build a KVCachePageTable from a KVCacheManagerV2. - Uses KVCacheManagerV2's public ``pool_group_descs`` layout API and - stamps each PoolView with the manager's native role-name strings - (``pool_role``) plus the closed-set ``mapper_kind`` discriminator used - by ``build_kv_mapper``. - - A physical pool group may be shared by several layer groups (life - cycles whose coalesced-buffer sizes are identical); each layer group - is exactly one ``SlotDescVariant`` of one pool group, so iterating - variants visits every layer group once. ``layer_groups`` stays indexed - by layer_group_id while ``pool_group_idx`` points at the shared - physical pool group entry, so per-window transfer logic keeps working. - """ - config = manager.impl.init_config - pool_group_descs = manager.impl.pool_group_descs - - # Every V2 manager declares how native roles map to the closed set of - # disaggregation mapper kinds; Role.ALL is the required fallback. - role_mapper_kinds = manager.get_disagg_role_mapper_kinds() - if Role.ALL not in role_mapper_kinds: - raise ValueError("Disaggregation role mapping must define Role.ALL") - for role, mapper_kind in role_mapper_kinds.items(): - if not isinstance(mapper_kind, MapperKind): - raise ValueError( - f"Invalid disaggregation mapper kind {mapper_kind!r} for role {role!s}" - ) - if mapper_kind not in _V2_ROLE_MAPPER_KINDS: - supported = ", ".join(kind.name for kind in sorted(_V2_ROLE_MAPPER_KINDS)) - raise ValueError( - f"Unsupported V2 disaggregation mapper kind {mapper_kind.name} " - f"for role {role!s}; supported kinds: {supported}" - ) - # INDEXED is the whole-manager legacy default, not a per-role - # choice: it may only appear as the Role.ALL fallback. Side-cache - # roles (e.g. INDEX_KEY) may declare their own non-INDEXED kind - # alongside it. - if mapper_kind is MapperKind.INDEXED and role != Role.ALL: - raise ValueError( - f"MapperKind.INDEXED is only valid as the Role.ALL mapping; " - f"got it for role {role!s}" - ) - default_mapper_kind = role_mapper_kinds[Role.ALL] + Uses the V2 storage layer APIs (pool.slot_address, pool.slot_size, + pool.num_slots) for accurate pool metadata, and stamps each PoolView + with the manager's native role-name strings (``pool_role``) plus the + closed-set ``mapper_kind`` discriminator used by ``build_kv_mapper``. - def _window_size_for_layer(internal_layer_id: int): - if internal_layer_id < len(config.layers): - return getattr(config.layers[internal_layer_id], "window_size", None) - - if hasattr(manager, "_layer_attn_to_layer_id"): - for (model_layer, _attn_type), layer_id in manager._layer_attn_to_layer_id.items(): - if layer_id != internal_layer_id: - continue - local_layer = manager.layer_offsets.get(model_layer) - if local_layer is not None and local_layer < len(config.layers): - return getattr(config.layers[local_layer], "window_size", None) - if model_layer < len(config.layers): - return getattr(config.layers[model_layer], "window_size", None) - - raise ValueError(f"Cannot resolve layer config for internal layer {internal_layer_id}") + Important: iterates over life cycles (layer groups), not storage pool + groups. Multiple life cycles with different sliding-window sizes may + share the same underlying storage pool group when their buffer sizes + are identical. The page table must reflect life cycles so that + per-window transfer logic works correctly. + """ + from collections import defaultdict + + from tensorrt_llm.runtime.kv_cache_manager_v2 import CacheTier + + storage = manager.impl._storage + config = manager.impl._init_config + + # Find GPU level + gpu_level = 0 + for level_idx, cache_tier_config in enumerate(config.cache_tiers): + if cache_tier_config.tier == CacheTier.GPU_MEM: + gpu_level = level_idx + break + + # Collect buffer entries keyed by (life_cycle_id, pool_idx). + # Also collect the set of native role-name strings per pool — used as + # ``PoolView.pool_role``, the manager-supplied equivalence label that + # disagg uses to match pools across peers without enumerating roles. + buffer_by_lc_pool: Dict[tuple, list] = defaultdict(list) + native_roles_by_pool: Dict[tuple, set] = defaultdict(set) + + for buffer_id, attr in storage._buffer_attr.items(): + layer_id, role = buffer_id + lc_id = attr.life_cycle_id + pool_idx = attr.pool_index + pool_key = (int(lc_id), pool_idx) + buffer_by_lc_pool[pool_key].append((layer_id, attr.offset, attr.size)) + native_roles_by_pool[pool_key].add(str(role)) + + # Iterate over life cycles (layer groups), not storage pool groups. + # Multiple layer_groups can share the same storage pool_group when their + # slot_size_list (coalesced buffer sizes) are identical. In that case, + # different layer_groups draw slots from the same physical pool, but a + # slot is exclusively allocated to one layer_group at a time (managed by + # SlotAllocator). Within a slot, each layer_group's buffer offsets start + # from 0 independently — the memory is reused, not concatenated. + # Therefore, slot_bytes / num_layers_for_this_layer_group correctly gives + # the per-layer size, and buffer offsets within a slot are contiguous for + # each layer_group. + num_life_cycles = storage.num_life_cycles + pool_group_storage = storage._levels[gpu_level].storage._pool_groups pool_groups: List[PhysicalPoolGroup] = [] storage_pg_to_list_idx: Dict[int, int] = {} - layer_groups_by_id: List[LayerGroup | None] = [None] * len(manager.impl.layer_grouping) - - for pg_desc in pool_group_descs: - storage_pg_idx = int(pg_desc.pool_group_index) - storage_pg_to_list_idx[storage_pg_idx] = len(pool_groups) - pool_groups.append( - PhysicalPoolGroup( - pools=[ - PhysicalPool( - base_address=int(pool.base_address), - slot_bytes=int(pool.slot_bytes), - num_slots=int(pg_desc.num_slots), - ) - for pool in pg_desc.pools - ] - ) - ) + layer_groups: List[LayerGroup] = [] - # Each variant is one layer group (life cycle) drawing slots from - # this pool group. Multiple layer groups share a pool group when - # their coalesced-buffer sizes are identical; within a slot, each - # layer group's buffer offsets start from 0 independently — the - # memory is reused, not concatenated. - for variant in pg_desc.slot_desc.variants: - layer_group_id = int(variant.layer_group_id) - all_internal_layer_ids = list(manager.impl.layer_grouping[layer_group_id]) - all_global_layer_ids = _compute_global_layer_ids(manager, layer_group_id) - - local_layers = [ - LocalLayer(local_layer_id=int(iid), global_layer_id=int(gid)) - for iid, gid in zip(all_internal_layer_ids, all_global_layer_ids) - ] - - # Bucket buffer entries by (pool, mapper kind). One PoolView is - # emitted per bucket and spans every layer of that role class, - # so the view count per layer group is bounded by the number of - # role classes — never by the layer count. A physical pool may - # hold several classes (V2 storage coalesces buffers purely by - # size within a layer group, so e.g. MiniMax M3's index-K shares - # the K/V pool when their per-block sizes coincide); each class - # still gets its own view, which keeps peer matching independent - # of that physical coalescing decision. ``pool_role`` stays the - # manager-supplied equivalence label used for peer matching - # without enumerating role names. Buffer offsets within a slot - # follow ``buffer_ids`` order: the i-th buffer of a coalesced - # buffer lives at ``i * single_buffer_size``. - bucket_entries: Dict[tuple, list] = defaultdict(list) - bucket_roles: Dict[tuple, set] = defaultdict(set) - for pool_idx, coalesced_buffer in enumerate(variant.coalesced_buffers): - single_buffer_size = int(coalesced_buffer.single_buffer_size) - offset = 0 - for buffer_id in coalesced_buffer.buffer_ids: - kind = role_mapper_kinds.get(buffer_id.role, default_mapper_kind) - bucket_key = (pool_idx, kind) - bucket_entries[bucket_key].append( - (int(buffer_id.layer_id), offset, single_buffer_size) - ) - bucket_roles[bucket_key].add(str(buffer_id.role)) - offset += single_buffer_size - - # Emit this layer group's views: one per (pool, mapper-kind - # class of roles). Roles sharing a kind share a view - # (KEY+VALUE); roles with different kinds in the same physical - # pool get separate views (M3 coalesced index-K). - # All ordering below is canonicalization — the page table is - # serialized and matched against peers, so view order (pool, - # then lowest slot offset), entry order (slot offset), and role - # text must not depend on dict/set iteration order. - pool_views = [] - lg_bucket_keys = sorted( - bucket_entries, - key=lambda key: (key[0], min(entry[1] for entry in bucket_entries[key])), - ) - for bucket_key in lg_bucket_keys: - pool_idx, mapper_kind = bucket_key - roles = frozenset(bucket_roles[bucket_key]) - entries = np.array( - sorted(bucket_entries[bucket_key], key=lambda entry: entry[1]), - dtype=BUFFER_ENTRY_DTYPE, + for lc_idx in range(num_life_cycles): + # Resolve the storage pool group for this life cycle. + # storage_pg_idx may be the same for multiple lc_idx values. + storage_pg_idx = storage.get_pool_group_index(lc_idx) + pool_group = pool_group_storage[storage_pg_idx] + num_pools = pool_group.num_pools + + # Build PhysicalPoolGroup once per unique storage pool group. + if storage_pg_idx not in storage_pg_to_list_idx: + storage_pg_to_list_idx[storage_pg_idx] = len(pool_groups) + pool_groups.append( + PhysicalPoolGroup( + pools=[ + PhysicalPool( + base_address=int(pool_group._pools[pi].slot_address(0)), + slot_bytes=int(pool_group._pools[pi].slot_size), + num_slots=int(pool_group._pools[pi].num_slots), + ) + for pi in range(num_pools) + ] ) - # Fail fast on invalid geometry and record the uniform - # per-layer region size on the wire. Every kind is - # entries-driven, so the contiguous-layer-region / - # uniform-size invariants apply to all views uniformly. - _, bytes_per_layer = compute_layer_byte_ranges( - entries, - context=( - f"View(layer_group={layer_group_id}, pool={pool_idx}, " - f"kind={mapper_kind.name}, role={sorted(roles)})" - ), - ) - pool_views.append( - PoolView( - pool_idx=pool_idx, - buffer_entries=entries, - pool_role=roles, - mapper_kind=mapper_kind, - bytes_per_layer=bytes_per_layer, - ) + ) + + # Compute group-level global layer IDs and internal layer IDs + all_internal_layer_ids = list(manager.impl.layer_grouping[lc_idx]) + all_global_layer_ids = _compute_global_layer_ids(manager, lc_idx) + + local_layers = [ + LocalLayer(local_layer_id=int(iid), global_layer_id=int(gid)) + for iid, gid in zip(all_internal_layer_ids, all_global_layer_ids) + ] + + pool_views = [] + for pool_idx in range(num_pools): + pool_key = (lc_idx, pool_idx) + buffers_info = buffer_by_lc_pool.get(pool_key, []) + + # Skip pools that have no buffers for this layer group. + # Multiple life cycles may share the same storage pool group; + # only include pools that actually belong to this life cycle. + if not buffers_info: + continue + + pool_views.append( + PoolView( + pool_idx=pool_idx, + buffer_entries=np.array(buffers_info, dtype=BUFFER_ENTRY_DTYPE), + pool_role=frozenset(native_roles_by_pool[pool_key]), + mapper_kind=MapperKind.INDEXED, ) + ) - # Determine layer group metadata. - # For managers with virtual layers, internal layer_ids - # may exceed the length of num_kv_heads_per_layer. Use index 0 as - # all layers within a pool group share the same kv_heads count. - first_local_layer = all_internal_layer_ids[0] - if first_local_layer < len(manager.num_kv_heads_per_layer): - num_kv_heads = manager.num_kv_heads_per_layer[first_local_layer] - else: - num_kv_heads = manager.num_kv_heads_per_layer[0] - sliding_window_size = _window_size_for_layer(first_local_layer) - - layer_groups_by_id[layer_group_id] = AttentionLayerGroup( + # Determine layer group metadata. + # For managers with virtual layers, internal layer_ids + # may exceed the length of num_kv_heads_per_layer. Use index 0 as all + # layers within a pool group share the same kv_heads count. + first_local_layer = all_internal_layer_ids[0] + if first_local_layer < len(manager.num_kv_heads_per_layer): + num_kv_heads = manager.num_kv_heads_per_layer[first_local_layer] + else: + num_kv_heads = manager.num_kv_heads_per_layer[0] + life_cycle = manager.impl._life_cycles[lc_idx] + sliding_window_size = life_cycle.window_size + + layer_groups.append( + AttentionLayerGroup( pool_group_idx=storage_pg_to_list_idx[storage_pg_idx], kv_head_num_per_rank=num_kv_heads, sliding_window_size=sliding_window_size, local_layers=local_layers, pool_views=pool_views, ) - - layer_groups: List[LayerGroup] = [] - for layer_group_id, layer_group in enumerate(layer_groups_by_id): - if layer_group is None: - raise ValueError(f"Missing V2 layer group descriptor for layer group {layer_group_id}") - layer_groups.append(layer_group) + ) if isinstance(manager, MambaHybridCacheManager): mamba_layer_group_idx = len(pool_groups) diff --git a/tensorrt_llm/_torch/disaggregation/resource/page.py b/tensorrt_llm/_torch/disaggregation/resource/page.py index 259561cfd354..81514c06d6a0 100644 --- a/tensorrt_llm/_torch/disaggregation/resource/page.py +++ b/tensorrt_llm/_torch/disaggregation/resource/page.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from __future__ import annotations from dataclasses import dataclass, field @@ -31,38 +16,23 @@ class MapperKind(IntEnum): - """Transfer semantics of one physical pool's bytes. - - Every PoolView carries ``buffer_entries`` listing - ``(local_layer_id, offset, size)`` per buffer; the view's exact layer - set always comes from those entries (a view may cover a subset of the - LG when V2 splits an LG into multiple pools by buffer-size class, or - when a role class exists only on some layers). The kind selects how - bytes move between heterogeneous topologies: - - INDEXED: Head-major (HND) K/V — the layout written by the TRTLLM - attention kernels and the default for V1 and standard V2 managers. - Heterogeneous-head transfer selects one contiguous head-major range - per K/V buffer. - REPLICATED: The pool holds bytes that are identical on every TP rank - (MiniMax M3 index-key, DSA indexer K). Copied without KV-head - remapping using per-layer strides; fan-in routing elects one owning - sender per destination so each peer receives exactly one copy. - NHD: Ordinary K/V whose per-buffer storage is token-major - ``[token, head, dim]``. Heterogeneous-head transfer must select the - corresponding head slice inside every token rather than a single - contiguous head-major range. - - A physical pool may hold roles of different kinds: V2 storage coalesces - buffers purely by ``(life_cycle, buffer size)``, so e.g. MiniMax M3's - replicated index-K shares the K/V pool at TP degrees where their - per-block sizes coincide. The page-table builder therefore emits one - PoolView per ``(physical pool, mapper kind)`` — a view covers exactly - the bytes of one role class, and its per-layer byte ranges come from - ``buffer_entries`` (offsets are per layer because another class may - interleave between layers; only the per-layer size is uniform, recorded - in ``bytes_per_layer``). View count per layer group is bounded by the - number of role classes, never by layer count. + """Slot metadata shape — selects how disagg derives the pool's layer set. + + INDEXED: PoolView.buffer_entries lists ``(local_layer_id, offset, size)`` + per buffer. Disagg reads ``local_layer_id`` to know *which* layers + from the LG live in this pool (a pool may cover a subset when V2 + splits an LG into multiple pools by buffer-size class). The + ``offset`` / ``size`` columns are carried for future use but are not + currently consumed at byte-transfer time. + FLAT: PoolView.buffer_entries is empty. Disagg assumes the pool + covers *all* layers of the LG, packed equal-sized in + ``local_layers`` order. Used today by the DSA (DeepSeek Sparse + Attention, v3.2) indexer K cache pool, whose slot layout is a dense + ``(numLayers, kvFactor, blockSize)`` array. + + Byte arithmetic is the same for both kinds: per-layer stride is + ``slot_bytes // num_layers``. The kind only affects how disagg discovers + the pool's layer set during pool matching. Mamba state pools do not use this enum: Mamba's transfer is dispatched through :class:`MambaPolicy` which hard-codes the ``is_conv`` switch and @@ -70,8 +40,7 @@ class MapperKind(IntEnum): """ INDEXED = 0 - REPLICATED = 1 - NHD = 2 + FLAT = 1 @dataclass @@ -138,7 +107,7 @@ class PoolView: pool_idx: Index of the physical pool within its pool group. buffer_entries: Structured array using ``BUFFER_ENTRY_DTYPE``. Each entry records a buffer's ``local_layer_id`` and its byte ``offset`` - and ``size`` within the pool slot. + and ``size`` within the pool slot. FLAT pools have no entries. pool_role: Set of native role-name strings (whatever the cache manager uses, e.g. ``"key"`` / ``"value"`` / ``"deepseek_v4_swa"``) that live in this pool. Used as the *equivalence label* for peer-to-peer @@ -146,19 +115,12 @@ class PoolView: are equal. Disagg never enumerates the role-name vocabulary — adding a new role on the manager side requires no disagg change. mapper_kind: Closed-set discriminator for picking the Mapper family. - bytes_per_layer: Uniform byte size of one layer's region within the - slot. The per-layer *offsets* live in ``buffer_entries``; only - the size is uniform, because a slot may interleave other role - classes between layers, making the layer stride non-uniform. - Set for every kind; ``None`` only in tables serialized by older - builders, where consumers re-derive it from the entries. """ pool_idx: int buffer_entries: np.ndarray # dtype=BUFFER_ENTRY_DTYPE pool_role: FrozenSet[str] = field(default_factory=frozenset) mapper_kind: MapperKind = MapperKind.INDEXED - bytes_per_layer: Optional[int] = None def to_dict(self) -> dict: return { @@ -166,9 +128,6 @@ def to_dict(self) -> dict: "buffer_entries": self.buffer_entries.tolist(), "pool_role": sorted(self.pool_role), "mapper_kind": int(self.mapper_kind), - "bytes_per_layer": ( - int(self.bytes_per_layer) if self.bytes_per_layer is not None else None - ), } @staticmethod @@ -184,9 +143,6 @@ def from_dict(data: dict) -> "PoolView": ), pool_role=frozenset(data["pool_role"]), mapper_kind=MapperKind(int(data["mapper_kind"])), - bytes_per_layer=( - int(data["bytes_per_layer"]) if data.get("bytes_per_layer") is not None else None - ), ) diff --git a/tensorrt_llm/_torch/disaggregation/resource/utils.py b/tensorrt_llm/_torch/disaggregation/resource/utils.py index 6f58bc2782e4..ee03e2171a22 100644 --- a/tensorrt_llm/_torch/disaggregation/resource/utils.py +++ b/tensorrt_llm/_torch/disaggregation/resource/utils.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from __future__ import annotations from typing import Dict, List, Set @@ -41,64 +26,6 @@ def get_slot_address(pool: PhysicalPool, slot_id: int) -> int: # ------------------------------------------------------------------------- -def compute_layer_byte_ranges( - buffer_entries, - *, - declared_bytes_per_layer: "int | None" = None, - context: str = "PoolView", -) -> tuple[Dict[int, int], int]: - """Per-layer slot-relative byte offsets from raw buffer entries. - - Returns ``({local_layer_id: start_offset}, bytes_per_layer)``. A layer's - region is the concatenation of its buffer entries, which must be - contiguous within the slot; the region size must be uniform across - layers (the slot may interleave other role classes between layers, so - only the *size* is uniform — offsets are per layer). ``context`` labels - error messages; ``declared_bytes_per_layer`` cross-checks a size that - was recorded elsewhere. - """ - starts: Dict[int, int] = {} - totals: Dict[int, int] = {} - entries_by_layer: Dict[int, list] = {} - for entry in buffer_entries: - entries_by_layer.setdefault(int(entry["local_layer_id"]), []).append( - (int(entry["offset"]), int(entry["size"])) - ) - if not entries_by_layer: - raise ValueError(f"{context} has no buffer entries; per-layer byte ranges are undefined") - for layer_id, spans in entries_by_layer.items(): - spans.sort() - for (off, size), (next_off, _) in zip(spans, spans[1:]): - if off + size != next_off: - raise ValueError( - f"{context} layer {layer_id} buffers are " - f"not contiguous: [{off}, {off + size}) is followed by offset {next_off}" - ) - starts[layer_id] = spans[0][0] - totals[layer_id] = sum(size for _, size in spans) - distinct_totals = set(totals.values()) - if len(distinct_totals) != 1: - raise ValueError( - f"{context} per-layer region sizes are not uniform: {sorted(totals.items())}" - ) - bytes_per_layer = distinct_totals.pop() - if declared_bytes_per_layer is not None and declared_bytes_per_layer != bytes_per_layer: - raise ValueError( - f"{context} declares bytes_per_layer={declared_bytes_per_layer} but buffer " - f"entries sum to {bytes_per_layer} per layer" - ) - return starts, bytes_per_layer - - -def get_layer_byte_ranges(pool_view: PoolView) -> tuple[Dict[int, int], int]: - """Per-layer byte ranges of a view; see :func:`compute_layer_byte_ranges`.""" - return compute_layer_byte_ranges( - pool_view.buffer_entries, - declared_bytes_per_layer=pool_view.bytes_per_layer, - context=f"PoolView(pool_idx={pool_view.pool_idx}, role={sorted(pool_view.pool_role)})", - ) - - def get_unique_layers(pool_view: PoolView) -> Set[int]: """Unique local layer IDs in *pool_view*.""" return {int(e["local_layer_id"]) for e in pool_view.buffer_entries} @@ -120,29 +47,15 @@ def get_pool_view_global_layer_ids( pool_view: PoolView, layer_group: AttentionLayerGroup ) -> List[int]: """ - Global layer IDs for the layers that appear in *pool_view*, ordered by their - physical offset within the coalesced buffer (ascending). - - The order is derived from the buffer entries' physical offsets rather than - from ``layer_group.local_layers`` order on purpose: the KV transfer maps - layers positionally (a layer's position in this list times the per-layer - slot size gives its byte offset), so the position must reflect the physical - slot layout. Deriving it from offsets keeps the transceiver decoupled from - the KV-cache manager's layer-grouping order (which is an implementation - detail, not an API contract). This mirrors ``get_aggregated_pages``, which - likewise sorts buffers by their offset inside the coalesced buffer. + Global layer IDs for the layers that appear in *pool_view*, ordered as in + *layer_group.local_layers*. """ - local_to_global = {ll.local_layer_id: ll.global_layer_id for ll in layer_group.local_layers} - # A layer may contribute several buffer entries (e.g. KEY and VALUE); use the - # smallest offset as that layer's position within the slot. - min_offset: dict[int, int] = {} - for entry in pool_view.buffer_entries: - local_layer_id = int(entry["local_layer_id"]) - offset = int(entry["offset"]) - if local_layer_id not in min_offset or offset < min_offset[local_layer_id]: - min_offset[local_layer_id] = offset - ordered_local_ids = sorted(min_offset, key=lambda lid: min_offset[lid]) - return [local_to_global[lid] for lid in ordered_local_ids] + local_ids_in_pool = get_unique_layers(pool_view) + return [ + ll.global_layer_id + for ll in layer_group.local_layers + if ll.local_layer_id in local_ids_in_pool + ] # ------------------------------------------------------------------------- @@ -217,24 +130,13 @@ def get_unique_pool_memory_descs( def get_layer_to_layer_group(page_table: KVCachePageTable) -> Dict[int, int]: """ - Build ``{global_layer_id: lg_idx}`` mapping. - - Layer groups must partition a rank's attention layers: every - global_layer_id belongs to exactly one group. Peer matching relies on - this, so a duplicate raises instead of silently keeping the last group. + Build ``{global_layer_id: lg_idx}`` mapping """ out: Dict[int, int] = {} for lg_idx, lg in enumerate(page_table.layer_groups): if isinstance(lg, AttentionLayerGroup): for ll in lg.local_layers: - gid = int(ll.global_layer_id) - if gid in out: - raise ValueError( - f"global_layer_id {gid} appears in layer groups " - f"{out[gid]} and {lg_idx}; layer groups must partition " - "a rank's attention layers" - ) - out[gid] = int(lg_idx) + out[int(ll.global_layer_id)] = int(lg_idx) return out diff --git a/tensorrt_llm/_torch/disaggregation/transceiver.py b/tensorrt_llm/_torch/disaggregation/transceiver.py index 2c257a39f731..5ff8b305596f 100644 --- a/tensorrt_llm/_torch/disaggregation/transceiver.py +++ b/tensorrt_llm/_torch/disaggregation/transceiver.py @@ -1,4 +1,3 @@ -import os import time import uuid from collections import defaultdict @@ -8,7 +7,6 @@ import numpy as np import torch -import tensorrt_llm.bindings from tensorrt_llm import logger from tensorrt_llm._torch.disaggregation.base.transfer import ( KVSlice, @@ -22,7 +20,6 @@ from tensorrt_llm._torch.disaggregation.native.bounce import ( config_from_size as bounce_config_from_size, ) -from tensorrt_llm._torch.disaggregation.native.perf_logger import perf_log_manager from tensorrt_llm._torch.disaggregation.native.transfer import TransferWorker, TransferWorkerConfig from tensorrt_llm._torch.disaggregation.resource.cache_reuse import ( CacheReuseAdapter, @@ -89,7 +86,7 @@ def __init__( max_concurrent_sessions=max(1, int(kv_cache_manager.max_batch_size)) * 20000, tx_timeout_s=self._sender_future_timeout_ms / 1000.0, rx_timeout_s=self.kv_transfer_timeout_ms / 1000.0, - # Size 0 turns bounce off; the block-count gate is internal (tuned via env). + # On/off switch via config (size 0 => None => per-block path). bounce=bounce_config_from_size(cache_transceiver_config.kv_cache_bounce_size_mb), ) ) @@ -185,16 +182,12 @@ def _create_kv_slice(self, req: LlmRequest) -> KVSlice: token_range = None if req.prompt_len > 0: - # end must match the trimmed block list below (ceil(prompt_len / tpb) - # blocks). num_extra_kv_tokens slots (speculative decoding) are not - # transferred. In the previously added support for ctx disabling - # speculative decoding while gen enables it, both sides currently - # use prompt_len as the transfer range, so the ranges stay - # consistent. - # TODO: the accuracy impact of not transferring num_extra_kv_tokens - # on MTP and other speculative decoding paths is currently unclear; - # revisit whether these extra KV slots need to be transferred. - token_range = TokenRange(start=0, end=req.prompt_len) + # Align with KV cache allocation (prepare_disagg_gen_init / + # _get_context_bytes), which reserves prompt_len + + # num_extra_kv_tokens slots for speculative decoding methods + # (e.g. EAGLE3) that consume extra KV positions per request. + num_extra_kv_tokens = getattr(self._kv_cache_manager, "num_extra_kv_tokens", 0) or 0 + token_range = TokenRange(start=0, end=req.prompt_len + num_extra_kv_tokens) groups = [] for idx, lg in enumerate(layer_groups): @@ -203,6 +196,8 @@ def _create_kv_slice(self, req: LlmRequest) -> KVSlice: continue block_ids = adapter.get_block_ids(req, idx, lg) # Limit to prompt_len blocks, matching C++ cacheFormatter behavior. + # Extra blocks from num_extra_kv_tokens (speculative decoding) have + # uninitialized KV data and must not be transferred. total_blocks = (req.prompt_len + tpb - 1) // tpb if block_ids.size > total_blocks: block_ids = block_ids[:total_blocks] @@ -347,6 +342,14 @@ def _gen_consensus(self, local_ids: list) -> list: all_ranks = self._gen_allgather(local_ids) if self._gen_need_sync else [local_ids] return _find_consensus_request_ids(all_ranks, sync_size) + @staticmethod + def _allgather_or_passthrough( + local_ids, allgather: Callable, need_sync: bool + ) -> List[List[int]]: + if not need_sync: + return [list(local_ids)] + return list(allgather(list(local_ids))) + @staticmethod def _union(all_lists: List[List[int]]) -> set: merged: set = set() @@ -368,14 +371,9 @@ def _consensus_outcome( self, to_process, cancelled, failed, completed, allgather: Callable, need_sync: bool ): # CANCELLED/FAILED on any rank → global; COMPLETED only when ALL ranks agree. - # Batch the three id lists into one allgather to cut the per-step collective count. - if not need_sync: - all_c, all_f, all_done = [list(cancelled)], [list(failed)], [list(completed)] - else: - packed = list(allgather([list(cancelled), list(failed), list(completed)])) - all_c = [p[0] for p in packed] - all_f = [p[1] for p in packed] - all_done = [p[2] for p in packed] + all_c = self._allgather_or_passthrough(cancelled, allgather, need_sync) + all_f = self._allgather_or_passthrough(failed, allgather, need_sync) + all_done = self._allgather_or_passthrough(completed, allgather, need_sync) n = len(all_c) global_cancelled = self._union(all_c) global_failed = self._union(all_f) @@ -409,57 +407,6 @@ def _ctx_consensus_outcome(self, to_process, cancelled, failed, completed, timed c, f, d = self._consensus_outcome(to_process, c, f, d, pp_allgather, True) return c, f, d, timed_out - def _sync_transfer_timing(self, reqs: list): - """Allgather timing for a batch of completed requests in one collective. - - Matches C++ ``batchUpdateKVCacheTransferBW()`` in ``cacheTransceiver.cpp``. - Only runs when ``TRTLLM_KVCACHE_TIME_OUTPUT_PATH`` is set (same gate - as C++) and multi-rank sync is needed. All ranks that participate in - the allgather update their local request objects. - """ - if not reqs: - return - if not os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH"): - return - if not self._gen_need_sync: - return - - # Pack local timing for all completed requests into one dict. - local_data = { - get_unique_rid(req): ( - req.get_kv_cache_transfer_start(), - req.get_kv_cache_transfer_end(), - req.kv_cache_size, - ) - for req in reqs - } - - # Single allgather for the whole batch. - all_data = self._gen_allgather(local_data) - - # Merge: per-rid min(start), max(end), sum(size) across ranks. - merged: dict = {} - for rank_data in all_data: - for rid, (start, end, size) in rank_data.items(): - if rid in merged: - prev = merged[rid] - merged[rid] = ( - min(prev[0], start), - max(prev[1], end), - prev[2] + size, - ) - else: - merged[rid] = (start, end, size) - - # Every rank updates its own local requests. - rid_to_req = {get_unique_rid(r): r for r in reqs} - for rid, (min_start, max_end, total_size) in merged.items(): - req = rid_to_req.get(rid) - if req is not None: - req.set_kv_cache_transfer_start(min_start) - req.set_kv_cache_transfer_end(max_end) - req.set_kv_cache_size(total_size) - def _collect_done(self, sessions: dict, reqs: dict): """Scan sessions and return (completed_rids, failed_rids).""" completed, failed = [], [] @@ -536,7 +483,6 @@ def _finalize_send(self, req: LlmRequest, session: TxSessionBase): @nvtx_range("KvCacheTransceiverV2.respond_and_send_async") def respond_and_send_async(self, req: LlmRequest): self._ever_had_send_session = True - req.set_kv_cache_transfer_start(tensorrt_llm.bindings.global_steady_clock_now()) session = self._get_or_create_send_session(req) req.state = LlmRequestState.DISAGG_CONTEXT_TRANS_IN_PROGRESS session.send(self._create_kv_slice(req)) @@ -581,7 +527,6 @@ def request_and_receive_sync(self, req: LlmRequest): @nvtx_range("KvCacheTransceiverV2.request_and_receive_async") def request_and_receive_async(self, req: LlmRequest): self._ever_had_recv_session = True - req.set_kv_cache_transfer_start(tensorrt_llm.bindings.global_steady_clock_now()) rid = get_unique_rid(req) if rid in self._recv_sessions: logger.warning( @@ -599,13 +544,9 @@ def request_and_receive_async(self, req: LlmRequest): def check_context_transfer_status( self, at_least_request_num: Optional[int], mark_complete: bool = False ): - # A worker that never sends KV has nothing to reconcile here, so skip the consensus. Safe - # because the flag flips together on every rank and never resets, so they all skip in step; - # gating on the live session dict instead would not be, since a cancel clears it per-rank. - # Keep the original sweep (only when tp/pp sync is on) so nothing is leaked. - if not self._ever_had_send_session: - if self._ctx_need_tp_sync or self._ctx_need_pp_sync: - self._transfer_worker.sweep_stale_req_infos() + if not self._ever_had_send_session and not ( + self._ctx_need_tp_sync or self._ctx_need_pp_sync + ): return [], [] block_all = at_least_request_num is None wait_num = at_least_request_num if not block_all else 0 @@ -689,11 +630,6 @@ def check_gen_transfer_status(self, at_least_request_num: Optional[int]): # distinguish the two cases and set the appropriate state. cancelled.append(rid) elif result == WaitResult.COMPLETED: - req = self._recv_reqs[rid] - if session.transfer_end_time is not None: - req.set_kv_cache_transfer_end(session.transfer_end_time) - if session.kv_cache_size_bytes > 0: - req.set_kv_cache_size(session.kv_cache_size_bytes) completed.append(rid) elif result == WaitResult.FAILED: failed.append(rid) @@ -711,20 +647,6 @@ def check_gen_transfer_status(self, at_least_request_num: Optional[int]): del self._recv_reqs[rid] del self._recv_sessions[rid] - # Log gen-side transfer summary after consensus. - if completed and os.getenv("TRTLLM_KVCACHE_TIME_OUTPUT_PATH"): - # Batch-sync timing for all completed requests in one allgather. - self._sync_transfer_timing([self._recv_reqs[rid] for rid in completed]) - for rid in completed: - req = self._recv_reqs[rid] - perf_log_manager.log_gen_transfer_summary( - unique_rid=rid, - instance_name=self._instance_name, - instance_rank=self._mapping.rank, - gen_side_transfer_time_ms=req.kv_cache_transfer_time_ms, - kv_cache_size=req.kv_cache_size, - ) - for rid in completed: session = self._recv_sessions[rid] req = self._recv_reqs[rid] @@ -860,11 +782,6 @@ def prepare_context_requests(self, requests: List[LlmRequest]): self._wait_reqs[rid] = req req.state = LlmRequestState.DISAGG_CONTEXT_WAIT_SCHEDULER - # Nothing waiting on any rank, so skip the consensus. The waiting set is the same on every - # rank, so they all skip together. - if not self._wait_reqs: - return - # Check which waiting requests have peer info locally, then allgather # consensus so all TP/PP ranks agree before promoting. # Without consensus, background peer info arriving at different times on diff --git a/tensorrt_llm/_torch/distributed/communicator.py b/tensorrt_llm/_torch/distributed/communicator.py index 2e0fc1e0e4c3..34acfc741947 100644 --- a/tensorrt_llm/_torch/distributed/communicator.py +++ b/tensorrt_llm/_torch/distributed/communicator.py @@ -17,10 +17,10 @@ MPI = None # deferred; functions will error if used when ENABLE_MULTI_DEVICE is True from tensorrt_llm._mnnvl_utils import init_helix_cp_comm -from tensorrt_llm._utils import (local_mpi_size, mpi_allgather, mpi_barrier, - mpi_comm, mpi_disabled, mpi_isend, - mpi_isend_object, mpi_recv, mpi_recv_object, - mpi_send, mpi_send_object, mpi_world_size, +from tensorrt_llm._utils import (mpi_allgather, mpi_barrier, mpi_comm, + mpi_disabled, mpi_isend, mpi_isend_object, + mpi_recv, mpi_recv_object, mpi_send, + mpi_send_object, mpi_world_size, torch_pybind11_abi) from tensorrt_llm.bindings.BuildInfo import ENABLE_MULTI_DEVICE from tensorrt_llm.bindings.internal.process_group import init_pg @@ -158,11 +158,6 @@ def has_cp_helix(self): def cp_config(self): return self.mapping.cp_config - @property - @abstractmethod - def local_world_size(self): - """Number of ranks co-located on this physical node.""" - @abstractmethod def barrier(self): pass @@ -673,10 +668,6 @@ def broadcast(self, obj, root=0, chunk_size: int = 4 * 1024 * 1024): def allgather(self, obj): return mpi_allgather(obj) - @property - def local_world_size(self): - return local_mpi_size() - def barrier(self): mpi_barrier() @@ -801,10 +792,6 @@ class TorchDist(Distributed): def rank(self): return torch.distributed.get_rank() - @property - def local_world_size(self): - return dist.get_world_size(group=self.local_comm) - def __init__(self, mapping: Mapping): super().__init__(mapping) assert dist.is_initialized( diff --git a/tensorrt_llm/_torch/kv_cache_compression/interface.py b/tensorrt_llm/_torch/kv_cache_compression/interface.py deleted file mode 100644 index cd4e7bf32068..000000000000 --- a/tensorrt_llm/_torch/kv_cache_compression/interface.py +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from enum import IntEnum, auto -from typing import Optional - - -class KvCacheCompressionMode(IntEnum): - """Algorithm-level traits of a KV-cache compression method. - - Configs map their ``algorithm`` string to a member here; callers read the - ``is_*`` predicates instead of comparing strings. - """ - - NONE = auto() - - def is_eviction_method(self): - """Whether this method physically evicts cached tokens. Evicting - algorithms add their member and extend this predicate.""" - return False - - @staticmethod - def from_string(name: Optional[str]) -> "KvCacheCompressionMode": - if name is None: - return KvCacheCompressionMode.NONE - try: - return KvCacheCompressionMode[name.upper()] - except KeyError: - return KvCacheCompressionMode.NONE diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 8f5641920c03..fd15550fd6e1 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -51,9 +51,8 @@ if TYPE_CHECKING: from tensorrt_llm.bindings import ModelConfig as ModelConfigCpp - from tensorrt_llm.llmapi.llm_args import (DecodingBaseConfig, - KvCacheCompressionConfig, - LoraConfig, SparseAttentionConfig, + from tensorrt_llm.llmapi.llm_args import (DecodingBaseConfig, LoraConfig, + SparseAttentionConfig, SpeculativeConfig) TConfig = TypeVar("TConfig", bound=transformers.PretrainedConfig) @@ -142,14 +141,8 @@ class ModelConfig(Generic[TConfig]): skip_create_weights_in_init: bool = False spec_config: Optional["DecodingBaseConfig"] = None - # When False, the column-parallel LM head keeps its vocab-sharded output - # instead of all-gathering to full vocab. Used for one-model speculative - # draft models so greedy draft sampling can do a lighter TP gather. Defaults - # to True to preserve behavior for every non-draft model. - lm_head_gather_output: bool = True lora_config: Optional["LoraConfig"] = None sparse_attention_config: Optional["SparseAttentionConfig"] = None - kv_cache_compression_config: Optional["KvCacheCompressionConfig"] = None is_generation: bool = True is_encoder_decoder: bool = False @@ -197,13 +190,6 @@ class ModelConfig(Generic[TConfig]): # If true, ONLY the vision encoder part of the full model is loaded/executed. mm_encoder_only: bool = False - # If true, the multimodal encoder of a multimodal checkpoint is NOT - # instantiated/loaded and the model serves text-only requests. This is - # opt-in per model: each model implementation must honor this flag when - # building its encoder (currently the Qwen3-VL / Qwen3.5-VL models); a - # model that does not check it simply ignores the flag (no-op). - disable_mm_encoder: bool = False - # Video pruning rate for VLM models (None = EVS disabled) video_pruning_rate: Optional[float] = None @@ -958,14 +944,9 @@ def update_sparse_attention_indexer_config(pretrained_config, kwargs): pretrained_config, 'compress_ratios', None) num_base_layers = pretrained_config.num_hidden_layers spec_config = kwargs.get('spec_config', None) - # ``num_nextn_predict_layers`` is MTP-specific (only read on - # the is_mtp_one_model path). Only set it on configs that - # actually declare the field; other DeepSeek-V4 spec modes - # (e.g. DSpark, which carries its own draft stage count) do - # not, and a blind setattr would fail pydantic validation. - if (spec_config is not None and 'num_nextn_predict_layers' - in type(spec_config).model_fields - and spec_config.num_nextn_predict_layers is None): + if (spec_config is not None + and getattr(spec_config, 'num_nextn_predict_layers', + None) is None): spec_config.num_nextn_predict_layers = getattr( pretrained_config, 'num_nextn_predict_layers', 1) mtp_enabled = (spec_config is not None and @@ -997,22 +978,6 @@ def update_sparse_attention_indexer_config(pretrained_config, kwargs): if window_size is None: window_size = pretrained_config.sliding_window - # DeepSeek-V4 needs explicit per-layer compress ratios. They - # must come from the checkpoint config or a user override; we - # intentionally do not synthesize a default list (it would - # silently change sparse-attention semantics). Fail fast with - # an actionable message instead of letting the normalization - # below raise an opaque TypeError on None. - if compress_ratios is None: - raise ValueError( - "DeepSeek-V4 requires per-layer `compress_ratios`, " - "but none were found in the checkpoint config and " - "none were provided via `sparse_attention_config`. " - "Set `compress_ratios` in the model's config.json, or " - "pass `sparse_attention_config=" - "DeepSeekV4SparseAttentionConfig(compress_ratios=[...])`" - " in --extra_llm_api_options.") - # Normalize checkpoint-facing ratio 0 (SWA-only/uncompressed) # to 1 internally so cache allocation math works. The # external config keeps the original semantics. diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/gemma4_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/gemma4_weight_mapper.py index 22cde4599c43..e336ef4b894b 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/gemma4_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/gemma4_weight_mapper.py @@ -23,10 +23,6 @@ _LANG_PREFIX = "model.language_model." _MODEL_PREFIX = "model." _LAYER_IDX_RE = re.compile(r"layers\.(\d+)$") -_LAYER_SCALAR_KEY_RE = re.compile(r"^(?:language_model\.)?model\.layers\.(\d+)\.layer_scalar$") -_K_PROJ_KEY_RE = re.compile( - r"^((?:language_model\.)?model\.layers\.(\d+)\.self_attn\.)k_proj(\..+)$" -) @register_mapper("HF", "Gemma4ForCausalLM") @@ -277,6 +273,9 @@ def _remap_moe_keys(self, weights: dict) -> dict: def _handle_buffers_and_kvdup(self, weights: dict) -> dict: """Load layer_scalar buffers and duplicate k_proj for k_eq_v layers.""" + # Determine the layer scalar key pattern and accessor based on + # whether any key starts with "language_model." (VLM sub-model + # weights after filter_weights) or "model." (text-only). # Navigate to decoder layers regardless of model structure # (multimodal wrapper has .llm.model.layers, text-only has .model.layers) _root = self.model @@ -290,9 +289,17 @@ def _handle_buffers_and_kvdup(self, weights: dict) -> dict: def get_layer(idx): return _layers[idx] if _layers else None + sample = next(iter(weights), "") + if sample.startswith("language_model.model."): + scalar_pattern = r"language_model\.model\.layers\.(\d+)\.layer_scalar" + key_tmpl = "language_model.model.layers.{}.self_attn.{}_proj.weight" + else: + scalar_pattern = r"model\.layers\.(\d+)\.layer_scalar" + key_tmpl = "model.layers.{}.self_attn.{}_proj.weight" + layer_scalar_keys = [k for k in weights if k.endswith(".layer_scalar")] for key in layer_scalar_keys: - m = _LAYER_SCALAR_KEY_RE.match(key) + m = re.match(scalar_pattern, key) if m: layer_idx = int(m.group(1)) try: @@ -305,17 +312,12 @@ def get_layer(idx): config = self.model.config if getattr(config, "attention_k_eq_v", False): layer_types = getattr(config, "layer_types", []) - for k_key, value in list(weights.items()): - match = _K_PROJ_KEY_RE.match(k_key) - if match is None: - continue - layer_idx = int(match.group(2)) - if layer_idx >= len(layer_types) or layer_types[layer_idx] != "full_attention": - continue - suffix = match.group(3) - suffix = {".k_scale": ".v_scale", ".k_bias": ".v_bias"}.get(suffix, suffix) - v_key = f"{match.group(1)}v_proj{suffix}" - weights.setdefault(v_key, value) + for layer_idx, lt in enumerate(layer_types): + if lt == "full_attention": + k_key = key_tmpl.format(layer_idx, "k") + v_key = key_tmpl.format(layer_idx, "v") + if k_key in weights and v_key not in weights: + weights[v_key] = weights[k_key] # KV shared layers: HF omits k_proj/v_proj for shared layers. # The model uses Q-only projection for these layers, so no dummy diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/minimaxm3_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/minimaxm3_weight_mapper.py deleted file mode 100644 index bb6d58957fd3..000000000000 --- a/tensorrt_llm/_torch/models/checkpoints/hf/minimaxm3_weight_mapper.py +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from torch import Tensor, nn - -from tensorrt_llm._torch.models.checkpoints.hf.weight_mapper import HfWeightMapper -from tensorrt_llm._torch.models.modeling_utils import register_mapper - -MINIMAX_M3_PARAMS_MAP = { - r"^(.*\.block_sparse_moe)\.e_score_correction_bias$": r"\1.gate.e_score_correction_bias", -} - - -@register_mapper("HF", "MiniMaxM3SparseForCausalLM") -@register_mapper("HF", "MiniMaxM3SparseForConditionalGeneration") -class MiniMaxM3HfWeightMapper(HfWeightMapper): - """Handle M3 gate naming and MXFP8 GQA duplication for loader v2.""" - - def __init__(self) -> None: - super().__init__() - self.params_map = MINIMAX_M3_PARAMS_MAP - - def _duplicate_kv_weights( - self, module: nn.Module, new_name: str, weights: dict[str, Tensor] - ) -> dict[str, Tensor]: - if new_name not in ["k_proj", "v_proj"]: - return weights - - duplicated_keys = ["weight", "bias"] - quant_config = getattr(module, "quant_config", None) - if quant_config is not None: - quant_mode = quant_config.quant_mode - if quant_mode.has_nvfp4(): - duplicated_keys.append("weight_scale") - if quant_mode.has_mxfp8(): - duplicated_keys.extend(["weight_scale", "weight_scale_inv"]) - - return { - key: self._duplicate_kv( - weight=value[:], num_kv_heads=self._num_kv_heads, tensor_parallel_size=self._tp_size - ) - if key in duplicated_keys - else value - for key, value in weights.items() - } diff --git a/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py b/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py index 91dbe496976b..ca5f47c7e4fc 100644 --- a/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/mx/checkpoint_loader.py @@ -35,7 +35,9 @@ from contextlib import contextmanager from enum import Enum from pathlib import Path -from typing import Any, Callable, Iterator, MutableMapping, Optional, Protocol, Type, Union +from typing import Any, Callable, Optional, Type, Union + +import grpc from tensorrt_llm._torch.models.checkpoints.base_config_loader import BaseConfigLoader from tensorrt_llm._torch.models.checkpoints.base_weight_loader import BaseWeightLoader @@ -56,13 +58,10 @@ # for a source. On a cold cluster (no donor up yet), this means the very # first replica blocks for an hour before falling back to disk. We cap # the default at 30 s so first-replica startup degrades gracefully; users -# can still override via the env var or the per-loader `query_timeout_s` setting. +# can still override via the env var or a future per-loader knob. # Tracked as MX-4 in §15 (non-blocking source-query API upstream). _MX_SOURCE_QUERY_TIMEOUT_DEFAULT_S = "30" -# ModelExpress 0.4.1 reads transfer configuration from process-wide -# environment variables and exposes a module-level identity builder. Keep all -# temporary mutation of that shared state in one critical section. -_MX_TRANSFER_STATE_LOCK = threading.Lock() +_MX_PUBLISH_ENV_LOCK = threading.Lock() _MX_SOURCE_IDENTITY_METADATA_KEY = "trtllm_source_identity" _MX_WEIGHT_LAYOUT_METADATA_KEY = "trtllm_weight_layout" _MX_TRANSFORM_PROTOCOL_VERSION_METADATA_KEY = "trtllm_transform_protocol_version" @@ -76,15 +75,9 @@ class _MxWeightLayoutStatus(Enum): UNSUPPORTED = "unsupported" -class _MxSourceIdentity(Protocol): - """Subset of ModelExpress's protobuf SourceIdentity used by this adapter.""" - - extra_parameters: MutableMapping[str, str] - - @contextmanager -def _temporary_env(key: str, value: Optional[str]) -> Iterator[None]: - """Temporarily set one environment variable when a value is provided.""" +def _temporary_env(key: str, value: Optional[str]): + """Temporarily set or clear one environment variable.""" if value is None: yield return @@ -99,95 +92,6 @@ def _temporary_env(key: str, value: Optional[str]) -> Iterator[None]: os.environ[key] = prior -def _serialize_source_identity(identity: SourceIdentity) -> str: - """Serialize TRT-LLM's layout identity for MX's identity map.""" - payload = identity.to_dict() - # `model_name` is a cleartext discovery descriptor and is deliberately - # excluded from SourceIdentity compatibility checks. The outer MX identity - # already carries the normalized model name; embedding a local checkpoint - # path here would make otherwise-compatible no-shards receivers hash to a - # different MX source. - payload.pop("model_name", None) - return json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ) - - -def _attach_trtllm_metadata_to_mx_identity( - mx_identity: _MxSourceIdentity, source_identity: Optional[SourceIdentity] -) -> _MxSourceIdentity: - """Attach TRT-LLM compatibility metadata to an MX SourceIdentity.""" - if source_identity is None: - return mx_identity - - extra_parameters = getattr(mx_identity, "extra_parameters", None) - if extra_parameters is None: - raise RuntimeError( - "MX SourceIdentity has no extra_parameters field; cannot attach " - "TRT-LLM SourceIdentity for compatibility filtering." - ) - - try: - for key, value in _build_mx_source_metadata(source_identity).items(): - extra_parameters[key] = value - except (AttributeError, TypeError, ValueError) as e: - raise RuntimeError( - "Failed to attach TRT-LLM compatibility metadata to MX " - "SourceIdentity; MX P2P compatibility filtering will reject " - "this source." - ) from e - return mx_identity - - -@contextmanager -def _patched_trtllm_identity_builder( - mx_transfer: Any, source_identity: Optional[SourceIdentity] -) -> Iterator[None]: - """Temporarily wrap upstream TRT-LLM identity construction.""" - original = getattr(mx_transfer, "_build_trtllm_identity", None) - if source_identity is None or not callable(original): - yield - return - - def _wrapped_build_identity(*args: Any, **kwargs: Any) -> _MxSourceIdentity: - return _attach_trtllm_metadata_to_mx_identity( - original(*args, **kwargs), - source_identity, - ) - - mx_transfer._build_trtllm_identity = _wrapped_build_identity - try: - yield - finally: - mx_transfer._build_trtllm_identity = original - - -def _close_mx_client(client: Any) -> None: - """Close a best-effort MX discovery client without masking its result.""" - if client is None: - return - close = getattr(client, "close", None) - if not callable(close): - return - try: - close() - except Exception: - logger.warning( - f"Failed to close MX discovery client; continuing with the " - f"completed probe result.\n{traceback.format_exc()}" - ) - - -def _synchronize_cuda_for_mx_publish() -> None: - """Finish pending CUDA writes before exposing source buffers through MX.""" - import torch - - if torch.cuda.is_initialized(): - torch.cuda.synchronize() - - @register_checkpoint_loader("MX") class MXCheckpointLoader(HfCheckpointLoader): """Checkpoint loader for MX (ModelExpress) P2P weight transfer. @@ -198,10 +102,9 @@ class MXCheckpointLoader(HfCheckpointLoader): publishes its weights after `post_load_weights()` runs, together with metadata that lets compatible targets skip one-shot post-load transforms. - When the MX server is unavailable, this loader transparently falls back - to standard HuggingFace checkpoint loading via the parent - `HfCheckpointLoader`. A missing MX client is treated as a configuration - error and reported with an actionable installation command. + When the MX server or library is unavailable, this loader + transparently falls back to standard HuggingFace checkpoint + loading via the parent `HfCheckpointLoader`. All transport-level mechanics (NIXL, dtype casts, source matching, fallback) are delegated to `modelexpress.trtllm_live_transfer` @@ -232,7 +135,7 @@ def __init__( # `model_name` is the human-readable identity to publish/look up # under on the MX server. Typically the user-supplied # `llm_args.model` (a Hub ID like `"Qwen/Qwen2.5-72B-Instruct"` - # or a local path). Transfer and publish paths resolve it via + # or a local path). `publish_as_source()` resolves it via # :func:`_resolve_mx_model_name` (with HF-snapshot path fallback). self._model_name = str(model_name) if model_name is not None else None self._query_timeout_s = query_timeout_s @@ -257,9 +160,9 @@ def model_name(self) -> Optional[str]: """Explicit model identity passed to the constructor (if any). Note this is the *as-configured* value (e.g. `llm_args.model`), - not the final resolved identity passed to ModelExpress as + not the final resolved identity that ends up in the published `MODEL_NAME`. The full resolution (with env var and basename - fallbacks) happens inside the transfer and publish paths. + fallbacks) happens inside :meth:`publish_as_source`. """ return self._model_name @@ -319,9 +222,6 @@ def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[ mapping: Distributed mapping configuration. **kwargs: Additional keyword arguments. When `model` is passed it is used as the target for direct P2P writes. - `prepare_post_transform_receiver`, when present, is called - after a post-transform source is qualified and before those - direct writes begin. Returns: A weights dict. Empty when MX P2P fully succeeded (weights @@ -332,7 +232,6 @@ def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[ # Popped here so it never leaks into the disk-fallback signature. self._local_source_identity = kwargs.pop("source_identity", None) allow_post_transform_weights = kwargs.pop("allow_post_transform_weights", False) - prepare_post_transform_receiver = kwargs.pop("prepare_post_transform_receiver", None) self._p2p_succeeded = False self._post_transform_weights_preloaded = False self._source_identity_compatible_for_last_load = False @@ -350,68 +249,23 @@ def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[ ) try: - from modelexpress import ( - trtllm_live_transfer as mx_transfer, # type: ignore[import-not-found] - ) - except ImportError as exc: - raise ImportError( - "ModelExpress checkpoint loading was explicitly requested, " - "but the ModelExpress client could not be imported. Install " - 'the MX dependencies with `pip install "tensorrt-llm[mx]"`, ' - "or select a different " - "`checkpoint_format` to continue without MX." - ) from exc - - try: - with _MX_TRANSFER_STATE_LOCK: - MxClient = mx_transfer.MxClient - MxLiveWeightLoader = mx_transfer.MxLiveWeightLoader - build_trtllm_identity = mx_transfer._build_trtllm_identity - # Resolve once so discovery and the released ModelExpress - # loader query the same source identity. The lock prevents a - # concurrent MX publish from temporarily changing MODEL_NAME or - # the identity builder while this state is captured. - resolved_name = self._resolve_publish_name(checkpoint_dir) - except AttributeError: - logger.warning( - "modelexpress TRT-LLM live-transfer symbols are missing; " - "cannot use MX P2P weight transfer. Falling back to disk " - "loading." - ) - return self._fallback_to_disk(checkpoint_dir, mapping, **kwargs) - - try: - source_metadata = self._fetch_source_metadata( - checkpoint_dir, + from modelexpress.trtllm_live_transfer import ( # type: ignore[import-not-found] MxClient, - build_trtllm_identity, - model_name=resolved_name, + MxLiveWeightLoader, + _build_trtllm_identity, ) - except Exception: - # Deliberately broad: source discovery is part of the optional MX - # fast path, so an upstream client failure must preserve disk - # loading as the correctness path. + except ImportError: logger.warning( - "MX source metadata fetch failed; falling back to disk " - f"loading.\n{traceback.format_exc()}" - ) - return self._fallback_to_disk( - checkpoint_dir, - mapping, - reason="MX source metadata probe failed", - **kwargs, + "modelexpress library not installed; cannot use MX P2P " + "weight transfer. Install from " + "https://github.com/ai-dynamo/modelexpress (Python client at " + "modelexpress_client/python). Falling back to disk loading." ) + return self._fallback_to_disk(checkpoint_dir, mapping, **kwargs) - source_registered = source_metadata is not None - if not source_registered and self._local_source_identity is not None: - # ModelExpress 0.4.1 hashes every SourceIdentity field, including - # extra_parameters. Proceed to MxLiveWeightLoader.load_weights() - # even though this immediate probe found no source: that method - # retries list_sources every five seconds until a source appears or - # query_timeout_s expires. It uses this same patched identity, so - # any source discovered later necessarily carries the expected - # TRT-LLM identity and layout metadata. - source_metadata = _build_mx_source_metadata(self._local_source_identity) + source_metadata = self._fetch_source_metadata( + checkpoint_dir, MxClient, _build_trtllm_identity + ) # Pre-transfer compatibility gate: on mismatch, skip the transfer # before any RDMA work starts and fall back to disk. self._source_identity_compatible_for_last_load = self._source_metadata_identity_compatible( @@ -446,53 +300,33 @@ def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[ mapping, reason=( "source publishes post-transform weights but this model is " - "not qualified for staged MX receiver loading" + "not allow-listed for staged MX receiver loading" ), **kwargs, ) - if self._post_transform_weights_preloaded: - if prepare_post_transform_receiver is None: - self._post_transform_weights_preloaded = False - self._source_identity_compatible_for_last_load = False - return self._fallback_to_disk( - checkpoint_dir, - mapping, - reason=( - "post-transform source requires receiver structure " - "preparation before exact-name P2P transfer" - ), - **kwargs, - ) - # Source-side setup_aliases() may change the first canonical name - # returned for aliased parameters. Mirror that structural state on - # the receiver before upstream MX matches tensors by exact name. - prepare_post_transform_receiver(model) timeout_override = self._resolve_query_timeout_override( - source_registered=source_registered, - model_name=resolved_name, + checkpoint_dir, + MxClient, + _build_trtllm_identity, ) - try: - with ( - _MX_TRANSFER_STATE_LOCK, - _temporary_env("MX_SOURCE_QUERY_TIMEOUT", timeout_override), - _temporary_env("MODEL_NAME", resolved_name), - _patched_trtllm_identity_builder(mx_transfer, self._local_source_identity), - ): + with _temporary_env("MX_SOURCE_QUERY_TIMEOUT", timeout_override): + try: mx_loader = MxLiveWeightLoader(mx_server=self._mx_server_url) fallback_weights = mx_loader.load_weights( checkpoint_dir, mapping=mapping, model=model, ) - except Exception: - # Deliberately broad: MX is an opportunistic fast path and HF - # disk loading remains the correctness path. Preserve the full - # traceback so unexpected upstream failures are diagnosable. - logger.warning( - f"MX P2P transfer failed; falling back to disk loading.\n{traceback.format_exc()}" - ) - return self._fallback_to_disk(checkpoint_dir, mapping, **kwargs) + except Exception: + # Deliberately broad: MX is an opportunistic fast path and HF + # disk loading remains the correctness path. Preserve the full + # traceback so unexpected upstream failures are diagnosable. + logger.warning( + "MX P2P transfer failed; falling back to disk loading.\n" + f"{traceback.format_exc()}" + ) + return self._fallback_to_disk(checkpoint_dir, mapping, **kwargs) if fallback_weights: fallback_bytes = sum( @@ -541,10 +375,7 @@ def load_weights(self, checkpoint_dir: str, mapping: Mapping, **kwargs) -> dict[ return {} def _resolve_query_timeout_override( - self, - *, - source_registered: bool, - model_name: str, + self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] ) -> Optional[str]: """Return temporary `MX_SOURCE_QUERY_TIMEOUT` override, if any.""" if self._query_timeout_s is not None: @@ -553,18 +384,64 @@ def _resolve_query_timeout_override( if os.environ.get("MX_SOURCE_QUERY_TIMEOUT"): return None - if source_registered: + if self._has_any_source_instance(checkpoint_dir, MxClient, build_identity): return None logger.warning( "No MX source is currently registered for " - f"{model_name}; " + f"{self._resolve_publish_name(checkpoint_dir)}; " f"using MX_SOURCE_QUERY_TIMEOUT={_MX_SOURCE_QUERY_TIMEOUT_DEFAULT_S} " "for fast disk fallback. Set mx_config.server_query_timeout_s or " "MX_SOURCE_QUERY_TIMEOUT for long-running donor-load deployments." ) return _MX_SOURCE_QUERY_TIMEOUT_DEFAULT_S + def _has_any_source_instance( + self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] + ) -> bool: + """Best-effort fast probe for registered MX source instances.""" + client = None + try: + identity = build_identity(model_name=self._resolve_publish_name(checkpoint_dir)) + client = MxClient(server_url=self._mx_server_url) + list_resp = client.list_sources(identity=identity) + return bool(getattr(list_resp, "instances", [])) + except (AttributeError, RuntimeError, TimeoutError, grpc.RpcError): + # If the probe cannot complete, prefer fast fallback over the + # upstream 1-hour default. The actual MxLiveWeightLoader call below + # remains the source of truth and may still succeed. + logger.warning( + f"MX source probe failed; using fast fallback timeout.\n{traceback.format_exc()}" + ) + return False + finally: + if client is not None and hasattr(client, "close"): + client.close() + + def _source_identity_compatible( + self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] + ) -> bool: + """Whether the MX source's identity is compatible with this receiver. + + Compares the receiver's local :class:`SourceIdentity` against the + publisher's via `check_weight_sharing_compatibility` with the `WARN_FALLBACK` + policy. + + Args: + checkpoint_dir: The checkpoint directory identifying the source. + MxClient: The MX discovery client type (forwarded to the fetch + seam). + build_identity: Builder used to derive the publisher identity + (forwarded to the fetch seam). + + Returns: + `True` to proceed with P2P only when both identities are present + and compatible. `False` when either identity is missing or the + identities mismatch, so the caller falls back to disk loading. + """ + source_identity = self._fetch_source_identity(checkpoint_dir, MxClient, build_identity) + return self._source_identity_compatible_with_source(source_identity) + def _source_metadata_identity_compatible(self, metadata: Optional[dict[str, Any]]) -> bool: source_identity = _source_identity_from_metadata(metadata) return self._source_identity_compatible_with_source(source_identity) @@ -580,23 +457,30 @@ def _source_identity_compatible_with_source( ) return decision.should_share + def _fetch_source_identity( + self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] + ) -> Optional[SourceIdentity]: + """Fetch the publisher's serialized :class:`SourceIdentity`. + + Args: + checkpoint_dir: The checkpoint directory identifying the source. + MxClient: The MX discovery client type. + build_identity: Builder used to derive the publisher identity. + + Returns: + The publisher's identity, or `None` when it cannot be fetched + yet (the compatibility gate then rejects P2P and falls back). + """ + metadata = self._fetch_source_metadata(checkpoint_dir, MxClient, build_identity) + return _source_identity_from_metadata(metadata) + def _fetch_source_metadata( - self, - checkpoint_dir: str, - MxClient: Type[Any], - build_identity: Callable[..., Any], - *, - model_name: Optional[str] = None, + self, checkpoint_dir: str, MxClient: Type[Any], build_identity: Callable[..., Any] ) -> Optional[dict[str, Any]]: """Fetch TRT-LLM metadata for the selected MX source, if available.""" client = None try: - identity = self._build_mx_identity( - checkpoint_dir, - build_identity, - self._local_source_identity, - model_name=model_name, - ) + identity = build_identity(model_name=self._resolve_publish_name(checkpoint_dir)) client = MxClient(server_url=self._mx_server_url) for method_name in ("get_source_metadata", "get_metadata", "get_worker_metadata"): method = getattr(client, method_name, None) @@ -605,13 +489,7 @@ def _fetch_source_metadata( try: metadata = method(identity=identity) except TypeError: - try: - metadata = method(identity) - except TypeError: - # modelexpress 0.4.1 get_metadata() takes - # mx_source_id/worker_id rather than an identity. Fall - # through to the exact-identity list_sources query. - continue + metadata = method(identity) metadata_dict = _metadata_to_dict(metadata) if _metadata_has_trtllm_key(metadata_dict): return metadata_dict @@ -623,36 +501,16 @@ def _fetch_source_metadata( metadata_dict = _source_instance_metadata(instance) if metadata_dict: metadata_candidates.append(metadata_dict) - selected_metadata = self._select_source_metadata(metadata_candidates) - if selected_metadata is not None: - return selected_metadata - - # modelexpress 0.4.1 SourceInstanceRef intentionally omits the - # queried SourceIdentity. A non-empty response still proves an - # exact match because list_sources hashes every identity field, - # including extra_parameters. Reconstruct the metadata that was - # embedded in the exact query so the compatibility/layout checks - # remain fail-closed without a second metadata channel. - if instances and self._local_source_identity is not None: - return _build_mx_source_metadata(self._local_source_identity) + return self._select_source_metadata(metadata_candidates) + except (AttributeError, RuntimeError, TimeoutError, TypeError, ValueError, grpc.RpcError): + logger.warning( + f"MX source metadata fetch failed; falling back to disk loading.\n" + f"{traceback.format_exc()}" + ) return None finally: - _close_mx_client(client) - - def _build_mx_identity( - self, - checkpoint_dir: str, - build_identity: Callable[..., _MxSourceIdentity], - source_identity: Optional[SourceIdentity], - *, - model_name: Optional[str] = None, - ) -> _MxSourceIdentity: - """Build the MX identity used for discovery and attach TRT-LLM identity.""" - resolved_name = model_name or self._resolve_publish_name(checkpoint_dir) - return _attach_trtllm_metadata_to_mx_identity( - build_identity(model_name=resolved_name), - source_identity, - ) + if client is not None and hasattr(client, "close"): + client.close() def _select_source_metadata( self, metadata_candidates: list[dict[str, Any]] @@ -696,7 +554,7 @@ def publish_as_source( Called by the integration in `model_loader.py` after `post_load_weights()` so targets receive the post-transform runtime - layout and, when qualified, can skip their own one-shot transforms. + layout and, when allow-listed, can skip their own one-shot transforms. Delegates to the upstream `modelexpress.trtllm_live_transfer.publish_model_params` @@ -723,71 +581,70 @@ def publish_as_source( return try: - from modelexpress import ( - trtllm_live_transfer as mx_transfer, # type: ignore[import-not-found] + from modelexpress.trtllm_live_transfer import ( + publish_model_params, # type: ignore[import-not-found] ) except ImportError: logger.debug("modelexpress library not installed; skipping MX publish.") return - try: - publish_model_params = mx_transfer.publish_model_params - except AttributeError: - logger.debug("modelexpress publish_model_params is missing; skipping MX publish.") - return # THREADSAFETY: upstream publish_model_params reads MODEL_EXPRESS_URL and # MODEL_NAME from the environment. Set both from our resolved # configuration so per-instance values (URL passed via # llm_args.mx_config.server_url, identity from llm_args.model) are - # respected, then restore prior state. MX transfer and publish calls in - # this interpreter are serialized while upstream requires process-wide - # state. Tracked as MX-2 in §15 (the env-var dance goes away when - # upstream exports a public identity builder / publish API). + # respected, then restore prior state. This is safe for the current + # sequential TRT-LLM worker path, but co-resident ranks in one Python + # interpreter would race on process-wide env. Tracked as MX-2 in §15 + # (the env-var dance goes away when upstream exports a public identity + # builder / publish API). + resolved_name = self._resolve_publish_name(checkpoint_dir) metadata = _build_mx_source_metadata(source_identity) - metadata_kwargs = _publish_metadata_kwargs(publish_model_params, metadata) or {} - identity_builder = getattr(mx_transfer, "_build_trtllm_identity", None) - if not metadata_kwargs and not callable(identity_builder): + metadata_kwargs = _publish_metadata_kwargs(publish_model_params, metadata) + if metadata_kwargs is None: logger.warning( "Skipping MX post-transform publish because " - "publish_model_params does not accept metadata and MX does " - "not expose its TRT-LLM identity builder; receivers cannot " - "safely verify transformed weights." + "publish_model_params does not accept metadata; receivers " + "cannot safely verify transformed weights." ) return + env_overrides = { + "MODEL_EXPRESS_URL": self._mx_server_url, + "MODEL_NAME": resolved_name, + } if threading.active_count() > 1: logger.warning_once( "MX publish uses process-wide MODEL_EXPRESS_URL/MODEL_NAME " - "environment variables; concurrent MX transfer and publish calls " - "in one Python process are serialized, but unrelated env readers " - "can still observe transient values. Tracked by MX-2.", + "environment variables; concurrent publish calls in one Python " + "process are serialized, but unrelated env readers can still " + "observe transient values. Tracked by MX-2.", key="mx_publish_env_threaded_warning", ) - try: - with _MX_TRANSFER_STATE_LOCK: - resolved_name = self._resolve_publish_name(checkpoint_dir) - # Post-load transforms may enqueue asynchronous writes. Make - # the source buffers globally ready before MX publishes their - # addresses and allows a receiver to issue RDMA reads. - _synchronize_cuda_for_mx_publish() - with ( - _temporary_env("MODEL_EXPRESS_URL", self._mx_server_url), - _temporary_env("MODEL_NAME", resolved_name), - _patched_trtllm_identity_builder(mx_transfer, source_identity), - ): - publish_model_params(model, **metadata_kwargs) + with _MX_PUBLISH_ENV_LOCK: + prior = {key: os.environ.get(key) for key in env_overrides} + for key, value in env_overrides.items(): + os.environ[key] = value + + try: + publish_model_params(model, **metadata_kwargs) logger.info( "Published post-transform weights to MX server at %s as model=%r", self._mx_server_url, resolved_name, ) - except Exception: - # Deliberately broad: publish is best-effort. A publish failure - # should not fail the local worker that already loaded weights. - logger.warning( - f"Failed to publish weights to MX server at {self._mx_server_url}.\n" - f"{traceback.format_exc()}" - ) + except Exception: + # Deliberately broad: publish is best-effort. A publish failure + # should not fail the local worker that already loaded weights. + logger.warning( + f"Failed to publish weights to MX server at {self._mx_server_url}.\n" + f"{traceback.format_exc()}" + ) + finally: + for key, prior_value in prior.items(): + if prior_value is None: + os.environ.pop(key, None) + else: + os.environ[key] = prior_value def post_load_publish( self, @@ -883,7 +740,9 @@ def _build_mx_source_metadata(source_identity: Optional[SourceIdentity]) -> dict _MX_TRANSFORM_PROTOCOL_VERSION_METADATA_KEY: str(_MX_STAGED_TRANSFORM_PROTOCOL_VERSION), } if source_identity is not None: - metadata[_MX_SOURCE_IDENTITY_METADATA_KEY] = _serialize_source_identity(source_identity) + metadata[_MX_SOURCE_IDENTITY_METADATA_KEY] = json.dumps( + source_identity.to_dict(), sort_keys=True + ) return metadata diff --git a/tensorrt_llm/_torch/models/dspark/__init__.py b/tensorrt_llm/_torch/models/dspark/__init__.py deleted file mode 100644 index b33d7553d877..000000000000 --- a/tensorrt_llm/_torch/models/dspark/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""DSpark draft-model components.""" diff --git a/tensorrt_llm/_torch/models/dspark/attention.py b/tensorrt_llm/_torch/models/dspark/attention.py deleted file mode 100644 index 7f63a40edfbe..000000000000 --- a/tensorrt_llm/_torch/models/dspark/attention.py +++ /dev/null @@ -1,462 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# The DSpark captured-context attention primitives are ported from DeepSeek's -# DeepSpec reference ``inference/kernel.py`` (``sparse_attn``) and -# ``inference/model.py`` (``get_dspark_topk_idxs``). The reference computes these -# with a TileLang kernel; this is a functional-first pure-PyTorch port with the -# same math (index-gather + online softmax + a learnable attention sink that -# contributes only to the softmax denominator). -"""DSpark draft captured-context attention primitives (hardware-agnostic). - -The DSpark draft uses *dense* sliding-window MLA (``compress_ratio == 0``): the -query comes from the block's draft tokens, while the keys/values are gathered -from a small per-request set of positions (a sliding window of the projected -captured context plus the current block's own positions). Two primitives capture -the parts that differ from the standard MLA path: - -* :func:`get_dspark_topk_idxs` — the (window-context + block) position list. -* :func:`dspark_sparse_attn` — index-gathered attention with an attention sink. -""" - -from functools import lru_cache - -import torch -import torch.nn.functional as F - -__all__ = [ - "get_dspark_topk_idxs", - "get_dspark_topk_idxs_batched", - "dspark_sparse_attn", - "precompute_dspark_freqs_cis", - "apply_dspark_rotary", - "apply_dspark_rotary_batched", - "dspark_attention_forward", - "dspark_attention_forward_batched", -] - - -def precompute_dspark_freqs_cis( - rope_head_dim: int, - seqlen: int, - rope_theta: float = 10000.0, - device: torch.device | str = "cpu", -) -> torch.Tensor: - """Plain (non-YaRN) RoPE complex exponentials for the DSpark draft. - - The dense draft attention (``compress_ratio == 0``) disables YaRN and uses the - base ``rope_theta`` (DeepSpec ``precompute_freqs_cis`` with - ``original_seq_len == 0``). - - Returns: - complex64 tensor ``[seqlen, rope_head_dim // 2]``. - """ - freqs = 1.0 / ( - rope_theta - ** (torch.arange(0, rope_head_dim, 2, dtype=torch.float32, device=device) / rope_head_dim) - ) - t = torch.arange(seqlen, dtype=torch.float32, device=device) - freqs = torch.outer(t, freqs) - return torch.polar(torch.ones_like(freqs), freqs) - - -def apply_dspark_rotary( - x: torch.Tensor, freqs_cis: torch.Tensor, inverse: bool = False -) -> torch.Tensor: - """Apply (or, with ``inverse``, de-apply) rotary embeddings, DeepSpec-style. - - Functional (non-in-place) port of DeepSpec ``apply_rotary_emb``: treats the - last dim as adjacent (re, im) pairs, rotates by ``freqs_cis`` indexed along the - sequence axis, and conjugates for the inverse (de-rotation applied to the - attention output). ``x`` is the rope-dim slice only: ``[b, s, rd]`` (3D) or - ``[b, s, h, rd]`` (4D), with ``freqs_cis`` of shape ``[s, rd // 2]``. - """ - orig_dtype = x.dtype - xc = torch.view_as_complex(x.float().unflatten(-1, (-1, 2))) - if inverse: - freqs_cis = freqs_cis.conj() - if xc.ndim == 3: - fc = freqs_cis.view(1, xc.size(1), xc.size(-1)) - else: - fc = freqs_cis.view(1, xc.size(1), 1, xc.size(-1)) - out = torch.view_as_real(xc * fc).flatten(-2) - return out.to(orig_dtype) - - -def apply_dspark_rotary_batched( - x: torch.Tensor, freqs_cis: torch.Tensor, inverse: bool = False -) -> torch.Tensor: - """Per-row (batched) variant of :func:`apply_dspark_rotary`. - - Identical math, but ``freqs_cis`` carries a leading batch axis so each row of - ``x`` is rotated by its own per-request phases (the generation draft runs each - request at a different absolute ``start_pos``). ``x`` is the rope-dim slice - only: ``[G, s, rd]`` (3D) or ``[G, s, h, rd]`` (4D), with ``freqs_cis`` of shape - ``[G, s, rd // 2]``. - """ - orig_dtype = x.dtype - xc = torch.view_as_complex(x.float().unflatten(-1, (-1, 2))) - if inverse: - freqs_cis = freqs_cis.conj() - g, s, half = freqs_cis.shape - if xc.ndim == 3: - fc = freqs_cis.view(g, s, half) - else: - fc = freqs_cis.view(g, s, 1, half) - out = torch.view_as_real(xc * fc).flatten(-2) - return out.to(orig_dtype) - - -@lru_cache(maxsize=64) -def _topk_matrix(window_size: int, block_size: int, start_pos: int) -> torch.Tensor: - # [min(window, start_pos+1)] context positions in the rolling KV window, - # followed by [block_size] positions for the current block's own K/V (which - # the caller appends to the window at offset ``window_size``). - ctx = torch.arange(min(window_size, start_pos + 1)) - blk = window_size + torch.arange(block_size) - return torch.cat([ctx, blk]).int() - - -def get_dspark_topk_idxs( - window_size: int, - bsz: int, - block_size: int, - start_pos: int, - device: torch.device | str = "cpu", -) -> torch.Tensor: - """Per-query attended-position indices for the DSpark draft block. - - Mirrors DeepSpec ``get_dspark_topk_idxs``: every one of the ``block_size`` - query positions attends to the same set — the ``min(window_size, start_pos+1)`` - most-recent context positions in the rolling KV window, then the - ``block_size`` positions of the current block (stored at offset - ``window_size`` in the concatenated KV). Note this is *non-causal* within the - block (every position sees every block position), matching the reference. - - Args: - window_size: sliding-window length of the captured-context KV cache. - bsz: batch size. - block_size: number of draft positions per request. - start_pos: absolute decode position (must be > 0); bounds the context. - device: device for the returned index tensor. - - Returns: - int32 tensor ``[bsz, block_size, topk]`` with - ``topk = min(window_size, start_pos+1) + block_size``. - """ - assert start_pos > 0, "DSpark draft attention runs at generation (start_pos > 0)" - matrix = _topk_matrix(int(window_size), int(block_size), int(start_pos)).to(device) - return matrix.view(1, 1, -1).expand(bsz, block_size, -1).contiguous() - - -def get_dspark_topk_idxs_batched( - window_size: int, - block_size: int, - start_pos: torch.Tensor, -) -> torch.Tensor: - """Sync-free, fixed-size (CUDA-graph-safe) batched ``get_dspark_topk_idxs``. - - Unlike the scalar :func:`get_dspark_topk_idxs` (whose ``topk`` width - ``min(window_size, start_pos+1) + block_size`` depends on the host int - ``start_pos``), this always returns the **fixed** width ``window_size + - block_size`` and masks the unfilled context slots with ``-1``. The masked - slots are excluded by :func:`dspark_sparse_attn` exactly as if they were - absent, so the result is numerically identical to gathering only the - ``min(window_size, start_pos+1)`` valid context positions — but the shape no - longer depends on the data, which is what CUDA-graph capture requires. - - Every query position attends to the same set: context window slots - ``0..window_size-1`` (slot ``c`` valid iff ``c <= start_pos[g]``, i.e. it has - been written) followed by the ``block_size`` block positions at offset - ``window_size`` (always valid). - - Args: - window_size: sliding-window length of the captured-context KV cache. - block_size: number of draft positions per request. - start_pos: ``[G]`` int tensor of per-request absolute decode positions. - - Returns: - int32 tensor ``[G, block_size, window_size + block_size]``. - """ - device = start_pos.device - g = start_pos.shape[0] - ctx_cols = torch.arange(window_size, device=device) # [win] - # Context slot c holds a written key iff c <= start_pos (slots 0..start_pos - # filled; for start_pos >= window_size-1 the whole rolling window is filled). - valid = ctx_cols.unsqueeze(0) <= start_pos.unsqueeze(1) # [G, win] - ctx_idx = torch.where( - valid, ctx_cols.unsqueeze(0).expand(g, -1), torch.full_like(valid, -1, dtype=torch.long) - ) - blk_idx = window_size + torch.arange(block_size, device=device) # [block] - blk_idx = blk_idx.unsqueeze(0).expand(g, -1) # [G, block] - row = torch.cat([ctx_idx, blk_idx], dim=1).to(torch.int32) # [G, win+block] - return row.unsqueeze(1).expand(g, block_size, -1).contiguous() - - -def dspark_sparse_attn( - q: torch.Tensor, - kv: torch.Tensor, - attn_sink: torch.Tensor, - topk_idxs: torch.Tensor, - softmax_scale: float, -) -> torch.Tensor: - """Index-gathered multi-query attention with an attention sink. - - Functional-first port of the DeepSpec ``sparse_attn`` TileLang kernel. For - each ``(batch, query, head)`` it gathers the ``topk`` KV rows named by - ``topk_idxs`` (an index of ``-1`` masks that slot), computes a scaled - dot-product softmax over them, and adds a per-head learnable *sink* logit that - participates only in the softmax denominator (i.e. an "attend-to-nothing" - option with a zero value vector). KV is shared across query heads (MQA). - - Args: - q: ``[b, m, h, d]`` query (``m`` = block_size, ``h`` = heads). - kv: ``[b, n, d]`` keys/values (shared across heads). - attn_sink: ``[h]`` per-head sink logits (fp32). - topk_idxs: ``[b, m, topk]`` int gather indices into ``kv`` (``-1`` masks). - softmax_scale: scalar applied to the q·k scores (``head_dim ** -0.5``). - - Returns: - ``[b, m, h, d]`` attention output, in ``q.dtype``. - """ - b, m, h, d = q.shape - idx = topk_idxs.long() # [b, m, topk] - valid = idx >= 0 - safe = idx.clamp(min=0) - - # Invalid slots read kv[0, :] (via safe.clamp), but masked_fill below - # zeros their softmax probs, so the einsum nullifies them. - kv_exp = kv.unsqueeze(1).expand(b, m, kv.shape[1], d) - gathered = torch.gather(kv_exp, 2, safe.unsqueeze(-1).expand(b, m, safe.shape[-1], d)).float() - - # Scores [b, m, h, topk]; mask invalid slots to -inf before the softmax. - scores = torch.einsum("bmhd,bmkd->bmhk", q.float(), gathered) * softmax_scale - scores = scores.masked_fill(~valid.unsqueeze(2), float("-inf")) - - # Online-softmax max is taken over gathered positions only (the sink is added - # to the denominator afterwards), matching the kernel's reduce order. - smax = scores.max(dim=-1, keepdim=True).values # [b, m, h, 1] - smax = torch.where(torch.isinf(smax), torch.zeros_like(smax), smax) - probs = torch.exp(scores - smax) # masked slots -> exp(-inf) = 0 - sink = torch.exp(attn_sink.to(torch.float32).view(1, 1, h) - smax.squeeze(-1)) - denom = probs.sum(dim=-1) + sink # [b, m, h] - out = torch.einsum("bmhk,bmkd->bmhd", probs, gathered) / denom.unsqueeze(-1) - return out.to(q.dtype) - - -def _rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: - """RMSNorm matching the DeepSpec reference (fp32 reduce, then * weight).""" - dtype = x.dtype - xf = x.float() - xf = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps) - return (weight.float() * xf).to(dtype) - - -def _rope_last_dims( - t: torch.Tensor, rope_head_dim: int, freqs_cis: torch.Tensor, inverse: bool = False -) -> torch.Tensor: - """Apply RoPE to the last ``rope_head_dim`` dims; pass the rest through.""" - nope = t[..., :-rope_head_dim] - rope = apply_dspark_rotary(t[..., -rope_head_dim:], freqs_cis, inverse=inverse) - return torch.cat([nope, rope], dim=-1) - - -def _rope_last_dims_batched( - t: torch.Tensor, rope_head_dim: int, freqs_cis: torch.Tensor, inverse: bool = False -) -> torch.Tensor: - """Per-row variant of :func:`_rope_last_dims` (``freqs_cis`` has a batch axis).""" - nope = t[..., :-rope_head_dim] - rope = apply_dspark_rotary_batched(t[..., -rope_head_dim:], freqs_cis, inverse=inverse) - return torch.cat([nope, rope], dim=-1) - - -def dspark_attention_forward( - x: torch.Tensor, - main_x: torch.Tensor, - start_pos: int, - kv_cache: torch.Tensor, - *, - wq_a: torch.Tensor, - q_norm_w: torch.Tensor, - wq_b: torch.Tensor, - wkv: torch.Tensor, - kv_norm_w: torch.Tensor, - wo_a: torch.Tensor, - wo_b: torch.Tensor, - attn_sink: torch.Tensor, - n_heads: int, - head_dim: int, - rope_head_dim: int, - n_groups: int, - o_lora_rank: int, - window_size: int, - eps: float, - softmax_scale: float, - freqs_cis: torch.Tensor, - persist: bool = False, -) -> torch.Tensor: - """Captured-context DSpark draft attention (generation path, ``start_pos > 0``). - - Functional port of DeepSpec ``DSparkAttention.forward`` for the dense - (``compress_ratio == 0``) draft: low-rank Q (``wq_a`` -> ``q_norm`` -> ``wq_b``) - with a per-head RMS + RoPE, MQA K/V from ``wkv`` (shared across heads), keys - gathered from a rolling captured-context window (``kv_cache``, into which the - projected ``main_x`` context is written at ``start_pos % window_size``) plus the - block's own positions, attention-sink softmax, inverse-RoPE on the output, and a - grouped low-rank O projection (``wo_a`` einsum + ``wo_b``). - - Weights are plain tensors for ``F.linear`` (the caller supplies the loaded / - dequantized projection weights); ``wo_a`` is the raw grouped weight matrix - ``[n_groups * o_lora_rank, n_heads * head_dim // n_groups]``. ``kv_cache`` is - ``[b, window_size, head_dim]`` and is updated functionally (cloned). - - Returns: - ``[b, block_size, dim]`` attention output (residual stream contribution). - """ - assert start_pos > 0, "DSpark draft attention runs at generation (start_pos > 0)" - b, block, _ = x.shape - rd = rope_head_dim - main_freqs = freqs_cis[start_pos : start_pos + 1] - blk_freqs = freqs_cis[start_pos + 1 : start_pos + 1 + block] - - # Captured-context K/V from main_x (MQA, shared across heads). - main_kv = _rmsnorm(F.linear(main_x, wkv), kv_norm_w, eps) # [b, 1, head_dim] - main_kv = _rope_last_dims(main_kv, rd, main_freqs) - - # Query: low-rank + per-head RMS + RoPE. - q = _rmsnorm(F.linear(x, wq_a), q_norm_w, eps) - q = F.linear(q, wq_b).unflatten(-1, (n_heads, head_dim)) # [b, block, h, head_dim] - # Per-head RMS in the query dtype (matches the reference inline normalization, - # which is NOT the fp32 RMSNorm path). - q = q * torch.rsqrt(q.square().mean(-1, keepdim=True) + eps) - q = _rope_last_dims(q, rd, blk_freqs) - - # Block K/V. - kv = _rmsnorm(F.linear(x, wkv), kv_norm_w, eps) # [b, block, head_dim] - kv = _rope_last_dims(kv, rd, blk_freqs) - - # Write the context K/V into the rolling window, then attend over - # [window context | block] with the sink. ``persist=True`` writes through - # to the caller's buffer (cross-step decode, worker-owned window); the - # default clones so single-shot callers (golden / unit tests) stay pure. - cache = kv_cache if persist else kv_cache.clone() - cache[:, start_pos % window_size] = main_kv.squeeze(1) - kv_full = torch.cat([cache, kv], dim=1) # [b, window + block, head_dim] - topk = get_dspark_topk_idxs(window_size, b, block, start_pos, device=x.device) - o = dspark_sparse_attn(q, kv_full, attn_sink, topk, softmax_scale) # [b, block, h, head_dim] - o = _rope_last_dims(o, rd, blk_freqs, inverse=True) - - # Grouped low-rank O projection. - o = o.reshape(b, block, n_groups, -1) - wo_a_v = wo_a.view(n_groups, o_lora_rank, -1) - o = torch.einsum("bsgd,grd->bsgr", o, wo_a_v) - return F.linear(o.flatten(2), wo_b) - - -def dspark_attention_forward_batched( - x: torch.Tensor, - main_x: torch.Tensor, - start_pos: torch.Tensor, - kv_cache: torch.Tensor, - slots: torch.Tensor, - *, - wq_a: torch.Tensor, - q_norm_w: torch.Tensor, - wq_b: torch.Tensor, - wkv: torch.Tensor, - kv_norm_w: torch.Tensor, - wo_a: torch.Tensor, - wo_b: torch.Tensor, - attn_sink: torch.Tensor, - n_heads: int, - head_dim: int, - rope_head_dim: int, - n_groups: int, - o_lora_rank: int, - window_size: int, - eps: float, - softmax_scale: float, - freqs_cis: torch.Tensor, - persist: bool = False, -) -> torch.Tensor: - """Batched, CUDA-graph-safe captured-context DSpark draft attention. - - Numerically identical, per request, to :func:`dspark_attention_forward`, but - free of host syncs and data-dependent shapes so it can be captured into a CUDA - graph (the one-engine drafter runs inside the target's graph). The differences - from the scalar path are purely mechanical: - - * ``start_pos`` is a ``[G]`` int tensor (one absolute decode position per gen - request) instead of a python int; RoPE phases are *gathered* per request from - the fixed ``freqs_cis`` table rather than sliced. - * the rolling-window context K/V is written/read through the ``slots`` index - into a shared ``kv_cache`` (``persist=True`` writes through to the caller's - worker-owned buffer; otherwise a clone is used), instead of mutating a - per-request cache in place. - * the attended-position list has the fixed width ``window_size + block_size`` - with ``-1`` masking (see :func:`get_dspark_topk_idxs_batched`). - - Args: - x: ``[G, block, dim]`` block layer input (per gen request). - main_x: ``[G, 1, hidden]`` projected captured context. - start_pos: ``[G]`` int tensor of absolute decode positions (> 0). - kv_cache: ``[N, window_size, head_dim]`` rolling captured-context windows - (``N`` rows indexed by ``slots``; ``N == G`` for single-shot callers). - slots: ``[G]`` int tensor mapping each request to its ``kv_cache`` row. - freqs_cis: ``[maxlen, rope_head_dim // 2]`` precomputed plain-RoPE table; - must satisfy ``maxlen > start_pos.max() + block_size``. - - Returns: - ``[G, block, dim]`` attention output (residual stream contribution). - """ - g, block, _ = x.shape - rd = rope_head_dim - # Per-request RoPE phases gathered from the fixed table (no host-int slicing). - main_freqs = freqs_cis[start_pos].unsqueeze(1) # [G, 1, rd//2] - blk_pos = start_pos.unsqueeze(1) + 1 + torch.arange(block, device=x.device) # [G, block] - blk_freqs = freqs_cis[blk_pos] # [G, block, rd//2] - - # Captured-context K/V from main_x (MQA, shared across heads). - main_kv = _rmsnorm(F.linear(main_x, wkv), kv_norm_w, eps) # [G, 1, head_dim] - main_kv = _rope_last_dims_batched(main_kv, rd, main_freqs) - - # Query: low-rank + per-head RMS + RoPE. - q = _rmsnorm(F.linear(x, wq_a), q_norm_w, eps) - q = F.linear(q, wq_b).unflatten(-1, (n_heads, head_dim)) # [G, block, h, head_dim] - q = q * torch.rsqrt(q.square().mean(-1, keepdim=True) + eps) - q = _rope_last_dims_batched(q, rd, blk_freqs) - - # Block K/V. - kv = _rmsnorm(F.linear(x, wkv), kv_norm_w, eps) # [G, block, head_dim] - kv = _rope_last_dims_batched(kv, rd, blk_freqs) - - # Write the context K/V into the rolling window at slot start_pos%window_size, - # then attend over [window context | block]. ``persist=True`` writes through to - # the worker-owned buffer (cross-step decode); otherwise clone so single-shot - # callers stay pure. Indexed scatter/gather by (slots, slot_pos) is graph-safe. - write_target = kv_cache if persist else kv_cache.clone() - slot_pos = start_pos % window_size # [G] - write_target[slots, slot_pos] = main_kv.squeeze(1).to(write_target.dtype) - cache_rows = write_target[slots] # [G, window, head_dim] - kv_full = torch.cat([cache_rows, kv], dim=1) # [G, window + block, head_dim] - topk = get_dspark_topk_idxs_batched(window_size, block, start_pos) - o = dspark_sparse_attn(q, kv_full, attn_sink, topk, softmax_scale) # [G, block, h, head_dim] - o = _rope_last_dims_batched(o, rd, blk_freqs, inverse=True) - - # Grouped low-rank O projection. - o = o.reshape(g, block, n_groups, -1) - wo_a_v = wo_a.view(n_groups, o_lora_rank, -1) - o = torch.einsum("bsgd,grd->bsgr", o, wo_a_v) - return F.linear(o.flatten(2), wo_b) diff --git a/tensorrt_llm/_torch/models/dspark/draft.py b/tensorrt_llm/_torch/models/dspark/draft.py deleted file mode 100644 index 1b47f4922965..000000000000 --- a/tensorrt_llm/_torch/models/dspark/draft.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DSpark draft I/O logic is ported from DeepSeek's DeepSeek-V4-Pro-DSpark -# reference (`inference/model.py`, DSparkBlock.forward_embed / forward_head). -"""DSpark draft I/O: block input and proposal stages. - -This module holds the *framework-agnostic* (pure-torch) input/output stages of -the DSpark draft block, separated from the heavy V4 backbone (MLA + MoE + mHC) so -they can be unit-tested in isolation: - - - ``build_draft_input_ids``: ``[bonus_token, noise, noise, ...]`` block input. - - ``dspark_propose``: given the per-position backbone ``base_logits`` and the - Markov / confidence heads, run the autoregressive Markov refinement to sample - the block tokens and apply the static confidence-threshold truncation. - -The backbone (3 V4 blocks producing ``block_hidden``) lives in the model module; -this file is the part fully specified by the reference and validated against it. -""" - -from typing import Optional - -import torch -from torch import nn - -from .heads import confident_prefix_length - - -def build_draft_input_ids( - bonus_token_ids: torch.Tensor, *, block_size: int, noise_token_id: int -) -> torch.Tensor: - """``[batch] -> [batch, block_size]`` = ``[bonus, noise, noise, ...]``. - - The first position is the verified bonus token (the target's last accepted - token); the rest are the DSpark noise/mask token (id 128799 for V4-Pro). - """ - batch = bonus_token_ids.shape[0] - out = bonus_token_ids.new_full((batch, block_size), int(noise_token_id)) - out[:, 0] = bonus_token_ids - return out - - -def dspark_propose( - base_logits: torch.Tensor, - *, - bonus_token_ids: torch.Tensor, - block_hidden: torch.Tensor, - markov_head: Optional[nn.Module], - confidence_head: Optional[nn.Module], - block_size: int, - temperature: float = 0.0, - confidence_threshold: float = 0.0, - return_logits: bool = False, -) -> tuple: - """Produce DSpark draft tokens for one block (functional-first, static length). - - Args: - base_logits: ``[batch, block_size, vocab]`` from the backbone + lm_head. - bonus_token_ids: ``[batch]`` the token preceding the first draft position. - block_hidden: ``[batch, block_size, hidden]`` backbone hidden (feeds the - confidence head, and the RNN-head variant). - markov_head / confidence_head: the validated DSpark heads (may be None). - Returns: - draft_tokens: ``[batch, block_size]`` sampled tokens (full block; callers - keep the tensor fixed-width for CUDA-graph safety). - num_proposed: ``[batch]`` int32 — how many leading tokens survive the - static confidence-threshold truncation (== block_size when no head / - threshold<=0). - """ - batch = base_logits.shape[0] - # ``draft_logits`` are the per-position distributions the draft token is drawn - # from (markov-corrected when a head is present, else the raw base logits). - # Surfaced under ``return_logits`` for the §7.9 probabilistic-acceptance - # (1-TV) measurement; the normal path ignores them. - draft_logits = base_logits - if markov_head is not None: - draft_tokens, corrected = markov_head.sample_block_tokens( - base_logits, - first_prev_token_ids=bonus_token_ids, - hidden_states=block_hidden, - temperature=temperature, - ) - draft_logits = corrected - else: - from .heads import greedy_or_sample - - draft_tokens = greedy_or_sample(base_logits, temperature) - - # Scaffolding: confidence-based dynamic drafting is NOT enabled in this PR. - # The worker always calls with confidence_threshold=0.0, so the block below is - # inert and num_proposed stays == block_size (the full block is proposed). The - # returned num_proposed is intentionally not yet consumed by the speculative - # scheduler/verifier; wiring it through is a follow-up (see PR description). - num_proposed = torch.full( - (batch,), int(block_size), dtype=torch.int32, device=base_logits.device - ) - if confidence_head is not None and confidence_threshold > 0.0: - # prev token at position k is [bonus, draft_0, ..., draft_{k-1}] - prev_ids = torch.cat([bonus_token_ids.unsqueeze(1), draft_tokens[:, :-1]], dim=1) - prev_emb = ( - markov_head.get_prev_embeddings(prev_ids) - if (markov_head is not None and getattr(confidence_head, "with_markov", False)) - else None - ) - conf_logits = ( - confidence_head(block_hidden, prev_embeddings=prev_emb) - if prev_emb is not None - else confidence_head(block_hidden) - ) - # Per-request prefix truncation (batch handled row-wise to stay simple; - # functional-first scope typically runs batch=1 for the draft). - for b in range(batch): - num_proposed[b] = confident_prefix_length( - conf_logits[b : b + 1], block_size=block_size, threshold=confidence_threshold - ) - if return_logits: - return draft_tokens, num_proposed, draft_logits - return draft_tokens, num_proposed diff --git a/tensorrt_llm/_torch/models/dspark/heads.py b/tensorrt_llm/_torch/models/dspark/heads.py deleted file mode 100644 index c49e35fbafa0..000000000000 --- a/tensorrt_llm/_torch/models/dspark/heads.py +++ /dev/null @@ -1,254 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# The DSpark Markov/RNN/confidence-head math is ported from DeepSeek's DeepSpec -# reference implementation (https://github.com/deepseek-ai/DeepSpec, MIT License). -"""DSpark draft-network heads (pure-torch, framework-agnostic). - -These modules implement the *sequential refinement* and *acceptance-confidence* -parts of DeepSeek's DSpark speculative-decoding draft network: - - - Markov head: a low-rank token-bigram logit bias ``logits_k += W2(W1[t_{k-1}])`` - applied autoregressively across the ``block_size`` draft positions (the cheap - "sequential" half of DSpark's "semi-parallel" drafting). RNN variant carries - a GRU-style recurrent state across positions. - - Confidence head: predicts a per-position acceptance probability; the cumulative - product over positions estimates prefix-acceptance and is used only to - *truncate* the proposed draft length (NOT to decide acceptance). - -This file deliberately depends on ``torch`` only so it can be unit-tested in -isolation (token-for-token) against the DeepSpec reference. -""" - -from typing import Optional - -import torch -from torch import nn - - -def greedy_or_sample(logits: torch.Tensor, temperature: float) -> torch.Tensor: - """Argmax for temperature<=0, else temperature-scaled multinomial. - - Args: - logits: ``[..., vocab]``. - Returns: - token ids with the trailing vocab dim reduced. - """ - if temperature <= 0.0: - return logits.argmax(dim=-1) - probs = torch.softmax(logits.float() / temperature, dim=-1) - flat = probs.reshape(-1, probs.shape[-1]) - sampled = torch.multinomial(flat, num_samples=1).squeeze(-1) - return sampled.view(probs.shape[:-1]) - - -class VanillaMarkov(nn.Module): - """Low-rank token-bigram logit bias: ``bias = W2(W1[token])``.""" - - markov_head_type = "vanilla" - - def __init__(self, *, vocab_size: int, markov_rank: int): - super().__init__() - self.vocab_size = int(vocab_size) - self.markov_rank = int(markov_rank) - assert self.markov_rank > 0, ( - f"VanillaMarkov requires markov_rank > 0, got {self.markov_rank}." - ) - self.markov_w1 = nn.Embedding(self.vocab_size, self.markov_rank) - self.markov_w2 = nn.Linear(self.markov_rank, self.vocab_size, bias=False) - - def get_prev_embeddings(self, token_ids: torch.Tensor) -> torch.Tensor: - return self.markov_w1(token_ids.long()) - - def project_bias(self, latent_states: torch.Tensor) -> torch.Tensor: - return self.markov_w2(latent_states) - - def compute_step_bias( - self, token_ids: torch.Tensor, hidden_states: Optional[torch.Tensor] - ) -> torch.Tensor: - del hidden_states - return self.project_bias(self.get_prev_embeddings(token_ids)) - - def apply_step_logits( - self, - logits: torch.Tensor, - *, - token_ids: torch.Tensor, - hidden_states: Optional[torch.Tensor], - ) -> torch.Tensor: - return logits + self.compute_step_bias(token_ids, hidden_states) - - def sample_block_tokens( - self, - base_logits: torch.Tensor, - *, - first_prev_token_ids: torch.Tensor, - hidden_states: Optional[torch.Tensor], - temperature: float = 0.0, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Autoregressive block sampling with the (memoryless) Markov bias. - - Args: - base_logits: ``[batch, block_size, vocab]`` from the backbone+lm_head. - first_prev_token_ids: ``[batch]`` token preceding the first position. - hidden_states: ``[batch, block_size, d]`` (unused by vanilla/gated). - Returns: - sampled_tokens ``[batch, block_size]``, corrected_logits ``[batch, block_size, vocab]``. - """ - batch_size, block_size = base_logits.shape[:2] - if block_size == 0: - empty = torch.empty(batch_size, 0, dtype=torch.long, device=base_logits.device) - return empty, base_logits - sampled, corrected = [], [] - prev = first_prev_token_ids.long() - for k in range(block_size): - step_hidden = None if hidden_states is None else hidden_states[:, k] - step_logits = self.apply_step_logits( - base_logits[:, k], token_ids=prev, hidden_states=step_hidden - ) - corrected.append(step_logits.unsqueeze(1)) - prev = greedy_or_sample(step_logits, temperature) - sampled.append(prev) - return torch.stack(sampled, dim=1), torch.cat(corrected, dim=1) - - -class GatedMarkovHead(VanillaMarkov): - """Markov bias gated by a sigmoid of [hidden, prev_embedding].""" - - markov_head_type = "gated" - - def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int): - super().__init__(vocab_size=vocab_size, markov_rank=markov_rank) - self.gate_proj = nn.Linear(hidden_size + markov_rank, markov_rank) - - def compute_step_bias( - self, token_ids: torch.Tensor, hidden_states: Optional[torch.Tensor] - ) -> torch.Tensor: - assert hidden_states is not None - prev_emb = self.get_prev_embeddings(token_ids) - gate = torch.sigmoid(self.gate_proj(torch.cat([hidden_states, prev_emb], dim=-1))).to( - dtype=prev_emb.dtype - ) - return self.project_bias(gate * prev_emb) - - -class RNNHead(VanillaMarkov): - """GRU-style head carrying recurrent state across block positions.""" - - markov_head_type = "rnn" - - def __init__(self, *, vocab_size: int, markov_rank: int, hidden_size: int): - super().__init__(vocab_size=vocab_size, markov_rank=markov_rank) - self.hidden_size = int(hidden_size) - # [s_{k-1}; W1[x_{k-1}]; h_k] -> [gate; candidate; output] - self.joint_proj = nn.Linear(2 * markov_rank + hidden_size, 3 * markov_rank) - - def _rnn_step(self, state, prev_embeddings, hidden_states): - z = torch.cat([state, prev_embeddings, hidden_states], dim=-1) - gate_raw, cand_raw, out_raw = self.joint_proj(z).chunk(3, dim=-1) - gate = torch.sigmoid(gate_raw) - candidate = torch.tanh(cand_raw) - new_state = gate * state + (1.0 - gate) * candidate - bias = self.project_bias(torch.tanh(out_raw)) - return new_state, bias - - def sample_block_tokens( - self, - base_logits: torch.Tensor, - *, - first_prev_token_ids: torch.Tensor, - hidden_states: Optional[torch.Tensor], - temperature: float = 0.0, - ) -> tuple[torch.Tensor, torch.Tensor]: - assert hidden_states is not None - batch_size, block_size = base_logits.shape[:2] - if block_size == 0: - empty = torch.empty(batch_size, 0, dtype=torch.long, device=base_logits.device) - return empty, base_logits - state = torch.zeros( - batch_size, self.markov_rank, device=base_logits.device, dtype=hidden_states.dtype - ) - sampled, corrected = [], [] - prev = first_prev_token_ids.long() - for k in range(block_size): - prev_emb = self.get_prev_embeddings(prev) - state, bias = self._rnn_step(state, prev_emb, hidden_states[:, k]) - step_logits = base_logits[:, k] + bias - corrected.append(step_logits.unsqueeze(1)) - prev = greedy_or_sample(step_logits, temperature) - sampled.append(prev) - return torch.stack(sampled, dim=1), torch.cat(corrected, dim=1) - - -def build_markov_head( - *, markov_head_type: str, vocab_size: int, markov_rank: int, hidden_size: int -) -> Optional[nn.Module]: - """Factory mirroring DeepSpec ``build_markov_head``; returns None if rank==0.""" - if int(markov_rank) <= 0: - return None - kind = str(markov_head_type).lower() - if kind == "vanilla": - return VanillaMarkov(vocab_size=vocab_size, markov_rank=markov_rank) - if kind == "gated": - return GatedMarkovHead( - vocab_size=vocab_size, markov_rank=markov_rank, hidden_size=hidden_size - ) - if kind == "rnn": - return RNNHead(vocab_size=vocab_size, markov_rank=markov_rank, hidden_size=hidden_size) - raise ValueError(f"Unsupported markov_head_type: {markov_head_type!r}") - - -class DSparkConfidenceHead(nn.Module): - """Per-position acceptance-confidence predictor (DeepSpec AcceptRatePredictor). - - Input features are the backbone hidden state, optionally concatenated with the - Markov head's previous-token embedding. Output is a single logit per position. - """ - - def __init__(self, *, hidden_size: int, markov_rank: int = 0, with_markov: bool = False): - super().__init__() - self.with_markov = bool(with_markov) - input_dim = int(hidden_size) + (int(markov_rank) if with_markov else 0) - # The checkpoint stores ``proj`` as a bias-free bf16 weight, but the - # confidence score is computed in fp32 (mirrors the DeepSpec reference - # ``Linear(input_dim, 1, dtype=torch.float32)`` with the fp32 matmul). - self.proj = nn.Linear(input_dim, 1, bias=False, dtype=torch.float32) - - def forward( - self, hidden_states: torch.Tensor, prev_embeddings: Optional[torch.Tensor] = None - ) -> torch.Tensor: - if self.with_markov: - assert prev_embeddings is not None - features = torch.cat([hidden_states, prev_embeddings.to(hidden_states.dtype)], dim=-1) - else: - features = hidden_states - # fp32 matmul for a stable confidence score (mirrors the reference). - return self.proj(features.float()).squeeze(-1) - - -def confident_prefix_length( - confidence_logits: torch.Tensor, *, block_size: int, threshold: float -) -> int: - """First position k where ``sigmoid(confidence_k) < threshold``. - - Returns ``block_size`` when threshold<=0 (no truncation) or all positions - are confident. Assumes batch size 1 (functional-first scope). - """ - if threshold <= 0.0: - return int(block_size) - below = confidence_logits.sigmoid() < threshold - if not bool(below[0].any().item()): - return int(block_size) - return int(torch.nonzero(below[0], as_tuple=False)[0].item()) diff --git a/tensorrt_llm/_torch/models/modeling_bart.py b/tensorrt_llm/_torch/models/modeling_bart.py index 314537b3ccbc..893f88f22dde 100644 --- a/tensorrt_llm/_torch/models/modeling_bart.py +++ b/tensorrt_llm/_torch/models/modeling_bart.py @@ -18,21 +18,20 @@ Key differences from T5: - LayerNorm instead of RMSNorm. - - BART uses post-norm; mBART uses pre-norm and final stack norms. + - Post-norm (residual → add → LayerNorm) instead of pre-norm. - Learned absolute positional embeddings (not relative bias). - - The checkpoint selects the MLP activation (typically GELU for BART and - ReLU for mBART). + - GELU activation (not ReLU / gated). - Bias in attention and MLP projections. - - mBART scales token embeddings by sqrt(d_model). + - Embedding scale = sqrt(d_model). """ import math from typing import Dict, Optional import torch +import torch.nn.functional as F from torch import nn from transformers import BartConfig -from transformers.activations import ACT2FN from ..attention_backend import AttentionMetadata from ..attention_backend.interface import PredefinedAttentionMask @@ -87,18 +86,6 @@ def _bart_head_dim(config: BartConfig) -> int: return config.d_model // config.encoder_attention_heads -def _bart_normalize_before(config: BartConfig) -> bool: - return getattr(config, "model_type", None) == "mbart" or bool( - getattr(config, "normalize_before", False) - ) - - -def _bart_add_final_layer_norm(config: BartConfig) -> bool: - return getattr(config, "model_type", None) == "mbart" or bool( - getattr(config, "add_final_layer_norm", False) - ) - - def _packed_position_ids( position_ids: Optional[torch.IntTensor], hidden_states: torch.Tensor, @@ -180,7 +167,7 @@ def __init__( class BartEncoderLayer(nn.Module): - """BART/mBART encoder layer with configurable pre- or post-norm.""" + """BART encoder layer: self-attention → add+LN → MLP → add+LN (post-norm).""" def __init__( self, @@ -192,7 +179,6 @@ def __init__( hidden_size = config.d_model ffn_dim = _bart_encoder_ffn_dim(config) num_heads = _bart_encoder_num_heads(config) - self.normalize_before = _bart_normalize_before(config) self.self_attn = BartSelfAttention(model_config, num_heads=num_heads, layer_idx=layer_idx) @@ -201,14 +187,13 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, - residual_in_fp32=False, ) self.mlp = MLP( hidden_size=hidden_size, intermediate_size=ffn_dim, bias=True, - activation=ACT2FN[config.activation_function], + activation=F.gelu, dtype=config.torch_dtype, config=model_config, layer_idx=layer_idx, @@ -219,7 +204,6 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, - residual_in_fp32=False, ) def forward( @@ -230,27 +214,19 @@ def forward( **kwargs, ) -> torch.Tensor: residual = hidden_states - if self.normalize_before: - hidden_states = self.self_attn_layer_norm(hidden_states) hidden_states = self.self_attn( position_ids=position_ids, hidden_states=hidden_states, attn_metadata=attn_metadata, attention_mask=PredefinedAttentionMask.FULL, ) - if self.normalize_before: - hidden_states = residual + hidden_states - else: - hidden_states, _ = self.self_attn_layer_norm(hidden_states, residual) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) residual = hidden_states - if self.normalize_before: - hidden_states = self.final_layer_norm(hidden_states) hidden_states = self.mlp(hidden_states) - if self.normalize_before: - hidden_states = residual + hidden_states - else: - hidden_states, _ = self.final_layer_norm(hidden_states, residual) + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) return hidden_states @@ -261,7 +237,7 @@ def forward( class BartDecoderLayer(nn.Module): - """BART/mBART decoder layer with configurable pre- or post-norm.""" + """BART decoder layer: self-attn → add+LN → cross-attn → add+LN → MLP → add+LN.""" def __init__( self, @@ -273,7 +249,6 @@ def __init__( hidden_size = config.d_model ffn_dim = _bart_decoder_ffn_dim(config) num_heads = _bart_decoder_num_heads(config) - self.normalize_before = _bart_normalize_before(config) self.self_attn = BartSelfAttention(model_config, num_heads=num_heads, layer_idx=layer_idx) @@ -282,7 +257,6 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, - residual_in_fp32=False, ) self.cross_attn = BartCrossAttention(model_config, layer_idx=layer_idx) @@ -292,14 +266,13 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, - residual_in_fp32=False, ) self.mlp = MLP( hidden_size=hidden_size, intermediate_size=ffn_dim, bias=True, - activation=ACT2FN[config.activation_function], + activation=F.gelu, dtype=config.torch_dtype, config=model_config, layer_idx=layer_idx, @@ -310,7 +283,6 @@ def __init__( eps=1e-5, dtype=config.torch_dtype, has_bias=True, - residual_in_fp32=False, ) def forward( @@ -323,25 +295,19 @@ def forward( skip_cross_kv_projection: bool = False, **kwargs, ) -> torch.Tensor: - # Self-attention + # Self-attention (post-norm) residual = hidden_states - if self.normalize_before: - hidden_states = self.self_attn_layer_norm(hidden_states) hidden_states = self.self_attn( position_ids=position_ids, hidden_states=hidden_states, attn_metadata=attn_metadata, attention_mask=PredefinedAttentionMask.CAUSAL, ) - if self.normalize_before: - hidden_states = residual + hidden_states - else: - hidden_states, _ = self.self_attn_layer_norm(hidden_states, residual) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) - # Cross-attention + # Cross-attention (post-norm) residual = hidden_states - if self.normalize_before: - hidden_states = self.cross_attn_layer_norm(hidden_states) hidden_states = self.cross_attn( hidden_states=hidden_states, encoder_hidden_states=encoder_hidden_states, @@ -349,20 +315,14 @@ def forward( cross_attn_metadata=cross_attn_metadata, skip_cross_kv_projection=skip_cross_kv_projection, ) - if self.normalize_before: - hidden_states = residual + hidden_states - else: - hidden_states, _ = self.cross_attn_layer_norm(hidden_states, residual) + hidden_states = residual + hidden_states + hidden_states = self.cross_attn_layer_norm(hidden_states) - # MLP + # MLP (post-norm) residual = hidden_states - if self.normalize_before: - hidden_states = self.final_layer_norm(hidden_states) hidden_states = self.mlp(hidden_states) - if self.normalize_before: - hidden_states = residual + hidden_states - else: - hidden_states, _ = self.final_layer_norm(hidden_states, residual) + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) return hidden_states @@ -373,15 +333,15 @@ def forward( class BartEncoder(nn.Module): - """BART/mBART encoder: positional embedding + encoder layers.""" + """BART encoder: positional embedding + encoder layers.""" def __init__(self, model_config: ModelConfig[BartConfig]): super().__init__() config = model_config.pretrained_config num_layers = _bart_encoder_num_layers(config) - # HF BART/mBART uses offset=2 for the padding token, so the actual - # embedding table has max_position_embeddings + 2 entries. + # HF BART uses offset=2 for the padding token, so the actual embedding + # table has max_position_embeddings + 2 entries. self.embed_positions = Embedding( config.max_position_embeddings + 2, config.d_model, @@ -393,16 +353,6 @@ def __init__(self, model_config: ModelConfig[BartConfig]): dtype=config.torch_dtype, has_bias=True, ) - self.layer_norm = ( - LayerNorm( - hidden_size=config.d_model, - eps=1e-5, - dtype=config.torch_dtype, - has_bias=True, - ) - if _bart_add_final_layer_norm(config) - else None - ) self.layers = nn.ModuleList( [BartEncoderLayer(model_config, layer_idx=i) for i in range(num_layers)] ) @@ -424,13 +374,11 @@ def forward( attn_metadata=attn_metadata, position_ids=position_ids, ) - if self.layer_norm is not None: - hidden_states = self.layer_norm(hidden_states) return hidden_states class BartDecoder(nn.Module): - """BART/mBART decoder: positional embedding + decoder layers.""" + """BART decoder: positional embedding + decoder layers.""" def __init__(self, model_config: ModelConfig[BartConfig]): super().__init__() @@ -448,16 +396,6 @@ def __init__(self, model_config: ModelConfig[BartConfig]): dtype=config.torch_dtype, has_bias=True, ) - self.layer_norm = ( - LayerNorm( - hidden_size=config.d_model, - eps=1e-5, - dtype=config.torch_dtype, - has_bias=True, - ) - if _bart_add_final_layer_norm(config) - else None - ) self.layers = nn.ModuleList( [BartDecoderLayer(model_config, layer_idx=i) for i in range(num_layers)] ) @@ -485,8 +423,6 @@ def forward( cross_attn_metadata=cross_attn_metadata, skip_cross_kv_projection=skip_cross_kv_projection, ) - if self.layer_norm is not None: - hidden_states = self.layer_norm(hidden_states) return hidden_states @@ -687,7 +623,6 @@ def _convert_hf_bart_weights( model.shared.weight model.encoder.embed_positions.weight model.encoder.layernorm_embedding.{weight,bias} - model.encoder.layer_norm.{weight,bias} # mBART model.encoder.layers.{i}.self_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias} model.encoder.layers.{i}.self_attn_layer_norm.{weight,bias} model.encoder.layers.{i}.fc1.{weight,bias} @@ -695,7 +630,6 @@ def _convert_hf_bart_weights( model.encoder.layers.{i}.final_layer_norm.{weight,bias} model.decoder.embed_positions.weight model.decoder.layernorm_embedding.{weight,bias} - model.decoder.layer_norm.{weight,bias} # mBART model.decoder.layers.{i}.self_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias} model.decoder.layers.{i}.self_attn_layer_norm.{weight,bias} model.decoder.layers.{i}.encoder_attn.{q_proj,k_proj,v_proj,out_proj}.{weight,bias} @@ -741,8 +675,6 @@ def _wb(prefix: str) -> dict: # Encoder positional embedding out["model.encoder.embed_positions"] = [{"weight": _get(f"{p}encoder.embed_positions.weight")}] out["model.encoder.layernorm_embedding"] = [_wb(f"{p}encoder.layernorm_embedding")] - if _maybe(f"{p}encoder.layer_norm.weight") is not None: - out["model.encoder.layer_norm"] = [_wb(f"{p}encoder.layer_norm")] # Encoder layers for i in range(enc_layers): @@ -768,8 +700,6 @@ def _wb(prefix: str) -> dict: # Decoder positional embedding out["model.decoder.embed_positions"] = [{"weight": _get(f"{p}decoder.embed_positions.weight")}] out["model.decoder.layernorm_embedding"] = [_wb(f"{p}decoder.layernorm_embedding")] - if _maybe(f"{p}decoder.layer_norm.weight") is not None: - out["model.decoder.layer_norm"] = [_wb(f"{p}decoder.layer_norm")] # Decoder layers for i in range(dec_layers): diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv3.py b/tensorrt_llm/_torch/models/modeling_deepseekv3.py index c95aac6c8147..57627b4650e7 100755 --- a/tensorrt_llm/_torch/models/modeling_deepseekv3.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv3.py @@ -28,7 +28,7 @@ import copy import math import os -from typing import Any, Dict, List, Literal, Optional, Tuple +from typing import Dict, List, Optional, Tuple import torch import triton @@ -66,8 +66,7 @@ from ..modules.fused_moe.routing import Deepseekv3RoutingImpl # isort: on from ..modules.gated_mlp import GatedMLP -from ..modules.linear import (Linear, TensorParallelMode, WeightsLoadingConfig, - is_static_nvfp4_input_eligible) +from ..modules.linear import Linear, TensorParallelMode, WeightsLoadingConfig from ..modules.multi_stream_utils import maybe_execute_in_parallel from ..modules.rms_norm import RMSNorm from ..peft.lora.layer import LoraLayer @@ -146,18 +145,6 @@ def moe_reduce_add_shared_output(routed_output, shared_output, out=None): return shared_output.add_(routed_reduced) -def _static_nvfp4_input_scale(linear): - """Return `linear`'s calibrated NVFP4 input_scale if it is eligible to be - folded into a producing RMSNorm's fused NVFP4 quantize, else None. - - Eligibility is the shared `is_static_nvfp4_input_eligible` predicate, also - used by MLA._resolve_qa_fused_scale, so every fold site agrees on what - "static-NVFP4 eligible" means.""" - if is_static_nvfp4_input_eligible(linear): - return linear.input_scale - return None - - class DeepseekV3WeightLoader: def __init__(self, model, is_draft_model: bool = False): @@ -1349,18 +1336,9 @@ def __init__(self, use_cute_dsl_blockscaling_mm, ) - # On NVFP4 models, construct the boundary/prologue norms as NVFP4 norms. - # post_load_weights attaches each norm's .nvfp4_scale (the consuming - # Linear's static input_scale); the fused (add+)RMSNorm+NVFP4-quant then - # fires automatically from that attribute (RMSNorm.forward) only when a - # scale was attached, kernel chosen by the size gate. input_layernorm - # serves both layer 0's residual-less prologue and, as the previous - # layer's next_layer_layernorm, the layer-boundary add+RMSNorm edge. - nvfp4_norm = "nvfp4" if self.is_nvfp4 else None self.input_layernorm = RMSNorm(hidden_size=config.hidden_size, eps=config.rms_norm_eps, - dtype=config.torch_dtype, - quantize_type=nvfp4_norm) + dtype=config.torch_dtype) # When enable_attention_dp is True, we normally skip attention all-reduce since each # DP rank works on different batch elements. However, with CP > 1, attention is split @@ -1372,14 +1350,9 @@ def __init__(self, or self.mapping.tp_size == 1 or can_skip_for_attention_dp) - # Dense (GatedMLP) layers fold post_attention_layernorm -> NVFP4 - # gate_up_proj; MoE layers route expert-input quant differently and - # never get an .nvfp4_scale attached, so the fused path stays inert for - # them even when constructed as an NVFP4 norm. self.post_attention_layernorm = RMSNorm(hidden_size=config.hidden_size, eps=config.rms_norm_eps, - dtype=config.torch_dtype, - quantize_type=nvfp4_norm) + dtype=config.torch_dtype) self.next_layer_layernorm: RMSNorm = None def _get_decoder_layer_quant_config( @@ -1446,16 +1419,7 @@ def forward( ) -> Tuple[torch.Tensor, torch.Tensor]: if residual is None: residual = hidden_states - if self.input_layernorm.nvfp4_scale is not None: - # Layer-0 prologue: the residual-less input_layernorm folds the - # NVFP4 kv_a_proj input-quant via its attached .nvfp4_scale - # (RMSNorm.forward), returning an Fp4QuantizedTensor. - # return_norm_out stashes the BF16 view DSA's pre_indexer_proj - # needs on that tensor's unquantized_hidden_states. - hidden_states = self.input_layernorm(hidden_states, - return_norm_out=True)[0] - else: - hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.input_layernorm(hidden_states) # Self Attention hidden_states = self.self_attn( position_ids=position_ids, @@ -1572,28 +1536,12 @@ def _run_MoE(hidden_states, hidden_states_fp4, do_finalize): self.layer_idx): spec_metadata.maybe_capture_hidden_states( self.layer_idx, hidden_states, residual) - hidden_states, residual = self._apply_next_layer_layernorm( - hidden_states, residual) + if self.next_layer_layernorm is not None: + hidden_states, residual = self.next_layer_layernorm( + hidden_states, residual) return hidden_states, residual - def _apply_next_layer_layernorm(self, hidden_states, residual): - """Apply the next layer's input_layernorm at a layer boundary on the - attention-DP / no-allreduce path. When that norm has an .nvfp4_scale - attached, RMSNorm.forward folds the next layer's NVFP4 kv_a_proj - input-quant into the same kernel and returns (Fp4QuantizedTensor, - residual_out) with the BF16 view stashed for DSA's pre_indexer_proj - (return_norm_out); otherwise it applies the plain add+RMSNorm. Shared by - forward_MoE and forward_mlp so the boundary fold stays identical for MoE - and dense layers.""" - if self.next_layer_layernorm is None: - return hidden_states, residual - if self.next_layer_layernorm.nvfp4_scale is not None: - return self.next_layer_layernorm(hidden_states, - residual, - return_norm_out=True) - return self.next_layer_layernorm(hidden_states, residual) - def forward_mlp( self, hidden_states: torch.Tensor, @@ -1625,13 +1573,6 @@ def forward_mlp( eps=self.post_attention_layernorm.variance_epsilon, ), ) - elif self.post_attention_layernorm.nvfp4_scale is not None: - # Attention-DP path (no allreduce): post_attention_layernorm folds - # the dense MLP's NVFP4 gate_up_proj input-quant into one kernel via - # its attached .nvfp4_scale (RMSNorm.forward), mirroring the - # PRE_MLP_FUSION quant but without the allreduce. - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual) else: # No fusion # We need to add twoshot allreduce here to avoid modifying MLA logic @@ -1659,8 +1600,9 @@ def forward_mlp( self.layer_idx): spec_metadata.maybe_capture_hidden_states( self.layer_idx, hidden_states, residual) - hidden_states, residual = self._apply_next_layer_layernorm( - hidden_states, residual) + if self.next_layer_layernorm is not None: + hidden_states, residual = self.next_layer_layernorm( + hidden_states, residual) return hidden_states, residual @@ -1893,30 +1835,6 @@ def forward( class DeepseekV3ForCausalLM(SpecDecOneEngineForCausalLM[DeepseekV3Model, PretrainedConfig]): - @classmethod - def get_preferred_transceiver_runtime(cls, - pretrained_config: Any = None - ) -> Optional[Literal["PYTHON"]]: - """GLM-5 family checkpoints default to the Python (v2) KV-cache transceiver. - - This implementation class is shared by DeepSeek-V3/V3.2 and the GLM-5 family — both - GLM-5 and GLM-5.2 declare ``GlmMoeDsaForCausalLM`` / ``glm_moe_dsa`` — so the preference - is differentiated per checkpoint: only GLM checkpoints opt into the Python transceiver. - The MLA backbone transfers a large latent KV, which the Python transceiver handles better - in disaggregated serving. This is only adopted when the user leaves - ``cache_transceiver_config.transceiver_runtime`` at 'auto' and the effective backend is - NIXL; otherwise the C++ transceiver is used. - """ - if pretrained_config is None: - return None - architectures = getattr(pretrained_config, 'architectures', None) or [] - # model_type is checked as a fallback: it is 'glm_moe_dsa' on GLM - # checkpoints until __init__ rewrites it to 'deepseek_v32'. - if ("GlmMoeDsaForCausalLM" in architectures or getattr( - pretrained_config, 'model_type', None) == 'glm_moe_dsa'): - return "PYTHON" - return None - def __init__(self, model_config: ModelConfig[PretrainedConfig]): self.mapping_with_cp = None # Note: Currently the usage of mapping is all over the place making its usage brittle @@ -2029,33 +1947,5 @@ def setup_aliases(self) -> None: if idx == self.config.num_hidden_layers - 1: layer.next_layer_layernorm = self.model.norm else: - next_layer = self.model.layers[idx + 1] - layer.next_layer_layernorm = next_layer.input_layernorm - - # Attach each norm's .nvfp4_scale (the consuming Linear's static - # input_scale) so RMSNorm.forward folds (add+)RMSNorm + NVFP4 - # input-quant into one kernel on the attention-DP path. The fold is - # self-gating: _static_nvfp4_input_scale returns None (no fold) - # unless that Linear is NVFP4 with a static (calibrated) input_scale, - # so this is safe to run unconditionally on every layer. - # - # input_layernorm -> this layer's kv_a_proj covers BOTH the layer-0 - # residual-less prologue AND the layer-boundary add+RMSNorm edge (the - # previous layer's next_layer_layernorm IS this input_layernorm). - # Both DSA (DSv3.2) and non-DSA (Kimi-K2.5) MLA are supported: the - # non-DSA forward_impl slices the Fp4QuantizedTensor by num_tokens - # via _slice_hidden_states_to_num_tokens, and kv_a_proj_with_mqa - # consumes the FP4 form directly. - self_attn = getattr(layer, "self_attn", None) - layer.input_layernorm.nvfp4_scale = _static_nvfp4_input_scale( - getattr(self_attn, "kv_a_proj_with_mqa", None)) - - # Intra-layer dense-MLP fold: post_attention_layernorm -> this - # layer's NVFP4 gate_up_proj. Only dense (GatedMLP) layers — MoE - # layers route their expert input quant differently and get no - # scale, so their fused path stays inert. - mlp = getattr(layer, "mlp", None) - if isinstance(mlp, GatedMLP): - layer.post_attention_layernorm.nvfp4_scale = ( - _static_nvfp4_input_scale(getattr(mlp, "gate_up_proj", - None))) + layer.next_layer_layernorm = self.model.layers[ + idx + 1].input_layernorm diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv4.py b/tensorrt_llm/_torch/models/modeling_deepseekv4.py index 5724c1052d8c..2fb611ecb999 100644 --- a/tensorrt_llm/_torch/models/modeling_deepseekv4.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv4.py @@ -1,6 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - # -------------------------------------------------- # Portions of this code were derived from DeepSeek‑V3: # https://github.com/deepseek-ai/DeepSeek-V3 @@ -231,66 +228,6 @@ def moe_reduce_add_shared_output(routed_output, shared_output): } -def _get_deepseek_v4_routed_moe_scale_name(weights: Dict, key_prefix: str) -> str: - """Return the model scale suffix for routed-expert checkpoint tensors.""" - for key, value in weights.items(): - if ( - key.startswith(key_prefix) - and ".ffn.experts." in key - and key.endswith(".weight") - and getattr(value, "ndim", 0) == 2 - and getattr(value, "dtype", None) in (torch.int8, torch.uint8) - ): - return "weight_scale" - return "weight_scale_inv" - - -def _rename_deepseek_v4_attn_subkey(rest: str) -> str: - """Rename a DeepSeek-V4 attention checkpoint subkey.""" - if rest == "attn_sink": - return "attn_sink" - if rest == "wo_a.weight": - return "o_a_proj" - if rest == "wo_a.scale": - return "o_a_proj.weight_scale_inv" - if rest.startswith("compressor.") or rest.startswith("indexer."): - return rest.replace(".scale", ".weight_scale_inv") - head, sep, tail = rest.partition(".") - new_head = _ATTN_PARAM_RENAME.get(head, head) - if tail == "scale": - tail = "weight_scale_inv" - return f"{new_head}.{tail}" if sep else new_head - - -def _rename_deepseek_v4_ffn_subkey(rest: str, routed_moe_scale_name: str) -> str: - """Rename a DeepSeek-V4 FFN checkpoint subkey.""" - if rest == "gate.bias": - return "gate.e_score_correction_bias" - if rest.startswith("experts.") and rest.endswith(".scale"): - return f"{rest[: -len('.scale')]}.{routed_moe_scale_name}" - rest = rest.replace(".scale", ".weight_scale_inv") - if rest.startswith("shared_experts."): - parts = rest.split(".") - if len(parts) >= 2 and parts[1] in _SHARED_EXPERT_RENAME: - parts[1] = _SHARED_EXPERT_RENAME[parts[1]] - rest = ".".join(parts) - return rest - - -def _maybe_view_deepseek_v4_routed_moe_tensor( - model_key: str, tensor: torch.Tensor, routed_moe_scale_name: str -) -> torch.Tensor: - """Expose packed MXFP4 routed-expert tensors through their uint8 view.""" - if ( - routed_moe_scale_name == "weight_scale" - and ".mlp.experts." in model_key - and (model_key.endswith(".weight") or model_key.endswith(".weight_scale")) - and tensor.dtype != torch.uint8 - ): - return tensor.view(torch.uint8) - return tensor - - def _resolve_enable_fused_hc(config: PretrainedConfig) -> bool: """Resolve the DeepSeek-V4 fused HC boundary-fusion knob.""" env = os.environ.get("TRTLLM_MHC_ENABLE_FUSED_HC") @@ -299,43 +236,6 @@ def _resolve_enable_fused_hc(config: PretrainedConfig) -> bool: return bool(getattr(config, "enable_fused_hc", True)) -def _normalize_deepseek_v4_nvfp4_mixed_precision_config( - model_config: ModelConfig[PretrainedConfig], -) -> ModelConfig[PretrainedConfig]: - """Resolve FP8 base layers in DeepSeek-V4 NVFP4 checkpoints.""" - quant_config = model_config.quant_config - hf_quant_config = getattr(model_config.pretrained_config, "quantization_config", None) - layer_quant_configs = model_config.quant_config_dict or {} - has_nvfp4_experts = any( - name.endswith(".mlp.experts") and config.quant_algo == QuantAlgo.NVFP4 - for name, config in layer_quant_configs.items() - ) - if ( - quant_config.quant_algo != QuantAlgo.MIXED_PRECISION - or not has_nvfp4_experts - or not isinstance(hf_quant_config, dict) - or hf_quant_config.get("quant_method") != "fp8" - or tuple(hf_quant_config.get("weight_block_size", ())) != (128, 128) - ): - return model_config - - default_exclude = ["*kv_b_proj*", "*k_b_proj*", "*eh_proj*"] - hf_exclude_modules = hf_quant_config.get("modules_to_not_convert") or [] - exclude_modules = list(dict.fromkeys(list(hf_exclude_modules) + default_exclude)) - fp8_quant_config = quant_config.model_copy( - deep=True, - update={ - "quant_algo": QuantAlgo.FP8_BLOCK_SCALES, - "group_size": 128, - "exclude_modules": exclude_modules, - }, - ) - fp8_quant_config.__dict__.pop("quant_mode", None) - fp8_quant_config.__dict__.pop("layer_quant_mode", None) - model_config.quant_config = fp8_quant_config - return model_config - - def _copy_deepseek_v4_fused_a_weight_scale( module: Linear, fused_a: torch.Tensor, fused_a_scale: torch.Tensor ) -> None: @@ -415,7 +315,66 @@ def _remap_deepseek_v4_checkpoint_keys( carries it but matches the main head, so we let the main head win. """ mtp_layer_prefix = f"model.layers.{num_hidden_layers}" - routed_moe_scale_name = _get_deepseek_v4_routed_moe_scale_name(weights, "layers.") + routed_moe_scale_name = "weight_scale_inv" + for key, value in weights.items(): + if ( + key.startswith("layers.") + and ".ffn.experts." in key + and key.endswith(".weight") + and getattr(value, "ndim", 0) == 2 + and value.dtype in (torch.int8, torch.uint8) + ): + routed_moe_scale_name = "weight_scale" + break + + def _rename_attn_subkey(rest: str) -> Optional[str]: + # rest examples: "wq_a.weight", "wq_a.scale", "wo_a.weight", + # "attn_sink", "compressor.wkv.weight", "indexer.wq_b.scale", + # "kv_norm.weight" + # ``attn_sink`` is loaded by the ``mqa`` branch in the per-module + # loader, which reads it under the parent ``self_attn.attn_sink`` + # key. Pass through unchanged. + if rest == "attn_sink": + return "attn_sink" + # `wo_a` is an nn.Parameter on the model side (not a Linear), so + # `wo_a.weight` carries the value directly into `o_a_proj` without + # a trailing ``.weight``. Retain `.scale` so the loader can dequantize + # FP8 block-scaled checkpoints before assigning the bf16 parameter. + if rest == "wo_a.weight": + return "o_a_proj" + if rest == "wo_a.scale": + return "o_a_proj.weight_scale_inv" + # Compressor / indexer paths — pass through with .scale rename, plus + # wkv+wgate fusion handled separately below. + if rest.startswith("compressor.") or rest.startswith("indexer."): + return rest.replace(".scale", ".weight_scale_inv") + head, sep, tail = rest.partition(".") + new_head = _ATTN_PARAM_RENAME.get(head, head) + if tail == "scale": + tail = "weight_scale_inv" + return f"{new_head}.{tail}" if sep else new_head + + def _rename_ffn_subkey(rest: str) -> str: + # Examples: + # gate.weight / gate.tid2eid → gate.weight / gate.tid2eid + # gate.bias → gate.e_score_correction_bias + # experts... → experts... + # shared_experts.. → shared_experts._proj. + if rest == "gate.bias": + return "gate.e_score_correction_bias" + if rest.startswith("experts.") and rest.endswith(".scale"): + return f"{rest[: -len('.scale')]}.{routed_moe_scale_name}" + rest = rest.replace(".scale", ".weight_scale_inv") + # Non-hashed layers carry the routing logit bias as `gate.bias`; the + # model wires it through `DeepseekV4Gate.e_score_correction_bias`. + if rest == "gate.bias": + return "gate.e_score_correction_bias" + if rest.startswith("shared_experts."): + parts = rest.split(".") + if len(parts) >= 2 and parts[1] in _SHARED_EXPERT_RENAME: + parts[1] = _SHARED_EXPERT_RENAME[parts[1]] + rest = ".".join(parts) + return rest def _rename_layer_subkey(rest: str) -> Optional[str]: # rest examples: "attn_norm.weight", "ffn_norm.weight", @@ -431,10 +390,10 @@ def _rename_layer_subkey(rest: str) -> Optional[str]: if rest.startswith("hc_attn_") or rest.startswith("hc_ffn_"): return rest if rest.startswith("attn."): - return f"self_attn.{_rename_deepseek_v4_attn_subkey(rest[len('attn.') :])}" + new_sub = _rename_attn_subkey(rest[len("attn.") :]) + return None if new_sub is None else f"self_attn.{new_sub}" if rest.startswith("ffn."): - new_sub = _rename_deepseek_v4_ffn_subkey(rest[len("ffn.") :], routed_moe_scale_name) - return f"mlp.{new_sub}" + return f"mlp.{_rename_ffn_subkey(rest[len('ffn.') :])}" return rest out: Dict[str, torch.Tensor] = {} @@ -458,7 +417,13 @@ def _emit_or_collect(model_key: str, tensor: torch.Tensor): part = "wkv" if model_key.endswith(".wkv.weight") else "wgate" _record_compressor_part(model_key, part, tensor) return - tensor = _maybe_view_deepseek_v4_routed_moe_tensor(model_key, tensor, routed_moe_scale_name) + if ( + routed_moe_scale_name == "weight_scale" + and ".mlp.experts." in model_key + and (model_key.endswith(".weight") or model_key.endswith(".weight_scale")) + and tensor.dtype != torch.uint8 + ): + tensor = tensor.view(torch.uint8) out[model_key] = tensor for k, v in weights.items(): @@ -1738,7 +1703,7 @@ def __init__( model_config: ModelConfig[PretrainedConfig], layer_idx: int, aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream], - attention_layer_idx: Optional[int] = None, + is_separate_draft_engine: bool = False, mapping_with_cp: Optional[Mapping] = None, disable_post_moe_fusion: bool = False, ): @@ -1767,12 +1732,14 @@ def __init__( post_mult_value=2.0, ) - if attention_layer_idx is None: - attention_layer_idx = layer_idx + layer_idx_for_attention = layer_idx + if is_separate_draft_engine: + # KVCacheManager only support 1 layer for separate draft engine + layer_idx_for_attention = layer_idx - model_config.pretrained_config.num_hidden_layers self.self_attn = DeepseekV4Attention( model_config, - layer_idx=attention_layer_idx, + layer_idx=layer_idx_for_attention, aux_stream=aux_stream_dict[AuxStreamType.Attention], reduce_output=not self.enable_attention_dp and self.mapping.tp_size > 1, ) @@ -1993,18 +1960,8 @@ def forward( # No engram concern here because engram only fires at layer entry. # When enable_fused_hc=False, fall back to the unfused chain. # ------------------------------------------------------------------- - capture_this_layer = spec_metadata is not None and spec_metadata.is_layer_capture( - self.layer_idx - ) - is_dspark_capture = capture_this_layer and spec_metadata.spec_dec_mode.is_dspark() - if capture_this_layer: + if spec_metadata is not None and spec_metadata.is_layer_capture(self.layer_idx): self.fusion_config.POST_MOE_FUSION = False - if is_dspark_capture: - # DSpark captures the FULL post-mapped mHC residual stream (reference - # `h.mean(dim=2)`), which is only materialized after - # hc_ffn.post_mapping -- so post_mapping must resolve in-layer rather - # than being deferred into the next layer's fused_hc. - self.defer_post_mapping = False if self.enable_fused_hc: residual, post_mix, comb_mix, layer_input = self.hc_ffn.fused_hc( x_prev=x_attn, @@ -2046,17 +2003,6 @@ def forward( post_layer_mix=post_mix, comb_res_mix=comb_mix, ) - if is_dspark_capture: - # Capture the full mHC residual stream [N, hc_mult*hidden]; the DSpark - # metadata means over the hc streams (reference `h.mean(dim=2)`) to - # form the draft's captured context (``main_x``). This is the correct - # representation -- NOT the pre-post_mapping MoE delta that the generic - # capture in forward_MoE records for other spec modes. - spec_metadata.maybe_capture_hidden_states( - self.layer_idx, - resolved_residual.reshape(resolved_residual.shape[0], -1), - None, - ) return HCState.resolved(resolved_residual) def _entry_boundary(self, hc_state, engram_embeddings, has_engram): @@ -2185,14 +2131,7 @@ def _run_MoE(hidden_states, hidden_states_fp4, do_finalize, input_ids): fc2_output, all_reduce_params=moe_all_reduce_params ) else: - # DSpark captures the post-mapped mHC residual stream after this layer - # (done in the decoder-layer forward), not the pre-post_mapping MoE - # output recorded here for other spec modes. - if ( - spec_metadata is not None - and spec_metadata.is_layer_capture(self.layer_idx) - and not spec_metadata.spec_dec_mode.is_dspark() - ): + if spec_metadata is not None and spec_metadata.is_layer_capture(self.layer_idx): spec_metadata.maybe_capture_hidden_states(self.layer_idx, hidden_states, None) return hidden_states @@ -2204,13 +2143,13 @@ def __init__( model_config: ModelConfig[PretrainedConfig], layer_idx: int, aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream], - attention_layer_idx: Optional[int] = None, + is_separate_draft_engine: bool = False, ): super().__init__( model_config, layer_idx, aux_stream_dict, - attention_layer_idx=attention_layer_idx, + is_separate_draft_engine, disable_post_moe_fusion=True, ) config = model_config.pretrained_config @@ -2523,7 +2462,6 @@ def get_model_defaults(cls, llm_args: "TorchLlmArgs") -> dict: } def __init__(self, model_config: ModelConfig[PretrainedConfig]): - model_config = _normalize_deepseek_v4_nvfp4_mixed_precision_config(model_config) self.mapping_with_cp = None # Note: Currently the usage of mapping is all over the place making its usage brittle # in this file. As a temporary WAR, we hold on to an original copy of mapping when CP diff --git a/tensorrt_llm/_torch/models/modeling_dspark.py b/tensorrt_llm/_torch/models/modeling_dspark.py deleted file mode 100644 index 3d38287d947a..000000000000 --- a/tensorrt_llm/_torch/models/modeling_dspark.py +++ /dev/null @@ -1,1113 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DSpark backbone ported from the DeepSeek-V4-Pro-DSpark reference -# (`inference/model.py`: DSparkBlock / Transformer.forward_spec). -"""DeepSeek-V4-Pro DSpark speculative-decoding draft backbone. - -The DSpark draft is ``n_mtp_layers`` (3 for V4-Pro) **full DeepSeek-V4 blocks** -stored under the ``mtp.*`` checkpoint namespace — it reuses the V4 decoder block -(MLA attention + MoE + manifold Hyper-Connections) and adds: - - - **stage 0**: ``main_proj`` (Linear, fp8) + ``main_norm`` (RMSNorm) — projects - the concatenation of captured target-layer hidden states ([58,59,60]) into the - draft's cross-attention context (``main_x``); replaces vanilla-MTP's - enorm/hnorm + e_proj/h_proj single-hidden mixing. - - **last stage**: ``norm`` + ``markov_head`` + ``confidence_head`` + - flat ``hc_head`` — the block-draft output head (see dspark_heads/dspark_draft). - -The per-stage *backbone* forward (block attention whose K/V derive from ``main_x``, -+ MoE + mHC) is brought up and numerically validated against the real fp8 weights -separately; ``forward_embed`` (capture) and ``forward_head`` (block draft) below -are the reference-faithful, unit-validated I/O stages. -""" - -import copy -import json -import os -import re -from typing import Dict, List, Optional - -import torch -import torch.nn.functional as F -from torch import nn - -from tensorrt_llm.logger import logger - -from ..distributed import AllReduceParams -from ..modules.linear import Linear -from ..modules.mhc.hyper_connection import HCHead -from ..modules.rms_norm import RMSNorm -from ..utils import AuxStreamType -from .dspark.attention import ( - _rmsnorm, - _rope_last_dims, - _rope_last_dims_batched, - dspark_attention_forward, - dspark_attention_forward_batched, - precompute_dspark_freqs_cis, -) -from .dspark.draft import build_draft_input_ids, dspark_propose -from .dspark.heads import DSparkConfidenceHead, build_markov_head -from .modeling_deepseekv4 import ( - DeepseekV4DecoderLayer, - DeepseekV4WeightLoader, - _get_deepseek_v4_routed_moe_scale_name, - _maybe_view_deepseek_v4_routed_moe_tensor, - _rename_deepseek_v4_attn_subkey, - _rename_deepseek_v4_ffn_subkey, -) - -# Matches the draft namespace ``mtp..`` in the V4-Pro-DSpark -# checkpoint. Each draft stage is a full DeepSeek-V4 block stored under this -# prefix; the main model's keys (``layers.*``, ``embed.weight``, ``head.weight``, -# top-level ``norm.weight`` / ``hc_head_*``) are loaded by the target model. -_DSPARK_MTP_RE = re.compile(r"^mtp\.(\d+)\.(.+)$") - - -def count_dspark_stages(ckpt_dir: str) -> Optional[int]: - """Count the DSpark draft stages (``mtp.{s}.*``) in a checkpoint index. - - The HF ``config.json`` does not expose ``n_mtp_layers`` (only the reference - ``inference/config.json`` does), so the authoritative draft stage count is - the number of distinct ``mtp.`` prefixes in the weight index. Returns - ``None`` if the index is missing or has no ``mtp.*`` keys (caller falls back - to the config-derived default). - """ - index = os.path.join(ckpt_dir, "model.safetensors.index.json") - if not os.path.isfile(index): - return None - with open(index, encoding="utf-8") as f: - weight_map = json.load(f).get("weight_map", {}) - stages = {int(m.group(1)) for k in weight_map if (m := _DSPARK_MTP_RE.match(k))} - return (max(stages) + 1) if stages else None - - -def _rename_dspark_stage_subkey(rest: str, routed_scale: str) -> str: - """Map a per-stage checkpoint subkey to the ``DSparkBlock`` param subkey.""" - if rest == "attn_norm.weight": - return "input_layernorm.weight" - if rest == "ffn_norm.weight": - return "post_attention_layernorm.weight" - # Flat manifold-Hyper-Connections / draft-head weights are loaded via - # ``load_flat_hc_weights`` (keyed by the parent module stem), so pass the - # flat-underscore form through unchanged: - # hc_attn_* / hc_ffn_* -> mHC on every block - # hc_head_* -> HCHead on the last stage - if rest.startswith(("hc_attn_", "hc_ffn_", "hc_head_")): - return rest - # DSpark capture projection (stage 0): fp8 Linear .scale -> .weight_scale_inv. - if rest == "main_proj.scale": - return "main_proj.weight_scale_inv" - if rest.startswith("attn."): - return f"self_attn.{_rename_deepseek_v4_attn_subkey(rest[len('attn.') :])}" - if rest.startswith("ffn."): - return f"mlp.{_rename_deepseek_v4_ffn_subkey(rest[len('ffn.') :], routed_scale)}" - # main_proj.weight, main_norm.weight, norm.weight, markov_head.*, - # confidence_head.* map 1:1 onto the DSparkBlock submodules. - return rest - - -def remap_dspark_draft_keys(weights: Dict, num_stages: int) -> Dict: - """Convert checkpoint ``mtp.{s}.*`` keys to ``mtp_layers.{s}.*`` model keys. - - Only the draft namespace is consumed (stages ``[0, num_stages)``); shared - ``embed_tokens`` / ``lm_head`` and other top-level keys belong to the target - model and are skipped here. The routed-expert scale suffix mirrors the V4 - loader: ``weight_scale`` for the packed MXFP4 layout, else ``weight_scale_inv``. - """ - routed_scale = _get_deepseek_v4_routed_moe_scale_name(weights, "mtp.") - out: Dict[str, torch.Tensor] = {} - for k, v in weights.items(): - m = _DSPARK_MTP_RE.match(k) - if not m: - continue - stage = int(m.group(1)) - if stage >= num_stages: - continue - sub = _rename_dspark_stage_subkey(m.group(2), routed_scale) - model_key = f"mtp_layers.{stage}.{sub}" - v = _maybe_view_deepseek_v4_routed_moe_tensor(model_key, v, routed_scale) - out[model_key] = v - return out - - -# The checkpoint stores -# ``mtp.{s}.attn.wo_a`` as fp8_e4m3 + a UE8M0 128x128 block scale (verified), and -# the reference (`inference/model.py`, ``self.wo_a`` is a bf16 ColumnParallelLinear -# loaded from the fp8 ckpt) uses the DEQUANTIZED bf16 ``wo_a`` (== ``wo_a_fp8 * -# scale`` ~ absmean 0.065). The bf16 captured-context path historically skipped -# this dequant (raw fp8-cast-to-bf16, ~993x too large); the correct behavior is to -# dequantize ``wo_a`` (cos 1.0 vs ``wo_a_fp8 * scale``). Always dequantize now. - - -class DSparkBlock(DeepseekV4DecoderLayer): - """One DSpark draft stage = a DeepSeek-V4 decoder block + DSpark extras. - - ``stage_id`` in ``[0, num_stages)``; only stage 0 owns the capture projection - and only the last stage owns the draft heads, matching the ``mtp.*`` schema. - """ - - def __init__( - self, - model_config, - layer_idx: int, - aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream], - *, - stage_id: int, - num_stages: int, - num_capture_layers: int, - ): - # The inherited attention uses a draft-local layer index, while the - # decoder layer keeps its model-level index for weights and captures. - super().__init__( - model_config, - layer_idx, - aux_stream_dict, - attention_layer_idx=stage_id, - disable_post_moe_fusion=True, - ) - config = model_config.pretrained_config - spec_cfg = getattr(model_config, "spec_config", None) - self.stage_id = int(stage_id) - self.num_stages = int(num_stages) - # mask_token_id is a user override on the speculative_config; None means - # fall back to the draft checkpoint's dspark_noise_token_id. - mask_token_id = getattr(spec_cfg, "mask_token_id", None) - self.noise_token_id = int( - mask_token_id - if mask_token_id is not None - else getattr(config, "dspark_noise_token_id", config.vocab_size) - ) - self.markov_rank = int(getattr(config, "dspark_markov_rank", 0)) - self.hc_mult = config.hc_mult - # markov_head_type is a user override on the speculative_config; None - # means fall back to the draft checkpoint's dspark_markov_head_type. - markov_head_type = getattr(spec_cfg, "markov_head_type", None) - if markov_head_type is None: - markov_head_type = getattr(config, "dspark_markov_head_type", "vanilla") - self.markov_head_type = markov_head_type - - # Stage 0: capture projection of the concatenated target-layer hiddens. - if self.has_capture: - self.main_proj = Linear( - config.hidden_size * num_capture_layers, - config.hidden_size, - bias=False, - dtype=config.torch_dtype, - quant_config=model_config.get_quant_config(), - skip_create_weights_in_init=model_config.skip_create_weights_in_init, - ) - self.main_norm = RMSNorm( - hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=config.torch_dtype - ) - - # Last stage: the block-draft output heads + mHC head + final norm. - if self.has_heads: - self.norm = RMSNorm( - hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=config.torch_dtype - ) - self.hc_head = HCHead(config.hc_mult, config.hidden_size) - self.markov_head = build_markov_head( - markov_head_type=self.markov_head_type, - vocab_size=config.vocab_size, - markov_rank=self.markov_rank, - hidden_size=config.hidden_size, - ) - self.confidence_head = DSparkConfidenceHead( - hidden_size=config.hidden_size, - markov_rank=self.markov_rank, - # Only concat the Markov prev-token embedding when a Markov head - # actually exists (build_markov_head returns None for - # markov_rank <= 0); otherwise dspark_propose passes no - # prev_embeddings and DSparkConfidenceHead.forward would assert. - with_markov=self.markov_rank > 0, - ) - - @property - def has_capture(self) -> bool: - return self.stage_id == 0 - - @property - def has_heads(self) -> bool: - return self.stage_id == self.num_stages - 1 - - -class DSparkDraftModel(nn.Module): - """The ``n_mtp_layers``-stage DSpark draft stacked on a DeepSeek-V4 target. - - Shares ``embed_tokens`` / ``lm_head`` with the target model. ``forward_embed`` - builds the block input from the captured context; the per-stage backbone runs - the 3 blocks; ``forward_head`` produces the block draft tokens + confidence. - """ - - def __init__( - self, - model_config, - aux_stream_dict: Dict[AuxStreamType, torch.cuda.Stream], - num_stages: Optional[int] = None, - block_size: Optional[int] = None, - ): - super().__init__() - config = model_config.pretrained_config - self.model_config = model_config - self.config = config - # The DSpark stage count is NOT the HF ``num_nextn_predict_layers`` (=1). - # It is ``n_mtp_layers`` (3 for V4-Pro), which lives in the draft - # sub-checkpoint config (inference/config.json) and is reflected by the - # ``mtp.{0..n-1}.*`` weight namespace. Resolve it from (in priority): - # an explicit override, the spec config's ``num_draft_layers``, a - # pretrained-config ``n_mtp_layers``, else fall back to nextn. - spec_cfg = getattr(model_config, "spec_config", None) - self.num_stages = int( - num_stages - if num_stages is not None - else getattr(spec_cfg, "num_draft_layers", None) - or getattr(config, "n_mtp_layers", None) - or config.num_nextn_predict_layers - ) - # Production passes the validated speculative-config value explicitly; - # direct construction falls back to the checkpoint's trained block size. - self.block_size = int( - block_size if block_size is not None else getattr(config, "dspark_block_size", 5) - ) - # mask_token_id is a user override on the speculative_config; None means - # fall back to the draft checkpoint's dspark_noise_token_id. - mask_token_id = getattr(spec_cfg, "mask_token_id", None) - self.noise_token_id = int( - mask_token_id - if mask_token_id is not None - else getattr(config, "dspark_noise_token_id", config.vocab_size) - ) - self.hc_mult = config.hc_mult - target_layer_ids = getattr(config, "dspark_target_layer_ids", []) - self.num_capture_layers = len(target_layer_ids) - base = config.num_hidden_layers - # Derive a draft-only model_config (a shallow copy so the shared config - # and the target model are untouched) carrying two draft-specific fixes: - # - # 1. compress_ratios SLICE — the draft runs as a separate engine, so the - # inherited DeepSeek-V4 block remaps each block's layer_idx to a - # draft-local index in [0, num_stages) (the 1-layer-style draft KV - # cache). Sparse-attention compress_ratios / RoPE are indexed by that - # draft-local id, so they must be the draft slice - # (compress_ratios[base : base + num_stages]); otherwise indices - # 0..n-1 resolve to the first *main* layers' sparse ratios — building - # a compressor the DSpark draft lacks and selecting YaRN over the - # dense path. For V4-Pro the draft slice is [1, 1, 1] (dense). - # - # 2. quant_config_dict EXTENSION — the checkpoint's per-module quant map - # only enumerates the base layers, so the draft layers' routed - # experts fall back to the global fp8 config and build fp8-shaped - # buffers. The draft experts are physically MXFP4 (same as the main - # MoE layers), so copy a main MoE layer's experts quant onto the - # draft layer keys. - draft_model_config = self._derive_draft_model_config(model_config, base, self.num_stages) - self.mtp_layers = nn.ModuleList( - [ - DSparkBlock( - draft_model_config, - base + s, - aux_stream_dict, - stage_id=s, - num_stages=self.num_stages, - num_capture_layers=self.num_capture_layers, - ) - for s in range(self.num_stages) - ] - ) - # Shared with target; wired by the spec wrapper after construction. - self.embed_tokens: Optional[nn.Module] = None - self.lm_head: Optional[nn.Module] = None - - # Scalar attention params for the captured-context draft attention. These - # are the dense (compress_ratio == 0) DSparkAttention constants — see the - # reference ``inference/model.py`` ``Attention.__init__``. ``head_dim`` is - # the MLA latent (MQA) dim; ``softmax_scale = head_dim ** -0.5``; the dense - # draft disables YaRN and uses the base ``rope_theta``. - self._attn_params = dict( - n_heads=int(config.num_attention_heads), - head_dim=int( - getattr(config, "head_dim", config.kv_lora_rank + config.qk_rope_head_dim) - ), - rope_head_dim=int(config.qk_rope_head_dim), - n_groups=int(config.o_groups), - o_lora_rank=int(config.o_lora_rank), - window_size=int(getattr(config, "window_size", 128)), - eps=float(config.rms_norm_eps), - ) - self._attn_params["softmax_scale"] = self._attn_params["head_dim"] ** -0.5 - self._rope_theta = float(getattr(config, "rope_theta", 10000.0)) - # Fixed-cap plain-RoPE table shared by the eager and CUDA-graph-safe - # batched paths. It is built once per device and gathered/sliced by the - # runtime decode positions, so the cache does not grow with sequence - # length and the batched consuming op's shape remains static. - self._freqs_cap = ( - int(getattr(config, "max_position_embeddings", 163840)) + self.block_size + 2 - ) - self._freqs_table_cache: Dict = {} - - def post_load_weights(self) -> None: - """Run the one-shot post-load transforms for the draft's quant linears. - - The fp8 UE8M0 linears we invoke as modules (``main_proj``, shared experts, - the heads) need ``resmooth_to_fp8_e8m0`` + ``transform_sf_into_required_layout`` - before the first forward, or the kernel reads raw scales and emits NaNs. - ``Linear.transform_weights`` is idempotent; the routed-expert MoE packs - itself in its own ``load_weights``. - - The bf16 captured-context attention does NOT use the MLA module's forward — - it runs ``dspark_attention_forward`` on dequantized bf16 weights cached via - :meth:`cache_attn_weights_from_checkpoint` — so the MLA projection linears are - skipped here (they would otherwise be transformed into the deep_gemm layout we - don't consume). - """ - attn_linear_ids = set() - for stage in self.mtp_layers: - for m in stage.self_attn.modules(): - if isinstance(m, Linear): - attn_linear_ids.add(id(m)) - - for module in self.modules(): - if isinstance(module, Linear) and id(module) not in attn_linear_ids: - module.transform_weights() - - @staticmethod - def _block_dequant(w_fp8: torch.Tensor, scale: torch.Tensor, block: int = 128) -> torch.Tensor: - """DeepSeek ``block``×``block`` block-scale dequant → bf16: ``real = fp8 * scale``. - - ``scale`` (possibly UE8M0) is broadcast over each ``block``×``block`` tile. - Pure-torch (matches the golden-validated reference dequant), robust to the - e8m0 scale dtype. - """ - wf = w_fp8.float() - out, inn = wf.shape - s = scale.float() - s_full = s.repeat_interleave(block, 0)[:out].repeat_interleave(block, 1)[:, :inn] - return (wf * s_full).to(torch.bfloat16) - - def _cache_attn_weights(self, src: Dict) -> None: - """Populate each stage's ``_dspark_attn`` from a dict of raw ``mtp.{s}.attn.*`` - tensors (source-agnostic core shared by the two public entry points). - - The captured-context attention runs the validated ``dspark_attention_forward`` - free function on reference-layout bf16 weights dequantized here. Sourcing the - separate ``wq_a``/``wkv`` (plain 128×128 block scale) sidesteps the TRT-LLM - ``MLA`` module's fused + interleaved fp8 storage (``kv_a_proj_with_mqa`` fuses - ``q_a``+``kv`` and stores the scale interleaved). This mirrors the - golden-validated dequant exactly. - """ - for s, stage in enumerate(self.mtp_layers): - pref = f"mtp.{s}.attn." - dev = stage.input_layernorm.weight.device - - def deq(name: str, fp8: bool) -> torch.Tensor: - w = src[f"{pref}{name}.weight"].to(dev) - if fp8: - return self._block_dequant(w, src[f"{pref}{name}.scale"].to(dev)) - return w.to(torch.bfloat16) - - stage._dspark_attn = dict( - wq_a=deq("wq_a", True), - q_norm_w=src[f"{pref}q_norm.weight"].to(dev).to(torch.bfloat16), - wq_b=deq("wq_b", True), - wkv=deq("wkv", True), - kv_norm_w=src[f"{pref}kv_norm.weight"].to(dev).to(torch.bfloat16), - # wo_a IS fp8+scale in the checkpoint (verified); always dequant. - wo_a=deq("wo_a", True), - wo_b=deq("wo_b", True), - attn_sink=src[f"{pref}attn_sink"].to(dev).float(), - ) - - def cache_attn_weights_from_checkpoint(self, ckpt_dir: str, weight_map: Dict[str, str]) -> None: - """Populate ``_dspark_attn`` by reading the ``mtp.{s}.attn.*`` tensors from the - checkpoint shards on disk, then dequantizing via :meth:`_cache_attn_weights`. - - TODO(step 3): source these from the loaded ``MLA`` modules instead, once the - fused/interleaved fp8 scale layout is decoded, to drop the checkpoint I/O. - """ - from safetensors import safe_open - - prefixes = tuple(f"mtp.{s}.attn." for s in range(len(self.mtp_layers))) - shards: Dict[str, list] = {} - for k in weight_map: - if k.startswith(prefixes): - shards.setdefault(weight_map[k], []).append(k) - raw: Dict[str, torch.Tensor] = {} - for shard, ks in shards.items(): - with safe_open(os.path.join(ckpt_dir, shard), framework="pt", device="cpu") as f: - for k in ks: - raw[k] = f.get_tensor(k) - self._cache_attn_weights(raw) - - def cache_attn_weights_from_state_dict(self, weights: Dict) -> None: - """Populate ``_dspark_attn`` from an already-loaded in-memory ``weights`` dict - (no extra disk I/O); used on the one-engine load path - (``DSparkForCausalLM.load_weights``). Delegates to :meth:`_cache_attn_weights`. - """ - self._cache_attn_weights(weights) - - def _dspark_freqs_table(self, device: torch.device) -> torch.Tensor: - """Return the fixed-size plain-RoPE table cached for ``device``.""" - key = str(device) - cached = self._freqs_table_cache.get(key) - if cached is None: - cached = precompute_dspark_freqs_cis( - self._attn_params["rope_head_dim"], - self._freqs_cap, - rope_theta=self._rope_theta, - device=device, - ) - self._freqs_table_cache[key] = cached - return cached - - @classmethod - def _derive_draft_model_config(cls, model_config, base: int, num_stages: int): - """Return a draft-only ``model_config`` copy with draft-specific fixes. - - Applies (1) the ``compress_ratios`` draft slice and (2) the - ``quant_config_dict`` MXFP4 extension for the draft layers' routed - experts. A single shallow copy is made (and only when something needs to - change) so the shared ``model_config`` and the target model are untouched. - - The draft MoE backend is **inherited** from the target's - ``model_config.moe_backend`` (carried by the shallow copy) — not pinned — - matching every other drafter (the MTP module reuses the V4 decoder layer, - whose MoE is built with ``moe_backend=model_config.moe_backend``; separate - Eagle3/DFlash drafts resolve it from their own config the same way). The - draft ``mtp.*`` stages are full V4 blocks, so they share the target's - MXFP4 ``n_routed_experts=384`` / ``n_group=8`` (= 48 experts/group) layout - and therefore the same backend constraints: pick a backend that supports - it (CUTLASS today, DeepGEMM megaMoE once available) on the target and the - draft follows. Note the TRTLLM-Gen ``blockScaleMoe`` routing kernel asserts - ``experts/group <= 32`` (warp size), so it is incompatible with this layout - for both the target and the draft. - """ - new_sa = cls._draft_sparse_config(model_config, base, num_stages) - new_qcd = cls._draft_quant_config_dict(model_config, base, num_stages) - if new_sa is None and new_qcd is None: - return model_config - draft_cfg = copy.copy(model_config) - # ModelConfig is a frozen dataclass; bypass the guard for these fields. - if new_sa is not None: - object.__setattr__(draft_cfg, "sparse_attention_config", new_sa) - if new_qcd is not None: - object.__setattr__(draft_cfg, "quant_config_dict", new_qcd) - return draft_cfg - - @staticmethod - def _draft_sparse_config(model_config, base: int, num_stages: int): - """Sparse-attention config sliced to the draft layers, or None if N/A. - - The inherited block remaps ``layer_idx`` to a draft-local index, so the - sparse config must expose the draft layers' per-layer ratios at indices - ``[0, num_stages)``. - """ - sa = getattr(model_config, "sparse_attention_config", None) - compress_ratios = getattr(sa, "compress_ratios", None) if sa is not None else None - if not compress_ratios or len(compress_ratios) < base + num_stages: - return None - draft_ratios = list(compress_ratios)[base : base + num_stages] - # Already draft-local (e.g. a draft-only checkpoint config); no slice. - if ( - draft_ratios == list(compress_ratios)[:num_stages] - and len(compress_ratios) == num_stages - ): - return None - return sa.model_copy(update={"compress_ratios": draft_ratios}) - - @staticmethod - def _draft_quant_config_dict(model_config, base: int, num_stages: int): - """quant_config_dict extended to cover draft-layer experts, or None. - - The checkpoint's per-module quant map only enumerates the base layers, so - ``model.layers.{base+s}.mlp.experts`` would fall back to the global fp8 - config and build fp8-shaped expert buffers. The draft routed experts are - physically MXFP4 (identical to the main MoE layers), so copy a - representative main MoE layer's experts quant onto the draft layer keys. - """ - qcd = getattr(model_config, "quant_config_dict", None) - if not qcd: - return None - src = next( - ( - qcd[f"model.layers.{li}.mlp.experts"] - for li in range(base) - if f"model.layers.{li}.mlp.experts" in qcd - ), - None, - ) - if src is None: - return None - new_qcd = dict(qcd) - changed = False - for s in range(num_stages): - key = f"model.layers.{base + s}.mlp.experts" - if new_qcd.get(key) is not src: - new_qcd[key] = src - changed = True - return new_qcd if changed else None - - @torch.inference_mode() - def write_context_windows( - self, - main_hidden: torch.Tensor, - positions: torch.Tensor, - stage_windows: torch.Tensor, - ) -> None: - """Write captured-context ``main_kv`` into the rolling per-stage KV windows. - - Replicates exactly the per-position context write that - :func:`dspark_attention_forward` performs each generation step - (``main_kv = RoPE_pos(rmsnorm(wkv @ main_x))`` written at - ``window[pos % window_size]``), but for an arbitrary set of - ``(captured-hidden, position)`` pairs. Used to (a) seed a request's - window from its prompt at prefill and (b) back-fill the intermediate - accepted tokens of a multi-accept step — both of which the per-step - generation path would otherwise leave as holes, starving the draft - attention of context (acceptance-rate only; verified decoding keeps - output correctness regardless). - - Args: - main_hidden: ``[M, num_capture * hidden]`` captured target hiddens. - positions: ``[M]`` absolute window positions (used for BOTH the RoPE - phase and the slot ``pos % window_size``). By the generation-path - convention this is ``committed_position + 1``. Must hold at most - ``window_size`` entries with distinct slots (the caller passes a - contiguous, deduplicated range) so the scatter is well defined. - stage_windows: ``[num_stages, window_size, head_dim]`` window for one - request's slot; updated in place. - """ - if getattr(self.mtp_layers[0], "_dspark_attn", None) is None: - return - M = int(main_hidden.shape[0]) - if M == 0: - return - win = int(self._attn_params["window_size"]) - rd = int(self._attn_params["rope_head_dim"]) - eps = float(self._attn_params["eps"]) - positions = positions.to(main_hidden.device).long() - # main_x is stage-invariant (stage 0's projection), matching forward_embed. - stage0 = self.mtp_layers[0] - main_x = stage0.main_norm(stage0.main_proj(main_hidden)) # [M, hidden] - freqs = self._dspark_freqs_table(main_x.device)[positions] - slots = positions % win # [M] - mx = main_x.unsqueeze(0) # [1, M, hidden] for the per-position RoPE layout - for s, stage in enumerate(self.mtp_layers): - a = stage._dspark_attn - kv = _rmsnorm(F.linear(mx, a["wkv"]), a["kv_norm_w"], eps) # [1, M, head_dim] - kv = _rope_last_dims(kv, rd, freqs) # [1, M, head_dim] - stage_windows[s, slots] = kv[0].to(stage_windows.dtype) - - def write_context_windows_batched( - self, - main_hidden: torch.Tensor, - positions: torch.Tensor, - slots: torch.Tensor, - mask: torch.Tensor, - kv_windows: torch.Tensor, - ) -> None: - """CUDA-graph-safe batched + masked variant of :meth:`write_context_windows`. - - Back-fills the intermediate accepted tokens of a multi-accept step into the - rolling per-stage KV windows for ALL gen requests at once, with a fixed - ``[G, M]`` shape (``M = max interim per request``) and a validity mask - (invalid entries are no-ops via a read-modify-write), so nothing depends on - the per-request accept count. Same per-position math as the scalar version - (``RoPE_pos(rmsnorm(wkv @ main_x))`` written at ``window[pos % window]``), - but indexed/scattered through ``slots`` into the shared persistent buffer. - - Args: - main_hidden: ``[G, M, num_capture * hidden]`` captured target hiddens - (rows beyond a request's interim count are masked). - positions: ``[G, M]`` absolute window positions (RoPE phase + slot). - slots: ``[G]`` row index of each request into ``kv_windows``. - mask: ``[G, M]`` bool — which ``(g, m)`` entries are real interim writes. - kv_windows: ``[N, num_stages, window_size, head_dim]`` persistent buffer; - updated in place. - """ - if getattr(self.mtp_layers[0], "_dspark_attn", None) is None: - return - G, M = positions.shape - if G == 0 or M == 0: - return - win = int(self._attn_params["window_size"]) - rd = int(self._attn_params["rope_head_dim"]) - eps = float(self._attn_params["eps"]) - positions = positions.long() - slots = slots.long() - freqs = self._dspark_freqs_table(main_hidden.device)[positions] # [G, M, rd//2] - cols = positions % win # [G, M] - rows = slots[:, None].expand(-1, M) # [G, M] - mask3 = mask.unsqueeze(-1) # [G, M, 1] - stage0 = self.mtp_layers[0] - main_x = stage0.main_norm(stage0.main_proj(main_hidden)) # [G, M, hidden] - for s, stage in enumerate(self.mtp_layers): - a = stage._dspark_attn - kv = _rmsnorm(F.linear(main_x, a["wkv"]), a["kv_norm_w"], eps) # [G, M, head_dim] - kv = _rope_last_dims_batched(kv, rd, freqs) # [G, M, head_dim] - win_s = kv_windows[:, s] # [N, win, head_dim] view onto the base buffer - # Read-modify-write so masked-out (g, m) entries keep their current - # value — a graph-safe masked scatter (no dynamic-shape compaction). - cur = win_s[rows, cols] # [G, M, head_dim] - win_s[rows, cols] = torch.where(mask3, kv.to(win_s.dtype), cur) - - def forward_embed( - self, main_hidden: torch.Tensor, bonus_token_ids: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Build the draft block input and the cross-attention context. - - Args: - main_hidden: ``[num_tokens, num_capture * hidden]`` captured context. - bonus_token_ids: ``[num_tokens]`` last accepted token per request. - Returns: - x: hc-expanded block embeddings ``[num_tokens, block_size, hc_mult, hidden]`` - main_x: projected context ``[num_tokens, hidden]`` - draft_ids: block input token ids ``[num_tokens, block_size]`` (the callers - reuse these as the per-position MoE routing ids, so we return them - rather than rebuild). - """ - stage0 = self.mtp_layers[0] - main_x = stage0.main_norm(stage0.main_proj(main_hidden)) - draft_ids = build_draft_input_ids( - bonus_token_ids, block_size=self.block_size, noise_token_id=self.noise_token_id - ) - x = self.embed_tokens(draft_ids) - x = x.unsqueeze(-2).repeat(1, 1, self.hc_mult, 1) - return x, main_x, draft_ids - - def _forward_stage( - self, - stage: "DSparkBlock", - h: torch.Tensor, - main_x: torch.Tensor, - start_pos, - freqs_cis: torch.Tensor, - moe_input_ids: torch.Tensor, - stage_window: Optional[torch.Tensor] = None, - slots: Optional[torch.Tensor] = None, - all_rank_num_tokens: Optional[List[int]] = None, - ) -> torch.Tensor: - """One DSpark stage = reference ``Block.forward`` with captured-context attn. - - ``h`` is the mHC residual stream ``[T, block, hc_mult, hidden]``. The mHC - ``pre_mapping``/``post_mapping`` preserve the leading ``[T, block]`` dims; - the captured-context attention and MoE run on the collapsed token axis. - Mirrors the reference (unfused) mHC boundaries exactly: - ``hc_pre → attn_norm → DSparkAttention → hc_post`` then - ``hc_pre → ffn_norm → MoE → hc_post``. - - ``stage_window`` is this stage's persistent rolling captured-context KV - window ``[T, window_size, head_dim]`` owned by the worker across decode - steps; the attention writes the current ``main_kv`` into it in place. When - ``None`` (golden / single-shot) a zero window is allocated per call. - - When ``slots`` (a ``[G]`` int tensor) is given, the CUDA-graph-safe batched - attention (:func:`dspark_attention_forward_batched`) is used instead: it - takes ``start_pos`` as a ``[G]`` tensor and writes/reads ``stage_window`` - (then shaped ``[N, window_size, head_dim]``) through the ``slots`` index. - """ - T, block, _, hidden = h.shape - - # --- attention sub-block (captured-context, not paged-KV MLA) --- - residual = h - post_mix, comb_mix, layer_input = stage.hc_attn.pre_mapping(residual) - layer_input = stage.input_layernorm(layer_input) # [T, block, hidden] - # Rolling-window cache: persist through the worker-owned ``stage_window`` - # for cross-step decode, else a fresh zero window for a single block. - persist = stage_window is not None - kv_cache = ( - stage_window - if persist - else torch.zeros( - T, - self._attn_params["window_size"], - self._attn_params["head_dim"], - dtype=torch.bfloat16, - device=h.device, - ) - ) - if slots is not None: - # Batched, CUDA-graph-safe path (start_pos is a [G] tensor; window is - # written/read through ``slots``). - attn = dspark_attention_forward_batched( - layer_input, - main_x, - start_pos, - kv_cache, - slots, - freqs_cis=freqs_cis, - persist=True, - **stage._dspark_attn, - **self._attn_params, - ) - else: - attn = dspark_attention_forward( - layer_input, - main_x, - start_pos, - kv_cache, - freqs_cis=freqs_cis, - persist=persist, - **stage._dspark_attn, - **self._attn_params, - ) - if stage.enable_fused_hc: - residual, post_mix, comb_mix, layer_input = stage.hc_ffn.fused_hc( - x_prev=attn, - residual_prev=residual, - post_mix_prev=post_mix, - comb_mix_prev=comb_mix, - norm_weight=stage.post_attention_layernorm.weight, - norm_eps=stage.post_attention_layernorm.variance_epsilon, - ) - else: - residual = stage.hc_attn.post_mapping( - x=attn, - residual=residual, - post_layer_mix=post_mix, - comb_res_mix=comb_mix, - ) - post_mix, comb_mix, layer_input = stage.hc_ffn.pre_mapping(residual) - layer_input = stage.post_attention_layernorm(layer_input) - num_tokens = T * block - # FUSED_COMM MoE backends (DeepGEMM MegaMoE) size their in-kernel - # NVLink-barrier chunk loop from ``max(all_rank_num_tokens)`` and index - # the local slice by ``moe_ep_rank``, so every EP rank must pass the same - # globally-gathered per-rank list (here: gen tokens = num_gens * block per - # rank). Passing only the local ``[num_tokens]`` desyncs the phase-flip - # barrier across ranks (hang / "unspecified launch failure"). Fall back to - # the local count for single-rank / non-ADP runs where no list is threaded. - moe_all_rank_num_tokens = ( - all_rank_num_tokens if all_rank_num_tokens is not None else [num_tokens] - ) - # The draft captured-context MoE must mirror the target DeepseekV4MoE's TP - # reduction policy. Under attention DP each rank is data-parallel (owns a - # distinct set of requests) and needs no cross-rank reduction; but under - # plain tensor parallelism (attention_dp off, tp_size > 1) the expert-sharded - # MoE output must be all-reduced across ranks -- exactly what the target MoE - # does (enable_allreduce = not (POST_MOE_FUSION or tp_size == 1)). Previously - # this was hard-coded to False, which dropped the reduction on the non-ADP - # path, corrupting the draft block proposals and roughly halving DSpark - # acceptance length whenever attention_dp was disabled. - moe_enable_allreduce = ( - not self.model_config.mapping.enable_attention_dp - and self.model_config.mapping.tp_size > 1 - ) - moe_out = stage.mlp( - layer_input.reshape(num_tokens, hidden), - input_ids=moe_input_ids, - all_rank_num_tokens=moe_all_rank_num_tokens, - final_all_reduce_params=AllReduceParams(enable_allreduce=moe_enable_allreduce), - do_finalize=True, - ).reshape(T, block, hidden) - h = stage.hc_ffn.post_mapping( - x=moe_out, residual=residual, post_layer_mix=post_mix, comb_res_mix=comb_mix - ) - return h - - def forward( - self, - main_hidden: torch.Tensor, - bonus_token_ids: torch.Tensor, - start_pos: int, - *, - kv_windows: Optional[torch.Tensor] = None, - temperature: float = 0.0, - confidence_threshold: float = 0.0, - return_logits: bool = False, - all_rank_num_tokens: Optional[List[int]] = None, - ) -> tuple: - """Full block-draft forward: chain the ``num_stages`` DSpark stages. - - Mirrors the reference ``Transformer.forward_spec`` (generation path, - ``start_pos > 0``): ``forward_embed`` builds the block input + captured - context, each stage runs the captured-context backbone, and - ``forward_head`` emits the block draft tokens + per-position confidence. - - Args: - main_hidden: ``[T, num_capture * hidden]`` captured target context. - bonus_token_ids: ``[T]`` last accepted token per request. - start_pos: absolute decode position (must be > 0). - kv_windows: optional persistent per-stage rolling captured-context KV - windows ``[T, num_stages, window_size, head_dim]`` owned by the - worker; updated in place each call. ``None`` allocates fresh zero - windows (single-shot golden / test path). - Returns: - ``(draft_tokens [T, block], num_proposed [T])`` from ``forward_head``. - """ - assert start_pos > 0, "DSpark draft runs at generation (start_pos > 0)" - if getattr(self.mtp_layers[0], "_dspark_attn", None) is None: - raise RuntimeError( - "DSpark attention weights not cached; call " - "cache_attn_weights_from_checkpoint(ckpt_dir, weight_map) after loading." - ) - x, main_x, draft_ids = self.forward_embed(main_hidden, bonus_token_ids) - main_x = main_x.unsqueeze(1) # [T, 1, hidden] for the MQA K/V projection - freqs_cis = self._dspark_freqs_table(x.device) - moe_input_ids = draft_ids.reshape(-1) - - h = x - for s, stage in enumerate(self.mtp_layers): - stage_window = kv_windows[:, s] if kv_windows is not None else None - h = self._forward_stage( - stage, - h, - main_x, - start_pos, - freqs_cis, - moe_input_ids, - stage_window, - all_rank_num_tokens=all_rank_num_tokens, - ) - - return self.forward_head( - h, - bonus_token_ids, - temperature=temperature, - confidence_threshold=confidence_threshold, - return_logits=return_logits, - ) - - def forward_batched( - self, - main_hidden: torch.Tensor, - bonus_token_ids: torch.Tensor, - start_pos: torch.Tensor, - *, - kv_windows: torch.Tensor, - slots: torch.Tensor, - temperature: float = 0.0, - confidence_threshold: float = 0.0, - return_logits: bool = False, - all_rank_num_tokens: Optional[List[int]] = None, - ) -> tuple: - """CUDA-graph-safe batched block-draft forward (all gen requests at once). - - Same computation as :meth:`forward`, but every host-int / data-dependent - operation is tensorized so the whole path can be captured into the target's - CUDA graph (DSpark is a one-engine drafter — its worker runs inside the - graph). ``start_pos`` is a ``[G]`` tensor (one absolute decode position per - gen request); the rolling captured-context windows are written/read through - ``slots`` into the worker-owned ``kv_windows`` buffer; RoPE phases are - gathered from a fixed table. ``forward_head`` is run with - ``confidence_threshold == 0`` (the worker proposes the full block), which is - the graph-safe branch of :func:`dspark_propose`. - - Args: - main_hidden: ``[G, num_capture * hidden]`` captured target context. - bonus_token_ids: ``[G]`` last accepted token per gen request. - start_pos: ``[G]`` int tensor of absolute decode positions (> 0). - kv_windows: ``[N, num_stages, window_size, head_dim]`` persistent rolling - windows; written in place through ``slots``. - slots: ``[G]`` int tensor mapping each request to its ``kv_windows`` row. - Returns: - ``(draft_tokens [G, block], num_proposed [G])`` from ``forward_head``. - """ - if getattr(self.mtp_layers[0], "_dspark_attn", None) is None: - raise RuntimeError( - "DSpark attention weights not cached; call " - "cache_attn_weights_from_checkpoint(ckpt_dir, weight_map) after loading." - ) - x, main_x, draft_ids = self.forward_embed(main_hidden, bonus_token_ids) - main_x = main_x.unsqueeze(1) # [G, 1, hidden] for the MQA K/V projection - freqs_cis = self._dspark_freqs_table(x.device) - moe_input_ids = draft_ids.reshape(-1) - - h = x - for s, stage in enumerate(self.mtp_layers): - stage_window = kv_windows[:, s] # [N, window_size, head_dim] - h = self._forward_stage( - stage, - h, - main_x, - start_pos, - freqs_cis, - moe_input_ids, - stage_window, - slots, - all_rank_num_tokens=all_rank_num_tokens, - ) - - return self.forward_head( - h, - bonus_token_ids, - temperature=temperature, - confidence_threshold=confidence_threshold, - return_logits=return_logits, - ) - - def run_moe_lockstep_noop( - self, all_rank_num_tokens: Optional[List[int]], device: torch.device - ) -> None: - """Cross the FUSED_COMM MoE NVLink barrier the same number of times as - gen-bearing ranks, for an EP rank whose local draft batch is empty. - - DeepGEMM MegaMoE (``scheduler_kind == FUSED_COMM``) synchronizes EP ranks - with an in-kernel phase-flip NVLink barrier that flips on every kernel - call, so every rank must invoke the MoE the same number of times or the - barrier desyncs (hang / "unspecified launch failure"). In the DSpark - draft only the MoE carries a cross-rank barrier (the captured-context - attention and the markov/confidence heads are per-rank), so a rank with - zero local generation requests replays just the per-stage MoE call with a - single 1-row dummy (its entry in ``all_rank_num_tokens`` is ``1``). The - scheduler runs its ``max``-derived chunk count, slicing this rank to the - 1 dummy row and zero-padding the remaining chunks, keeping the barrier - lockstep. No-op when there is no cross-rank work (single-rank / non-ADP, - or every rank is empty). - """ - if all_rank_num_tokens is None or max(all_rank_num_tokens) == 0: - return - hidden = self.config.hidden_size - # Use a 1-row dummy, NOT a 0-row tensor: DeepseekV4MoE's router / - # shared-expert dense GEMMs reject a 0-row input (cuBLAS - # CUBLAS_STATUS_INVALID_VALUE). The paired ``all_rank_num_tokens`` encodes - # 1 for this rank, so the FUSED_COMM scheduler slices to this 1 dummy row - # and still launches ``num_chunks`` cross-rank barrier crossings in - # lockstep with the gen-bearing ranks. - dummy_x = torch.zeros((1, hidden), dtype=torch.bfloat16, device=device) - dummy_ids = torch.zeros((1,), dtype=torch.long, device=device) - for stage in self.mtp_layers: - stage.mlp( - dummy_x, - input_ids=dummy_ids, - all_rank_num_tokens=all_rank_num_tokens, - final_all_reduce_params=AllReduceParams(enable_allreduce=False), - do_finalize=True, - ) - - def forward_head( - self, - block_hidden: torch.Tensor, - bonus_token_ids: torch.Tensor, - *, - temperature: float = 0.0, - confidence_threshold: float = 0.0, - return_logits: bool = False, - ) -> tuple: - """Block-draft head: hc_head + norm + lm_head -> markov refine + confidence. - - ``block_hidden`` is the last stage's mHC residual ``[*, block, hc_mult, hidden]``. - Returns (draft_tokens [*, block], num_proposed [*]); with ``return_logits`` - also returns the per-position draft logits [*, block, vocab] (§7.9 1-TV). - """ - last = self.mtp_layers[-1] - h = last.hc_head(block_hidden) - h = last.norm(h) - base_logits = self.lm_head(h) - return dspark_propose( - base_logits, - bonus_token_ids=bonus_token_ids, - block_hidden=h, - markov_head=last.markov_head, - confidence_head=last.confidence_head, - block_size=self.block_size, - temperature=temperature, - confidence_threshold=confidence_threshold, - return_logits=return_logits, - ) - - -class DSparkForCausalLM(nn.Module): - """One-engine draft wrapper for DSpark (mirrors ``DFlashForCausalLM``). - - Wraps :class:`DSparkDraftModel` (the ``n_mtp_layers``-stage ``mtp.*`` backbone) - for the single-engine external-drafter flow: created by ``get_draft_model``, - appended to the target's epilogue, and driven by ``DSparkWorker``. - - ``embed_tokens`` / ``lm_head`` are shared with the target model - (:meth:`load_weights_from_target_model`). The draft weights live in the SAME - checkpoint under ``mtp.*``; :meth:`load_weights` remaps them - (``remap_dspark_draft_keys``), loads via ``DeepseekV4WeightLoader``, runs the - fp8 ``post_load_weights`` transforms, and caches the bf16 captured-context - attention weights from the in-memory state dict. - """ - - def __init__(self, draft_config, aux_stream_dict=None, num_stages=None, block_size=None): - super().__init__() - self.dspark_model = DSparkDraftModel( - draft_config, - aux_stream_dict, - num_stages=num_stages, - block_size=block_size, - ) - # Generic handles expected by the loader / weight mappers. - self.model = self.dspark_model - self.model_config = draft_config - self.config = draft_config.pretrained_config - # Worker-facing interface (the worker receives this wrapper as - # ``draft_model`` and calls forward()/reads these properties and scalars). - self.num_stages = self.dspark_model.num_stages - self._attn_params = self.dspark_model._attn_params - self.lm_head = None # shared from the target (load_weights_from_target_model) - self.logits_processor = None # set by the caller after construction - - @property - def block_size(self): - return self.dspark_model.block_size - - @property - def embed_tokens(self): - return self.dspark_model.embed_tokens - - def forward(self, main_hidden, bonus_token_ids, start_pos, **kwargs): - return self.dspark_model.forward(main_hidden, bonus_token_ids, start_pos, **kwargs) - - def forward_batched(self, main_hidden, bonus_token_ids, start_pos, **kwargs): - """CUDA-graph-safe batched draft forward (delegates to the draft model).""" - return self.dspark_model.forward_batched(main_hidden, bonus_token_ids, start_pos, **kwargs) - - def run_moe_lockstep_noop(self, all_rank_num_tokens, device): - """Empty-batch MoE barrier lockstep (delegates to the draft model).""" - return self.dspark_model.run_moe_lockstep_noop(all_rank_num_tokens, device) - - def write_context_windows(self, main_hidden, positions, stage_windows): - """Seed / back-fill the rolling KV windows (delegates to the draft model).""" - return self.dspark_model.write_context_windows(main_hidden, positions, stage_windows) - - def write_context_windows_batched(self, main_hidden, positions, slots, mask, kv_windows): - """Batched + masked window back-fill (delegates to the draft model).""" - return self.dspark_model.write_context_windows_batched( - main_hidden, positions, slots, mask, kv_windows - ) - - def load_weights(self, weights: Dict, weight_mapper=None, **kwargs): - """Load the ``mtp.*`` draft weights from the (full) checkpoint dict. - - ``weight_mapper`` is accepted for interface parity with the draft-weight - loader but unused: DSpark does its own ``mtp.{s}.* -> mtp_layers.{s}.*`` - remap (``remap_dspark_draft_keys``) onto the shared V4 weight loader. - """ - remapped = remap_dspark_draft_keys(weights, num_stages=self.num_stages) - logger.info( - f"[DSpark] loading {len(remapped)} draft params across {self.num_stages} stages" - ) - DeepseekV4WeightLoader(self.dspark_model).load_weights(remapped) - self.dspark_model.post_load_weights() - # bf16 captured-context attention path: dequantize the raw mtp.{s}.attn.* - # tensors for ``dspark_attention_forward``. - self.dspark_model.cache_attn_weights_from_state_dict(weights) - logger.info("[DSpark] draft weight load complete") - - def load_weights_from_target_model(self, target_model): - """Share the target's embed_tokens / lm_head (DSpark has neither).""" - if self.dspark_model.embed_tokens is None: - self.dspark_model.embed_tokens = target_model.model.embed_tokens - if self.lm_head is None: - self.lm_head = target_model.lm_head - self.dspark_model.lm_head = target_model.lm_head - - -__all__ = ["DSparkBlock", "DSparkDraftModel", "DSparkForCausalLM"] diff --git a/tensorrt_llm/_torch/models/modeling_gemma4_vision.py b/tensorrt_llm/_torch/models/modeling_gemma4_vision.py index 40c595dfe5de..b921a442b6e9 100644 --- a/tensorrt_llm/_torch/models/modeling_gemma4_vision.py +++ b/tensorrt_llm/_torch/models/modeling_gemma4_vision.py @@ -629,11 +629,11 @@ def _position_embeddings( self, pixel_position_ids: torch.Tensor, padding_positions: torch.Tensor ) -> torch.Tensor: clamped = pixel_position_ids.clamp(min=0) - position_embeddings = ( - self.position_embedding_table[0, clamped[..., 0]] - + self.position_embedding_table[1, clamped[..., 1]] - ) - return position_embeddings.masked_fill(padding_positions.unsqueeze(-1), 0.0) + one_hot = F.one_hot(clamped, num_classes=self.position_embedding_size) + one_hot = one_hot.permute(0, 2, 1, 3).to(self.position_embedding_table) + position_embeddings = one_hot @ self.position_embedding_table + position_embeddings = position_embeddings.sum(dim=1) + return torch.where(padding_positions.unsqueeze(-1), 0.0, position_embeddings) def forward( self, diff --git a/tensorrt_llm/_torch/models/modeling_gpt_oss.py b/tensorrt_llm/_torch/models/modeling_gpt_oss.py index d48d454715e1..00b5c77c8951 100644 --- a/tensorrt_llm/_torch/models/modeling_gpt_oss.py +++ b/tensorrt_llm/_torch/models/modeling_gpt_oss.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Literal, Optional +from typing import Dict, Optional import torch from torch import nn @@ -60,10 +60,7 @@ def __init__( beta_fast=pretrained_config.rope_scaling['beta_fast'], beta_slow=pretrained_config.rope_scaling['beta_slow'], duplicate_data=False), - # GPT-OSS applies NeoX-style (rotate-half) RoPE, matching the HF - # reference. The fused kernel ignores this flag for yarn (which - # masked the wrong value); the unfused path honors it. - is_neox=True, + is_neox=False, ) super().__init__( @@ -552,13 +549,6 @@ def forward( @register_auto_model("GptOssForCausalLM") class GptOssForCausalLM(SpecDecOneEngineForCausalLM[Transformer, GptOssConfig]): - @classmethod - def get_preferred_transceiver_runtime( - cls, - pretrained_config: Any = None, - ) -> Optional[Literal["CPP", "PYTHON"]]: - return "PYTHON" - params_map = { # TRTLLM module name : GptOss module name "qkv_proj": "qkv", diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k25.py b/tensorrt_llm/_torch/models/modeling_kimi_k25.py index a216e916df0f..e4706cb79502 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k25.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k25.py @@ -34,7 +34,7 @@ import os import tempfile from datetime import datetime, timezone -from typing import Any, Dict, List, Literal, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np import torch @@ -1517,21 +1517,6 @@ class KimiK25ForConditionalGeneration(PreTrainedModel): _LANG_PREFIX = "language_model." - @classmethod - def get_preferred_transceiver_runtime( - cls, - pretrained_config: Any = None, - ) -> Literal["PYTHON"]: - """Kimi-K2.5 defaults to the Python (v2) KV-cache transceiver. - - The DeepSeek-V3 MLA backbone transfers a large latent KV, which the - Python transceiver handles better in disaggregated serving. This is - only adopted when the user leaves - ``cache_transceiver_config.transceiver_runtime`` at 'auto' and the - effective backend is NIXL; otherwise the C++ transceiver is used. - """ - return "PYTHON" - def __init__( self, model_config: ModelConfig[PretrainedConfig], diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3.py b/tensorrt_llm/_torch/models/modeling_minimaxm3.py index f10696c97bb9..e38b9f8be115 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3.py @@ -35,17 +35,7 @@ from tensorrt_llm.models.modeling_utils import QuantConfig from ..attention_backend import AttentionMetadata -from ..attention_backend.interface import ( - AttentionForwardArgs, - PositionalEmbeddingParams, - RopeParams, -) -from ..attention_backend.sparse.minimax_m3 import ( - MiniMaxM3MsaSparseAttention, - MiniMaxM3SparseRuntimeBackend, - _gather_paged_batched, - _write_main_kv_slots_to_pool, -) +from ..attention_backend.interface import PositionalEmbeddingParams, RopeParams from ..distributed import AllReduce, AllReduceParams, MiniMaxAllReduceRMS from ..modules.attention import Attention from ..modules.decoder_layer import DecoderLayer @@ -62,8 +52,6 @@ get_model_extra_attrs, is_torch_compiling, ) -from .checkpoints.base_weight_mapper import BaseWeightMapper -from .checkpoints.hf.minimaxm3_weight_mapper import MINIMAX_M3_PARAMS_MAP, MiniMaxM3HfWeightMapper from .modeling_utils import DecoderModel, DecoderModelForCausalLM, ModelConfig, register_auto_model # Dense layers use SDPA with non-contiguous Q/K/V and a bool attn_mask. @@ -923,6 +911,11 @@ def _dense_attention_core( output: torch.Tensor, ) -> torch.Tensor: """Run dense cache updates and attention into ``output``.""" + from ..attention_backend.sparse.minimax_m3 import ( + _gather_paged_batched, + _write_main_kv_slots_to_pool, + ) + kv_cache_manager = getattr(attn_metadata, "kv_cache_manager", None) if kv_cache_manager is None: raise RuntimeError( @@ -1124,27 +1117,11 @@ def _attention_core( attn_metadata: AttentionMetadata, output: torch.Tensor, ) -> torch.Tensor: - # The MSA backend runs the sparse GQA or dense paged GQA through its - # inherited forward; this layer selects the top-k blocks (sparse only) - # and builds the forward_args the FMHA reads. - if isinstance(self.attn, MiniMaxM3MsaSparseAttention): - if self.is_sparse_attention_layer: - assert idx_q is not None and idx_k is not None - # Publish the selected blocks so the FMHA runs the sparse path. - kv_block_indexes = self.attn.run_indexer(idx_q, idx_k, attn_metadata) - forward_args = AttentionForwardArgs(output=output, topk_indices=kv_block_indexes) - else: - assert idx_q is None and idx_k is None - # No top-k selection means the FMHA attends the full page table. - forward_args = AttentionForwardArgs(output=output) - self.attn.forward(q, k, v, attn_metadata, forward_args=forward_args) - return output - else: - if self.is_sparse_attention_layer: - assert idx_q is not None and idx_k is not None - return self._sparse_attention_core(q, k, v, idx_q, idx_k, attn_metadata, output) - assert idx_q is None and idx_k is None - return self._dense_attention_core(q, k, v, attn_metadata, output) + if self.is_sparse_attention_layer: + assert idx_q is not None and idx_k is not None + return self._sparse_attention_core(q, k, v, idx_q, idx_k, attn_metadata, output) + assert idx_q is None and idx_k is None + return self._dense_attention_core(q, k, v, attn_metadata, output) def _sparse_forward( self, @@ -1164,7 +1141,7 @@ def _sparse_forward( 4. Pull paged main K/V cache (reshaped to flat-slot view) and paged side index-K cache from the :class:`MiniMaxM3KVCacheManagerV2`. - 5. Build a :class:`MiniMaxM3TritonSparseAttentionMetadata` from the + 5. Build a :class:`MiniMaxM3SparseAttentionMetadata` from the standard :class:`AttentionMetadata` (using ``request_ids`` + ``seq_lens`` + ``num_cached_tokens_per_seq``). 6. Write the new token's K/V/idx_K to the slots named by the @@ -1177,7 +1154,7 @@ def _sparse_forward( Production callers (the LLM API path) drive :meth:`MiniMaxM3AttentionMetadata.prepare` outside any CUDA-graph capture window; that method attaches a pre-built - :class:`MiniMaxM3TritonSparseAttentionMetadata` and an + :class:`MiniMaxM3SparseAttentionMetadata` and an ``out_cache_loc`` tensor as ``attn_metadata.minimax_m3``. Test callers attach the same dict manually. This forward path always reads the pre-built attachment and never builds metadata @@ -1221,7 +1198,7 @@ def _sparse_attention_core( attn_metadata: AttentionMetadata, output: torch.Tensor, ) -> torch.Tensor: - """Run sparse cache updates and attention into ``output`` (Triton path).""" + """Run sparse cache updates and attention into ``output``.""" kv_cache_manager = getattr(attn_metadata, "kv_cache_manager", None) if kv_cache_manager is None: raise RuntimeError( @@ -1291,7 +1268,10 @@ def _sparse_attention_core( # registers :class:`MiniMaxM3SparseRuntimeBackend` as # ``self.attn``; any other backend on a sparse layer is a # configuration error. - if not isinstance(self.attn, MiniMaxM3SparseRuntimeBackend): + from ..attention_backend.sparse.minimax_m3 import get_minimax_m3_attention_backend_cls + + m3_backend_cls = get_minimax_m3_attention_backend_cls() + if not isinstance(self.attn, m3_backend_cls): raise RuntimeError( f"MiniMax-M3 sparse forward (layer {self.layer_idx}) requires " f"self.attn to be a MiniMaxM3SparseRuntimeBackend, got " @@ -1473,6 +1453,20 @@ def forward( return hidden_states +# HF MiniMax-M3 stores the routed score-correction bias one level above +# the router weight (``block_sparse_moe.e_score_correction_bias``, +# sibling of ``block_sparse_moe.gate.weight``). The TRT-LLM module tree +# binds it to :class:`MiniMaxM3Gate`, so the generic loader expects to +# see it at ``block_sparse_moe.gate.e_score_correction_bias``. The +# regex below moves the key into the gate's prefix before the loader +# dispatches; this lets ``mark_consumed("...gate")`` cleanly remove +# both the weight and the bias together without disturbing the sibling +# ``block_sparse_moe.experts.*`` backend subtree. +_M3_GATE_BIAS_RENAME_MAP = { + r"^(.*\.block_sparse_moe)\.e_score_correction_bias$": (r"\1.gate.e_score_correction_bias"), +} + + @register_auto_model("MiniMaxM3SparseForCausalLM") class MiniMaxM3ForCausalLM(DecoderModelForCausalLM[MiniMaxM3Model, PretrainedConfig]): """Text-only M3 model.""" @@ -1488,23 +1482,13 @@ def __init__(self, model_config: "ModelConfig[PretrainedConfig]"): vocab_size=model_config.pretrained_config.vocab_size, ) - def load_weights( - self, - weights: Dict, - weight_mapper: Optional[BaseWeightMapper] = None, - params_map: Optional[Dict[str, str]] = None, - allow_partial_loading: bool = False, - ) -> None: - if weight_mapper is None: - weight_mapper = MiniMaxM3HfWeightMapper() - weight_mapper.init_model_and_config(self, self.model_config) - merged_params_map = {**MINIMAX_M3_PARAMS_MAP, **(params_map or {})} - super().load_weights( - weights=weights, - weight_mapper=weight_mapper, - params_map=merged_params_map, - allow_partial_loading=allow_partial_loading, - ) + def load_weights(self, weights, *args, **kwargs): + # Merge the M3-specific gate-bias rename into any caller- + # supplied ``params_map`` so the VL wrapper and any downstream + # tooling that already passes one keep working. + params_map = kwargs.pop("params_map", None) or {} + merged = {**_M3_GATE_BIAS_RENAME_MAP, **params_map} + return super().load_weights(weights, *args, params_map=merged, **kwargs) def _strip_language_model_prefix( @@ -1640,13 +1624,7 @@ def __init__(self, model_config: "ModelConfig[PretrainedConfig]"): self.last_loaded_vision_keys = [] self.last_missing_vision_keys = [] - def load_weights( - self, - weights: Dict, - weight_mapper: Optional[BaseWeightMapper] = None, - params_map: Optional[Dict[str, str]] = None, - allow_partial_loading: bool = False, - ) -> None: + def load_weights(self, weights, *args, **kwargs): text_cfg = self.config if is_minimax_m3_vl_config(text_cfg): text_cfg = get_text_config(text_cfg) @@ -1669,12 +1647,7 @@ def load_weights( self.last_loaded_vision_keys = loaded self.last_missing_vision_keys = missing - super().load_weights( - weights=text_weights, - weight_mapper=weight_mapper, - params_map=params_map, - allow_partial_loading=allow_partial_loading, - ) + return super().load_weights(text_weights, *args, **kwargs) def forward( self, diff --git a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py index 55ce331d176c..fce64e1a561c 100644 --- a/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py +++ b/tensorrt_llm/_torch/models/modeling_minimaxm3_vl.py @@ -2071,14 +2071,9 @@ def get_minimax_m3_vl_input_processor_cls(): path. Re-type by dynamic subclassing here so the registered class inherits from the base. """ - from tensorrt_llm.inputs.registry import ( - BaseMultimodalDummyInputsBuilder, - BaseMultimodalInputProcessor, - ) + from tensorrt_llm.inputs.registry import BaseMultimodalInputProcessor - class _Registered( - MiniMaxM3VLInputProcessor, BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder - ): + class _Registered(MiniMaxM3VLInputProcessor, BaseMultimodalInputProcessor): pass _Registered.__name__ = "MiniMaxM3VLInputProcessor" diff --git a/tensorrt_llm/_torch/models/modeling_mistral.py b/tensorrt_llm/_torch/models/modeling_mistral.py index 2246a73f8fdd..b7dcd7806cb2 100644 --- a/tensorrt_llm/_torch/models/modeling_mistral.py +++ b/tensorrt_llm/_torch/models/modeling_mistral.py @@ -669,8 +669,6 @@ class Mistral3VLM(MultimodalModelMixin, PreTrainedModel): `tensorrt_llm/inputs/utils.py`). """ - supports_encoder_cache = True - def __init__( self, model_config: ModelConfig[Mistral3Config], @@ -788,17 +786,9 @@ def multimodal_token_ids(self) -> torch.Tensor: return self._image_token_ids @property - def text_embedding_layer(self) -> Embedding: + def text_embedding_layer(self) -> torch.nn.Module: return self.llm.model.embed_tokens - @property - def embedding_dim(self) -> int: - return self.text_embedding_layer.embedding_dim - - @property - def embedding_dtype(self) -> torch.dtype: - return self.text_embedding_layer.weight.dtype - @property def draft_config(self): return self.llm.draft_config diff --git a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py index b1a2c5df6845..8088e5d89470 100644 --- a/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py +++ b/tensorrt_llm/_torch/models/modeling_multimodal_mixin.py @@ -16,22 +16,10 @@ import contextlib from dataclasses import dataclass -from typing import ( - TYPE_CHECKING, - Any, - ClassVar, - Dict, - Hashable, - Iterable, - Iterator, - Optional, - Sequence, -) +from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, Optional, Sequence import torch -from tensorrt_llm._torch.model_config import ModelConfig -from tensorrt_llm._torch.tensor_lru_cache import TensorLRUCache from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.inputs.multimodal import MultimodalParams, MultimodalRuntimeData from tensorrt_llm.logger import logger @@ -49,7 +37,6 @@ _MM_DATA_INPUT_MODALITY_KEYS = frozenset({"audio", "image", "video"}) _MM_AUX_STREAM: Optional[tuple[int, torch.cuda.Stream]] = None -_MM_ENCODER_CACHE_LOG_NAME = "mm_encoder_cache" def _get_mm_aux_stream(max_prefetch_ahead: int = 0) -> Optional[torch.cuda.Stream]: @@ -118,24 +105,11 @@ class PreparedLlmInputs: class MultimodalModelMixin: """Template-method mixin for PyTorch multimodal causal LM models. - Concrete model forwards can call `prepare_multimodal_inputs` while keeping their explicit - language-model delegation. - - Current limitations: - - * For the time being, the persistent multimodal encoder cache stores per-item embeddings for - single-modality `MultimodalParams` objects. - * Cache reuse is all-or-nothing for each `MultimodalParams` object: every item in that object - hit the cache before cached embeddings are reused. Mixed-modality `MultimodalParams` objects - bypass the persistent cache. + Concrete model forwards can call `prepare_multimodal_inputs` while + keeping their explicit language-model delegation. A future optional + mixin-owned forward can build on the same template method. """ - supports_encoder_cache: ClassVar[bool] = False - """Whether the model's production forward path uses the persistent encoder cache.""" - - model_config: ModelConfig - _multimodal_encoder_cache: Optional[TensorLRUCache] = None - @classmethod def _cast_multimodal_encoder_dtype( cls, @@ -180,16 +154,6 @@ def text_embedding_layer(self): """Return the token embedding layer used by `fuse_input_embeds`.""" raise NotImplementedError - @property - def embedding_dim(self) -> int: - """Return the width of each cached multimodal embedding row.""" - raise NotImplementedError - - @property - def embedding_dtype(self) -> torch.dtype: - """Return the dtype of each cached multimodal embedding row.""" - raise NotImplementedError - def select_multimodal_params( self, multimodal_params: Sequence[MultimodalParams], @@ -240,7 +204,6 @@ def after_active_multimodal_embeddings( # them as extra embeds without changing the base flow. return active_embeddings, () - # A future optional mixin-owned forward can build on the same template method. def prepare_multimodal_inputs( self, *, @@ -311,262 +274,15 @@ def _get_or_encode_multimodal_embeddings( Delegates cache lookup and gather behavior to `get_multimodal_embeddings`, then validates the single tensor contract for both encoded and cached-only paths. """ - encoder_cache = self._get_multimodal_encoder_cache() - cache_misses = [] - if encoder_cache is not None: - for param in multimodal_params: - if param.multimodal_data.get("multimodal_embedding") is not None: - # The forward that attached this request-local embedding already populated the - # persistent cache. - continue - if not self._attach_encoder_cache_hit(param, encoder_cache): - cache_misses.append(param) - embeddings = get_multimodal_embeddings( encoder_forward_fn=self.encode_multimodal_inputs, multimodal_params=list(multimodal_params), ) - if encoder_cache is not None: - for param in cache_misses: - self._write_encoder_cache_entries(param, encoder_cache) - # Validate post-gather so cached-only paths (KV reuse, all-cached chunked prefill) are also # checked, not just paths that ran the encoder. self._validate_embeddings(embeddings, multimodal_params) return embeddings[0] - def _get_multimodal_encoder_cache(self) -> Optional[TensorLRUCache]: - """Return the per-model encoder cache, if enabled. - - The cache stores per-item embeddings for params that can be represented by one modality. - See `_encoder_cache_keys` for the mixed-modality skip path and its technical limitation. - """ - multimodal_config = self.model_config.multimodal_config - if multimodal_config is None: - return None - - max_bytes = multimodal_config.encoder_cache_max_bytes - if max_bytes <= 0: - logger.debug_once( - f"{_MM_ENCODER_CACHE_LOG_NAME}: disabled because " - "multimodal_config.encoder_cache_max_bytes=0.", - key="mm_encoder_cache_disabled", - ) - return None - - if self._multimodal_encoder_cache is None: - # Per-item embeddings are views produced by splitting a request-level encoder output. - # Clone them so a cached item neither aliases mutable caller output nor retains the - # entire batch allocation while cache accounting charges only its logical size. This - # briefly needs source and clone memory during insertion, but preserves existing cache - # entries when the copy cannot be allocated. - self._multimodal_encoder_cache = TensorLRUCache( - max_bytes, - name=_MM_ENCODER_CACHE_LOG_NAME, - ) - try: - embedding_dim = self.embedding_dim - embedding_dtype = self.embedding_dtype - except NotImplementedError: - logger.info( - f"{_MM_ENCODER_CACHE_LOG_NAME}: created with max_bytes={max_bytes}, " - "embedding row capacity unavailable because the model does not implement " - "embedding_dim and embedding_dtype." - ) - else: - bytes_per_embedding_row = ( - embedding_dim * torch.empty((), dtype=embedding_dtype).element_size() - ) - max_embedding_rows = max_bytes // bytes_per_embedding_row - logger.info( - f"{_MM_ENCODER_CACHE_LOG_NAME}: created with max_bytes={max_bytes}, " - f"max_embedding_rows={max_embedding_rows}, embedding_dim={embedding_dim}, " - f"embedding_dtype={embedding_dtype}" - ) - return self._multimodal_encoder_cache - - @staticmethod - def _encoder_cache_modality(param: MultimodalParams) -> Optional[str]: - """Return the single modality represented by `param`, if cacheable. - - `None` means the params either do not identify a modality or contain - multiple modality inputs. The persistent encoder cache deliberately does - not cache mixed-modality params today. - """ - mm_data = param.multimodal_data or {} - modalities = [key for key in _MM_DATA_INPUT_MODALITY_KEYS if key in mm_data] - - modality = mm_data.get("modality_type") - if isinstance(modality, str): - # Trust the explicit `modality_type` only when it agrees with the actual data keys. - # Otherwise fall through to the mixed-modality skip so an inconsistent producer (e.g. - # `modality_type="image"` while both image and audio data are present) cannot bypass the - # safety check below and have the cache serve embeddings for the wrong modality. - if modalities == [modality]: - return modality - - if len(modalities) != 1: - # Mixed-modality params are skipped because the cache key metadata is request-item - # oriented: `multimodal_hashes` and `multimodal_embedding_lengths` are parallel per - # item, but there is no parallel per-item modality list. Without that, a cache key - # cannot unambiguously distinguish, for example, an image item from an audio item inside - # the same params object. - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: skipping params with {len(modalities)} detected " - "modalities." - ) - return None - return modalities[0] - - @classmethod - def _encoder_cache_keys( - cls, - param: MultimodalParams, - ) -> Optional[list[Hashable]]: - """Build per-item encoder cache keys for single-modality params. - - The returned keys split one request's concatenated encoder output by - `multimodal_embedding_lengths`, using the same modality for every item. - - Mixed-modality params are not cacheable until runtime metadata carries a - modality per item alongside `multimodal_hashes` and embedding lengths. - """ - mm_input = param.multimodal_input - mm_data = param.multimodal_data or {} - if mm_input is None: - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: skipping params without multimodal hashes." - ) - return None - - modality = cls._encoder_cache_modality(param) - embedding_lengths = mm_data.get("multimodal_embedding_lengths") - kwargs_hash = mm_data.get("mm_processor_kwargs_hash") - if modality is None or not isinstance(embedding_lengths, list) or kwargs_hash is None: - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: skipping unkeyable params, " - f"has_modality={modality is not None}, " - f"has_embedding_lengths={isinstance(embedding_lengths, list)}, " - f"has_processor_kwargs_hash={kwargs_hash is not None}" - ) - return None - if len(mm_input.multimodal_hashes) != len(embedding_lengths): - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: skipping params with mismatched " - "multimodal_hashes and multimodal_embedding_lengths counts" - ) - return None - - # The item hash, embedding row count, processor kwargs, and modality fully describe a - # reusable item embedding. Request order is excluded so the same item can be reused from a - # different request layout; the current request order is restored when cached item tensors - # are concatenated below. - return [ - ( - modality, - tuple(item_hash), - int(embedding_length), - kwargs_hash, - ) - for item_hash, embedding_length in zip( - mm_input.multimodal_hashes, - embedding_lengths, - strict=True, - ) - ] - - @classmethod - def _attach_encoder_cache_hit( - cls, - param: MultimodalParams, - encoder_cache: TensorLRUCache, - ) -> bool: - """Attach a full persistent-cache hit and report whether one was found.""" - if param.multimodal_data.get("multimodal_embedding") is not None: - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: request-local multimodal embedding present; " - "skipping persistent cache lookup" - ) - return False - - keys = cls._encoder_cache_keys(param) - if not keys: - return False - - cached_embeddings = [] - for key in keys: - cached_embedding = encoder_cache.get(key) - if cached_embedding is None: - # TODO(TRTLLM-13996): allow re-computing only the uncached items. - # `get_multimodal_embeddings` treats a param as either fully cached or uncached. - # Attaching partial hits would make the later concatenated tensor ambiguous because - # there is no placeholder for missing item rows inside `multimodal_embedding`. - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: cache miss; hit_items={len(cached_embeddings)}," - f" total_items={len(keys)}." - ) - return False - cached_embeddings.append(cached_embedding) - - if len(cached_embeddings) == 1: - param.multimodal_data["multimodal_embedding"] = cached_embeddings[0] - else: - param.multimodal_data["multimodal_embedding"] = torch.cat(cached_embeddings, dim=0) - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: full cache hit for {len(keys)} item entries, " - f"rows={param.multimodal_data['multimodal_embedding'].shape[0]}." - ) - return True - - @classmethod - def _write_encoder_cache_entries( - cls, - param: MultimodalParams, - encoder_cache: TensorLRUCache, - ) -> None: - keys = cls._encoder_cache_keys(param) - if not keys: - return - - embedding = param.multimodal_data.get("multimodal_embedding") - if isinstance(embedding, list): - embedding = torch.cat(embedding, dim=0) - param.multimodal_data["multimodal_embedding"] = embedding - if not isinstance(embedding, torch.Tensor): - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: skipping write because no tensor embedding was " - "attached after encoder execution." - ) - return - - embedding_lengths = param.multimodal_data["multimodal_embedding_lengths"] - if sum(embedding_lengths) != embedding.shape[0]: - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: skipping write because embedding row count " - "does not match multimodal_embedding_lengths." - ) - return - - # Encoder outputs are concatenated per params object. Splitting by item length lets future - # requests reuse matching items independently, even when their request-level item order - # differs. - inserted_entries = 0 - rejected_entries = 0 - for key, item_embedding in zip( - keys, - torch.split(embedding, embedding_lengths, dim=0), - strict=True, - ): - if encoder_cache.put(key, item_embedding): - inserted_entries += 1 - else: - rejected_entries += 1 - logger.debug( - f"{_MM_ENCODER_CACHE_LOG_NAME}: wrote {inserted_entries} item entries, " - f"rejected={rejected_entries}, rows={embedding.shape[0]}." - ) - encoder_cache.log_stats("multimodal encoder cache write.") - def _fuse_multimodal_embeddings( self, *, diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h.py b/tensorrt_llm/_torch/models/modeling_nemotron_h.py index 3bb491fd1b91..d931c3889d91 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h.py @@ -553,7 +553,7 @@ def __init__( def post_load_weights(self): """Post-process after loading weights.""" - if self.norm.is_nvfp4 and self.norm.nvfp4_scale is None: + if self.norm.is_nvfp4 and not hasattr(self.norm, "nvfp4_scale"): self._try_attach_nvfp4_scale() def _try_attach_nvfp4_scale(self): @@ -602,7 +602,7 @@ def forward( if hasattr(self, 'pre_allreduce'): norm = self.norm - has_nvfp4_scale = norm.nvfp4_scale is not None + has_nvfp4_scale = hasattr(norm, 'nvfp4_scale') if norm.is_nvfp4 and has_nvfp4_scale and norm.return_hp_output: fusion_op = AllReduceFusionOp.RESIDUAL_RMS_NORM_OUT_QUANT_NVFP4 elif norm.is_nvfp4 and has_nvfp4_scale: @@ -1275,8 +1275,6 @@ def forward( residual=residual, attn_metadata=attn_metadata, all_rank_num_tokens=all_rank_num_tokens, - spec_metadata=spec_metadata, - mamba_metadata=attn_metadata.mamba_metadata, lora_params=lora_params, ) return hidden_states diff --git a/tensorrt_llm/_torch/models/modeling_qwen2vl.py b/tensorrt_llm/_torch/models/modeling_qwen2vl.py index 3e945693902d..0a4107bd3d9b 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen2vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen2vl.py @@ -154,14 +154,7 @@ def _prepare_qwen_vl_mrope_config( if len(delta_tensors) != num_seq_slots: raise RuntimeError( "Missing MRoPE position deltas for seq-slot cache update") - # `delta_tensors` originate from per-request `multimodal_data` and may - # be CPU-resident when the owning model's `multimodal_data_device_paths` - # does not cover `mrope_config.*` (or a path skips the engine's H2D - # move), while the seq-slot cache and `seq_slots` live on the model - # device. `index_copy_` requires all tensors on the same device. - deltas = torch.cat(delta_tensors, - dim=0).to(device=mrope_position_deltas_cache.device, - non_blocking=True) + deltas = torch.cat(delta_tensors, dim=0) mrope_position_deltas_cache.index_copy_(0, seq_slots, deltas) if position_ids is not None \ @@ -1167,10 +1160,7 @@ def apply_rope(self, # uses head_dim=80 (e.g. 1280 hidden / 16 heads), so use PyTorch RoPE. if IS_FLASHINFER_AVAILABLE and self.head_dim % 64 == 0 and position_ids is not None: try: - # flashinfer requires cos_sin_cache in float32; upstream may cache - # cos/sin in the vision tower dtype (e.g. bf16) as a perf hint. - cos_sin_cache = torch.cat([cos, sin], dim=-1).to( - torch.float32).contiguous() + cos_sin_cache = torch.cat([cos, sin], dim=-1).contiguous() flashinfer_apply_rope_with_cos_sin_cache_inplace( position_ids, q, @@ -1180,7 +1170,7 @@ def apply_rope(self, is_neox=True, ) return q, k, v - except (RuntimeError, ValueError) as err: + except RuntimeError as err: logger.warning( "Qwen2.5-VL vision RoPE: FlashInfer failed (%s); " "falling back to PyTorch RotaryEmbedding.apply_rotary_pos_emb.", diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_5.py b/tensorrt_llm/_torch/models/modeling_qwen3_5.py index 6fc43129a353..2137aa7ac6fc 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_5.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_5.py @@ -34,6 +34,7 @@ from ..pyexecutor.config_utils import get_qwen3_hybrid_layer_types from .checkpoints.base_weight_mapper import BaseWeightMapper from .checkpoints.hf.qwen3_5_weight_mapper import Qwen3_5MoeHfWeightMapper +from .modeling_multimodal_utils import _is_mm_disagg from .modeling_qwen3_next import Qwen3NextForCausalLM from .modeling_qwen3vl import ( Qwen3VisionModel, @@ -690,8 +691,7 @@ def multimodal_data_device_paths(self) -> List[str]: ] def load_weights(self, weights: Dict[str, torch.Tensor], weight_mapper: BaseWeightMapper): - # None under MM E/P disagg or disable_mm_encoder. - if self.mm_encoder is not None: + if not _is_mm_disagg(): self.mm_encoder.load_weights(weights) weight_mapper = Qwen3_5MoeHfWeightMapper() diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_next.py b/tensorrt_llm/_torch/models/modeling_qwen3_next.py index 6a8a2a9495c3..0d91b4ebad74 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_next.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_next.py @@ -1,7 +1,5 @@ # Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py # Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/configs/qwen3_next.py -# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 # coding=utf-8 # Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. # @@ -62,58 +60,6 @@ from .modeling_utils import DecoderModel, EagerFusionConfig, register_auto_model -def _fused_norm_weight(norm: RMSNorm) -> torch.Tensor: - """Weight to feed the fused AllReduce+RMSNorm op for ``norm``. - - Gemma RMSNorm scales by ``(1 + weight)`` (see RMSNorm.forward), but the - fused AR+RMSNorm kernels (and the NCCL / NCCL_SYMMETRIC fallbacks the - AUTO strategy may pick) only apply ``weight``. Baking the ``+1`` into the - weight makes EVERY allreduce backend produce the correct gemma result - without any backend-specific flag. - - For gemma norms the ``(1 + weight)`` tensor is precomputed once in - ``Qwen3NextForCausalLM.cache_derived_state`` and cached on the module as - ``_fused_norm_weight``. Computing it inline here would re-run a cast+add - elementwise kernel every forward inside the CUDA graph. The inline path - below is only a correctness fallback if the cache is absent. - """ - cached = getattr(norm, "_fused_norm_weight", None) - if cached is not None: - return cached - w = norm.weight - if getattr(norm, "use_gemma", False): - return (w.float() + 1.0).to(w.dtype) - return w - - -def _precompute_fused_norm_weights(module: nn.Module) -> None: - """Bake ``(1 + weight)`` once for every gemma RMSNorm under ``module``. - - Caches the result on each norm as ``_fused_norm_weight`` so the fused - AllReduce+RMSNorm path reads a ready tensor instead of recomputing the - cast+add every forward. Non-gemma norms are left untouched (the fused op - uses their ``weight`` directly). Must run after weights are loaded onto the - device; the cached tensor is a plain attribute, not a registered buffer, so - it stays out of the state dict. - - Norms whose ``weight`` was stripped are skipped: the layer-wise benchmark - runs ``remove_weights`` on unused layers (``skip_forward``), leaving a - ``use_gemma`` norm without a ``weight`` parameter; those layers never run - the fused path, so there is nothing to precompute. - """ - for norm in module.modules(): - if isinstance(norm, RMSNorm) and getattr(norm, "use_gemma", False): - w = getattr(norm, "weight", None) - if w is None: - continue - norm._fused_norm_weight = (w.float() + 1.0).to(w.dtype) - - -def _eager_fusion_enabled(enable_attention_dp: bool) -> bool: - return (os.environ.get("TRTLLM_QWEN3_EAGER_FUSION_DISABLED", "0") == "0" - and not enable_attention_dp) - - class Qwen3NextGate(nn.Module): def __init__( @@ -420,15 +366,17 @@ def __init__( self.next_layer_layernorm: RMSNorm = None self.fusion_config = EagerFusionConfig() - self.enable_fusion = _eager_fusion_enabled(self.enable_attention_dp) + ### TODO: enable eager_fusion by default + self.enable_fusion = os.environ.get( + "TRTLLM_QWEN3_EAGER_FUSION_DISABLED", "1") == "0" + self.enable_fusion &= not self.enable_attention_dp has_tp = self.mapping.has_tp() has_pp = self.mapping.has_pp() self.fusion_config.PRE_MOE_FUSION = self.enable_fusion and has_tp - self.fusion_config.POST_MOE_FUSION = self.fusion_config.PRE_MOE_FUSION and not has_pp - self.disable_attn_allreduce = (self.fusion_config.PRE_MOE_FUSION - or self.mapping.tp_size == 1 + self.fusion_config.POST_MOE_FUSION = self.fusion_config.PRE_MOE_FUSION and not has_pp and self.enable_attention_dp + self.disable_attn_allreduce = (self.mapping.tp_size == 1 or self.enable_attention_dp) self.moe_allreduce = MoEAllReduce(mapping=model_config.mapping) @@ -465,18 +413,21 @@ def forward( all_reduce_params=AllReduceParams( fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, residual=residual, - norm_weight=_fused_norm_weight( - self.post_attention_layernorm), + norm_weight=self.post_attention_layernorm.weight, eps=self.post_attention_layernorm.variance_epsilon, + enable_allreduce=not self.disable_attn_allreduce, )) else: # No fusion hidden_states, residual = self.post_attention_layernorm( hidden_states, residual) - # Qwen3NextSparseMoeBlock does not implement do_finalize=False. Defer - # only its final all-reduce so the decoder can fuse it with RMSNorm. - do_finalize = True + # Note: this fusion pattern is only supported for TRTLLM-nvfp4 backend now + do_finalize = not (self.fusion_config.POST_MOE_FUSION + and hidden_states.shape[0] + <= self.moe_allreduce.max_token + and self.model_config.moe_backend == 'TRTLLM' + and self.mlp.experts.has_nvfp4) hidden_states = self.mlp( hidden_states, @@ -495,8 +446,7 @@ def forward( all_reduce_params=AllReduceParams( fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, residual=residual, - norm_weight=_fused_norm_weight( - self.next_layer_layernorm), + norm_weight=self.next_layer_layernorm.weight, eps=self.next_layer_layernorm.variance_epsilon, )) else: @@ -539,7 +489,6 @@ def __init__(self, model_config: ModelConfig[Qwen3NextConfig], fuse_qk_norm_rope=fuse_qk_norm_rope, attn_output_gate=True, use_gemma_rms_norm=True) - self._fuse_qk_norm_rope_gate = True class Qwen3NextFullAttentionDecoderLayer(DecoderLayer): @@ -582,24 +531,17 @@ def __init__(self, model_config: ModelConfig[Qwen3NextConfig], self.next_layer_layernorm: RMSNorm = None self.fusion_config = EagerFusionConfig() - self.enable_fusion = _eager_fusion_enabled(self.enable_attention_dp) + self.enable_fusion = os.environ.get( + "TRTLLM_QWEN3_EAGER_FUSION_DISABLED", "0") == "0" + self.enable_fusion &= not self.enable_attention_dp has_tp = self.mapping.has_tp() has_pp = self.mapping.has_pp() self.fusion_config.PRE_MOE_FUSION = self.enable_fusion and has_tp - # POST_MOE_FUSION fuses the MoE-output all-reduce with the next layer's - # RMSNorm. It is a tensor-parallel (TEP) optimization: it is only valid - # when ranks share the same tokens (not attention_dp, where each rank holds - # different tokens and the MoE block does no cross-rank all-reduce). This - # mirrors the DeepSeek-V3 pattern (POST == PRE in the non-attention_dp path). - self.fusion_config.POST_MOE_FUSION = self.fusion_config.PRE_MOE_FUSION and not has_pp - # When PRE_MOE_FUSION is on, the attention all-reduce is deferred to the - # fused PRE all-reduce+RMSNorm, so disable the in-attention all-reduce to - # avoid reducing twice. - self.disable_attn_allreduce = (self.fusion_config.PRE_MOE_FUSION - or self.mapping.tp_size == 1 + self.fusion_config.POST_MOE_FUSION = self.fusion_config.PRE_MOE_FUSION and not has_pp and self.enable_attention_dp + self.disable_attn_allreduce = (self.mapping.tp_size == 1 or self.enable_attention_dp) self.moe_allreduce = MoEAllReduce(mapping=model_config.mapping) @@ -632,14 +574,13 @@ def forward( **kwargs, ) - if self.fusion_config.PRE_MOE_FUSION: + if self.fusion_config.PRE_MOE_FUSION and self.enable_attention_dp: hidden_states, residual = self.allreduce( hidden_states, all_reduce_params=AllReduceParams( fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, residual=residual, - norm_weight=_fused_norm_weight( - self.post_attention_layernorm), + norm_weight=self.post_attention_layernorm.weight, eps=self.post_attention_layernorm.variance_epsilon, )) else: @@ -647,12 +588,12 @@ def forward( hidden_states, residual = self.post_attention_layernorm( hidden_states, residual) - # The fully-fused do_finalize=False MoE path (MoEAllReduce on the - # unfinalized expert output) is not implemented by Qwen3NextSparseMoeBlock - # (it raises NotImplementedError). Keep do_finalize=True so POST_MOE_FUSION - # still fuses the *finalized* MoE all-reduce with the next layer's RMSNorm - # via the do_finalize branch below, without hitting the unimplemented path. - do_finalize = True + # Note: this fusion pattern is only supported for TRTLLM-nvfp4 backend now + do_finalize = not (hidden_states.shape[0] + <= self.moe_allreduce.max_token + and self.fusion_config.POST_MOE_FUSION + and self.model_config.moe_backend == 'TRTLLM' + and self.mlp.experts.has_nvfp4) hidden_states = self.mlp( hidden_states, attn_metadata, @@ -670,8 +611,7 @@ def forward( all_reduce_params=AllReduceParams( fusion_op=AllReduceFusionOp.RESIDUAL_RMS_NORM, residual=residual, - norm_weight=_fused_norm_weight( - self.next_layer_layernorm), + norm_weight=self.next_layer_layernorm.weight, eps=self.next_layer_layernorm.variance_epsilon, )) else: @@ -837,9 +777,6 @@ def __init__(self, model_config: ModelConfig[Qwen3NextConfig], use_cute_dsl_blockscaling_mm=False, ) self.shared_head = Qwen3NextMTPHead(mtp_model_config) - # MTP applies shared_head.norm after the base decoder forward, so its - # MoE-output all-reduce cannot consume next_layer_layernorm. - self.fusion_config.POST_MOE_FUSION = False @staticmethod def _is_mtp_excluded_from_quant( @@ -1076,7 +1013,3 @@ def setup_aliases(self) -> None: else: layer.next_layer_layernorm = self.model.layers[ idx + 1].input_layernorm - - def cache_derived_state(self) -> None: - super().cache_derived_state() - _precompute_fused_norm_weights(self) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3vl.py b/tensorrt_llm/_torch/models/modeling_qwen3vl.py index 2ae083d77355..95da74e5d3bd 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3vl.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3vl.py @@ -1062,27 +1062,6 @@ def load_weights(self, weights: Dict[str, torch.Tensor]): def _parse_and_batch_multimodal_data( self, multimodal_params: List[MultimodalParams] ) -> Tuple[Dict[str, Any], Dict[str, List[Any]]]: - # Only the request itself knows how its image and video items - # interleave in the prompt, so a mixed request must carry a manifest. - for mp in multimodal_params: - data = mp.multimodal_data or {} - if ( - data.get("image") is not None - and data.get("video") is not None - and not mp.mm_item_order - ): - raise ValueError( - "Qwen3-VL mixed-modality requests must carry mm_item_order on MultimodalParams." - ) - - # Any batch that contains both modalities — whether within a single - # request or spread across heterogeneous single-modality requests — - # goes through the modality-blind ViT via one cat'd stream. - has_image = any(mp.multimodal_data.get("image") is not None for mp in multimodal_params) - has_video = any(mp.multimodal_data.get("video") is not None for mp in multimodal_params) - if has_image and has_video: - return self._interleave_multimodal_data(multimodal_params) - pixel_values_list = [] pixel_values_videos_list = [] image_grid_thw_list = [] @@ -1132,71 +1111,19 @@ def _parse_and_batch_multimodal_data( return mm_content_dict, mm_extra_data - def _interleave_multimodal_data( - self, multimodal_params: List[MultimodalParams] - ) -> Tuple[Dict[str, Any], Dict[str, Any]]: - """Build one prompt-order pixel_values + grid_thw for mixed batches. - - The ViT is modality-blind (image = grid_thw row with t=1, video = t>1), - so we cat everything into a single stream in prompt order. Emits only - the "pixel_values" + "image_grid_thw" keys so forward() takes its - image branch uniformly over both modalities. - """ - pixels: List[torch.Tensor] = [] - grids: List[torch.Tensor] = [] - for mp in multimodal_params: - data = mp.multimodal_data - order = mp.mm_item_order or [] - img = data.get("image") or {} - vid = data.get("video") or {} - img_pv, img_thw = img.get("pixel_values"), img.get("image_grid_thw") - vid_pv = vid.get("pixel_values_videos") - vid_thw = vid.get("video_grid_thw") - - # Single-modality request in a mixed batch — no interleave - # needed, just append its rows. - if not order or img_pv is None or vid_pv is None: - if img_pv is not None: - pixels.append(img_pv) - grids.append(img_thw) - if vid_pv is not None: - pixels.append(vid_pv) - grids.append(vid_thw) - continue - - img_off = vid_off = 0 - for entry in order: - modality = entry["modality"] - idx = entry["index"] - if modality == "image": - thw = img_thw[idx] - n = int(thw.prod().item()) - pixels.append(img_pv[img_off : img_off + n]) - img_off += n - elif modality == "video": - thw = vid_thw[idx] - n = int(thw.prod().item()) - pixels.append(vid_pv[vid_off : vid_off + n]) - vid_off += n - else: - raise ValueError(f"Unknown modality in mm_item_order: {modality}") - grids.append(thw.unsqueeze(0)) - - return ( - {"pixel_values": torch.cat(pixels, dim=0)}, - {"image_grid_thw": torch.cat(grids, dim=0)}, - ) - @torch.inference_mode() def forward(self, multimodal_params: List[MultimodalParams]) -> List[torch.Tensor]: mm_content_data, mm_extra_data = self._parse_and_batch_multimodal_data(multimodal_params) pixel_values = mm_content_data.get("pixel_values", None) pixel_values_videos = mm_content_data.get("pixel_values_videos", None) + if pixel_values is not None and pixel_values_videos is not None: + raise ValueError("Currently only support single modality per request") + image_grid_thw = mm_extra_data.get("image_grid_thw", None) video_grid_thw = mm_extra_data.get("video_grid_thw", None) - embeds: List[torch.Tensor] = [] + embeds = [] if pixel_values is not None: pixel_values = pixel_values.to(self.model_dtype) image_embeds, deepstack_image_embeds = self.visual( @@ -1326,18 +1253,11 @@ def __init__( self.llm = AutoModelForCausalLM.from_config(llm_model_config) self.mm_encoder = None - # Normal workers own the encoder. MM E/P handoff uses attached - # embeddings; disable_mm_encoder serves the checkpoint text-only and - # saves the encoder's GPU memory for the KV cache pool. - if not (_is_mm_disagg() or model_config.disable_mm_encoder): + # Normal workers own the encoder. MM E/P handoff uses attached embeddings. + if not _is_mm_disagg(): self.mm_encoder = Qwen3VisionModelBase( copy.deepcopy(model_config), kwargs.get("vision_model_class", None) ).eval() - elif model_config.disable_mm_encoder: - logger.info( - f"{type(self).__name__}: multimodal encoder disabled " - "(disable_mm_encoder=True); serving text-only requests." - ) self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes") self.deepstack_num_level = ( diff --git a/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py b/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py index 061d21bc0df0..92d3cccb5550 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py +++ b/tensorrt_llm/_torch/models/modeling_qwen_image_bench.py @@ -40,16 +40,10 @@ class _QwenImageBenchModelMixin: @property def multimodal_data_device_paths(self) -> List[str]: - # Keep the mrope_config entries in sync with `_Qwen3_5VLModel` - # (modeling_qwen3_5.py): the shared Qwen-VL mRoPE seq-slot cache path - # consumes `mrope_position_deltas` on the model device, so the engine - # must move them H2D along with the rest of the multimodal payload. return [ "image.pixel_values", "video.pixel_values_videos", "multimodal_embedding", - "mrope_config.mrope_position_ids", - "mrope_config.mrope_position_deltas", ] @property diff --git a/tensorrt_llm/_torch/models/modeling_speculative.py b/tensorrt_llm/_torch/models/modeling_speculative.py index a52534de4f50..dd4aa0bb5f30 100755 --- a/tensorrt_llm/_torch/models/modeling_speculative.py +++ b/tensorrt_llm/_torch/models/modeling_speculative.py @@ -777,9 +777,7 @@ def __init__(self, draft_config): draft_config.pretrained_config) # Remove spec_config to prevent recursive spec-dec initialization - draft_config_no_spec = replace(draft_config, - spec_config=None, - lm_head_gather_output=False) + draft_config_no_spec = replace(draft_config, spec_config=None) # Weights will be loaded later by ModelLoader.load_draft_weights() self.draft_model_full = DraftModelClass(draft_config_no_spec) @@ -865,9 +863,7 @@ def __init__(self, draft_config): pretrained_cfg.architectures = original_archs # Remove spec_config to prevent recursive spec-dec initialization - draft_config_no_spec = replace(draft_config, - spec_config=None, - lm_head_gather_output=False) + draft_config_no_spec = replace(draft_config, spec_config=None) # Weights will be loaded later by ModelLoader.load_draft_weights() self.draft_model_full = DraftModelClass(draft_config_no_spec) @@ -1863,27 +1859,7 @@ def get_draft_model(model_config, draft_config, lm_head, model): if any("Laguna" in arch for arch in draft_arches): return DFlashLagunaForCausalLM(draft_config) return DFlashForCausalLM(draft_config) - elif spec_dec_mode.is_dspark(): - # Lazy import to avoid a cycle (modeling_dspark -> modeling_deepseekv4 -> - # modeling_speculative). The DSpark draft reuses the target's aux streams. - # The draft stage count (n_mtp_layers) is not in the HF config, so derive - # it from the checkpoint's mtp.* namespace. - from .modeling_dspark import DSparkForCausalLM, count_dspark_stages - num_stages = count_dspark_stages( - model_config.spec_config.speculative_model) - return DSparkForCausalLM( - draft_config, - getattr(model, "aux_stream_dict", None), - num_stages=num_stages, - block_size=model_config.spec_config.block_size, - ) elif spec_dec_mode.is_draft_target_one_model(): - # Keep the draft LM head vocab-sharded so greedy draft sampling uses the - # lighter TP gather (see SpecWorkerBase.greedy_sample_draft_with_tp_gather). - was_frozen = draft_config._frozen - draft_config._frozen = False - draft_config.lm_head_gather_output = False - draft_config._frozen = was_frozen return AutoModelForCausalLM.from_config(draft_config) else: raise NotImplementedError( @@ -1972,10 +1948,6 @@ def __init__(self, model: TModel, model_config: ModelConfig[TConfig]): model_config.mapping, use_separate_draft_kv_cache=self.use_separate_draft_kv_cache) if self.spec_worker is not None: - # Cache the static draft->target vocab map now that the draft - # model is loaded, so workers read self._d2t instead of probing - # draft_model.model.d2t on every forward. - self.spec_worker.set_draft_model(self.draft_model) self.epilogue.append(self.spec_worker) self.layer_idx = -1 @@ -2071,8 +2043,7 @@ def load_draft_weights(self, if self.spec_config and ( not self.spec_config.spec_dec_mode.is_external_drafter() - or self.spec_config.spec_dec_mode.is_dflash() - or self.spec_config.spec_dec_mode.is_dspark()): + or self.spec_config.spec_dec_mode.is_dflash()): self.draft_model.load_weights_from_target_model(self) def set_guided_decoder(self, diff --git a/tensorrt_llm/_torch/models/modeling_t5.py b/tensorrt_llm/_torch/models/modeling_t5.py index 07891f605af4..5a8a58083346 100644 --- a/tensorrt_llm/_torch/models/modeling_t5.py +++ b/tensorrt_llm/_torch/models/modeling_t5.py @@ -31,7 +31,7 @@ """ import math -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Optional import torch import torch.nn.functional as F @@ -43,7 +43,6 @@ from ..attention_backend import AttentionMetadata from ..attention_backend.interface import PositionalEmbeddingParams, PredefinedAttentionMask -from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE from ..model_config import ModelConfig from ..modules.attention import Attention from ..modules.cross_attention import CrossAttention @@ -113,12 +112,8 @@ def _gelu_new(x: torch.Tensor) -> torch.Tensor: def _t5_gated_act_fn(config: T5Config): act_fn = _t5_dense_act_fn(config) - act_name = getattr(config, "dense_act_fn", None) or "relu" - use_fused_gelu = act_name == "gelu_new" and IS_FLASHINFER_AVAILABLE def gated_act_fn(hidden_states: torch.Tensor) -> torch.Tensor: - if use_fused_gelu and hidden_states.dtype in (torch.float16, torch.bfloat16): - return torch.ops.trtllm.flashinfer_gelu_tanh_and_mul(hidden_states) gate, up = hidden_states.chunk(2, dim=-1) return act_fn(gate) * up @@ -157,36 +152,19 @@ def __init__( super().__init__(hidden_size=hidden_size, eps=eps, dtype=dtype) self._use_hopper_rms_norm: Optional[bool] = None - def forward( - self, - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor] = None, - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: if self._use_hopper_rms_norm is None and hidden_states.is_cuda: sm_version = get_sm_version() self._use_hopper_rms_norm = 90 <= sm_version < 100 - if residual is not None and hidden_states.dtype == torch.float16: - hidden_states = _clamp_fp16_infs(hidden_states + residual) - return self.forward(hidden_states), hidden_states - if self._use_hopper_rms_norm and hidden_states.dtype in (torch.float16, torch.bfloat16): - if residual is None: - return super().forward(hidden_states) - return super().forward(hidden_states, residual) - - if residual is not None: - hidden_states = hidden_states + residual - residual = hidden_states + return super().forward(hidden_states) variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) if self.weight.dtype in (torch.float16, torch.bfloat16): hidden_states = hidden_states.to(self.weight.dtype) - hidden_states = self.weight * hidden_states - if residual is None: - return hidden_states - return hidden_states, residual + return self.weight * hidden_states def _t5_encoder_num_layers(config: T5Config) -> int: @@ -494,14 +472,10 @@ def forward( attn_metadata: AttentionMetadata, position_ids: Optional[torch.IntTensor] = None, position_bias: Optional[torch.Tensor] = None, - residual: Optional[torch.Tensor] = None, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) hidden_states = self.self_attn( position_ids=position_ids, @@ -510,10 +484,16 @@ def forward( attention_mask=PredefinedAttentionMask.FULL, position_bias=position_bias, ) - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = residual + hidden_states + hidden_states = _clamp_fp16_infs(hidden_states) + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + hidden_states = _clamp_fp16_infs(hidden_states) - return hidden_states, residual + return hidden_states # --------------------------------------------------------------------------- @@ -590,15 +570,11 @@ def forward( position_bias: Optional[torch.Tensor] = None, relative_attention_bias: Optional[torch.Tensor] = None, relative_attention_max_distance: int = 0, - residual: Optional[torch.Tensor] = None, **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: # Self-attention (pre-norm) - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) hidden_states = self.self_attn( position_ids=position_ids, hidden_states=hidden_states, @@ -608,9 +584,12 @@ def forward( relative_attention_bias=relative_attention_bias, relative_attention_max_distance=relative_attention_max_distance, ) + hidden_states = residual + hidden_states + hidden_states = _clamp_fp16_infs(hidden_states) # Cross-attention (pre-norm) - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.cross_attn( hidden_states=hidden_states, encoder_hidden_states=encoder_hidden_states, @@ -618,12 +597,17 @@ def forward( cross_attn_metadata=cross_attn_metadata, skip_cross_kv_projection=skip_cross_kv_projection, ) + hidden_states = residual + hidden_states + hidden_states = _clamp_fp16_infs(hidden_states) # MLP (pre-norm) - hidden_states, residual = self.cross_attn_layernorm(hidden_states, residual) + residual = hidden_states + hidden_states = self.cross_attn_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + hidden_states = _clamp_fp16_infs(hidden_states) - return hidden_states, residual + return hidden_states # --------------------------------------------------------------------------- @@ -670,16 +654,14 @@ def forward( seq_len = hidden_states.shape[0] if seq_lens is None else int(seq_lens.max().item()) position_bias = self.relative_position_bias(seq_len, seq_len, hidden_states.device) - residual = None for layer in self.layers: - hidden_states, residual = layer( + hidden_states = layer( hidden_states=hidden_states, attn_metadata=attn_metadata, position_ids=position_ids, position_bias=position_bias, - residual=residual, ) - hidden_states, _ = self.final_layernorm(hidden_states, residual) + hidden_states = self.final_layernorm(hidden_states) return hidden_states @@ -734,9 +716,8 @@ def forward( ) relative_attention_max_distance = self.relative_position_bias.max_distance - residual = None for layer in self.layers: - hidden_states, residual = layer( + hidden_states = layer( position_ids=position_ids, hidden_states=hidden_states, attn_metadata=attn_metadata, @@ -746,9 +727,8 @@ def forward( position_bias=position_bias, relative_attention_bias=relative_attention_bias, relative_attention_max_distance=relative_attention_max_distance, - residual=residual, ) - hidden_states, _ = self.final_layernorm(hidden_states, residual) + hidden_states = self.final_layernorm(hidden_states) return hidden_states diff --git a/tensorrt_llm/_torch/models/modeling_utils.py b/tensorrt_llm/_torch/models/modeling_utils.py index 94d290d4f192..5355bb7d3193 100755 --- a/tensorrt_llm/_torch/models/modeling_utils.py +++ b/tensorrt_llm/_torch/models/modeling_utils.py @@ -1,14 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - import contextlib import inspect import math import os import time from dataclasses import dataclass -from typing import (Any, Dict, Generic, List, Literal, Optional, Tuple, Type, - TypeVar, Union) +from typing import Dict, Generic, List, Optional, Tuple, Type, TypeVar, Union import torch from torch import nn @@ -387,46 +383,11 @@ def __init__(self, model: TModel, *, config: ModelConfig[TConfig], self.pp_size = config.mapping.pp_size self.has_custom_lm_head = False - # Per-layer quant entry for lm_head (e.g. ModelOpt MIXED_PRECISION - # checkpoints that quantize lm_head to NVFP4). Model-specific - # config normalizers opt in by keeping/synthesizing this entry; - # exclude_modules still wins. Applies to both the attention-DP - # (replicated) and TP lm_head below. - lm_head_quant_config = None - if config.quant_config_dict is not None: - lm_head_quant_config = config.quant_config_dict.get("lm_head") - if (lm_head_quant_config is not None - and config.quant_config is not None and config.quant_config. - is_module_excluded_from_quantization("lm_head")): - lm_head_quant_config = None - if (lm_head_quant_config is not None and getattr( - config.pretrained_config, 'tie_word_embeddings', False)): - # Tied embeddings replace lm_head.weight with the dense - # bf16 embedding weight below, which would silently clash - # with a quantized (packed) weight and its quant method. - logger.info( - "Ignoring lm_head quant entry: tie_word_embeddings " - "shares the dense embedding weight, so lm_head stays " - "unquantized") - lm_head_quant_config = None - if (lm_head_quant_config is not None - and config.mapping.enable_attention_dp - and config.mapping.enable_lm_head_tp_in_adp): - # lm_head TP in ADP slices the dense weight at forward time - # for the spec-decoding head (see LMHead.forward), which is - # incompatible with quantized (packed) weights — LMHead - # rejects that combination at construction. - logger.info( - "Ignoring lm_head quant entry: lm_head TP in ADP " - "slices the dense weight, so lm_head stays unquantized") - lm_head_quant_config = None - if config.mapping.enable_attention_dp and not config.mapping.enable_lm_head_tp_in_adp: self.lm_head = LMHead( vocab_size, hidden_size, dtype=config.pretrained_config.torch_dtype, - quant_config=lm_head_quant_config, ) else: if (hasattr(config, 'lora_config') @@ -440,10 +401,29 @@ def __init__(self, model: TModel, *, config: ModelConfig[TConfig], self.has_custom_lm_head = True vocab_size = lora_loader.vocab_size - # A custom LoRA lm_head replaces the checkpoint weight with a - # dense bf16 tensor, so the quant entry must not apply. - if self.has_custom_lm_head: - lm_head_quant_config = None + # Per-layer quant entry for lm_head (e.g. ModelOpt MIXED_PRECISION + # checkpoints that quantize lm_head to NVFP4). Model-specific + # config normalizers opt in by keeping/synthesizing this entry; + # exclude_modules still wins. + lm_head_quant_config = None + if not self.has_custom_lm_head and config.quant_config_dict is not None: + lm_head_quant_config = config.quant_config_dict.get("lm_head") + if (lm_head_quant_config is not None + and config.quant_config is not None + and config.quant_config. + is_module_excluded_from_quantization("lm_head")): + lm_head_quant_config = None + if (lm_head_quant_config is not None + and getattr(config.pretrained_config, + 'tie_word_embeddings', False)): + # Tied embeddings replace lm_head.weight with the dense + # bf16 embedding weight below, which would silently clash + # with a quantized (packed) weight and its quant method. + logger.info( + "Ignoring lm_head quant entry: tie_word_embeddings " + "shares the dense embedding weight, so lm_head stays " + "unquantized") + lm_head_quant_config = None self.lm_head = LMHead( vocab_size, @@ -451,7 +431,7 @@ def __init__(self, model: TModel, *, config: ModelConfig[TConfig], dtype=config.pretrained_config.torch_dtype, mapping=config.mapping, tensor_parallel_mode=TensorParallelMode.COLUMN, - gather_output=config.lm_head_gather_output, + gather_output=True, reduce_output=False, use_custom_cublas_mm=getattr(model, 'use_custom_cublas_mm', False), @@ -625,41 +605,9 @@ def get_model_defaults(cls, llm_args: 'TorchLlmArgs') -> dict: The returned dict is deep-merged with the user's llm_args, with user-set values taking priority over these defaults. - - Note: ``cache_transceiver_config`` is rejected here (enforced at - load time) — the deep-merge could materialize or silently enable a - transceiver config the user did not turn on. Use - :meth:`get_preferred_transceiver_runtime` instead. """ return {} - @classmethod - def get_preferred_transceiver_runtime( - cls, - pretrained_config: Any = None - ) -> Optional[Literal["CPP", "PYTHON"]]: - """Return the model's preferred KV-cache transceiver runtime. - - Subclasses can override this to opt into a specific transceiver - implementation ('CPP' or 'PYTHON') that is adopted when the user - leaves ``cache_transceiver_config.transceiver_runtime`` at its - default 'auto'. Return None to defer to the global default (C++). - - Args: - pretrained_config: the loaded HF pretrained config (may be None - on paths where no config was loaded). Implementation classes - shared by several architectures can inspect e.g. - ``pretrained_config.architectures`` to differentiate per - checkpoint. - - This preference is intentionally kept out of the generic - :meth:`get_model_defaults` deep-merge: it must not materialize a - ``cache_transceiver_config`` when disaggregated serving is disabled, - and it is only honored when the effective backend supports it (the - Python transceiver requires NIXL). - """ - return None - @property def config(self): return self.model_config.pretrained_config @@ -928,7 +876,7 @@ def get_model_architecture( model_config.architectures) > 0: cls = MODEL_CLASS_MAPPING.get(model_config.architectures[0]) else: - raise RuntimeError("Model architecture is not provided.") + raise RuntimeError(f"Model architecture is not provided.") if cls is None: arch = model_config.architectures[0] diff --git a/tensorrt_llm/_torch/modules/attention.py b/tensorrt_llm/_torch/modules/attention.py index e902a892af3e..80e9c67d74d3 100644 --- a/tensorrt_llm/_torch/modules/attention.py +++ b/tensorrt_llm/_torch/modules/attention.py @@ -1,6 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - import math import weakref from typing import List, Optional, Tuple, Union @@ -699,42 +696,6 @@ def split_qkv(self, q, k=None, v=None): q, k, v = q.split([self.q_size, self.kv_size, self.kv_size], dim=-1) return q, k, v - def preprocess_qkv( - self, qkv: torch.Tensor, position_ids: Optional[torch.Tensor] - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], - Optional[torch.Tensor]]: - """Transform the fused QKV projection into attention inputs. - - Splits out the optional attention output gate and applies RoPE (plus - any subclass-specific processing such as QK norm, via apply_rope). - Subclasses may override this to fuse these steps into a single kernel. - - Returns: - tuple: (q, k, v, gate). k and v are None when q holds the fused - QKV tensor; gate is None when attn_output_gate is disabled. - """ - gate = None - if self.attn_output_gate: - q_gate, k, v = qkv.split( - [self.q_size * 2, self.kv_size, self.kv_size], dim=-1) - orig_shape = q_gate.shape[:-1] - # Single line: view -> chunk -> reshape both q and gate - q, gate = [ - t.reshape(*orig_shape, -1) for t in torch.chunk( - q_gate.view(*orig_shape, self.num_heads, -1), 2, dim=-1) - ] - else: - q, k, v = qkv, None, None - q, k, v = self.apply_rope(q, k, v, position_ids) - return q, k, v, gate - - def apply_output_gate(self, attention_output: torch.Tensor, - gate: torch.Tensor) -> torch.Tensor: - """Apply the attention output gate.""" - if gate.shape != attention_output.shape: - gate = gate.reshape(attention_output.shape) - return attention_output * torch.sigmoid(gate) - def convert_qkv(self, q, k, v): if k is None and v is None and not self.support_fused_qkv: q, k, v = self.split_qkv(q) @@ -1022,6 +983,18 @@ def forward( if qkv_lora is not None: qkv = qkv + qkv_lora + if self.attn_output_gate: + q_gate, k, v = qkv.split( + [self.q_size * 2, self.kv_size, self.kv_size], dim=-1) + orig_shape = q_gate.shape[:-1] + # Single line: view -> chunk -> reshape both q and gate + q, gate = [ + t.reshape(*orig_shape, -1) for t in torch.chunk( + q_gate.view(*orig_shape, self.num_heads, -1), 2, dim=-1) + ] + else: + q, k, v = qkv, None, None + # For dynamic tree spec decoding with Python RoPE, adjust position_ids # to use tree offsets (same as C++ kernel: past_seq_len + offset). if (not self.rope_fusion @@ -1035,7 +1008,7 @@ def forward( position_ids = self._adjust_position_ids_for_spec_dec( position_ids, attn_metadata) - q, k, v, gate = self.preprocess_qkv(qkv, position_ids) + q, k, v = self.apply_rope(q, k, v, position_ids) q, k, v = self.convert_qkv(q, k, v) if attention_sinks is not None: @@ -1061,7 +1034,8 @@ def forward( ) if self.attn_output_gate: - attn_output = self.apply_output_gate(attn_output, gate) + gate = torch.sigmoid(gate) + attn_output = attn_output * gate attn_output = _helix_cp_output_projection(self.o_proj, attn_output, attn_metadata, diff --git a/tensorrt_llm/_torch/modules/fla/cached_replay.py b/tensorrt_llm/_torch/modules/fla/cached_replay.py deleted file mode 100644 index 1eba282eaba4..000000000000 --- a/tensorrt_llm/_torch/modules/fla/cached_replay.py +++ /dev/null @@ -1,815 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from typing import Optional - -import torch -import triton -import triton.language as tl -import triton.language.extra.libdevice as tldevice - -from tensorrt_llm._torch.modules.fla.op import exp -from tensorrt_llm._torch.modules.fla.utils import input_guard -from tensorrt_llm._utils import get_sm_version - -_SMALL_GRID_HEAD_TILES = 512 -_EIGHT_WARP_COMMIT_HEAD_TILES = 1024 -_PIPELINED_COMMIT_HEAD_TILES = 2048 -_TWO_STAGE_REPLAY_HEAD_TILES = 4096 -_L2_STREAMING_HEAD_TILES = 8192 - -CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE = 16 - - -@triton.jit -def _gdc_wait_with_memory_clobber(): - tl.inline_asm_elementwise( - "griddepcontrol.wait; // dummy $0", - "=r,~{memory}", - [], - dtype=tl.int32, - is_pure=False, - pack=1, - ) - - -@triton.jit -def _cached_replay_kernel( - q, - k, - v, - packed_qkv, - g, - beta, - o, - h0_source, - h0_indices, - old_u, - old_k, - old_G, - cache_buf_idx, - pnat, - scale, - pool_stride_slot, - A_log, - dt_bias, - T: tl.constexpr, - HIST: tl.constexpr, - BT: tl.constexpr, - BH: tl.constexpr, - H: tl.constexpr, - HV: tl.constexpr, - K: tl.constexpr, - V: tl.constexpr, - BK: tl.constexpr, - BV: tl.constexpr, - USE_QK_L2NORM_IN_KERNEL: tl.constexpr, - STATE_FP32: tl.constexpr, - FUSED_GATING: tl.constexpr, - USE_L2_STATE_CACHE: tl.constexpr, - USE_L2_STREAMING_INPUTS: tl.constexpr, - USE_L2_SHARED_INPUTS: tl.constexpr, - USE_PACKED_QKV: tl.constexpr, - LAUNCH_WITH_PDL: tl.constexpr, - ENABLE_STATE_COMMIT: tl.constexpr, -): - """Replay new tokens from cached causal cached updates. - - old_u/old_k/old_G contain the prefix-invariant cached update vectors, - normalized keys, and cumulative log-decay from the current checkpoint. - Only the T new cached updates are solved on each verify step. - """ - i_v, i_nh = tl.program_id(0), tl.program_id(1) - i_n, i_hv = i_nh // HV, i_nh % HV - i_h = i_hv // (HV // H) - - o_k = tl.arange(0, BK) - o_v = i_v * BV + tl.arange(0, BV) - o_t = tl.arange(0, BT) - o_hist = tl.arange(0, BH) - mask_k = o_k < K - mask_v = o_v < V - mask_t = o_t < T - mask_hist_capacity = o_hist < HIST - - slot = tl.load(h0_indices + i_n) - if slot >= 0: - b_pnat = tl.load(pnat + slot) - b_buf = tl.load(cache_buf_idx + slot) - is_write = (b_pnat + T) > HIST - w_buf = tl.where(is_write, 1 - b_buf, b_buf) - w_off = tl.where(is_write, 0, b_pnat) - is_hist = (o_hist < b_pnat) & mask_hist_capacity - - hk_base = old_k + (slot.to(tl.int64) * 2 + b_buf) * HIST * H * K + i_h * K - hu_base = old_u + (slot.to(tl.int64) * 2 + b_buf) * HIST * HV * V + i_hv * V - hG_base = old_G + ((slot.to(tl.int64) * 2 + b_buf) * HV + i_hv) * HIST - wk_base = old_k + (slot.to(tl.int64) * 2 + w_buf) * HIST * H * K + i_h * K - wu_base = old_u + (slot.to(tl.int64) * 2 + w_buf) * HIST * HV * V + i_hv * V - wG_base = old_G + ((slot.to(tl.int64) * 2 + w_buf) * HV + i_hv) * HIST - - if USE_L2_SHARED_INPUTS: - b_kh = tl.load( - hk_base + o_hist[:, None] * H * K + o_k[None, :], - mask=is_hist[:, None] & mask_k[None, :], - other=0, - cache_modifier=".cg", - ) - else: - b_kh = tl.load( - hk_base + o_hist[:, None] * H * K + o_k[None, :], - mask=is_hist[:, None] & mask_k[None, :], - other=0, - ) - if USE_L2_STREAMING_INPUTS: - b_uh = tl.load( - hu_base + o_hist[:, None] * HV * V + o_v[None, :], - mask=is_hist[:, None] & mask_v[None, :], - other=0, - cache_modifier=".cg", - ) - else: - b_uh = tl.load( - hu_base + o_hist[:, None] * HV * V + o_v[None, :], - mask=is_hist[:, None] & mask_v[None, :], - other=0, - ) - b_Gh = tl.load(hG_base + o_hist, mask=is_hist, other=0.0).to(tl.float32) - g_start = tl.load( - hG_base + b_pnat - 1, - mask=b_pnat > 0, - other=0.0, - ).to(tl.float32) - - if LAUNCH_WITH_PDL: - _gdc_wait_with_memory_clobber() - - if USE_PACKED_QKV: - qkv_row = (i_n * T + o_t[:, None]) * (2 * H * K + HV * V) - p_k = packed_qkv + qkv_row + H * K + i_h * K + o_k[None, :] - p_q = packed_qkv + qkv_row + i_h * K + o_k[None, :] - p_v = packed_qkv + qkv_row + 2 * H * K + i_hv * V + o_v[None, :] - else: - p_k = k + ((i_n * T + o_t[:, None]) * H + i_h) * K + o_k[None, :] - p_q = q + ((i_n * T + o_t[:, None]) * H + i_h) * K + o_k[None, :] - p_v = v + ((i_n * T + o_t[:, None]) * HV + i_hv) * V + o_v[None, :] - - if USE_L2_SHARED_INPUTS: - b_k = tl.load( - p_k, - mask=mask_t[:, None] & mask_k[None, :], - other=0, - cache_modifier=".cg", - ) - b_q = tl.load( - p_q, - mask=mask_t[:, None] & mask_k[None, :], - other=0, - cache_modifier=".cg", - ) - else: - b_k = tl.load( - p_k, - mask=mask_t[:, None] & mask_k[None, :], - other=0, - ) - b_q = tl.load( - p_q, - mask=mask_t[:, None] & mask_k[None, :], - other=0, - ) - if USE_L2_STREAMING_INPUTS: - b_v = tl.load( - p_v, - mask=mask_t[:, None] & mask_v[None, :], - other=0, - cache_modifier=".cg", - ) - else: - b_v = tl.load( - p_v, - mask=mask_t[:, None] & mask_v[None, :], - other=0, - ) - b_g = tl.load( - g + (i_n * T + o_t) * HV + i_hv, - mask=mask_t, - other=0.0, - ).to(tl.float32) - b_beta = tl.load( - beta + (i_n * T + o_t) * HV + i_hv, - mask=mask_t, - other=0.0, - ).to(tl.float32) - if FUSED_GATING: - g_A_exp = tl.exp(tl.load(A_log + i_hv).to(tl.float32)) - g_dt_bias = tl.load(dt_bias + i_hv).to(tl.float32) - x = b_g + g_dt_bias - softplus = tl.where( - x <= 20.0, - 0.6931471805599453 * tldevice.fast_log2f(1.0 + tldevice.fast_expf(x)), - x, - ) - b_g = tl.where(mask_t, -g_A_exp * softplus, 0.0) - b_beta = tl.where( - mask_t, - tldevice.fast_dividef(1.0, 1.0 + tldevice.fast_expf(-b_beta)), - 0.0, - ) - - if USE_QK_L2NORM_IN_KERNEL: - b_kf = b_k.to(tl.float32) - b_qf = b_q.to(tl.float32) - inv_k = 1.0 / (tl.sqrt(tl.sum(b_kf * b_kf, 1)) + 1e-6) - inv_q = scale / (tl.sqrt(tl.sum(b_qf * b_qf, 1)) + 1e-6) - b_kn = (b_kf * inv_k[:, None]).to(b_k.dtype) - b_qn = (b_qf * inv_q[:, None]).to(b_q.dtype) - else: - b_kn = b_k - b_qn = (b_q.to(tl.float32) * scale).to(b_q.dtype) - - b_G_local = tl.cumsum(b_g, 0) - b_G = g_start + b_G_local - - p_h0 = ( - h0_source - + slot.to(tl.int64) * pool_stride_slot - + i_hv * V * K - + o_k[:, None] - + o_v[None, :] * K - ) - if STATE_FP32: - if USE_L2_STATE_CACHE: - b_h = tl.load( - p_h0, - mask=mask_k[:, None] & mask_v[None, :], - other=0, - cache_modifier=".cg", - ) - else: - b_h = tl.load( - p_h0, - mask=mask_k[:, None] & mask_v[None, :], - other=0, - ) - b_h_hi = b_h.to(b_kn.dtype) - b_h_lo = (b_h - b_h_hi.to(tl.float32)).to(b_kn.dtype) - b_kh0 = tl.dot(b_kn, b_h_hi) + tl.dot(b_kn, b_h_lo) - b_qh0 = tl.dot(b_qn, b_h_hi) + tl.dot(b_qn, b_h_lo) - else: - if USE_L2_STATE_CACHE: - b_h = tl.load( - p_h0, - mask=mask_k[:, None] & mask_v[None, :], - other=0, - cache_modifier=".cg", - ).to(b_kn.dtype) - else: - b_h = tl.load( - p_h0, - mask=mask_k[:, None] & mask_v[None, :], - other=0, - ).to(b_kn.dtype) - b_kh0 = tl.dot(b_kn, b_h) - b_qh0 = tl.dot(b_qn, b_h) - - hist_decay = exp(b_G[:, None] - b_Gh[None, :]) - b_kk_hist = tl.dot(b_kn, tl.trans(b_kh)) - b_qk_hist = tl.dot(b_qn, tl.trans(b_kh)) - b_k_hist_coeff = tl.where(is_hist[None, :], b_kk_hist * hist_decay, 0.0) - b_q_hist_coeff = tl.where(is_hist[None, :], b_qk_hist * hist_decay, 0.0) - b_k_hist = tl.dot(b_k_hist_coeff.to(b_uh.dtype), b_uh) - b_q_hist = tl.dot(b_q_hist_coeff.to(b_uh.dtype), b_uh) - - b_rhs = b_beta[:, None] * (b_v.to(tl.float32) - exp(b_G)[:, None] * b_kh0 - b_k_hist) - lower = o_t[:, None] > o_t[None, :] - b_kk_new = tl.dot(b_kn, tl.trans(b_kn)) - new_decay = exp(b_G[:, None] - b_G[None, :]) - b_A = tl.where( - lower, - b_beta[:, None] * b_kk_new * new_decay, - 0.0, - ) - - # T is at most eight for the cached replay path. Forward substitution - # avoids constructing/inverting the full HIST+T triangular system. - b_U = tl.zeros([BT, BV], dtype=tl.float32) - for row in range(T): - row_mask = o_t == row - rhs_row = tl.sum(tl.where(row_mask[:, None], b_rhs, 0.0), axis=0) - a_row = tl.sum(tl.where(row_mask[:, None], b_A, 0.0), axis=0) - correction = tl.sum(a_row[:, None] * b_U, axis=0) - u_row = rhs_row - correction - b_U += tl.where(row_mask[:, None], u_row[None, :], 0.0) - - incl = o_t[:, None] >= o_t[None, :] - b_qk_new = tl.dot(b_qn, tl.trans(b_kn)) - b_q_new_coeff = tl.where(incl, b_qk_new * new_decay, 0.0) - # BT is only four for MTP draft-3 verification, while tl.dot requires - # a reduction dimension of at least 16 on this architecture. Keep - # this genuinely small operation scalar instead of padding it into a - # mostly-empty tensor-core GEMM. - b_q_new = tl.zeros([BT, BV], dtype=tl.float32) - for row in range(T): - row_mask = o_t == row - coeff_row = tl.sum(tl.where(row_mask[:, None], b_q_new_coeff, 0.0), axis=0) - q_new_row = tl.sum(coeff_row[:, None] * b_U, axis=0) - b_q_new += tl.where(row_mask[:, None], q_new_row[None, :], 0.0) - b_o = exp(b_G)[:, None] * b_qh0 + b_q_hist + b_q_new - p_o = o + ((i_n * T + o_t[:, None]) * HV + i_hv) * V + o_v[None, :] - tl.store( - p_o, - b_o.to(p_o.dtype.element_ty), - mask=mask_t[:, None] & mask_v[None, :], - ) - - write_pos = w_off + o_t - tl.store( - wu_base + write_pos[:, None] * HV * V + o_v[None, :], - b_U.to(wu_base.dtype.element_ty), - mask=mask_t[:, None] & mask_v[None, :], - ) - if i_v == 0: - stored_G = tl.where(is_write, b_G_local, b_G) - tl.store(wG_base + write_pos, stored_G, mask=mask_t) - if i_hv % (HV // H) == 0: - tl.store( - wk_base + write_pos[:, None] * H * K + o_k[None, :], - b_kn.to(wk_base.dtype.element_ty), - mask=mask_t[:, None] & mask_k[None, :], - ) - - if ENABLE_STATE_COMMIT: - if is_write: - commit_decay = exp(g_start - b_Gh) - b_Uc = b_uh.to(tl.float32) * tl.where(is_hist, commit_decay, 0.0)[:, None] - b_Uc_hi = b_Uc.to(b_kh.dtype) - b_Uc_lo = (b_Uc - b_Uc_hi.to(tl.float32)).to(b_kh.dtype) - b_hc = ( - b_h.to(tl.float32) * exp(g_start) - + tl.dot(tl.trans(b_kh), b_Uc_hi) - + tl.dot(tl.trans(b_kh), b_Uc_lo) - ) - tl.store( - p_h0, - b_hc.to(p_h0.dtype.element_ty), - mask=mask_k[:, None] & mask_v[None, :], - ) - - -@triton.jit -def _cached_replay_layered_commit_kernel( - h0_source, - old_u, - old_k, - old_G, - replay_work_items, - n_writes, - pool_stride_layer, - pool_stride_slot, - old_u_stride_layer, - old_k_stride_layer, - old_G_stride_layer, - HIST: tl.constexpr, - BH: tl.constexpr, - NUM_LAYERS: tl.constexpr, - H: tl.constexpr, - HV: tl.constexpr, - K: tl.constexpr, - V: tl.constexpr, - BK: tl.constexpr, - BV: tl.constexpr, - NV: tl.constexpr, - NUM_PERSISTENT: tl.constexpr, - USE_L2_STATE_CACHE: tl.constexpr, - USE_L2_STREAMING_INPUTS: tl.constexpr, - USE_L2_SHARED_INPUTS: tl.constexpr, - PIPE_STAGES: tl.constexpr, -): - """Advance every local GDN layer from one cached-history snapshot.""" - pid = tl.program_id(0) - total_work = tl.load(n_writes) * NUM_LAYERS * HV * NV - for tile_id in tl.range( - pid, - total_work, - NUM_PERSISTENT, - num_stages=PIPE_STAGES, - flatten=True, - ): - i_v = tile_id % NV - tile_id = tile_id // NV - i_hv = tile_id % HV - tile_id = tile_id // HV - layer = tile_id % NUM_LAYERS - work_idx = tile_id // NUM_LAYERS - i_h = i_hv // (HV // H) - - work_base = replay_work_items + work_idx * 4 - slot = tl.load(work_base + 1).to(tl.int64) - b_pnat = tl.load(work_base + 2) - b_buf = tl.load(work_base + 3) - layer_i64 = layer.to(tl.int64) - - o_k = tl.arange(0, BK) - o_v = i_v * BV + tl.arange(0, BV) - o_hist = tl.arange(0, BH) - mask_k = o_k < K - mask_v = o_v < V - is_hist = (o_hist < b_pnat) & (o_hist < HIST) - - hk_base = ( - old_k + layer_i64 * old_k_stride_layer + (slot * 2 + b_buf) * HIST * H * K + i_h * K - ) - hu_base = ( - old_u + layer_i64 * old_u_stride_layer + (slot * 2 + b_buf) * HIST * HV * V + i_hv * V - ) - hG_base = old_G + layer_i64 * old_G_stride_layer + ((slot * 2 + b_buf) * HV + i_hv) * HIST - - if USE_L2_SHARED_INPUTS: - b_kh = tl.load( - hk_base + o_hist[:, None] * H * K + o_k[None, :], - mask=is_hist[:, None] & mask_k[None, :], - other=0, - cache_modifier=".cg", - ) - else: - b_kh = tl.load( - hk_base + o_hist[:, None] * H * K + o_k[None, :], - mask=is_hist[:, None] & mask_k[None, :], - other=0, - ) - if USE_L2_STREAMING_INPUTS: - b_uh = tl.load( - hu_base + o_hist[:, None] * HV * V + o_v[None, :], - mask=is_hist[:, None] & mask_v[None, :], - other=0, - cache_modifier=".cg", - ) - else: - b_uh = tl.load( - hu_base + o_hist[:, None] * HV * V + o_v[None, :], - mask=is_hist[:, None] & mask_v[None, :], - other=0, - ) - b_Gh = tl.load(hG_base + o_hist, mask=is_hist, other=0.0).to(tl.float32) - g_start = tl.load(hG_base + b_pnat - 1).to(tl.float32) - - p_h0 = ( - h0_source - + layer_i64 * pool_stride_layer - + slot * pool_stride_slot - + i_hv * V * K - + o_k[:, None] - + o_v[None, :] * K - ) - if USE_L2_STATE_CACHE: - b_h = tl.load( - p_h0, - mask=mask_k[:, None] & mask_v[None, :], - other=0, - cache_modifier=".cg", - ) - else: - b_h = tl.load( - p_h0, - mask=mask_k[:, None] & mask_v[None, :], - other=0, - ) - commit_decay = exp(g_start - b_Gh) - b_Uc = b_uh.to(tl.float32) * tl.where(is_hist, commit_decay, 0.0)[:, None] - b_Uc_hi = b_Uc.to(b_kh.dtype) - b_Uc_lo = (b_Uc - b_Uc_hi.to(tl.float32)).to(b_kh.dtype) - b_hc = ( - b_h.to(tl.float32) * exp(g_start) - + tl.dot(tl.trans(b_kh), b_Uc_hi) - + tl.dot(tl.trans(b_kh), b_Uc_lo) - ) - tl.store( - p_h0, - b_hc.to(p_h0.dtype.element_ty), - mask=mask_k[:, None] & mask_v[None, :], - ) - - -def commit_gdn_cached_replay_history_layers( - *, - ssm_states: torch.Tensor, - old_u: torch.Tensor, - old_k: torch.Tensor, - old_G: torch.Tensor, - replay_work_items: torch.Tensor, - n_writes: torch.Tensor, - history_size: int, - persistent_waves: int = 2, - commit_block_v: Optional[int] = None, - commit_num_warps: Optional[int] = None, - commit_pipeline_stages: Optional[int] = None, -) -> None: - """Advance all local layer checkpoints from cached replay histories.""" - num_layers, _, HV, V, K = ssm_states.shape - assert old_u.ndim == 6 and old_k.ndim == 6 and old_G.ndim == 5 - H = old_k.shape[-2] - assert old_u.shape[0] == num_layers and old_u.shape[-2:] == (HV, V) - assert old_k.shape[0] == num_layers and old_k.shape[-1] == K - assert old_G.shape[0] == num_layers and old_G.shape[-2:] == (HV, history_size) - assert old_u.is_contiguous() and old_k.is_contiguous() and old_G.is_contiguous() - assert replay_work_items.ndim == 2 and replay_work_items.shape[1] == 4 - assert replay_work_items.dtype == torch.int32 - assert n_writes.dtype == torch.int32 and n_writes.numel() == 1 - assert persistent_waves > 0 - - N = replay_work_items.shape[0] - if N == 0 or num_layers == 0: - return - BK = triton.next_power_of_2(K) - BH = triton.next_power_of_2(history_size) - assert BK == K and BH <= 16 - per_layer_head_tiles = N * HV - use_tuned_bf16_mapping = ( - history_size <= 16 - and HV == 4 * H - and K == 128 - and V == 128 - and ssm_states.dtype == torch.bfloat16 - ) - use_small_grid_mapping = ( - use_tuned_bf16_mapping and per_layer_head_tiles <= _SMALL_GRID_HEAD_TILES - ) - if commit_block_v is None: - commit_block_v = 64 if use_small_grid_mapping else triton.next_power_of_2(V) - if commit_num_warps is None: - commit_num_warps = ( - 8 - if use_tuned_bf16_mapping - and per_layer_head_tiles >= _EIGHT_WARP_COMMIT_HEAD_TILES - and commit_block_v == 128 - else 2 - if use_tuned_bf16_mapping - else 4 - ) - use_large_workload_mapping = ( - per_layer_head_tiles >= _L2_STREAMING_HEAD_TILES - if use_tuned_bf16_mapping - else N >= 128 and ssm_states.dtype == torch.bfloat16 - ) - if commit_pipeline_stages is None: - commit_pipeline_stages = ( - 5 - if use_tuned_bf16_mapping and per_layer_head_tiles >= _PIPELINED_COMMIT_HEAD_TILES - else 5 - if not use_tuned_bf16_mapping and use_large_workload_mapping - else 1 - ) - assert triton.next_power_of_2(commit_block_v) == commit_block_v - assert commit_block_v <= V and commit_pipeline_stages > 0 - - commit_nv = triton.cdiv(V, commit_block_v) - num_sms = torch.cuda.get_device_properties(ssm_states.device).multi_processor_count - total_tiles = N * num_layers * HV * commit_nv - num_persistent = min(num_sms * persistent_waves, total_tiles) - _cached_replay_layered_commit_kernel[(num_persistent,)]( - h0_source=ssm_states, - old_u=old_u, - old_k=old_k, - old_G=old_G, - replay_work_items=replay_work_items, - n_writes=n_writes, - pool_stride_layer=ssm_states.stride(0), - pool_stride_slot=ssm_states.stride(1), - old_u_stride_layer=old_u.stride(0), - old_k_stride_layer=old_k.stride(0), - old_G_stride_layer=old_G.stride(0), - HIST=history_size, - BH=BH, - NUM_LAYERS=num_layers, - H=H, - HV=HV, - K=K, - V=V, - BK=BK, - BV=commit_block_v, - NV=commit_nv, - NUM_PERSISTENT=num_persistent, - USE_L2_STATE_CACHE=use_large_workload_mapping, - USE_L2_STREAMING_INPUTS=use_large_workload_mapping, - USE_L2_SHARED_INPUTS=use_large_workload_mapping, - PIPE_STAGES=commit_pipeline_stages, - num_warps=commit_num_warps, - num_stages=commit_pipeline_stages, - ) - - -@input_guard( - exclude_args=[ - "q", - "k", - "v", - "packed_qkv", - "ssm_states", - "old_u", - "old_k", - "old_G", - "old_beta", - "cache_buf_idx", - "prev_num_accepted_tokens", - "replay_work_items", - "n_writes", - "output", - ] -) -def fused_recurrent_gated_delta_rule_cached_replay_update( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - g: torch.Tensor, - beta: torch.Tensor, - ssm_states: torch.Tensor, - state_indices: torch.Tensor, - old_u: torch.Tensor, - old_k: torch.Tensor, - old_G: torch.Tensor, - old_beta: torch.Tensor, - cache_buf_idx: torch.Tensor, - prev_num_accepted_tokens: torch.Tensor, - history_size: int, - scale: Optional[float] = None, - use_qk_l2norm_in_kernel: bool = False, - A_log: Optional[torch.Tensor] = None, - dt_bias: Optional[torch.Tensor] = None, - launch_with_pdl: bool = False, - replay_work_items: Optional[torch.Tensor] = None, - n_writes: Optional[torch.Tensor] = None, - block_v: Optional[int] = None, - num_warps: Optional[int] = None, - use_l2_state_cache: Optional[bool] = None, - use_l2_streaming_inputs: Optional[bool] = None, - use_l2_shared_inputs: Optional[bool] = None, - main_num_stages: Optional[int] = None, - packed_qkv: Optional[torch.Tensor] = None, - use_all_layer_commit: bool = False, - output: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """GDN replay using cached causal updates rather than raw history.""" - del old_beta # Signature-compatible placeholder; cached updates embed beta. - N, T, H, K = k.shape - HV, V = v.shape[2], v.shape[3] - assert q.shape == k.shape - assert v.shape[:2] == (N, T) - use_packed_qkv = packed_qkv is not None - if use_packed_qkv: - qkv_width = 2 * H * K + HV * V - assert packed_qkv is not None - assert packed_qkv.shape == (N * T, qkv_width) - assert packed_qkv.dtype == q.dtype - assert packed_qkv.device == q.device - assert packed_qkv.is_contiguous() - else: - q = q.contiguous() - k = k.contiguous() - v = v.contiguous() - packed_qkv = q - BK = triton.next_power_of_2(K) - # GB200 dispatch for the production Qwen3.5 MTP per-CTA shape. Balanced - # DEP and TEP runs at the same global batch have the same N * HV head-tile - # count, so use that workload measure instead of topology-specific H/HV - # values or the per-rank batch alone. - use_tuned_bf16_mapping = ( - T == 4 - and history_size <= 16 - and HV == 4 * H - and K == 128 - and V == 128 - and ssm_states.dtype == torch.bfloat16 - ) - head_tiles = N * HV - use_small_grid_mapping = use_tuned_bf16_mapping and head_tiles <= _SMALL_GRID_HEAD_TILES - if block_v is None: - block_v = 64 if use_small_grid_mapping else triton.next_power_of_2(V) - if num_warps is None: - num_warps = ( - 2 if use_tuned_bf16_mapping and (use_small_grid_mapping or launch_with_pdl) else 4 - ) - BV = block_v - use_large_workload_mapping = ( - head_tiles >= _L2_STREAMING_HEAD_TILES - if use_tuned_bf16_mapping - else N >= 128 and ssm_states.dtype == torch.bfloat16 - ) - if use_l2_state_cache is None: - use_l2_state_cache = use_large_workload_mapping - if use_l2_streaming_inputs is None: - use_l2_streaming_inputs = use_large_workload_mapping - if use_l2_shared_inputs is None: - use_l2_shared_inputs = use_large_workload_mapping - if main_num_stages is None: - if use_tuned_bf16_mapping: - main_num_stages = 2 if head_tiles >= _TWO_STAGE_REPLAY_HEAD_TILES else 1 - else: - main_num_stages = 3 if use_large_workload_mapping else 1 - BT = triton.next_power_of_2(T) - BH = triton.next_power_of_2(history_size) - assert BK == K and triton.next_power_of_2(BV) == BV and BV <= V - assert main_num_stages > 0 - assert T <= 8 - assert history_size >= T - assert BH <= 16 - if scale is None: - scale = K**-0.5 - - fused_gating = A_log is not None - if fused_gating: - assert dt_bias is not None - assert A_log.numel() == HV and dt_bias.numel() == HV - else: - A_log = q - dt_bias = q - - if launch_with_pdl and get_sm_version() < 90: - launch_with_pdl = False - - s_h0_0, s_h0_1, s_h0_2, s_h0_3 = ssm_states.stride() - assert s_h0_3 == 1 and s_h0_2 == K and s_h0_1 == V * K - for name, tensor in (("old_u", old_u), ("old_k", old_k), ("old_G", old_G)): - assert tensor.is_contiguous(), f"{name} must be contiguous" - if (replay_work_items is None) != (n_writes is None): - raise ValueError("replay_work_items and n_writes must either both be set or both be None") - if replay_work_items is not None: - assert replay_work_items.shape == (N, 4) - assert replay_work_items.dtype == torch.int32 - assert replay_work_items.is_contiguous() - assert n_writes is not None and n_writes.dtype == torch.int32 - assert n_writes.numel() == 1 - if not use_all_layer_commit: - raise ValueError("Partitioned replay requires the all-layer commit") - elif use_all_layer_commit: - raise ValueError("use_all_layer_commit requires replay work items") - - if output is None: - output = q.new_empty(N, T, HV, V) - else: - assert output.is_contiguous(), "output must be contiguous" - NV = triton.cdiv(V, BV) - grid = (NV, N * HV) - - def launch( - *, - enable_state_commit: bool, - use_pdl: bool, - ): - _cached_replay_kernel[grid]( - q=q, - k=k, - v=v, - packed_qkv=packed_qkv, - g=g, - beta=beta, - o=output, - h0_source=ssm_states, - h0_indices=state_indices, - old_u=old_u, - old_k=old_k, - old_G=old_G, - cache_buf_idx=cache_buf_idx, - pnat=prev_num_accepted_tokens, - scale=scale, - pool_stride_slot=s_h0_0, - A_log=A_log, - dt_bias=dt_bias, - T=T, - HIST=history_size, - BT=BT, - BH=BH, - H=H, - HV=HV, - K=K, - V=V, - BK=BK, - BV=BV, - USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, - STATE_FP32=ssm_states.dtype == torch.float32, - FUSED_GATING=fused_gating, - USE_L2_STATE_CACHE=use_l2_state_cache, - USE_L2_STREAMING_INPUTS=use_l2_streaming_inputs, - USE_L2_SHARED_INPUTS=use_l2_shared_inputs, - USE_PACKED_QKV=use_packed_qkv, - LAUNCH_WITH_PDL=use_pdl, - ENABLE_STATE_COMMIT=enable_state_commit, - num_warps=num_warps, - num_stages=main_num_stages, - launch_pdl=use_pdl, - ) - - if replay_work_items is None: - launch( - enable_state_commit=True, - use_pdl=launch_with_pdl, - ) - else: - # Large batches keep replay free of the checkpoint-state expression. - # The cache manager advances every local GDN layer in one launch after - # all layers have populated their history caches. - launch( - enable_state_commit=False, - use_pdl=launch_with_pdl, - ) - return output diff --git a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py index e819f1690728..f9f21a911a95 100644 --- a/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py +++ b/tensorrt_llm/_torch/modules/fla/flashinfer_chunk.py @@ -16,13 +16,10 @@ (the FlashInfer prefill kernel does NOT apply L2 norm internally; the ``use_qk_l2norm_in_kernel`` parameter on ``flashinfer.chunk_gated_delta_rule`` is currently a dead arg, see ``flashinfer/gdn_prefill.py:317-356``). - * Pre-gather and post-scatter of indexed SSM state into FlashInfer's packed - ``[num_seqs, H, V, K]`` layout. TRT-LLM's GDN state pool uses the same - ``[N, H, V, K]`` logical layout, so the adapter gathers/scatters without - transposing the last two dims. The SM100/SM103 kernel carries the recurrent - state in fp32 in TMEM regardless of the initial/output-state I/O dtype, so - the round-trip stays in the native pool dtype (bf16/fp16) there with no - precision change; only SM90/SM120 need an fp32 up-cast/down-cast. + * Pre-gather and post-scatter of indexed SSM state (FlashInfer requires + packed ``[num_seqs, H, V, K]`` fp32 initial/output state). TRT-LLM's GDN + state pool uses the same ``[N, H, V, K]`` logical layout, so the adapter + casts/gathers/scatters without transposing the last two dims. This module is only imported when ``TLLM_USE_FLASHINFER_GDN_PREFILL=1`` is set at process start; do not import it lazily inside hot paths. @@ -37,7 +34,6 @@ gather_cast_vk_to_fp32_vk, ) from tensorrt_llm._torch.modules.fla.l2norm import l2norm_fwd -from tensorrt_llm._utils import is_sm_100f # Mirror the @torch.compiler.disable on the legacy Triton wrapper @@ -107,18 +103,10 @@ def chunk_gated_delta_rule( q3 = l2norm_fwd(q3) k3 = l2norm_fwd(k3) - # --- Step 4: gather initial state (+ cast dtype only when required) --- + # --- Step 4: gather initial state and cast dtype --------------------- # TRT-LLM's GDN kernels and FlashInfer both use [N, H, V, K] state layout. - # The SM100/SM103 kernel carries the recurrent state in fp32 in TMEM - # regardless of the initial/output-state I/O dtype (the state tensors are - # only the gmem load/store format), so passing bf16/fp16 state there is - # numerically identical to the fp32 round-trip while moving half the bytes. - # SM90/SM120 still require fp32 state. Fuse gather (+ optional cast) and - # contiguous into a single Triton kernel. - state_dtype = initial_state.dtype if is_sm_100f() else torch.float32 - gathered_init = gather_cast_vk_to_fp32_vk( - initial_state, initial_state_indices, out_dtype=state_dtype - ) + # Fuse gather + cast-to-fp32 + contiguous into a single Triton kernel. + gathered_init = gather_cast_vk_to_fp32_vk(initial_state, initial_state_indices) # --- Step 5+6: call FlashInfer with pre-allocated output/state buffers # FI 0.6.10 accepts `output=` / `output_state=`; pre-allocating skips its @@ -138,10 +126,7 @@ def chunk_gated_delta_rule( ) if need_state: num_seqs = cu_seqlens.shape[0] - 1 - # Match the initial-state dtype (native bf16/fp16 on SM100/SM103, else - # fp32); FlashInfer writes the final state in this dtype and the scatter - # below adapts to the destination pool dtype without an extra cast. - state_buf = q3.new_empty(num_seqs, num_o_heads, head_size, head_size, dtype=state_dtype) + state_buf = q3.new_empty(num_seqs, num_o_heads, head_size, head_size, dtype=torch.float32) out_packed, out_state = flashinfer.chunk_gated_delta_rule( q=q3, k=k3, @@ -173,9 +158,8 @@ def chunk_gated_delta_rule( ) out_state = None - # --- Step 7: cast state back (if needed), scatter / return --------- - # Fuse cast (out_state.dtype -> destination dtype; a no-op on SM100/SM103 - # where both are the native pool dtype) + optional indexed scatter into a + # --- Step 7: cast state back, scatter / return --------------------- + # Fuse cast (fp32 -> initial_state.dtype) + optional indexed scatter into a # single Triton pass, mirroring Step 4. The inplace branch writes only the # slots named by ``initial_state_indices`` and leaves the rest untouched. if inplace_indexed_state_update: diff --git a/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py b/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py index a2ede4bb54ad..961189d97346 100644 --- a/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py +++ b/tensorrt_llm/_torch/modules/fla/fused_sigmoid_gating_recurrent.py @@ -265,24 +265,6 @@ def _flashinfer_gdn_decode( HV = v.shape[2] V = v.shape[3] - # The FlashInfer CuTe-DSL kernel requires every input tensor's data pointer - # to be 32-byte aligned (enforced in build_memref_desc). ``a`` and ``b`` are - # per-head-scalar slices of the fused ``in_proj_ba`` output: ``b`` starts at - # offset 0 (aligned) but ``a`` starts ``num_v_heads_per_tp`` bf16 elements in, - # so when ``num_v_heads_per_tp`` is not a multiple of 16 (e.g. Qwen3.6-35B-A3B - # TEP4: 32 v-heads / 4 = 8 -> 16-byte offset) the slice base is not 32-byte - # aligned and the kernel aborts. ``.contiguous()`` is NOT enough: at decode - # the token dim is 1, so the strided/offset slice already reports as - # contiguous (size-1 dims are ignored by is_contiguous) and ``.contiguous()`` - # is a no-op that keeps the misaligned pointer. Clone into fresh (allocator- - # aligned) storage instead, and only when misaligned so the common aligned - # case (e.g. Qwen3.5-397B TEP4: 64 / 4 = 16 -> 32-byte offset) stays zero-copy. - # q/k/v are sliced on 128-element head boundaries (>=256 B), always aligned. - if a.data_ptr() % 32 != 0: - a = a.clone(memory_format=torch.contiguous_format) - if b.data_ptr() % 32 != 0: - b = b.clone(memory_format=torch.contiguous_format) - # Reshape from packed varlen [1, N*T, ...] to batched [N, T, ...]. q_bat = q.view(N, T_per_seq, q.shape[2], q.shape[3]) k_bat = k.view(N, T_per_seq, k.shape[2], k.shape[3]) diff --git a/tensorrt_llm/_torch/modules/fla/fused_state_io.py b/tensorrt_llm/_torch/modules/fla/fused_state_io.py index ee420993fec3..c5885f5b6011 100644 --- a/tensorrt_llm/_torch/modules/fla/fused_state_io.py +++ b/tensorrt_llm/_torch/modules/fla/fused_state_io.py @@ -79,16 +79,8 @@ def _gather_cast_vk_to_fp32_vk_kernel( def gather_cast_vk_to_fp32_vk( initial_state: torch.Tensor, initial_state_indices: Optional[torch.Tensor], - out_dtype: Optional[torch.dtype] = None, ) -> torch.Tensor: - """Fused ``initial_state[indices].to(out_dtype).contiguous()`` for ``[N, H, V, K]`` state. - - ``out_dtype`` defaults to ``torch.float32``, the dtype the SM90/SM120 - FlashInfer GDN prefill kernels require. On the SM100/SM103 kernel, which - reads native bf16/fp16 state and casts to fp32 internally, pass - ``initial_state.dtype`` to gather without an up-cast (paired with a matching - scatter that skips the down-cast). - """ + """Fused ``initial_state[indices].to(fp32).contiguous()`` for ``[N, H, V, K]`` state.""" assert initial_state.dim() == 4, f"initial_state must be 4D, got {initial_state.shape}" n_pool, h, v, k = initial_state.shape if initial_state_indices is not None: @@ -101,9 +93,7 @@ def gather_cast_vk_to_fp32_vk( # K and V are typically 128 in GDN; one (BLOCK_K, BLOCK_V) tile covers the full K and V dimensions. # entire (K, V) plane per (seq, head). Larger tiles save grid overhead; # smaller tiles improve occupancy at small num_seqs * H. - if out_dtype is None: - out_dtype = torch.float32 - output = torch.empty(num_seqs, h, v, k, dtype=out_dtype, device=initial_state.device) + output = torch.empty(num_seqs, h, v, k, dtype=torch.float32, device=initial_state.device) block_v = min(v, 128) block_k = min(k, 128) num_v_blocks = triton.cdiv(v, block_v) diff --git a/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md b/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md index 77895edb07ff..b84f0e5c4577 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md +++ b/tensorrt_llm/_torch/modules/fused_moe/MOE_DEVELOPER_GUIDE.md @@ -150,7 +150,7 @@ Still on old path (standalone, with embedded communication): | `fused_moe_cute_dsl_b12x.py` | `CuteDslB12xFusedMoE` | SM120/SM121 | NVFP4 hybrid CUTLASS-prefill / FlashInfer NVFP4 MoE decode — best perf on RTX PRO 6000 (SM120) and DGX Spark (SM121); select via the `CUTEDSL` backend path (auto-promoted when flashinfer is importable) | `EXTERNAL_COMM` | | `mega_moe/mega_moe_deepgemm.py` | `MegaMoEDeepGemm` | SM100/SM103 | W4A8_MXFP4_MXFP8 via DeepGEMM `fp8_fp4_mega_moe` fused dispatch+GEMM+act+GEMM+combine kernel; requires `hidden_size % 512 == 0` | `FUSED_COMM` | | `mega_moe/mega_moe_cute_dsl.py` | `MegaMoECuteDsl` | SM100/SM103 | NVFP4 via ported CuteDSL `Sm100MegaMoEKernel` fused dispatch+FC1+act+FC2+combine kernel; requires CUDA 13 Cutlass DSL runtime (PR #14354) and NVSHMEM provider (hard gate); threads per-expert `fc31_alpha`/`fc2_alpha`/`fc1_norm_const` through the kernel ABI and supports SwiGLU clamp via `swiglu_limit`; default deepgemm graph (topk score folded before fc1-out quant, host `combine_output.sum(dim=1)`) | `FUSED_COMM` | -| `fused_moe_marlin.py` | `MarlinFusedMoE` | SM90 only | W4A16 NVFP4 on Hopper (BF16 activations + FP4 weights, fused single-launch `marlin_nvfp4_moe_gemm` kernel); supports attention-DP + EP via external comm (scheduler precomputes routing; dispatch payload is plain BF16, no activation scales); non-NVFP4 layers (e.g. unquantized MTP draft layers) fall back to Cutlass in `get_moe_cls`; no dynamic EPLB | `EXTERNAL_COMM` | +| `fused_moe_marlin.py` | `MarlinFusedMoE` | SM90 only | W4A16 NVFP4 on Hopper (BF16 activations + FP4 weights, fused single-launch `marlin_nvfp4_moe_gemm` kernel); no dynamic EPLB | `EXTERNAL_COMM` | | `fused_moe_triton.py` | `TritonFusedMoE` | SM90 only | GPT-OSS on Hopper (requires `swiglu_gptoss_style=True`) | (legacy path) | | `fused_moe_wide_ep.py` | `WideEPMoE` | All GPUs | Deprecating — use ConfigurableMoE instead | (legacy path) | | `fused_moe_vanilla.py` | `VanillaMoE` | All devices | Reference / debugging only | (legacy path) | diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/__init__.py b/tensorrt_llm/_torch/modules/fused_moe/communication/__init__.py index 9858693c5f37..0d44ecd2df1e 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/communication/__init__.py +++ b/tensorrt_llm/_torch/modules/fused_moe/communication/__init__.py @@ -34,7 +34,6 @@ from .communication_factory import CommunicationFactory from .deep_ep import DeepEP from .deep_ep_low_latency import DeepEPLowLatency -from .nccl_ep import NcclEP from .nvlink_one_sided import NVLinkOneSided from .nvlink_two_sided import NVLinkTwoSided @@ -47,7 +46,6 @@ "NVLinkOneSided", "DeepEP", "DeepEPLowLatency", - "NcclEP", # Factory "CommunicationFactory", ] diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py b/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py index a514291bdca5..17a67deee219 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py +++ b/tensorrt_llm/_torch/modules/fused_moe/communication/communication_factory.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,7 +33,6 @@ from .base import Communication from .deep_ep import DeepEP from .deep_ep_low_latency import DeepEPLowLatency -from .nccl_ep import NcclEP from .nvlink_one_sided import NVLinkOneSided from .nvlink_two_sided import NVLinkTwoSided from .nvlink_two_sided_flashinfer import NVLinkTwoSidedFlashinfer @@ -69,7 +68,6 @@ def create_strategy( 2. Auto-selection (tries in order): - NVLinkOneSided (highest priority for throughput) - NVLinkTwoSided (high priority for latency) - - NcclEP (if nccl-ep is available) - DeepEP (if enabled via TRTLLM_CAN_USE_DEEP_EP) - DeepEPLowLatency (if enabled via TRTLLM_CAN_USE_DEEP_EP) - AllGather + ReduceScatter (fallback, always works) @@ -132,7 +130,7 @@ def create_strategy( ) # Auto-selection: Try strategies in priority order using try-catch - # Priority: NVLinkOneSided > NVLinkTwoSided > NcclEP > DeepEP > DeepEPLowLatency > AllGather + # Priority: NVLinkOneSided > NVLinkTwoSided > DeepEP > DeepEPLowLatency > AllGather try: enable_eplb = model_config.moe_load_balancer is not None @@ -190,34 +188,6 @@ def create_strategy( except Exception as e: logger.info(f"NVLinkTwoSided not available: {e}") - # Try NCCL EP (rank-major LL). Falls through to DeepEP/AllGather if - # prerequisites are not met or libnccl_ep.so is not available. - nccl_ep_unavailable_reason = CommunicationFactory._get_nccl_ep_unavailable_reason( - act_dtype, - quant_config, - num_slots, - hidden_size, - max_num_tokens, - moe_max_num_tokens, - top_k, - ) - if nccl_ep_unavailable_reason is None: - try: - strategy = NcclEP( - mapping, - num_slots, - hidden_size, - max_num_tokens, - moe_max_num_tokens, - top_k=top_k, - ) - logger.info("Selected communication strategy: NcclEP") - return strategy - except RuntimeError as e: - logger.debug(f"NcclEP not available: {e}") - else: - logger.debug(f"NcclEP not available: {nccl_ep_unavailable_reason}") - # Try DeepEP (if enabled and weight dtype is bfloat16) if os.environ.get("TRTLLM_CAN_USE_DEEP_EP", "1") == "1" and act_dtype == torch.bfloat16: try: @@ -361,56 +331,7 @@ def _create_forced_method( use_low_precision_combine, moe_max_num_tokens, ) - elif method == "NCCL_EP": - nccl_ep_unavailable_reason = CommunicationFactory._get_nccl_ep_unavailable_reason( - act_dtype, - quant_config, - num_slots, - hidden_size, - max_num_tokens, - moe_max_num_tokens, - top_k, - ) - if nccl_ep_unavailable_reason is not None: - raise ValueError(nccl_ep_unavailable_reason) - return NcclEP( - mapping, - num_slots, - hidden_size, - max_num_tokens, - moe_max_num_tokens, - top_k=top_k, - ) elif method == "ALLGATHER": return AllGatherReduceScatter(mapping) else: raise ValueError(f"Unknown communication method: {method}") - - @staticmethod - def _get_nccl_ep_unavailable_reason( - act_dtype: torch.dtype, - quant_config, - num_slots: int, - hidden_size: int, - max_num_tokens: int, - moe_max_num_tokens: Optional[int], - top_k: int, - ) -> Optional[str]: - if act_dtype != torch.bfloat16: - return f"NcclEP requires act_dtype=torch.bfloat16, got {act_dtype}." - if quant_config is not None: - quant_mode = getattr(quant_config, "layer_quant_mode", None) - if quant_mode is not None and quant_mode.has_any_quant(exclude_kv_cache=True): - return "NcclEP v0.1 does not support quantized MoE communication." - if num_slots <= 0 or hidden_size <= 0 or max_num_tokens <= 0: - return ( - "NcclEP requires positive num_slots, hidden_size, and max_num_tokens, got " - f"{num_slots=}, {hidden_size=}, {max_num_tokens=}." - ) - if moe_max_num_tokens is not None and moe_max_num_tokens <= 0: - return ( - f"NcclEP requires moe_max_num_tokens > 0 when provided, got {moe_max_num_tokens}." - ) - if top_k <= 0 or top_k > num_slots: - return f"NcclEP requires 0 < top_k <= num_slots, got {top_k=}, {num_slots=}." - return None diff --git a/tensorrt_llm/_torch/modules/fused_moe/communication/nccl_ep.py b/tensorrt_llm/_torch/modules/fused_moe/communication/nccl_ep.py deleted file mode 100644 index 53cf15b2469c..000000000000 --- a/tensorrt_llm/_torch/modules/fused_moe/communication/nccl_ep.py +++ /dev/null @@ -1,397 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""NCCL EP (Expert Parallelism) Communication Strategy for MoE -- LL rank-major. - -Targets the ``nccl.ep`` Python package shipped in the nccl4py wheel (built -against an NCCL master tree containing ``contrib/nccl_ep``). The dispatch -returns rank-major LL outputs: - - * ``recv_x`` : 3D ``[ep_size, max_tokens_per_rank, hidden]`` bf16, - reshaped to 2D for the downstream MoE pipeline. - * ``recv_topk_idx`` : 2D ``[..., top_k]`` int32 with real expert IDs (-1 for invalid rows) - * ``recv_topk_weights`` : 2D ``[..., top_k]`` float32 (the original router weights) - -This matches NVLinkOneSided's contract directly, so NO -``_modify_output_to_adapt_fused_moe`` adapter is needed. The MoE backend's -``fused_moe`` runs top_k experts per row, applies the weights, and produces one -reduced output per row. ``handle.combine`` then sums per-source-rank -contributions back to the home rank. - -Persistent handle: ``Group.create_handle`` is called ONCE (first dispatch); -subsequent dispatches call ``handle.update(topk_idx, ...)`` to rebind routing. -CUDA-graph capture is supported once the handle exists. -""" - -from typing import List, Optional, Tuple - -import torch - -from tensorrt_llm.logger import logger -from tensorrt_llm.mapping import Mapping - -from .base import Communication - -_NCCL_RUNTIME_ERRORS = (RuntimeError, OSError) - - -class NcclEP(Communication): - """NCCL EP Low-Latency rank-major communication strategy for MoE expert parallelism.""" - - def __init__( - self, - mapping: Mapping, - num_slots: int, - hidden_size: int, - max_num_tokens: int = 1024, - moe_max_num_tokens: Optional[int] = None, - top_k: int = 8, - ): - super().__init__(mapping) - - from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import is_nccl_ep_installed - - if not is_nccl_ep_installed(): - raise RuntimeError("nccl-ep is not installed.") - - if self.ep_size <= 0: - raise ValueError(f"NcclEP requires moe_ep_size > 0, got {self.ep_size}") - if not 0 <= self.ep_rank < self.ep_size: - raise ValueError( - f"NcclEP requires 0 <= moe_ep_rank < moe_ep_size, " - f"got {self.ep_rank=}, {self.ep_size=}" - ) - if num_slots <= 0: - raise ValueError(f"NcclEP requires num_slots > 0, got {num_slots}") - if num_slots % self.ep_size != 0: - raise ValueError( - f"NcclEP requires num_slots divisible by moe_ep_size, " - f"got {num_slots=}, {self.ep_size=}" - ) - if hidden_size <= 0: - raise ValueError(f"NcclEP requires hidden_size > 0, got {hidden_size}") - if top_k <= 0 or top_k > num_slots: - raise ValueError(f"NcclEP requires 0 < top_k <= num_slots, got {top_k=}, {num_slots=}") - if max_num_tokens <= 0: - raise ValueError(f"NcclEP requires max_num_tokens > 0, got {max_num_tokens}") - if moe_max_num_tokens is not None and moe_max_num_tokens <= 0: - raise ValueError( - f"NcclEP requires moe_max_num_tokens > 0 when provided, got {moe_max_num_tokens}" - ) - - self.num_slots = num_slots - self.num_experts = num_slots - self.hidden_size = hidden_size - self.num_local_experts = num_slots // self.ep_size - self.max_top_k = top_k - - self.max_tokens_per_rank = ( - max_num_tokens - if moe_max_num_tokens is None - else min(max_num_tokens, moe_max_num_tokens) - ) - self.max_recv_tokens = self.ep_size * self.max_tokens_per_rank - - # Singleton NCCL EP context: owns the EP group, RDMA buffers, and - # persistent OUTPUT Tensor descriptors. Allocate it lazily on first - # dispatch because full-model construction runs under MetaInitMode, - # which redirects torch.empty to the meta device even when a CUDA - # device is passed explicitly. - self._ctx = None - - # Persistent dispatch handle. Created on first dispatch via - # group.create_handle; reused thereafter via handle.update so - # subsequent dispatches are CUDA-graph-safe. - self._handle = None # nccl.ep.Handle | None - self._dispatch_state: dict = {} - - @staticmethod - def is_platform_supported() -> bool: - from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import is_nccl_ep_installed - - return is_nccl_ep_installed() - - def is_workload_feasible(self, all_rank_num_tokens: List[int], num_chunks: int) -> bool: - if num_chunks > 1: - return False - if max(all_rank_num_tokens) > self.max_tokens_per_rank: - return False - return True - - def _get_context(self): - if self._ctx is None: - if torch.cuda.is_current_stream_capturing(): - raise RuntimeError( - "NcclEP context must be initialized before CUDA graph capture. " - "Run an eager warmup forward before enabling or capturing CUDA graphs." - ) - from nccl.ep import Layout - - from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import get_nccl_ep_context - - self._ctx = get_nccl_ep_context( - self.mapping, - self.num_experts, - self.max_tokens_per_rank, - self.hidden_size, - self.max_top_k, - Layout.RANK_MAJOR, - ) - return self._ctx - - def _setup_handle(self, ctx, topk_nd, stream): - """Ensure self._handle exists; rebind topk via handle.update on subsequent calls.""" - if self._handle is None: - if torch.cuda.is_current_stream_capturing(): - raise RuntimeError( - "NcclEP dispatch handle must be initialized before CUDA graph capture. " - "Run an eager warmup forward before enabling or capturing CUDA graphs." - ) - self._handle = ctx.ep_group.create_handle( - ctx.layout, - topk_nd, - stream=stream, - ) - else: - self._handle.update(topk_nd, stream=stream) - return self._handle - - # ------------------------------------------------------------------ - # Dispatch -- rank-major LL - # ------------------------------------------------------------------ - - def dispatch( - self, - hidden_states: torch.Tensor, - hidden_states_sf: Optional[torch.Tensor], - token_selected_slots: torch.Tensor, - token_final_scales: Optional[torch.Tensor], - all_rank_num_tokens: List[int], - use_dp_padding: Optional[bool] = None, - **kwargs, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], torch.Tensor, Optional[torch.Tensor]]: - """Dispatch tokens via NCCL EP LL rank-major. - - Returns rank-major-shaped tensors directly: - (recv_hs [N, H], recv_sf [N, H/128] or None, recv_slots [N, top_k] int32, - recv_scales [N, top_k] float32) - - where N = ep_size * max_tokens_per_rank. Rows beyond - ``recv_rank_counter[r]`` for source rank r have recv_slots = -1 - (sentinel), naturally skipped by the MoE backend. - """ - from nccl.ep import DispatchConfig, DispatchInputs, DispatchOutputs, LayoutInfo, Tensor - - ctx = self._get_context() - - all_rank_max_num_tokens = max(all_rank_num_tokens) - if all_rank_max_num_tokens > self.max_tokens_per_rank: - raise ValueError( - f"all_rank_max_num_tokens={all_rank_max_num_tokens} > " - f"max_tokens_per_rank={self.max_tokens_per_rank}" - ) - - num_tokens = hidden_states.shape[0] - top_k = token_selected_slots.shape[1] - if top_k > self.max_top_k: - raise ValueError(f"top_k={top_k} exceeds configured max_top_k={self.max_top_k}") - if token_final_scales is None: - raise RuntimeError( - "NcclEP rank-major dispatch requires token_final_scales " - "(router weights) -- it is an INPUT to handle.dispatch." - ) - - stream = ctx.get_stream() - - # TODO(NCCL): topk_weights still requires float32; once bf16/native - # weights are accepted upstream, drop this conversion too. - weights_f32 = ( - token_final_scales - if token_final_scales.dtype == torch.float32 - else token_final_scales.to(torch.float32) - ) - hidden_states_c = hidden_states.contiguous() - weights_f32_c = weights_f32.contiguous() - - input_tokens_nd = Tensor(hidden_states_c) - input_topk_weights_nd = Tensor(weights_f32_c) - - # Mark padding rows with the -1 sentinel so fused_moe skips them. - # The dispatch kernel only writes recv_topk_idx for slots that - # received tokens; rows beyond `recv_rank_counter[r]` keep stale - # data from prior dispatches. recv_rank_counter is written fresh - # by the dispatch kernel (low_latency.cu:877) so it does not need - # pre-zeroing, and recv_topk_weights on -1 rows is don't-care - # (fused_moe ignores the weight when the expert id is -1). - ctx.recv_topk_idx_buf.fill_(-1) - - outputs = DispatchOutputs( - tokens=ctx.output_tokens_nd, - topk_weights=ctx.recv_topk_weights_nd, - topk_idx=ctx.recv_topk_idx_nd, - scales=None, - ) - layout_info = LayoutInfo(src_rank_counters=ctx.recv_rank_counter_nd) - # The v0.2-gated capability path asks the kernel to emit global - # expert ids directly; v0.1 retains the default local-id contract - # and uses the translation below. - if ctx._expert_id_kind_global is not None: - layout_info._lowpp.recv_topk_idx_kind = ctx._expert_id_kind_global - - topk_idx_dev = token_selected_slots.to(ctx.topk_idx_dtype).contiguous() - topk_nd = Tensor(topk_idx_dev) - handle = self._setup_handle(ctx, topk_nd, stream) - inputs = DispatchInputs( - tokens=input_tokens_nd, - topk_weights=input_topk_weights_nd, - ) - handle.dispatch( - inputs, - outputs, - layout_info=layout_info, - config=DispatchConfig(round_scales=0), - stream=stream, - ) - - # The handle internally references topk_nd; keep both the Tensor - # descriptor and its backing torch tensor alive until combine completes. - self._dispatch_state = { - "num_tokens": num_tokens, - "topk_nd": topk_nd, - "topk_idx_dev": topk_idx_dev, - } - - # Match NVLinkOneSided's contract: token_selected_slots in - # [0, num_experts) for valid rows, -1 for invalid. When the kernel - # writes GLOBAL ids directly (opt-in detected at ctx init), the - # buffer is already in the right space and we pass it through. - # Otherwise the kernel writes LOCAL ids in [0, num_local_experts) - # and we add ep_rank * num_local_experts to restore the global - # numbering downstream consumers expect. - # The dispatch buffer is 3D [ep_size, max_tokens_per_rank, max_top_k] - # per the LL rank-major contract; flatten to 2D for downstream. - recv_topk_idx_flat = ctx.recv_topk_idx_buf.view(self.max_recv_tokens, self.max_top_k) - if ctx.kernel_writes_global_ids: - recv_slots_global = recv_topk_idx_flat - else: - recv_slots_global = torch.where( - recv_topk_idx_flat >= 0, - recv_topk_idx_flat + self.ep_rank * self.num_local_experts, - recv_topk_idx_flat, - ) - - # Output buffers are 3D [ep_size, max_tokens_per_rank, ...] per the - # LL rank-major contract; downstream MoE pipeline expects 2D -- - # flatten via view. - return ( - ctx.output_tokens_buf.view(self.max_recv_tokens, self.hidden_size), - None, - recv_slots_global, - ctx.recv_topk_weights_buf.view(self.max_recv_tokens, self.max_top_k), - ) - - # ------------------------------------------------------------------ - # Combine -- rank-major LL - # ------------------------------------------------------------------ - - def combine( - self, - final_hidden_states: torch.Tensor, - **kwargs, - ) -> torch.Tensor: - """Combine MoE-reduced rank-major output back to the home rank. - - Input: [max_recv_tokens, hidden] -- already weighted per-row by fused_moe. - Output: [num_tokens, hidden] -- combined to original token order. - """ - from nccl.ep import CombineInputs, CombineOutputs, Tensor - - ctx = self._ctx - if ctx is None: - raise RuntimeError("NcclEP.combine called before dispatch.") - state = self._dispatch_state - stream = ctx.get_stream() - - num_tokens = state["num_tokens"] - - # NCCL-EP LL combine consumes rank-major tokens with shape - # [ep_size, max_tokens_per_rank, hidden]. The scheduler normally - # provides the equivalent 2D [max_recv_tokens, hidden] view. - if final_hidden_states.dim() == 2: - expected_shape = (self.max_recv_tokens, self.hidden_size) - if tuple(final_hidden_states.shape) != expected_shape: - raise ValueError( - f"combine input shape={tuple(final_hidden_states.shape)} " - f"expected={expected_shape}" - ) - final_hidden_states = final_hidden_states.view( - self.ep_size, - self.max_tokens_per_rank, - self.hidden_size, - ) - elif final_hidden_states.dim() == 3: - expected_shape = (self.ep_size, self.max_tokens_per_rank, self.hidden_size) - if tuple(final_hidden_states.shape) != expected_shape: - raise ValueError( - f"combine input shape={tuple(final_hidden_states.shape)} " - f"expected={expected_shape}" - ) - else: - raise ValueError( - "NcclEP combine input must be 2D [max_recv_tokens, hidden] or " - "3D [ep_size, max_tokens_per_rank, hidden], got " - f"shape={tuple(final_hidden_states.shape)}" - ) - - combine_input_c = final_hidden_states.contiguous() - combine_output = torch.empty( - num_tokens, - self.hidden_size, - dtype=torch.bfloat16, - device=combine_input_c.device, - ) - - combine_input_nd = Tensor(combine_input_c) - combine_output_nd = Tensor(combine_output) - - # Rank-major combine: no layout_info, no config required (send_only=0 - # is the default; defaults round-trip fine). - self._handle.combine( - CombineInputs(tokens=combine_input_nd), - CombineOutputs(tokens=combine_output_nd), - stream=stream, - ) - - self._dispatch_state = {} - return combine_output - - def destroy(self): - """Release per-instance NCCL EP resources (handle). - - NcclEpContext is shared across instances and released through a - refcounted cache. - """ - if self._handle is not None: - try: - self._handle.destroy() - except _NCCL_RUNTIME_ERRORS as e: - logger.warning(f"Handle.destroy error during destroy: {e}") - self._handle = None - - from tensorrt_llm._torch.modules.fused_moe.nccl_ep_utils import release_nccl_ep_context - - if self._ctx is not None: - release_nccl_ep_context(self._ctx) - self._ctx = None - self._dispatch_state = {} diff --git a/tensorrt_llm/_torch/modules/fused_moe/create_moe.py b/tensorrt_llm/_torch/modules/fused_moe/create_moe.py index 3761fab5a06e..1f3e9b112dd0 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/create_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/create_moe.py @@ -64,15 +64,10 @@ def get_moe_cls( quant_config = override_quant_config layer_prefix = f"[layer_idx={layer_idx}] " if layer_idx is not None else "" if moe_backend.upper() == "MARLIN": - # Marlin MoE is a Hopper-specific NVFP4 W4A16 backend. Layers without - # NVFP4 quantization (e.g. deliberately-unquantized MTP draft layers in - # MIXED_PRECISION checkpoints) fall back to CutlassFusedMoE, matching - # the CUTEDSL / DENSEGEMM / MEGAMOE_* fallback behavior below. + # Marlin MoE is a Hopper-specific NVFP4 W4A16 backend. Require nvfp4 + # quantization explicitly so a misconfigured model fails fast. if quant_config is None or not quant_config.quant_mode.has_nvfp4(): - logger.warning(f"{layer_prefix}MarlinFusedMoE only supports NVFP4 " - "quantization. Check out details in quant_config: " - f"{quant_config}. Using CutlassFusedMoE instead.") - return CutlassFusedMoE + raise ValueError("MarlinFusedMoE only supports NVFP4 quantization.") return MarlinFusedMoE if moe_backend.upper() == "CUTLASS": return CutlassFusedMoE diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_marlin.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_marlin.py index c4728be2b140..e6e9f3c7ea7c 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_marlin.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_marlin.py @@ -104,6 +104,31 @@ def _get_quant_method(self): def _supports_load_balancer(self) -> bool: return False + def validate_configurable_moe(self, moe) -> None: + """Reject configs that require external-communication MoE dispatch. + + Marlin is W4A16 (``quantize_input`` produces no activation scale) and + routes internally inside ``run_moe``. The host-side all-to-all + dispatch/combine path that ConfigurableMoE uses for attention-DP + expert parallelism needs the routing decided *before* dispatch and a + per-token scale payload, so it is incompatible with Marlin and fails + inside ``moe_a2a_dispatch``. + + ConfigurableMoE only creates an external communication strategy when + ``enable_attention_dp and dp_size > 1`` (see CommunicationFactory); + ``moe.comm`` is not assigned yet when this hook runs, so check that + same condition via the mapping. Single-GPU, and TP/EP without + attention DP, are supported. + """ + if moe.use_dp and moe.mapping.dp_size > 1: + raise ValueError( + "MarlinFusedMoE does not support external-communication MoE " + "(attention data parallelism combined with expert " + "parallelism): its W4A16 layout and internal routing are " + "incompatible with the all-to-all dispatch path. Use Marlin " + "single-node, or with TP/EP without attention DP." + ) + def _apply_activation(self, gemm1_out: torch.Tensor) -> torch.Tensor: """Apply the activation function to the gemm1 output. @@ -177,15 +202,6 @@ def run_moe( local_n = self.expert_size_per_partition if local_n != self.num_experts: - # EP: non-local token-expert pairs are clamped to local expert 0 - # with a zero final scale, so they still run through both GEMMs - # and are discarded at the combine. - # TODO(perf): skip them instead — e.g. mark non-local pairs with a - # sentinel expert id that moe_align_block_size drops. Requires - # bounds guards in moeAlignKernels.cu (the histogram and - # count_and_sort kernels index shared/cumsum buffers with the raw - # id) and zero-initializing unscheduled GEMM output rows before - # the index_add_ combine. slot_start = self.slot_start is_local = (token_selected_experts >= slot_start) & ( token_selected_experts < slot_start + local_n diff --git a/tensorrt_llm/_torch/modules/fused_moe/interface.py b/tensorrt_llm/_torch/modules/fused_moe/interface.py index 1e7ecd06845f..82b9e5f2df5c 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/interface.py +++ b/tensorrt_llm/_torch/modules/fused_moe/interface.py @@ -103,8 +103,6 @@ class AlltoallMethodType(IntEnum): DeepEP = 3 # DeepEP low latency: CUDA Graphs are supported, IBGDA is required DeepEPLowLatency = 4 - # NCCL EP: Low-latency expert parallelism via NCCL EP library - NcclEP = 5 class MoESchedulerKind(Enum): diff --git a/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py b/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py index 283be7276fa7..fe0153386763 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py +++ b/tensorrt_llm/_torch/modules/fused_moe/moe_scheduler.py @@ -51,7 +51,7 @@ from tensorrt_llm._torch.utils import EventType, Fp4QuantizedTensor from tensorrt_llm.tools.layer_wise_benchmarks import get_calibrator -from .communication import DeepEP, DeepEPLowLatency, NcclEP, NVLinkOneSided, NVLinkTwoSided +from .communication import DeepEP, DeepEPLowLatency, NVLinkOneSided, NVLinkTwoSided from .communication.nvlink_two_sided_flashinfer import NVLinkTwoSidedFlashinfer from .fused_moe_cute_dsl import CuteDslFusedMoE from .fused_moe_cutlass import CutlassFusedMoE, raise_moe_lora_multichunk_unsupported @@ -374,14 +374,8 @@ def _forward_chunk_impl( moe._load_balancer_start_wait_gpu_stage(is_first_call) # ========== Step 2: Apply routing ========== - # External dispatch (Step 5) sends per-token expert/scale payloads, so - # routing must be precomputed whenever a comm strategy is active — even - # for backends whose run_moe can otherwise route internally from - # router_logits (e.g. MarlinFusedMoE under attention-DP + EP). requires_separated_routing = ( - moe.backend._supports_load_balancer() - or moe.routing_method.requires_separated_routing - or moe.comm is not None + moe.backend._supports_load_balancer() or moe.routing_method.requires_separated_routing ) if requires_separated_routing: # Separated routing: ConfigurableMoE calls routing_method @@ -407,9 +401,10 @@ def _forward_chunk_impl( "Current workaround for apply_router_weight_on_input does not support fp8 input" ) x = x * token_final_scales.to(x.dtype) - # These strategies need non-None token_final_scales, so feed - # all-ones after folding the real weights into x. - if isinstance(moe.comm, (DeepEP, DeepEPLowLatency, NcclEP)): + # DeepEP variants need a non-None token_final_scales tensor + # (they don't tolerate None), so feed all-ones; other strategies + # accept None and skip the multiply. + if isinstance(moe.comm, (DeepEP, DeepEPLowLatency)): token_final_scales = torch.ones_like(token_final_scales) else: token_final_scales = None diff --git a/tensorrt_llm/_torch/modules/fused_moe/nccl_ep_utils.py b/tensorrt_llm/_torch/modules/fused_moe/nccl_ep_utils.py deleted file mode 100644 index 9822a0f90a2a..000000000000 --- a/tensorrt_llm/_torch/modules/fused_moe/nccl_ep_utils.py +++ /dev/null @@ -1,421 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""NCCL EP utilities backed by the nccl4py wheel's ``nccl.ep`` package. - -Owns the long-lived NCCL EP resources (communicator, group, persistent receive -NDTensors) for the MoE NcclEP communication strategy. Per-step dispatch handles -are created in ``communication/nccl_ep.py``. -""" - -from typing import Optional - -import torch - -from tensorrt_llm._utils import TensorWrapper, convert_to_torch_tensor -from tensorrt_llm.logger import logger -from tensorrt_llm.mapping import Mapping - -_MIN_NCCL_RUNTIME_VERSION = "2.30.4" -_MIN_NCCL_EP_INT32_TOPK_VERSION = "0.2" -_NCCL_RUNTIME_ERRORS = (RuntimeError, OSError) -_NCCL_AVAILABILITY_ERRORS = (ImportError,) + _NCCL_RUNTIME_ERRORS - -_nccl_ep_installed: Optional[bool] = None - - -def is_nccl_ep_installed() -> bool: - """Return True iff ``nccl.ep`` is usable. - - Requires that ``nccl.ep`` imports cleanly AND the loaded ``libnccl.so`` - runtime version is >= 2.30.4. - """ - global _nccl_ep_installed - if _nccl_ep_installed is not None: - return _nccl_ep_installed - try: - import nccl - from packaging.version import Version - - runtime = nccl.get_version().nccl.version - if runtime < Version(_MIN_NCCL_RUNTIME_VERSION): - logger.info( - f"NCCL EP disabled: libnccl runtime {runtime} " - f"< required {_MIN_NCCL_RUNTIME_VERSION}" - ) - _nccl_ep_installed = False - return False - import nccl.ep # noqa: F401 - - _nccl_ep_installed = True - except _NCCL_AVAILABILITY_ERRORS as e: - logger.info(f"NCCL EP disabled: nccl.ep is not usable ({e!r})") - _nccl_ep_installed = False - return _nccl_ep_installed - - -def _nccl_ep_supports_int32_topk_idx() -> bool: - """Return True when the loaded libnccl_ep supports int32 input topk_idx.""" - try: - import nccl - from packaging.version import Version - - nccl_ep_info = nccl.get_version().nccl_ep - nccl_ep_version = nccl_ep_info.version if nccl_ep_info is not None else None - except (ImportError, AttributeError, RuntimeError, OSError) as e: - logger.info( - f"NCCL EP int32 topk_idx disabled: could not determine libnccl_ep version ({e!r})" - ) - return False - - if nccl_ep_version is None: - logger.info("NCCL EP int32 topk_idx disabled: libnccl_ep version is not available") - return False - - if nccl_ep_version < Version(_MIN_NCCL_EP_INT32_TOPK_VERSION): - logger.info( - f"NCCL EP int32 topk_idx disabled: libnccl_ep {nccl_ep_version} " - f"< required {_MIN_NCCL_EP_INT32_TOPK_VERSION}" - ) - return False - - return True - - -# Singleton EP context keyed by (ep_size, ep_rank, max_tokens, num_experts, -# hidden, max_top_k, layout). -_ep_group_cache: dict = {} -_ep_group_refcounts: dict = {} - - -class NcclEpContext: - """Long-lived NCCL EP group + receive buffers, shared across NcclEP instances. - - Owns the :class:`nccl.ep.Group`, the source :class:`nccl.core.Communicator`, - and the rank-major LL persistent receive buffers (tokens, top-k idx / weights, - per-source-rank counter) wrapped as - :class:`nccl.ep.Tensor` descriptors. - - Per-step routing handles (``Handle``) are created in ``NcclEP``, not here. - """ - - def __init__( - self, - mapping: Mapping, - num_experts: int, - max_tokens_per_rank: int, - hidden_size: int, - max_top_k: int, - layout: Optional[int] = None, - ): - import nccl.core as nccl_core - from nccl.ep import Algorithm, Group, GroupConfig, Layout, Tensor - - from tensorrt_llm._utils import mpi_comm - - self.mapping = mapping - self.ep_size = mapping.moe_ep_size - self.ep_rank = mapping.moe_ep_rank - self.num_experts = num_experts - self.num_local_experts = num_experts // self.ep_size - self.max_tokens_per_rank = max_tokens_per_rank - self.max_top_k = max_top_k - self.hidden_size = hidden_size - self.layout = Layout.RANK_MAJOR if layout is None else Layout(layout) - self.max_recv_tokens = self.ep_size * max_tokens_per_rank - - # topk_idx dtype passed to the EP runtime. NCCL-EP < 0.2 asserts - # int64 in ncclEpUpdateHandle; 0.2+ supports TRT-LLM's native int32 - # routing ids and avoids the per-iter widening conversion. - self.topk_idx_dtype = torch.int32 if _nccl_ep_supports_int32_topk_idx() else torch.int64 - self._v0_2_features_enabled = self.topk_idx_dtype == torch.int32 - - # NCCL-EP v0.2+ may expose a configurable receive expert-id kind. - # Within that version-gated path, detect whether the linked binding supports a - # configurable recv_topk_idx kind on LayoutInfo. When the field - # is present we set it to GLOBAL and skip the post-dispatch - # local->global rewrite; otherwise the kernel writes LOCAL ids - # unconditionally (older nccl-ep builds) and the dispatch - # wrapper applies torch.where to restore the global contract - # NVLinkOneSided also advertises. - self.kernel_writes_global_ids = False - self._expert_id_kind_global = None - if self._v0_2_features_enabled: - try: - from nccl.bindings.nccl_ep import ExpertIdKind as _ExpertIdKind - from nccl.bindings.nccl_ep import LayoutInfo as _LowLayoutInfo - - self.kernel_writes_global_ids = hasattr(_LowLayoutInfo(), "recv_topk_idx_kind") - self._expert_id_kind_global = ( - int(_ExpertIdKind.GLOBAL) if self.kernel_writes_global_ids else None - ) - except (ImportError, AttributeError): - pass - - # NCCL-EP v0.2+ may support opportunistic zero-copy dispatch. - # When the Pythonic GroupConfig facade exposes `zero_copy` (i.e., - # the wheel was built against a libnccl_ep.so that has the field - # in ncclEpGroupConfig_t), we allocate a VMM-backed, - # window-registered dispatch output buffer; the LL dispatch - # opportunistically picks zero-copy when recv_x->win_hdl is set - # (nvlink-only + rank-major). The config flag itself stays - # AUTO/OFF -- strict zero_copy=ON requires combine inputs to be - # windowed too, which would force a caller-side interface change - # (the MLP output is caller-owned). The C-side strict-ON check - # remains in the library for future use. - self.zerocopy_enabled = self._v0_2_features_enabled and "zero_copy" in getattr( - GroupConfig, "__dataclass_fields__", {} - ) - - # MPI sub-communicator scoped to the EP group. Mirrors the - # DeepEPLowLatency pattern (see deep_ep_utils.py:104): split - # MPI_COMM_WORLD by pp_rank so each pipeline stage gets its own EP - # comm, keyed by moe_ep_rank. Avoids the wheel's - # nccl.ep.get_nccl_comm_from_group() helper which requires - # torch.distributed.init_process_group() -- the test infrastructure - # (mpi_pool_executor) and microbenchmarks use MPI4PY only. - self._ep_mpi_comm = mpi_comm().Split(mapping.pp_rank, mapping.moe_ep_rank) - ep_world_rank = self._ep_mpi_comm.Get_rank() - ep_world_size = self._ep_mpi_comm.Get_size() - unique_id = nccl_core.get_unique_id() if ep_world_rank == 0 else None - unique_id = self._ep_mpi_comm.bcast(unique_id, root=0) - self.comm = nccl_core.Communicator.init( - nranks=ep_world_size, - rank=ep_world_rank, - unique_id=unique_id, - ) - - cfg = GroupConfig( - algorithm=Algorithm.LOW_LATENCY, - num_experts=num_experts, - max_dispatch_tokens_per_rank=max_tokens_per_rank, - max_recv_tokens_per_rank=self.max_recv_tokens, - max_token_bytes=hidden_size * 2, # bfloat16 - ) - self.ep_group = Group.create(self.comm, cfg) - - logger.info( - f"NCCL EP group created: ep_size={self.ep_size}, " - f"num_experts={num_experts}, max_tokens_per_rank={max_tokens_per_rank}, " - f"hidden_size={hidden_size}, max_top_k={max_top_k}, " - f"layout={self.layout.name}" - ) - - device_id = torch.cuda.current_device() - device = torch.device("cuda", device_id) - - # Dispatch output tokens: 3D [ep_size, max_tokens_per_rank, hidden] - # for LL rank-major. When zerocopy is enabled the buffer must be - # VMM-backed (cuMemMap) so ncclCommWindowRegister's internal - # cuMemGetAddressRange call succeeds -- torch's caching - # allocator returns plain cudaMalloc memory which fails that - # check with CUDA_ERROR_INVALID_VALUE. Allocate via - # nccl.core.mem_alloc (VMM-backed) then build a zero-copy torch - # view over the raw pointer via the TRT-LLM CAI wrapper. - token_shape = (self.ep_size, max_tokens_per_rank, hidden_size) - token_nbytes = self.ep_size * max_tokens_per_rank * hidden_size * 2 - self._output_tokens_alloc = None - self._recv_x_window = None - if self.zerocopy_enabled: - self._output_tokens_alloc = nccl_core.mem_alloc( - token_nbytes, - device=device_id, - ) - self.output_tokens_buf = convert_to_torch_tensor( - TensorWrapper( - int(self._output_tokens_alloc.handle), - dtype=torch.bfloat16, - shape=token_shape, - ) - ) - self._recv_x_window = self.comm.register_window( - self._output_tokens_alloc, - ) - else: - self.output_tokens_buf = torch.empty( - *token_shape, - dtype=torch.bfloat16, - device=device, - ) - # Received topk indices: int32 [ep_size, max_tokens_per_rank, max_top_k] - # for the LL rank-major dispatch contract. -1 marks invalid rows. - # Downstream consumers want 2D [max_recv, max_top_k]; flatten via view. - self.recv_topk_idx_buf = torch.empty( - self.ep_size, - max_tokens_per_rank, - max_top_k, - dtype=torch.int32, - device=device, - ) - # Received topk weights: float32 [ep_size, max_tokens_per_rank, max_top_k] - self.recv_topk_weights_buf = torch.empty( - self.ep_size, - max_tokens_per_rank, - max_top_k, - dtype=torch.float32, - device=device, - ) - # Per-source-rank received-token counter (passed via - # LayoutInfo.src_rank_counters at dispatch time). - self.recv_rank_counter_buf = torch.empty( - self.ep_size, - dtype=torch.int32, - device=device, - ) - # Wrap each persistent buffer as a Tensor descriptor. Torch owns the - # storage; the descriptor only carries shape + a pointer (+ window - # handle on dispatch output when zerocopy is on, so libnccl_ep's - # opportunistic LL zero-copy path can fire). - if self.zerocopy_enabled and self._recv_x_window is not None: - self.output_tokens_nd = Tensor( - self.output_tokens_buf, - window=self._recv_x_window, - window_offset=0, - ) - else: - self.output_tokens_nd = Tensor(self.output_tokens_buf) - self.recv_topk_idx_nd = Tensor(self.recv_topk_idx_buf) - self.recv_topk_weights_nd = Tensor(self.recv_topk_weights_buf) - self.recv_rank_counter_nd = Tensor(self.recv_rank_counter_buf) - - def get_stream(self) -> int: - """Current CUDA stream as a raw int handle (accepted by ``nccl.ep`` APIs).""" - return torch.cuda.current_stream().cuda_stream - - def destroy(self): - """Release EP group, NCCL comm, and MPI sub-comm in LIFO order. - - Avoids relying on Python GC ordering between the group, the comm it - was built from, and the MPI sub-comm seeding it: the group must go - first (uses the comm), then ``finalize`` + ``destroy`` on the comm - (the recommended nccl4py pattern), then ``Free`` on the MPI comm. - """ - if self.ep_group is not None: - try: - self.ep_group.destroy() - except _NCCL_RUNTIME_ERRORS as e: - logger.warning(f"NCCL EP group destroy error: {e}") - self.ep_group = None - - # Deregister windows before the comm goes away. close() is - # idempotent and local; the comm would auto-close any leftover - # windows on destroy, but explicit LIFO release matches the rest - # of this teardown path. Both dispatch-output and combine-input - # windows are registered only when zerocopy is on. - for attr in ("_combine_input_window", "_recv_x_window"): - w = getattr(self, attr, None) - if w is not None: - try: - w.close() - except _NCCL_RUNTIME_ERRORS as e: - logger.warning(f"NCCL EP window close error ({attr}): {e}") - setattr(self, attr, None) - - # Drop torch view + EP descriptor before freeing the underlying - # NCCL-allocated Buffer (CAI view doesn't refcount the source). - # close() the cuda.core.Buffer to call nccl.core.mem_free; the - # alloc is only populated when zerocopy is enabled. - self.output_tokens_nd = None - self.output_tokens_buf = None - if getattr(self, "_output_tokens_alloc", None) is not None: - try: - self._output_tokens_alloc.close() - except _NCCL_RUNTIME_ERRORS as e: - logger.warning(f"NCCL EP recv_x buffer free error: {e}") - self._output_tokens_alloc = None - - if self.comm is not None: - try: - self.comm.finalize() - self.comm.destroy() - except _NCCL_RUNTIME_ERRORS as e: - logger.warning(f"NCCL EP comm destroy error: {e}") - self.comm = None - - if self._ep_mpi_comm is not None: - from mpi4py import MPI - - try: - self._ep_mpi_comm.Free() - except MPI.Exception as e: - logger.warning(f"EP MPI sub-comm free error: {e}") - self._ep_mpi_comm = None - - -def get_nccl_ep_context( - mapping: Mapping, - num_experts: int, - max_tokens_per_rank: int, - hidden_size: int, - max_top_k: int, - layout: Optional[int] = None, -) -> NcclEpContext: - """Get or create a singleton :class:`NcclEpContext` for the given configuration.""" - from nccl.ep import Layout - - if layout is None: - layout = Layout.RANK_MAJOR - key = ( - mapping.moe_ep_size, - mapping.moe_ep_rank, - max_tokens_per_rank, - num_experts, - hidden_size, - max_top_k, - int(layout), - ) - if key not in _ep_group_cache: - _ep_group_cache[key] = NcclEpContext( - mapping, - num_experts, - max_tokens_per_rank, - hidden_size, - max_top_k, - layout, - ) - _ep_group_refcounts[key] = _ep_group_refcounts.get(key, 0) + 1 - return _ep_group_cache[key] - - -def release_nccl_ep_context(ctx: Optional[NcclEpContext]) -> None: - """Release one reference to a cached :class:`NcclEpContext`.""" - if ctx is None: - return - - key = next((key for key, cached_ctx in _ep_group_cache.items() if cached_ctx is ctx), None) - if key is None: - return - - refcount = _ep_group_refcounts.get(key, 0) - 1 - if refcount > 0: - _ep_group_refcounts[key] = refcount - return - - _ep_group_refcounts.pop(key, None) - cached_ctx = _ep_group_cache.pop(key) - try: - cached_ctx.destroy() - except _NCCL_RUNTIME_ERRORS as e: - logger.warning(f"Error destroying NCCL EP context: {e}") - - -def destroy_all_nccl_ep_contexts(): - """Destroy all cached NCCL EP contexts (call at process teardown).""" - for ctx in list(_ep_group_cache.values()): - try: - ctx.destroy() - except _NCCL_RUNTIME_ERRORS as e: - logger.warning(f"Error destroying NCCL EP context: {e}") - _ep_group_cache.clear() - _ep_group_refcounts.clear() diff --git a/tensorrt_llm/_torch/modules/fused_ops/fused_qk_norm_rope_gate.py b/tensorrt_llm/_torch/modules/fused_ops/fused_qk_norm_rope_gate.py deleted file mode 100644 index d1f51326a3eb..000000000000 --- a/tensorrt_llm/_torch/modules/fused_ops/fused_qk_norm_rope_gate.py +++ /dev/null @@ -1,348 +0,0 @@ -# Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/kernels/ops/attention/fused_qk_rmsnorm_rope_gate.py -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Fused Qwen3.5 full-attention preprocessing and output gating. - -The projection produces interleaved ``[q0, gate0, q1, gate1, ..., K, V]``. -The preprocessing kernel reads that layout directly and writes packed -``[Q, K, V]`` while applying per-head Gemma RMSNorm and NeoX RoPE to Q/K. -Gate deinterleave is fused into the same pass. Keeping the checkpoint layout -unchanged avoids coupling the optimization to weight loading, quantization, -LoRA, or attention-backend fallback behavior. -""" - -from typing import Optional, Tuple - -import torch -import triton -import triton.language as tl - - -@triton.jit -def _fused_qkv_gemma_rmsnorm_rope_gate_kernel( - qkv_ptr, - qkv_out_ptr, - gate_out_ptr, - q_weight_ptr, - k_weight_ptr, - cos_sin_ptr, - positions_ptr, - num_tokens, - max_positions, - qkv_stride_token, - qkv_out_stride_token, - gate_out_stride_token, - cos_sin_stride_position, - num_q_heads: tl.constexpr, - num_kv_heads: tl.constexpr, - head_dim: tl.constexpr, - rotary_dim: tl.constexpr, - half_rotary: tl.constexpr, - head_block: tl.constexpr, - rotary_block: tl.constexpr, - eps: tl.constexpr, - output_fp16: tl.constexpr, - has_pass_through: tl.constexpr, - use_mrope: tl.constexpr, - mrope_section1: tl.constexpr, - mrope_section2: tl.constexpr, -): - token = tl.program_id(0).to(tl.int64) - head = tl.program_id(1) - q_size = num_q_heads * head_dim - kv_size = num_kv_heads * head_dim - output_dtype = tl.float16 if output_fp16 else tl.bfloat16 - - if head < num_q_heads + num_kv_heads: - is_k = head >= num_q_heads - local_head = tl.where(is_k, head - num_q_heads, head) - if is_k: - input_base = qkv_ptr + token * qkv_stride_token + 2 * q_size + local_head * head_dim - output_base = ( - qkv_out_ptr + token * qkv_out_stride_token + q_size + local_head * head_dim - ) - weight_ptr = k_weight_ptr - else: - input_base = qkv_ptr + token * qkv_stride_token + local_head * 2 * head_dim - output_base = qkv_out_ptr + token * qkv_out_stride_token + local_head * head_dim - weight_ptr = q_weight_ptr - - head_offsets = tl.arange(0, head_block) - head_mask = head_offsets < head_dim - x = tl.load(input_base + head_offsets, mask=head_mask, other=0.0).to(tl.float32) - weight = tl.load(weight_ptr + head_offsets, mask=head_mask, other=0.0).to(tl.float32) - inverse_rms = tl.rsqrt(tl.sum(x * x, axis=0) / head_dim + eps) - normalized = (x * inverse_rms * (weight + 1.0)).to(output_dtype).to(tl.float32) - - if has_pass_through: - pass_mask = head_mask & (head_offsets >= rotary_dim) - tl.store(output_base + head_offsets, normalized, mask=pass_mask) - - rotary_offsets = tl.arange(0, rotary_block) - rotary_mask = rotary_offsets < half_rotary - x_first = tl.load(input_base + rotary_offsets, mask=rotary_mask, other=0.0).to(tl.float32) - x_second = tl.load( - input_base + half_rotary + rotary_offsets, mask=rotary_mask, other=0.0 - ).to(tl.float32) - weight_first = tl.load(weight_ptr + rotary_offsets, mask=rotary_mask, other=0.0).to( - tl.float32 - ) - weight_second = tl.load( - weight_ptr + half_rotary + rotary_offsets, mask=rotary_mask, other=0.0 - ).to(tl.float32) - x_first = (x_first * inverse_rms * (weight_first + 1.0)).to(output_dtype).to(tl.float32) - x_second = (x_second * inverse_rms * (weight_second + 1.0)).to(output_dtype).to(tl.float32) - - if use_mrope: - section = tl.where( - (rotary_offsets % 3 == 1) & (rotary_offsets < mrope_section1 * 3), - 1, - tl.where( - (rotary_offsets % 3 == 2) & (rotary_offsets < mrope_section2 * 3), - 2, - 0, - ), - ) - position = tl.load( - positions_ptr + section * num_tokens + token, - mask=rotary_mask, - other=0, - ).to(tl.int64) - else: - position = tl.load(positions_ptr + token).to(tl.int64) - position_is_valid = (position >= 0) & (position < max_positions) - tl.device_assert(position_is_valid, "position is outside the RoPE table") - safe_position = tl.where(position_is_valid, position, 0) - cos_sin_base = cos_sin_ptr + safe_position * cos_sin_stride_position - cos = tl.load(cos_sin_base + rotary_offsets, mask=rotary_mask, other=0.0).to(tl.float32) - sin = tl.load(cos_sin_base + half_rotary + rotary_offsets, mask=rotary_mask, other=0.0).to( - tl.float32 - ) - cos = tl.where(position_is_valid, cos, float("nan")) - sin = tl.where(position_is_valid, sin, float("nan")) - tl.store( - output_base + rotary_offsets, - x_first * cos - x_second * sin, - mask=rotary_mask, - ) - tl.store( - output_base + half_rotary + rotary_offsets, - x_second * cos + x_first * sin, - mask=rotary_mask, - ) - - if not is_k: - gate_input = input_base + head_dim - gate_output = gate_out_ptr + token * gate_out_stride_token + local_head * head_dim - gate = tl.load(gate_input + head_offsets, mask=head_mask, other=0.0) - tl.store(gate_output + head_offsets, gate, mask=head_mask) - else: - local_head = head - num_q_heads - num_kv_heads - head_offsets = tl.arange(0, head_block) - head_mask = head_offsets < head_dim - input_base = ( - qkv_ptr + token * qkv_stride_token + 2 * q_size + kv_size + local_head * head_dim - ) - output_base = ( - qkv_out_ptr + token * qkv_out_stride_token + q_size + kv_size + local_head * head_dim - ) - value = tl.load(input_base + head_offsets, mask=head_mask, other=0.0) - tl.store(output_base + head_offsets, value, mask=head_mask) - - -def fused_qkv_gemma_rmsnorm_rope_gate( - qkv: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - cos_sin: torch.Tensor, - positions: torch.Tensor, - eps: float, - num_q_heads: int, - num_kv_heads: int, - head_dim: int, - rotary_dim: int, - mrope_section: Optional[Tuple[int, int, int]] = None, -) -> Tuple[torch.Tensor, torch.Tensor]: - """Prepare packed QKV and gate with one Triton kernel. - - Args: - qkv: ``[num_tokens, 2 * q_size + 2 * kv_size]`` BF16/FP16 projection - output in per-head interleaved Q/G layout. - q_weight: ``[head_dim]`` raw Gemma RMSNorm Q weights. - k_weight: ``[head_dim]`` raw Gemma RMSNorm K weights. - cos_sin: ``[max_positions, 2, rotary_dim // 2]`` FP32 NeoX table. - positions: Flattenable ``[num_tokens]`` plain-RoPE positions, or - ``[3, ..., num_tokens]`` interleaved-MRoPE positions. - eps: RMSNorm epsilon. - num_q_heads: Local query-head count. - num_kv_heads: Local key/value-head count. - head_dim: Per-head Q/K/V dimension. - rotary_dim: Prefix dimension receiving NeoX RoPE. - mrope_section: Temporal/height/width rotary-half dimensions. ``None`` - selects plain RoPE; a tuple selects Qwen-style interleaved MRoPE. - - Returns: - Packed QKV ``[num_tokens, q_size + 2 * kv_size]`` and gate - ``[num_tokens, num_q_heads, head_dim]``. - """ - assert qkv.dim() == 2 and qkv.stride(-1) == 1 - assert qkv.dtype in (torch.bfloat16, torch.float16) - assert q_weight.shape == (head_dim,) and k_weight.shape == (head_dim,) - assert q_weight.device == qkv.device and k_weight.device == qkv.device - assert cos_sin.dtype == torch.float32 and cos_sin.is_contiguous() - assert cos_sin.device == qkv.device and positions.device == qkv.device - assert rotary_dim > 0 and rotary_dim <= head_dim and rotary_dim % 2 == 0 - assert cos_sin.shape == (cos_sin.shape[0], 2, rotary_dim // 2) - assert cos_sin.shape[0] > 0 - - num_tokens = qkv.shape[0] - use_mrope = mrope_section is not None - if use_mrope: - assert len(mrope_section) == 3 - assert sum(mrope_section) == rotary_dim // 2 - assert positions.numel() == 3 * num_tokens - positions = positions.reshape(3, num_tokens) - else: - assert positions.numel() == num_tokens - positions = positions.reshape(-1) - assert positions.is_contiguous() - assert positions.dtype in (torch.int32, torch.int64) - - q_size = num_q_heads * head_dim - kv_size = num_kv_heads * head_dim - assert qkv.shape[1] == 2 * q_size + 2 * kv_size - - qkv_out = torch.empty((num_tokens, q_size + 2 * kv_size), dtype=qkv.dtype, device=qkv.device) - gate_out = torch.empty((num_tokens, num_q_heads, head_dim), dtype=qkv.dtype, device=qkv.device) - if num_tokens == 0: - return qkv_out, gate_out - - half_rotary = rotary_dim // 2 - head_block = triton.next_power_of_2(head_dim) - rotary_block = triton.next_power_of_2(half_rotary) - grid = (num_tokens, num_q_heads + 2 * num_kv_heads) - _fused_qkv_gemma_rmsnorm_rope_gate_kernel[grid]( - qkv, - qkv_out, - gate_out, - q_weight, - k_weight, - cos_sin, - positions, - num_tokens, - cos_sin.shape[0], - qkv.stride(0), - qkv_out.stride(0), - gate_out.stride(0), - cos_sin.stride(0), - num_q_heads=num_q_heads, - num_kv_heads=num_kv_heads, - head_dim=head_dim, - rotary_dim=rotary_dim, - half_rotary=half_rotary, - head_block=head_block, - rotary_block=rotary_block, - eps=eps, - output_fp16=qkv.dtype == torch.float16, - has_pass_through=rotary_dim < head_dim, - use_mrope=use_mrope, - mrope_section1=mrope_section[1] if use_mrope else 0, - mrope_section2=mrope_section[2] if use_mrope else 0, - num_warps=4, - ) - return qkv_out, gate_out - - -@triton.jit -def _fused_sigmoid_mul_kernel( - output_ptr, - attention_ptr, - gate_ptr, - output_stride_token, - attention_stride_token, - gate_stride_token, - gate_stride_head, - hidden_size: tl.constexpr, - head_dim: tl.constexpr, - block_size: tl.constexpr, -): - token = tl.program_id(0).to(tl.int64) - block = tl.program_id(1) - offsets = block * block_size + tl.arange(0, block_size) - mask = offsets < hidden_size - head = offsets // head_dim - dim = offsets - head * head_dim - - attention = tl.load( - attention_ptr + token * attention_stride_token + offsets, - mask=mask, - other=0.0, - ).to(tl.float32) - gate = tl.load( - gate_ptr + token * gate_stride_token + head * gate_stride_head + dim, - mask=mask, - other=0.0, - ).to(tl.float32) - tl.store( - output_ptr + token * output_stride_token + offsets, - attention * tl.sigmoid(gate), - mask=mask, - ) - - -def fused_sigmoid_mul( - attention_output: torch.Tensor, - gate: torch.Tensor, - *, - inplace: bool = False, -) -> torch.Tensor: - """Compute ``attention_output * sigmoid(gate)`` in one kernel.""" - assert attention_output.dim() == 2 and attention_output.stride(-1) == 1 - num_tokens, hidden_size = attention_output.shape - if gate.dim() == 3: - assert gate.shape[0] == num_tokens - assert gate.shape[1] * gate.shape[2] == hidden_size - assert gate.stride(-1) == 1 - head_dim = gate.shape[2] - gate_stride_token = gate.stride(0) - gate_stride_head = gate.stride(1) - else: - assert gate.dim() == 2 and gate.shape == attention_output.shape - assert gate.stride(-1) == 1 - head_dim = hidden_size - gate_stride_token = gate.stride(0) - gate_stride_head = hidden_size - - output = attention_output if inplace else torch.empty_like(attention_output) - if num_tokens == 0: - return output - - max_block_size = 1024 if num_tokens < 1024 else 2048 - block_size = min(triton.next_power_of_2(hidden_size), max_block_size) - grid = (num_tokens, triton.cdiv(hidden_size, block_size)) - _fused_sigmoid_mul_kernel[grid]( - output, - attention_output, - gate, - output.stride(0), - attention_output.stride(0), - gate_stride_token, - gate_stride_head, - hidden_size=hidden_size, - head_dim=head_dim, - block_size=block_size, - num_warps=4, - ) - return output diff --git a/tensorrt_llm/_torch/modules/gated_mlp.py b/tensorrt_llm/_torch/modules/gated_mlp.py index e04debbfe5cc..fb7f43f689b1 100644 --- a/tensorrt_llm/_torch/modules/gated_mlp.py +++ b/tensorrt_llm/_torch/modules/gated_mlp.py @@ -67,33 +67,13 @@ def __init__( # Calculate local intermediate size after tensor parallel sharding tp_size = mapping.tp_size + local_intermediate_size = self.intermediate_size // tp_size - local_intermediate_start = Linear._calc_shard(self.intermediate_size, - mapping.tp_size, - mapping.tp_rank) - local_intermediate_end = Linear._calc_shard(self.intermediate_size, - mapping.tp_size, - mapping.tp_rank + 1) - local_intermediate_size = local_intermediate_end - local_intermediate_start - - self._uneven_tp_blocks_lora = (mapping.tp_size > 1 - and self.intermediate_size % - mapping.tp_size != 0) - - # gateup_shard_indices_mapping is the local offset and size for each sub-weight - # in this rank's concatenated (gate || up) buffer. - # override_tp_sharding is the absolute range of the global weight from which - # this rank pulls each sub-weight. gateup_shard_indices_mapping = { 'gate': (0, local_intermediate_size), 'up': (local_intermediate_size, local_intermediate_size), } - override_tp_sharding = { - 'gate': (local_intermediate_start, local_intermediate_end), - 'up': (local_intermediate_start, local_intermediate_end), - } - self.gate_up_proj = Linear( self.hidden_size, self.intermediate_size * 2, @@ -113,7 +93,6 @@ def __init__( disable_deep_gemm=disable_deep_gemm, fused_weight_shard_indices_mapping=gateup_shard_indices_mapping, use_custom_cublas_mm=use_custom_cublas_mm, - override_tp_sharding=override_tp_sharding, ) if is_shared_expert: @@ -327,10 +306,6 @@ def forward_lora( ) -> torch.Tensor: assert lora_params is not None assert self.layer_idx is not None, "layer_idx is required for lora" - if self._uneven_tp_blocks_lora: - raise NotImplementedError( - "LoRA is not supported with uneven TP for GatedMLP " - "(intermediate_size not divisible by tp_size).") h1 = self.gate_up_proj(x) diff --git a/tensorrt_llm/_torch/modules/layer_norm.py b/tensorrt_llm/_torch/modules/layer_norm.py index 743966ea103c..811067952c52 100644 --- a/tensorrt_llm/_torch/modules/layer_norm.py +++ b/tensorrt_llm/_torch/modules/layer_norm.py @@ -34,9 +34,6 @@ class LayerNorm(nn.Module): device: Optional device for parameters. has_weights: Whether to include learnable weight parameters. has_bias: Whether to include learnable bias parameters. - residual_in_fp32: Whether to accumulate the residual in FP32 before - normalization. If false, preserve the input dtype for the residual - addition and convert the result to FP32 for normalization. """ def __init__( @@ -48,7 +45,6 @@ def __init__( device: Optional[torch.device] = None, has_weights: bool = True, has_bias: bool = True, - residual_in_fp32: bool = True, ): super().__init__() if has_weights: @@ -70,7 +66,6 @@ def __init__( device=device), persistent=False) self.variance_epsilon = eps - self.residual_in_fp32 = residual_in_fp32 @maybe_compile(dynamic=True) def forward( @@ -89,15 +84,10 @@ def forward( """ input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) if isinstance(residual, torch.Tensor): - if self.residual_in_fp32: - hidden_states = hidden_states.to(torch.float32) + residual.to( - torch.float32) - else: - hidden_states = (hidden_states + residual).to(torch.float32) + hidden_states = hidden_states + residual.to(torch.float32) residual = hidden_states.to(input_dtype) - else: - hidden_states = hidden_states.to(torch.float32) hidden_states = nn.functional.layer_norm( hidden_states, diff --git a/tensorrt_llm/_torch/modules/linear.py b/tensorrt_llm/_torch/modules/linear.py index 10504cf6ecd7..e8da57a36c0e 100644 --- a/tensorrt_llm/_torch/modules/linear.py +++ b/tensorrt_llm/_torch/modules/linear.py @@ -107,12 +107,6 @@ def load_weight_shard( device: torch.device = torch.device('cpu'), return_slice_indices: bool = False, ) -> torch.Tensor: - """Legacy weight shard helper using ceil-divide sharding. - - `Linear.load_shard` is preferred — it respects uneven-TP overrides, fused - QKV/gate-up sharding dicts, and quant-specific scale/packing semantics that - this function does not. - """ # Skip device transfers on integrated GPUs to conserve shared memory if weight.device.type != device.type and is_device_integrated(): # For integrated GPU systems (e.g., DGX Spark), CPU and GPU share limited physical memory. @@ -184,8 +178,7 @@ def load_weights_vanilla_helper(module: Linear, weights: List[Dict], weight_transform=lambda x: x, bias_transform=lambda x: x, - allow_partial_loading: bool = False, - elm_packing: int = 1): + allow_partial_loading: bool = False): assert len(weights) == 1 if not allow_partial_loading: assert "weight" in weights[0] @@ -193,10 +186,9 @@ def load_weights_vanilla_helper(module: Linear, assert "bias" in weights[0] device = torch.device('cuda') - weight = module.load_shard(weights[0], - 'weight', - device=device, - elm_packing=elm_packing) + weight = load_weight_shard(weights[0]['weight'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "weight" in weights[0] else None if weight is not None: if module.has_weight_only_quant: @@ -211,7 +203,9 @@ def load_weights_vanilla_helper(module: Linear, copy_weight(module.weight, weight_transform(weight)) if module.bias is not None: - bias = module.load_shard(weights[0], 'bias', device=device) + bias = load_weight_shard(weights[0]['bias'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "bias" in weights[0] else None if bias is not None: copy_weight(module.bias, bias_transform(bias)) @@ -221,8 +215,7 @@ def load_weights_fused_qkv_helper( weights: List[Dict], weight_transform=lambda x: x, bias_transform=lambda x: x, - allow_partial_loading: bool = False, - elm_packing: int = 1, + allow_partial_loading: bool = False ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: if not allow_partial_loading: assert all('weight' in weights[i] for i in range(3)) @@ -234,27 +227,26 @@ def load_weights_fused_qkv_helper( ) is not None, "Fused weight shard indices mapping is required in partial loading" device = torch.device('cuda') - q_weight = module.load_shard(weights[0], - 'weight', - device=device, - name='q', - elm_packing=elm_packing) - k_weight = module.load_shard(weights[1], - 'weight', - device=device, - name='k', - elm_packing=elm_packing) - v_weight = module.load_shard(weights[2], - 'weight', - device=device, - name='v', - elm_packing=elm_packing) + q_weight = load_weight_shard(weights[0]['weight'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "weight" in weights[0] else None + k_weight = load_weight_shard(weights[1]['weight'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "weight" in weights[1] else None + v_weight = load_weight_shard(weights[2]['weight'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "weight" in weights[2] else None if module.bias is not None: - q_bias = module.load_shard(weights[0], 'bias', device=device, name='q') - k_bias = module.load_shard(weights[1], 'bias', device=device, name='k') - v_bias = module.load_shard(weights[2], 'bias', device=device, name='v') - + q_bias = load_weight_shard(weights[0]['bias'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "bias" in weights[0] else None + k_bias = load_weight_shard(weights[1]['bias'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "bias" in weights[1] else None + v_bias = load_weight_shard(weights[2]['bias'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "bias" in weights[2] else None if not allow_partial_loading: copy_weight(module.bias, bias_transform(torch.cat((q_bias, k_bias, v_bias)))) @@ -272,12 +264,11 @@ def load_weights_fused_qkv_helper( def load_weights_fused_gate_up_helper( - module: Linear, - weights: List[Dict], - weight_transform=lambda x: x, - bias_transform=lambda x: x, - allow_partial_loading: bool = False, - elm_packing: int = 1, + module: Linear, + weights: List[Dict], + weight_transform=lambda x: x, + bias_transform=lambda x: x, + allow_partial_loading: bool = False ) -> tuple[torch.Tensor, torch.Tensor]: if not allow_partial_loading: assert all('weight' in weights[i] for i in range(2)) @@ -289,26 +280,19 @@ def load_weights_fused_gate_up_helper( ) is not None, "Fused weight shard indices mapping is required in partial loading" device = torch.device('cuda') - gate_weight = module.load_shard(weights[0], - 'weight', - device=device, - name='gate', - elm_packing=elm_packing) - up_weight = module.load_shard(weights[1], - 'weight', - device=device, - name='up', - elm_packing=elm_packing) - + gate_weight = load_weight_shard(weights[0]['weight'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "weight" in weights[0] else None + up_weight = load_weight_shard(weights[1]['weight'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "weight" in weights[1] else None if module.bias is not None: - gate_bias = module.load_shard(weights[0], - 'bias', - device=device, - name='gate') - up_bias = module.load_shard(weights[1], - 'bias', - device=device, - name='up') + gate_bias = load_weight_shard(weights[0]['bias'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "bias" in weights[0] else None + up_bias = load_weight_shard(weights[1]['bias'], module.tp_size, + module.tp_rank, module.tp_mode, + device) if "bias" in weights[1] else None if not allow_partial_loading: copy_weight(module.bias, bias_transform(torch.cat((gate_bias, up_bias)))) @@ -486,12 +470,6 @@ def pre_reload_weights(self, module: Linear) -> None: requires_grad=False) module.register_parameter(param_name, param) - def get_tp_alignment(self, - tp_mode: Optional[TensorParallelMode], - quant_config: Optional[QuantConfig] = None) -> int: - """ Alignment required for TP shard boundaries. """ - return 1 - class UnquantizedLinearMethod(LinearMethodBase): @@ -632,11 +610,6 @@ class FP8QDQLinearMethod(UnquantizedLinearMethod): supports_nccl_symmetric_memory_window_output: ClassVar[bool] = True - def get_static_input_scale(self, module: Linear) -> Optional[torch.Tensor]: - if module.input_scale is not None and not module.force_dynamic_quantization: - return module.input_scale - return None - def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): weight_shape = (out_features, in_features) @@ -670,13 +643,12 @@ def apply(self, module: Linear, input: torch.Tensor, if input.dim() > 2: input = input.reshape(-1, input.shape[-1]) - static_input_scale = self.get_static_input_scale(module) cur_input_scale = module.input_scale if input.dtype != torch.float8_e4m3fn: - if static_input_scale is not None: + if module.input_scale is not None and not module.force_dynamic_quantization: # Static quantization qinput, _ = torch.ops.tensorrt_llm.static_quantize_e4m3_per_tensor( - input, static_input_scale) + input, module.input_scale) else: # Dynamic quantization qinput, cur_input_scale = torch.ops.tensorrt_llm.quantize_e4m3_per_tensor( @@ -743,12 +715,18 @@ def load_weight_scales(self, shard_keys: list[str] = None): input_scales, weight_scales = {}, {} if shard_keys is None: - shard_keys = [None] - for shard_key, w in zip(shard_keys, weights): - if "input_scale" in w: - input_scales[shard_key] = w["input_scale"][...].reshape([]) - if "weight_scale" in w: - weight_scales[shard_key] = w["weight_scale"][...].reshape([]) + for w in weights: + if "input_scale" in w: + input_scales[None] = w["input_scale"][...].reshape([]) + if "weight_scale" in w: + weight_scales[None] = w["weight_scale"][...].reshape([]) + else: + for shard_key, w in zip(shard_keys, weights): + if "input_scale" in w: + input_scales[shard_key] = w["input_scale"][...].reshape([]) + if "weight_scale" in w: + weight_scales[shard_key] = w["weight_scale"][...].reshape( + []) return input_scales, weight_scales def load_weights_vanilla(self, @@ -1034,7 +1012,9 @@ def load_weights_vanilla(self, module, weights, allow_partial_loading=allow_partial_loading) scale_name = self._get_scale_name(weights) if scale_name in weights[0]: - weight_scale = module.load_shard(weights[0], scale_name) + weight_scale = load_weight_shard(weights[0][scale_name], + module.tp_size, module.tp_rank, + module.tp_mode) # compressed-tensors stores per-channel weight scales as [out, 1]; # the weight_scale buffer is 1-D [out] (ModelOpt/DS recipes store # it 1-D). Flatten so copy_ does not broadcast to [out, out]. @@ -1051,12 +1031,16 @@ def load_weights_fused_qkv_linear(self, allow_partial_loading: bool = False): super().load_weights_fused_qkv_linear( module, weights, allow_partial_loading=allow_partial_loading) - scale_name = self._get_scale_name(weights) - q_scale = module.load_shard(weights[0], scale_name, name='q') - k_scale = module.load_shard(weights[1], scale_name, name='k') - v_scale = module.load_shard(weights[2], scale_name, name='v') - + q_scale = load_weight_shard( + weights[0][scale_name], module.tp_size, module.tp_rank, + module.tp_mode) if scale_name in weights[0] else None + k_scale = load_weight_shard( + weights[1][scale_name], module.tp_size, module.tp_rank, + module.tp_mode) if scale_name in weights[1] else None + v_scale = load_weight_shard( + weights[2][scale_name], module.tp_size, module.tp_rank, + module.tp_mode) if scale_name in weights[2] else None for shard_key, scale in zip( module.fused_weight_shard_indices_mapping.keys(), [q_scale, k_scale, v_scale]): @@ -1076,11 +1060,13 @@ def load_weights_fused_gate_up_linear( allow_partial_loading: bool = False) -> None: super().load_weights_fused_gate_up_linear( module, weights, allow_partial_loading=allow_partial_loading) - scale_name = self._get_scale_name(weights) - gate_scale = module.load_shard(weights[0], scale_name, name='gate') - up_scale = module.load_shard(weights[1], scale_name, name='up') - + gate_scale = load_weight_shard( + weights[0][scale_name], module.tp_size, module.tp_rank, + module.tp_mode) if scale_name in weights[0] else None + up_scale = load_weight_shard( + weights[1][scale_name], module.tp_size, module.tp_rank, + module.tp_mode) if scale_name in weights[1] else None for shard_key, scale in zip( module.fused_weight_shard_indices_mapping.keys(), [gate_scale, up_scale]): @@ -1099,9 +1085,6 @@ class FP8BlockScalesLinearMethod(UnquantizedLinearMethod): # fp8_block_scaling_gemm does not support writing into an NCCL window buffer. supports_nccl_symmetric_memory_window_output: ClassVar[bool] = False - def get_tp_alignment(self, tp_mode, quant_config=None): - return 128 - def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): weight_shape = (out_features, in_features) @@ -1194,8 +1177,8 @@ def load_weights_vanilla(self, # modelopt fp8_pb_wo can have 2 extra singleton dimensions if full_weight_scale.dim() == 4: full_weight_scale = full_weight_scale.squeeze(1).squeeze(-1) - - weight_scale = module.load_shard(full_weight_scale, scale_span=128) + weight_scale = load_weight_shard(full_weight_scale, module.tp_size, + module.tp_rank, module.tp_mode) copy_weight(module.weight_scale, weight_scale) if "input_scale" in weights[0]: copy_weight(module.input_scale, weights[0]["input_scale"]) @@ -1239,9 +1222,8 @@ def load_weights_fused_qkv_linear( ] scales = [ - module.load_shard(s, scale_span=128, name=name) - if s is not None else None - for s, name in zip(full_scales_squeezed, ('q', 'k', 'v')) + load_weight_shard(s, module.tp_size, module.tp_rank, module.tp_mode) + if s is not None else None for s in full_scales_squeezed ] processed_mapping = self.remap_fused_shard_indices_by_divisible_factor( module.fused_weight_shard_indices_mapping, 128) @@ -1268,11 +1250,9 @@ def load_weights_fused_gate_up_linear( for s in full_scales ] scales = [ - module.load_shard(s, scale_span=128, name=name) - if s is not None else None - for s, name in zip(full_scales_squeezed, ('gate', 'up')) + load_weight_shard(s, module.tp_size, module.tp_rank, module.tp_mode) + if s is not None else None for s in full_scales_squeezed ] - processed_mapping = self.remap_fused_shard_indices_by_divisible_factor( module.fused_weight_shard_indices_mapping, 128) for shard_key, scale in zip(processed_mapping.keys(), scales): @@ -1283,40 +1263,11 @@ def load_weights_fused_gate_up_linear( def transform_weights(self, module: Linear) -> None: super().transform_weights(module) - use_deep_gemm_layout = ( - is_sm_100f() - and not (module.use_cute_dsl_blockscaling_mm - or module.disable_deep_gemm)) or get_sm_version() == 120 - use_indexer_q_cutedsl_layout = (use_deep_gemm_layout and getattr( - module, "use_indexer_q_cutedsl_fusion", False)) - if use_deep_gemm_layout or use_indexer_q_cutedsl_layout: + if (is_sm_100f() and not (module.use_cute_dsl_blockscaling_mm + or module.disable_deep_gemm)) or \ + get_sm_version() == 120: weight, weight_scale = resmooth_to_fp8_e8m0(module.weight, module.weight_scale) - - if use_indexer_q_cutedsl_layout: - # Native SM100 MXF8 MMA consumes one scale per 32 K values. - # The checkpoint/production quantization contract remains - # 128x128: expand each row block and repeat each K scale four - # times, then materialize CUTLASS/CuTe's 128x4 swizzle once at - # weight-load time. - n = weight.shape[0] - scale_cutedsl = weight_scale.repeat_interleave(128, dim=0)[:n] - scale_cutedsl = scale_cutedsl.repeat_interleave(4, dim=1) - scale_cutedsl = torch.ops.trtllm.block_scale_interleave( - scale_cutedsl.to(torch.float8_e8m0fnu).view(torch.uint8)) - module.register_buffer( - "indexer_q_weight_scale_cutedsl", - scale_cutedsl, - persistent=False, - ) - module.register_buffer( - "indexer_q_alpha_cutedsl", - torch.ones((1, ), dtype=torch.float32, - device=weight.device), - persistent=False, - ) - - if use_deep_gemm_layout: transformed_scale = transform_sf_into_required_layout( weight_scale, mn=weight.shape[0], @@ -1342,13 +1293,6 @@ class NVFP4LinearMethod(LinearMethodBase): # construction; LLM paths leave it False to avoid host overhead. use_tunable_quantize: bool = False - def get_tp_alignment(self, tp_mode, quant_config=None): - # 32-element alignment for both modes. ROW shards in_features which - # is packed 2:1, so 32 → 16 packed, meeting GEMM col_alignment=16. - # COLUMN must also be 32 because column output feeds row input, and - # row's packed weight K dimension needs 16-alignment. - return 32 - def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): module.scaling_vector_size = 16 @@ -1558,17 +1502,17 @@ def load_weight_scales(self, """ device = torch.device("cuda") - scale_span = 16 if module.tp_mode == TensorParallelMode.ROW else 1 # Per-shard weight_scale: load, TP-shard, store in tmp dict keyed by shard if shard_keys is not None: if not hasattr(module, "tmp_nvfp4_weight_scales"): module.tmp_nvfp4_weight_scales = {} for shard_key, w in zip(shard_keys, weights): if "weight_scale" in w: - ws = module.load_shard(w["weight_scale"], - device=device, - scale_span=scale_span, - name=shard_key).contiguous() + ws = load_weight_shard(w["weight_scale"], + module.tp_size, + module.tp_rank, + module.tp_mode, + device=device).contiguous() assert ws.dtype == torch.float8_e4m3fn module.tmp_nvfp4_weight_scales[shard_key] = ws.view( fp4_utils.float4_sf_dtype) @@ -1576,9 +1520,11 @@ def load_weight_scales(self, # Vanilla: single weight_scale, load + interleave directly w = weights[0] if "weight_scale" in w: - ws = module.load_shard(w["weight_scale"], - device=device, - scale_span=scale_span).contiguous() + ws = load_weight_shard(w["weight_scale"], + module.tp_size, + module.tp_rank, + module.tp_mode, + device=device).contiguous() ws = ws.view(fp4_utils.float4_sf_dtype) ws = torch.ops.trtllm.block_scale_interleave(ws) copy_weight(module.weight_scale, ws) @@ -1674,12 +1620,9 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict], allow_partial_loading: bool = False) -> None: - - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 load_weights_vanilla_helper(module, weights, - allow_partial_loading=allow_partial_loading, - elm_packing=elm_packing) + allow_partial_loading=allow_partial_loading) # Load scales (vanilla = no shard_keys) self.load_weight_scales(module, weights, shard_keys=None) @@ -1687,14 +1630,13 @@ def load_weights_vanilla(self, # Load pre_quant_scale if it exists (for NVFP4_AWQ) if "pre_quant_scale" in weights[0]: device = module.weight.device - # scale_span is flipped because flip_tp=True - act_scale_span = 1 if module.tp_mode == TensorParallelMode.ROW else 16 - pre_quant_scale = module.load_shard( + pre_quant_scale = load_weight_shard( weights[0]["pre_quant_scale"], + module.tp_size, + module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - flip_tp=True, - device=device, - scale_span=act_scale_span, + TensorParallelMode.flip(module.tp_mode), + device, ) module.pre_quant_scale = Parameter( @@ -1708,12 +1650,8 @@ def load_weights_fused_qkv_linear( module: Linear, weights: List[Dict], allow_partial_loading: bool = False) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, - weights, - allow_partial_loading=allow_partial_loading, - elm_packing=elm_packing) + module, weights, allow_partial_loading=allow_partial_loading) weight_mode = module.weights_loading_config.weight_mode @@ -1785,12 +1723,8 @@ def load_weights_fused_gate_up_linear( module: Linear, weights: List[Dict], allow_partial_loading: bool = False) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, - weights, - allow_partial_loading=allow_partial_loading, - elm_packing=elm_packing) + module, weights, allow_partial_loading=allow_partial_loading) weight_mode = module.weights_loading_config.weight_mode device = torch.device("cuda") @@ -1809,13 +1743,14 @@ def load_weights_fused_gate_up_linear( # Load pre_quant_scale if it exists (for NVFP4_AWQ) # NOTE: pre_quant_scale is the same for gate and up since modelopt checks which layer shared the same input if "pre_quant_scale" in weights[0]: - act_scale_span = 1 if module.tp_mode == TensorParallelMode.ROW else 16 - pre_quant_scale = module.load_shard( + device = module.weight.device + pre_quant_scale = load_weight_shard( weights[0]["pre_quant_scale"], + module.tp_size, + module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - flip_tp=True, - device=module.weight.device, - scale_span=act_scale_span, + TensorParallelMode.flip(module.tp_mode), + device, ) module.pre_quant_scale = Parameter( @@ -2043,10 +1978,6 @@ def apply(self, module: Linear, input: torch.Tensor, class W4A8NVFP4FP8LinearMethod(LinearMethodBase): - def get_tp_alignment(self, tp_mode, quant_config=None): - # Same as NVFP4: 32-element alignment for both modes. - return 32 - def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): module.epilogue_tile_m = 128 @@ -2115,9 +2046,10 @@ def apply(self, module: Linear, input: torch.Tensor, def load_weight_scales( self, - module: Linear, weights: List[Dict], - shard_keys: Optional[List[str]] = None, + tp_size: int = 1, + tp_rank: int = 0, + tp_mode: Optional[TensorParallelMode] = None, ): # For concatenated weights (qkv_proj / up_gate_proj), the global scaling factors and input scaling factors should be shared. input_scale = None @@ -2125,27 +2057,6 @@ def load_weight_scales( weight_scale = [] device = torch.device("cuda") - scale_span = 32 if module.tp_mode == TensorParallelMode.ROW else 1 - - if shard_keys is not None: - for shard_key, w in zip(shard_keys, weights): - if "weight_scale" in w: - ws = module.load_shard(w["weight_scale"], - device=device, - scale_span=scale_span, - name=shard_key).contiguous() - assert ws.dtype == torch.float8_e4m3fn - weight_scale.append( - ws.view(dtype=fp4_utils.float4_sf_dtype)) - else: - for w in weights: - if "weight_scale" in w: - ws = module.load_shard(w["weight_scale"], - device=device, - scale_span=scale_span).contiguous() - assert ws.dtype == torch.float8_e4m3fn - weight_scale.append( - ws.view(dtype=fp4_utils.float4_sf_dtype)) for w in weights: if "input_scale" in w: @@ -2154,6 +2065,14 @@ def load_weight_scales( else: assert input_scale == w["input_scale"][ ...], "The input_scale should be same for all the weights" + if "weight_scale" in w: + ws = load_weight_shard(w["weight_scale"], + tp_size, + tp_rank, + tp_mode, + device=device).contiguous() + assert ws.dtype == torch.float8_e4m3fn + weight_scale.append(ws.view(dtype=fp4_utils.float4_sf_dtype)) if "weight_scale_2" in w: if weight_scale_2 is None: weight_scale_2 = w["weight_scale_2"][...] @@ -2169,16 +2088,16 @@ def load_weight_scales( return input_scale, weight_scale, weight_scale_2, alpha def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 # FIXME: this depends on the kernel internals load_weights_vanilla_helper( - module, - weights, - lambda w: fp4_utils.shuffle_matrix_a(w, module.epilogue_tile_m), - elm_packing=elm_packing) + module, weights, + lambda w: fp4_utils.shuffle_matrix_a(w, module.epilogue_tile_m)) input_scale, weight_scale, weight_scale_2, alpha = self.load_weight_scales( - module, weights) + weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) assert len(weights) == 1 weight_scale = weight_scale[0] @@ -2194,13 +2113,14 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights, elm_packing=elm_packing) + module, weights) - weight_mode = module.weights_loading_config.weight_mode input_scale, weight_scales, weight_scale_2, alpha = self.load_weight_scales( - module, weights, shard_keys=weight_mode.shard_keys) + weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) # Swizzle weight scales after concatenation weight_scale = torch.cat(weight_scales, 0) # Shuffle and Swizzle weight scale @@ -2220,17 +2140,18 @@ def load_weights_fused_qkv_linear(self, module: Linear, def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((gate_weight, up_weight)) fused_weight = fp4_utils.shuffle_matrix_a(fused_weight, module.epilogue_tile_m) copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode input_scale, weight_scales, weight_scale_2, alpha = self.load_weight_scales( - module, weights, shard_keys=weight_mode.shard_keys) + weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) # Swizzle weight scales after concatenation weight_scale = torch.cat(weight_scales, 0) # Shuffle and Swizzle weight scale @@ -2246,10 +2167,6 @@ def load_weights_fused_gate_up_linear(self, module: Linear, class W4A8MXFP4FP8LinearMethod(LinearMethodBase): - def get_tp_alignment(self, tp_mode, quant_config=None): - # Same as NVFP4: 32-element alignment for both modes. - return 32 - def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): module.scaling_vector_size = 32 @@ -2296,30 +2213,32 @@ def apply(self, module: Linear, input: torch.Tensor, return output def load_weight_scales(self, - module: Linear, weights: List[Dict], - shard_keys: Optional[List[str]] = None): + tp_size: int = 1, + tp_rank: int = 0, + tp_mode: Optional[TensorParallelMode] = None): + # For concatenated weights (qkv_proj / up_gate_proj), the global scaling factors and input scaling factors should be shared. weight_scale = [] device = torch.device("cuda") - scale_span = 32 if module.tp_mode == TensorParallelMode.ROW else 1 - - if shard_keys is None: - shard_keys = [None] - for shard_key, w in zip(shard_keys, weights): + for w in weights: if "weight_scale" in w: - ws = module.load_shard(w["weight_scale"], - device=device, - scale_span=scale_span, - name=shard_key).contiguous() + ws = load_weight_shard(w["weight_scale"], + tp_size, + tp_rank, + tp_mode, + device=device).contiguous() + # Should be E8M0 for MXFP4 assert ws.dtype == torch.uint8 weight_scale.append(ws.view(fp4_utils.float4_sf_dtype)) return weight_scale def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 - load_weights_vanilla_helper(module, weights, elm_packing=elm_packing) + load_weights_vanilla_helper(module, weights) - weight_scale = self.load_weight_scales(module, weights) + weight_scale = self.load_weight_scales(weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) assert len(weights) == 1 weight_scale = weight_scale[0] # Swizzle weight scale @@ -2328,30 +2247,31 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((q_weight, k_weight, v_weight)) copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode - weight_scale = self.load_weight_scales( - module, weights, shard_keys=weight_mode.shard_keys) + weight_scale = self.load_weight_scales(weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) weight_scale = torch.cat(weight_scale, 0) weight_scale = torch.ops.trtllm.block_scale_interleave(weight_scale) copy_weight(module.weight_scale, weight_scale) def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.ROW else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((gate_weight, up_weight)) copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode - weight_scale = self.load_weight_scales( - module, weights, shard_keys=weight_mode.shard_keys) + weight_scale = self.load_weight_scales(weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) + # Swizzle weight scales after concatenation weight_scale = torch.cat(weight_scale, 0) weight_scale = torch.ops.trtllm.block_scale_interleave(weight_scale) copy_weight(module.weight_scale, weight_scale) @@ -2359,17 +2279,6 @@ def load_weights_fused_gate_up_linear(self, module: Linear, class WeightOnlyQuantLinearMethod(LinearMethodBase): - def get_tp_alignment(self, tp_mode, quant_config=None): - # preprocess_weights_for_mixed_gemm requires: - # - ROW (in_features): % B_ROWS_PER_MMA (32 for INT4, 16 for INT8) - # - COLUMN (out_features): % MMA_SHAPE_N (8) - # COLUMN must also satisfy ROW of next layer in a column->row pipeline. - if quant_config is not None and quant_config.layer_quant_mode.is_int4_weight_only( - ): - return 32 - else: - return 16 - def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype) -> None: @@ -2408,61 +2317,69 @@ def apply(self, module: Linear, input: torch.Tensor, def load_weight_scales( self, - module: Linear, weights: List[Dict], - shard_keys: Optional[List[Optional[str]]] = None - ) -> List[torch.Tensor]: + tp_size: int = 1, + tp_rank: int = 0, + tp_mode: Optional[TensorParallelMode] = None) -> List[torch.Tensor]: device = torch.device("cuda") - if shard_keys is None: - shard_keys = [None] - weight_scales = [] - for shard_key, w in zip(shard_keys, weights): - ws = module.load_shard(w, - 'weight_scale', - device=device, - name=shard_key) - weight_scales.append(ws) + q_weight_scale = load_weight_shard(weights[0]['weight_scale'], + tp_size, + tp_rank, + tp_mode, + device=device) + k_weight_scale = load_weight_shard(weights[1]['weight_scale'], + tp_size, + tp_rank, + tp_mode, + device=device) + v_weight_scale = load_weight_shard(weights[2]['weight_scale'], + tp_size, + tp_rank, + tp_mode, + device=device) + weight_scales = [q_weight_scale, k_weight_scale, v_weight_scale] + return weight_scales def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: - weight_dtype, weight_id = get_weight_dtype_and_id(module) - # INT4 checkpoint tensors are packed 2:1 along the output dimension - # before preprocessing, so COLUMN logical shard boundaries must be - # converted to packed coordinates. ROW shards the unpacked K dimension. - elm_packing = weight_id if module.tp_mode == TensorParallelMode.COLUMN else 1 - load_weights_vanilla_helper(module, weights, elm_packing=elm_packing) + load_weights_vanilla_helper(module, weights) + + device = torch.device('cuda') + weight_scale = load_weight_shard(weights[0]['weight_scale'], + module.tp_size, module.tp_rank, + module.tp_mode, device) - weight_scales = self.load_weight_scales(module, weights) - assert len(weight_scales) == 1 - copy_weight(module.weight_scale, weight_scales[0]) + copy_weight(module.weight_scale, weight_scale) def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]) -> None: - weight_dtype, weight_id = get_weight_dtype_and_id(module) - elm_packing = weight_id if module.tp_mode == TensorParallelMode.COLUMN else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((q_weight, k_weight, v_weight)) + weight_dtype, _ = get_weight_dtype_and_id(module) fused_weight = preprocess_weights_for_mixed_gemm( fused_weight.to(torch.int8).T.contiguous().cpu(), weight_dtype, torch.float16).cuda().contiguous() copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode - weight_scales = self.load_weight_scales( - module, weights, shard_keys=weight_mode.shard_keys) + weight_scales = self.load_weight_scales(weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) + + # Create concatenated weight scale tensor cat_weight_scale = torch.cat(weight_scales, dim=0) copy_weight(module.weight_scale, cat_weight_scale) def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]) -> None: - weight_dtype, weight_id = get_weight_dtype_and_id(module) - elm_packing = weight_id if module.tp_mode == TensorParallelMode.COLUMN else 1 + device = torch.device('cuda') + weight_dtype, _ = get_weight_dtype_and_id(module) gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((gate_weight, up_weight)) @@ -2472,22 +2389,18 @@ def load_weights_fused_gate_up_linear(self, module: Linear, copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode - weight_scales = self.load_weight_scales( - module, weights, shard_keys=weight_mode.shard_keys) - fused_scale = torch.cat(weight_scales, dim=0) + left_scale = load_weight_shard(weights[0]['weight_scale'], + module.tp_size, module.tp_rank, + module.tp_mode, device).contiguous() + right_scale = load_weight_shard(weights[1]['weight_scale'], + module.tp_size, module.tp_rank, + module.tp_mode, device).contiguous() + fused_scale = torch.cat([left_scale, right_scale], dim=0) copy_weight(module.weight_scale, fused_scale) class W4A16_AWQ_LinearMethod(LinearMethodBase): - def get_tp_alignment(self, tp_mode, quant_config=None): - if quant_config is None: - return 1 - # ROW shards input groups directly. COLUMN shards output features, which - # feed the next row-parallel layer's grouped input dimension in MLPs. - return quant_config.group_size - def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype) -> None: @@ -2538,51 +2451,60 @@ def apply(self, module: Linear, input: torch.Tensor, def load_weight_scales( self, - module: Linear, weights: List[Dict], - shard_keys: Optional[List[str]] = None) -> List[torch.Tensor]: + tp_size: int = 1, + tp_rank: int = 0, + tp_mode: Optional[TensorParallelMode] = None) -> List[torch.Tensor]: device = torch.device("cuda") - scale_span = module.quant_config.group_size if module.tp_mode == TensorParallelMode.ROW else 1 - weight_scales = [] - if shard_keys is None: - shard_keys = [None] - for shard_key, w in zip(shard_keys, weights): - weight_scales.append( - module.load_shard(w, - 'weight_scale', - device=device, - name=shard_key, - scale_span=scale_span)) + q_weight_scale = load_weight_shard(weights[0]['weight_scale'], + tp_size, + tp_rank, + tp_mode, + device=device) + k_weight_scale = load_weight_shard(weights[1]['weight_scale'], + tp_size, + tp_rank, + tp_mode, + device=device) + v_weight_scale = load_weight_shard(weights[2]['weight_scale'], + tp_size, + tp_rank, + tp_mode, + device=device) + weight_scales = [q_weight_scale, k_weight_scale, v_weight_scale] + return weight_scales def load_weights_vanilla(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 - load_weights_vanilla_helper(module, weights, elm_packing=elm_packing) + load_weights_vanilla_helper(module, weights) # Use the same device as the weight tensor # as we register pre_quant_scale after sharded model weights are moved to respective gpus device = module.weight.device - pre_quant_scale = module.load_shard( + pre_quant_scale = load_weight_shard( weights[0]["pre_quant_scale"], + module.tp_size, + module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - flip_tp=True, - device=device, + TensorParallelMode.flip(module.tp_mode), + device, ) module.pre_quant_scale = Parameter( torch.ones((module.in_features, ), dtype=pre_quant_scale.dtype), requires_grad=False).to(device=device) - weight_scale = self.load_weight_scales(module, weights)[0] + weight_scale = load_weight_shard(weights[0]['weight_scale'], + module.tp_size, module.tp_rank, + module.tp_mode, device) copy_weight(module.pre_quant_scale, pre_quant_scale) copy_weight(module.weight_scale, weight_scale.T.contiguous()) def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((q_weight, k_weight, v_weight)) fused_weight = preprocess_weights_for_mixed_gemm( @@ -2591,9 +2513,8 @@ def load_weights_fused_qkv_linear(self, module: Linear, copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode - weight_scales = self.load_weight_scales( - module, weights, shard_keys=weight_mode.shard_keys) + weight_scales = self.load_weight_scales(weights, module.tp_size, + module.tp_rank, module.tp_mode) # Create concatenated weight scale tensor cat_weight_scale = torch.cat(weight_scales, dim=0).T.contiguous() @@ -2601,9 +2522,9 @@ def load_weights_fused_qkv_linear(self, module: Linear, def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]) -> None: - elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 + device = torch.device('cuda') gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((gate_weight, up_weight)) fused_weight = preprocess_weights_for_mixed_gemm( @@ -2612,22 +2533,18 @@ def load_weights_fused_gate_up_linear(self, module: Linear, copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode - weight_scales = self.load_weight_scales( - module, weights, shard_keys=weight_mode.shard_keys) - fused_scale = torch.cat(weight_scales, dim=0).T.contiguous() + left_scale = load_weight_shard(weights[0]['weight_scale'], + module.tp_size, module.tp_rank, + module.tp_mode, device).contiguous() + right_scale = load_weight_shard(weights[1]['weight_scale'], + module.tp_size, module.tp_rank, + module.tp_mode, device).contiguous() + fused_scale = torch.cat([left_scale, right_scale], dim=0).T.contiguous() copy_weight(module.weight_scale, fused_scale) class W4A8_AWQ_LinearMethod(LinearMethodBase): - def get_tp_alignment(self, tp_mode, quant_config=None): - if quant_config is None: - return 1 - # Same grouped INT4 weight layout as W4A16_AWQ. ROW must keep input - # groups intact, and COLUMN output shards feed row-parallel grouped K. - return quant_config.group_size - def create_weights(self, module: Linear, in_features: int, out_features: int, bias: bool, dtype: torch.dtype): # Quantized weights @@ -2707,20 +2624,18 @@ def apply(self, module: Linear, input: torch.Tensor, return output def load_weight_scales_w4a8(self, - module: Linear, weights: List[Dict], - shard_keys: Optional[List[str]] = None): + tp_size: int = 1, + tp_rank: int = 0, + tp_mode: Optional[TensorParallelMode] = None): # For concatenated weights (qkv_proj / up_gate_proj), the global scaling factors and input scaling factors should be shared. input_scale = None weight_scale_2 = None weight_scale = [] device = torch.device("cuda") - scale_span = module.quant_config.group_size if module.tp_mode == TensorParallelMode.ROW else 1 - if shard_keys is None: - shard_keys = [None] - for shard_key, w in zip(shard_keys, weights): + for w in weights: if "input_scale" in w: if input_scale is None: input_scale = w["input_scale"][...] @@ -2728,11 +2643,11 @@ def load_weight_scales_w4a8(self, assert input_scale == w["input_scale"][ ...], "The input_scale should be same for all the weights" if "weight_scale" in w: - ws = module.load_shard(w, - 'weight_scale', - device=device, - name=shard_key, - scale_span=scale_span) + ws = load_weight_shard(w["weight_scale"], + tp_size, + tp_rank, + tp_mode, + device=device) weight_scale.append(ws.to(torch.float16)) if "weight_scale_2" in w: @@ -2748,17 +2663,18 @@ def load_weight_scales_w4a8(self, return input_scale, weight_scale, alpha, weight_scale_2 def load_weights_vanilla(self, module: Linear, weights: List[Dict]): - elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 - load_weights_vanilla_helper(module, weights, elm_packing=elm_packing) + load_weights_vanilla_helper(module, weights) # Use the same device as the weight tensor # as we register pre_quant_scale after sharded model weights are moved to respective gpus device = module.weight.device - pre_quant_scale = module.load_shard( + pre_quant_scale = load_weight_shard( weights[0]["pre_quant_scale"], + module.tp_size, + module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - flip_tp=True, - device=device, + TensorParallelMode.flip(module.tp_mode), + device, ) assert pre_quant_scale.dtype == module.dtype @@ -2770,7 +2686,10 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict]): copy_weight(module.pre_quant_scale, pre_quant_scale) input_scale, weight_scale, alpha, weight_scale_2 = self.load_weight_scales_w4a8( - module, weights) + weights=weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) assert len(weight_scale) == 1, "there should be only one weight scale" @@ -2787,9 +2706,8 @@ def load_weights_vanilla(self, module: Linear, weights: List[Dict]): def load_weights_fused_qkv_linear(self, module: Linear, weights: List[Dict]): - elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 q_weight, k_weight, v_weight = load_weights_fused_qkv_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((q_weight, k_weight, v_weight)) fused_weight = preprocess_weights_for_mixed_gemm( @@ -2798,9 +2716,11 @@ def load_weights_fused_qkv_linear(self, module: Linear, copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode input_scale, weight_scales, alpha, weight_scale_2 = self.load_weight_scales_w4a8( - module, weights, shard_keys=weight_mode.shard_keys) + weights=weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) # Create concatenated weight scale tensor cat_weight_scale = (torch.cat(weight_scales, dim=0).T / @@ -2816,11 +2736,13 @@ def load_weights_fused_qkv_linear(self, module: Linear, # Use the same device as the weight tensor # as we register pre_quant_scale after sharded model weights are moved to respective gpus device = module.weight.device - pre_quant_scale = module.load_shard( + pre_quant_scale = load_weight_shard( weights[0]["pre_quant_scale"], + module.tp_size, + module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - flip_tp=True, - device=device, + TensorParallelMode.flip(module.tp_mode), + device, ) module.pre_quant_scale = Parameter( @@ -2832,9 +2754,8 @@ def load_weights_fused_qkv_linear(self, module: Linear, def load_weights_fused_gate_up_linear(self, module: Linear, weights: List[Dict]): - elm_packing = 2 if module.tp_mode == TensorParallelMode.COLUMN else 1 gate_weight, up_weight = load_weights_fused_gate_up_helper( - module, weights, elm_packing=elm_packing) + module, weights) fused_weight = torch.cat((gate_weight, up_weight)) fused_weight = preprocess_weights_for_mixed_gemm( @@ -2843,9 +2764,11 @@ def load_weights_fused_gate_up_linear(self, module: Linear, copy_weight(module.weight, fused_weight) - weight_mode = module.weights_loading_config.weight_mode input_scale, weight_scale, alpha, weight_scale_2 = self.load_weight_scales_w4a8( - module, weights, shard_keys=weight_mode.shard_keys) + weights=weights, + tp_size=module.tp_size, + tp_rank=module.tp_rank, + tp_mode=module.tp_mode) fused_scale = (torch.cat(weight_scale, dim=0).T / weight_scale_2).contiguous() @@ -2859,11 +2782,13 @@ def load_weights_fused_gate_up_linear(self, module: Linear, # Use the same device as the weight tensor # as we register pre_quant_scale after sharded model weights are moved to respective gpus device = module.weight.device - pre_quant_scale = module.load_shard( + pre_quant_scale = load_weight_shard( weights[0]["pre_quant_scale"], + module.tp_size, + module.tp_rank, # pre_quant_scale applies to activation as opposed to weight, so flip tp_mode the other way around - flip_tp=True, - device=device, + TensorParallelMode.flip(module.tp_mode), + device, ) # NOTE:Create this tensor in load_weights, since not all layer have this tensor and memory is not allocated for it (same as W4A16) @@ -3258,9 +3183,6 @@ def __init__( fused_weight_shard_indices_mapping: Optional[dict] = None, nvfp4_allowed_backends: Optional[List[str]] = None, enable_gemm_allreduce_fusion: bool = True, - override_tp_sharding: Optional[Union[tuple[int, int], - Dict[str, tuple[int, - int]]]] = None, ): """ Args: @@ -3300,42 +3222,25 @@ def __init__( 'cutlass', 'cublaslt', 'cuda_core' ] - if self.tp_mode not in (TensorParallelMode.ROW, - TensorParallelMode.COLUMN, None): - raise ValueError( - f"Invalid tp_mode {self.tp_mode!r}; expected ROW, COLUMN, or None." - ) + local_in_features = in_features + local_out_features = out_features - # Init TP sharding either from override or auto generated - _uneven_tp_unsupported = {QuantAlgo.NVFP4_ARC} - _quant_algo = quant_config.quant_algo if quant_config else None - if override_tp_sharding is not None: - if _quant_algo in _uneven_tp_unsupported: - raise ValueError( - f"Uneven TP is not supported with QuantAlgo {_quant_algo}") - self.tp_sharding = override_tp_sharding - elif self.tp_size > 1 and self.tp_mode is not None \ - and self.weights_loading_config.weight_mode == WeightMode.VANILLA \ - and _quant_algo not in _uneven_tp_unsupported \ - and not skip_create_weights_in_init: - features = in_features if self.tp_mode == TensorParallelMode.ROW else out_features - self.tp_sharding = self._auto_tp_sharding(features, quant_config) + if self.tp_mode == TensorParallelMode.ROW: + assert in_features % self.tp_size == 0, ( + f'in_features {in_features} must be divisible by tp_size {self.tp_size}' + ) + local_in_features = in_features // self.tp_size + elif self.tp_mode == TensorParallelMode.COLUMN: + assert out_features % self.tp_size == 0, ( + f'out_features {out_features} must be divisible by tp_size {self.tp_size}' + ) + local_out_features = out_features // self.tp_size + reduce_output = False if self.mapping.enable_attention_dp else reduce_output else: - self.tp_sharding = None - if self.tp_size > 1 and self.tp_mode is not None: - features = in_features if self.tp_mode == TensorParallelMode.ROW else out_features - assert features % self.tp_size == 0, ( - f"Uneven TP not supported for this configuration " - f"(weight_mode={self.weights_loading_config.weight_mode}, " - f"quant_algo={_quant_algo}). " - f"features={features} must be divisible by tp_size={self.tp_size}." - ) + assert self.tp_mode is None, f'unsupported tensor parallel mode: {self.tp_mode}' - self.in_features = self.calculate_local_in_features(in_features) - self.out_features = self.calculate_local_out_features(out_features) - - if self.tp_mode == TensorParallelMode.COLUMN: - reduce_output = False if self.mapping.enable_attention_dp else reduce_output + self.in_features = local_in_features + self.out_features = local_out_features self.all_reduce = AllReduce(mapping=self.mapping, strategy=allreduce_strategy, @@ -3383,164 +3288,6 @@ def __init__( def get_quant_method(self, quant_config: Optional[QuantConfig] = None): return get_quant_method(quant_config) - @staticmethod - def _calc_shard(total, tp_size, rank): - return (total // tp_size) * rank + min(total % tp_size, rank) - - def _auto_tp_sharding(self, features, quant_config): - """Auto-generate tp_sharding tuple based on quant alignment requirements. - - VANILLA mode only. Fused modes (FUSED_QKV, FUSED_GATE_UP) require explicit - override_tp_sharding because individual sub-weight sizes (Q vs K vs V; gate - vs up) are not knowable here — they aren't always equal (e.g. GQA), and - cross-rank consistency must be decided by the caller. - """ - assert self.weights_loading_config.weight_mode == WeightMode.VANILLA, ( - f"_auto_tp_sharding only supports VANILLA mode, got " - f"{self.weights_loading_config.weight_mode}. Fused modes require " - f"explicit override_tp_sharding.") - alignment = get_quant_method(quant_config).get_tp_alignment( - self.tp_mode, quant_config) - if alignment <= 1: - # No alignment constraint — use standard element-level distribution - start = self._calc_shard(features, self.tp_size, self.tp_rank) - end = self._calc_shard(features, self.tp_size, self.tp_rank + 1) - else: - # Distribute whole alignment-sized blocks across ranks - assert features % alignment == 0, ( - f"Feature dim ({features}) must be divisible by quant alignment " - f"({alignment}) for TP sharding") - num_blocks = features // alignment - block_start = self._calc_shard(num_blocks, self.tp_size, - self.tp_rank) - block_end = self._calc_shard(num_blocks, self.tp_size, - self.tp_rank + 1) - start = block_start * alignment - end = block_end * alignment - return (start, end) - - def _calculate_local_features_helper(self, features): - if isinstance(self.tp_sharding, tuple): - assert self.weights_loading_config.weight_mode == WeightMode.VANILLA - start, end = self.tp_sharding - return end - start - elif isinstance(self.tp_sharding, dict): - assert self.weights_loading_config.weight_mode in ( - WeightMode.FUSED_GATE_UP_LINEAR, WeightMode.FUSED_QKV_LINEAR) - return sum(end - start for start, end in self.tp_sharding.values()) - else: - assert features % self.tp_size == 0 - return features // self.tp_size - - def calculate_local_in_features(self, in_features): - """Local input feature count after TP sharding (full size if not row-parallel).""" - if self.tp_mode != TensorParallelMode.ROW: - return in_features - - return self._calculate_local_features_helper(in_features) - - def calculate_local_out_features(self, out_features): - """Local output feature count after TP sharding (full size if not column-parallel).""" - if self.tp_mode != TensorParallelMode.COLUMN: - return out_features - - return self._calculate_local_features_helper(out_features) - - def load_shard( - self, - weights: Union[Dict, torch.Tensor], - label: Optional[str] = None, - device: torch.device = torch.device('cpu'), - name: Optional[str] = None, - flip_tp: bool = False, # for input activation scales - scale_span: Optional[int] = None, - # number of elms in a given "slot", used for fp4 since - # 2 are packed in each 8 bit element of the tensor - elm_packing: int = 1, - ) -> torch.Tensor: - """Slice a weight tensor for this rank's TP shard. - - Unified entry point for module-aware weight loading: respects - `self.tp_sharding` (uneven TP overrides, fused QKV/gate-up dicts) and - quant-specific knobs (`scale_span`, `elm_packing`). Pass `weights` as a - dict with `label` to pick the entry, or as a bare tensor when there's - only one. Supersedes the free function `load_weight_shard`. - """ - if label: - if label not in weights: - return None - weight = weights[label] - else: - weight = weights - - # Skip device transfers on integrated GPUs to conserve shared memory - if weight.device.type != device.type and is_device_integrated(): - # For integrated GPU systems (e.g., DGX Spark), CPU and GPU share limited physical memory. - # Avoiding device transfers reduces memory consumption and unnecessary data copies, - # enabling support for larger models on memory-constrained systems. - logger.warning_once( - f"[Linear.load_shard] Skipping device transfer from {weight.device} to {device} on integrated GPU to conserve shared memory.", - key="load_weight_shard_skip_device_transfer_with_integrated_gpu" - ) - device = weight.device - if isinstance(weight, torch.Tensor): - tensor_shape = weight.shape - - def maybe_convert_to_torch_tensor(tensor: torch.Tensor, - indices: list[slice] - | None = None): - if indices is None: - # Avoid unnecessary copy - return tensor.to(device) - else: - return tensor[indices].to(device) - - # WAR to check whether it is a safetensor slice since safetensor didn't register the type to the module - # safetensors slice, supports lazy loading, type(weight) is `builtin.PySafeSlice` - elif hasattr(weight, "get_shape"): - tensor_shape = weight.get_shape() - - def maybe_convert_to_torch_tensor( - tensor, indices: Union[slice, tuple[slice]] = slice(None)): - return tensor[indices].to(device) - else: - raise ValueError(f'unsupported weight type: {type(weight)}') - if self.tp_mode is None or self.tp_size <= 1: - return maybe_convert_to_torch_tensor(weight) - - tp_mode = TensorParallelMode.flip( - self.tp_mode) if flip_tp else self.tp_mode - split_dim = TensorParallelMode.split_dim(tp_mode) - - if len(tensor_shape) == 1 and split_dim == 1: - return maybe_convert_to_torch_tensor(weight) - - width = tensor_shape[split_dim] - if width == 1: - return maybe_convert_to_torch_tensor(weight) - - if self.tp_sharding is None: - slice_start = self._calc_shard(width, self.tp_size, self.tp_rank) - slice_end = self._calc_shard(width, self.tp_size, self.tp_rank + 1) - else: - if isinstance(self.tp_sharding, tuple): - slice_start, slice_end = self.tp_sharding - elif isinstance(self.tp_sharding, dict): - slice_start, slice_end = self.tp_sharding[name] - - if scale_span: - assert slice_end % scale_span == 0 and slice_start % scale_span == 0 - slice_start //= scale_span - slice_end //= scale_span - - assert slice_start % elm_packing == 0 and slice_end % elm_packing == 0 - slice_start //= elm_packing - slice_end //= elm_packing - - slice_obj = [slice(d) for d in tensor_shape] - slice_obj[split_dim] = slice(slice_start, slice_end) - return maybe_convert_to_torch_tensor(weight, tuple(slice_obj)) - def create_weights(self): if self._weights_created: return @@ -3750,25 +3497,6 @@ def pre_reload_weights(self): self.quant_method.pre_reload_weights(self) -def is_static_nvfp4_input_eligible(linear) -> bool: - """Whether `linear` consumes a static (calibrated) NVFP4 input, making it - eligible to have its input-quantize folded into a producing RMSNorm. - - Eligible iff the Linear has NVFP4 weights, a calibrated (static) - `input_scale`, no AWQ `pre_quant_scale`, and is not forced to dynamic - quantization. This is the single canonical definition shared by every - NVFP4-fold site (the layer-boundary / dense folds in modeling_deepseekv3.py - and the q_a_layernorm -> q_b_proj fold in attention.py's MLA) so the gate - cannot drift between them. - """ - if linear is None: - return False - return (getattr(linear, "has_nvfp4", False) - and not getattr(linear, "force_dynamic_quantization", False) - and getattr(linear, "input_scale", None) is not None - and getattr(linear, "pre_quant_scale", None) is None) - - class NVFP4ARCLinearMethod(NVFP4LinearMethod): supports_nccl_symmetric_memory_window_output: ClassVar[bool] = True diff --git a/tensorrt_llm/_torch/modules/mamba/fuse_elementwise_ops.py b/tensorrt_llm/_torch/modules/mamba/fuse_elementwise_ops.py index efe6b1acfbdd..1499f606689a 100644 --- a/tensorrt_llm/_torch/modules/mamba/fuse_elementwise_ops.py +++ b/tensorrt_llm/_torch/modules/mamba/fuse_elementwise_ops.py @@ -203,368 +203,235 @@ def fused_split_rearrange_after_conv1d( @triton.jit -def _fused_gdn_post_conv_kernel( - prefill_ptr, +def _transpose_and_split_qkv_kernel( + prefill_t_ptr, decode_ptr, - a_ptr, - b_ptr, - A_log_ptr, - dt_bias_ptr, q_ptr, k_ptr, v_ptr, - g_ptr, - beta_ptr, - num_prefill_tokens, - num_decode_tokens, - prefill_stride_token, - prefill_stride_dim, - decode_stride_token, - decode_stride_dim, - a_stride_token, - a_stride_head, - b_stride_token, - b_stride_head, - l2_norm_eps, - softplus_threshold, - NUM_K_HEADS: tl.constexpr, - NUM_V_HEADS: tl.constexpr, - HEAD_K_DIM: tl.constexpr, - HEAD_V_DIM: tl.constexpr, - HAS_DECODE: tl.constexpr, - BLOCK_TOKENS: tl.constexpr, - BLOCK_K: tl.constexpr, - BLOCK_V: tl.constexpr, + num_prefill, + num_decode, + decode_stride_seq, + q_dim: tl.constexpr, + k_dim: tl.constexpr, + v_dim: tl.constexpr, + BLOCK_SEQ: tl.constexpr, + BLOCK_DIM: tl.constexpr, ): - """Prepare contiguous Q/K/V and GDN gates from causal-conv output.""" - token_block = tl.program_id(0) - head_idx = tl.program_id(1) - - token_offsets = token_block * BLOCK_TOKENS + tl.arange(0, BLOCK_TOKENS) - total_tokens = num_prefill_tokens + num_decode_tokens - token_mask = token_offsets < total_tokens - is_prefill = token_offsets < num_prefill_tokens - if HAS_DECODE: - is_decode = ~is_prefill & token_mask - - if head_idx < NUM_K_HEADS: - dim_offsets = tl.arange(0, BLOCK_K) - dim_mask = dim_offsets < HEAD_K_DIM - load_mask_prefill = is_prefill[:, None] & dim_mask[None, :] - store_mask = token_mask[:, None] & dim_mask[None, :] - - q_feature_offsets = head_idx * HEAD_K_DIM + dim_offsets - - prefill_q_offsets = ( - token_offsets[:, None].to(tl.int64) * prefill_stride_token - + q_feature_offsets[None, :].to(tl.int64) * prefill_stride_dim - ) - prefill_k_offsets = prefill_q_offsets + NUM_K_HEADS * HEAD_K_DIM * prefill_stride_dim - q_prefill = tl.load(prefill_ptr + prefill_q_offsets, mask=load_mask_prefill, other=0.0) - k_prefill = tl.load(prefill_ptr + prefill_k_offsets, mask=load_mask_prefill, other=0.0) - - if HAS_DECODE: - load_mask_decode = is_decode[:, None] & dim_mask[None, :] - decode_rows = token_offsets - num_prefill_tokens - decode_q_offsets = ( - decode_rows[:, None].to(tl.int64) * decode_stride_token - + q_feature_offsets[None, :].to(tl.int64) * decode_stride_dim - ) - decode_k_offsets = decode_q_offsets + NUM_K_HEADS * HEAD_K_DIM * decode_stride_dim - q_decode = tl.load(decode_ptr + decode_q_offsets, mask=load_mask_decode, other=0.0) - k_decode = tl.load(decode_ptr + decode_k_offsets, mask=load_mask_decode, other=0.0) - q_values = tl.where(is_prefill[:, None], q_prefill, q_decode).to(tl.float32) - k_values = tl.where(is_prefill[:, None], k_prefill, k_decode).to(tl.float32) - else: - q_values = q_prefill.to(tl.float32) - k_values = k_prefill.to(tl.float32) - - q_inv_norm = 1.0 / tl.sqrt(tl.sum(q_values * q_values, axis=1) + l2_norm_eps) - k_inv_norm = 1.0 / tl.sqrt(tl.sum(k_values * k_values, axis=1) + l2_norm_eps) - q_values *= q_inv_norm[:, None] - k_values *= k_inv_norm[:, None] - - output_offsets = ( - token_offsets[:, None].to(tl.int64) * (NUM_K_HEADS * HEAD_K_DIM) - + q_feature_offsets[None, :] - ) - tl.store(q_ptr + output_offsets, q_values, mask=store_mask) - tl.store(k_ptr + output_offsets, k_values, mask=store_mask) - else: - value_head_idx = head_idx - NUM_K_HEADS - dim_offsets = tl.arange(0, BLOCK_V) - dim_mask = dim_offsets < HEAD_V_DIM - load_mask_prefill = is_prefill[:, None] & dim_mask[None, :] - store_mask = token_mask[:, None] & dim_mask[None, :] - - value_feature_offsets = ( - 2 * NUM_K_HEADS * HEAD_K_DIM + value_head_idx * HEAD_V_DIM + dim_offsets - ) - prefill_v_offsets = ( - token_offsets[:, None].to(tl.int64) * prefill_stride_token - + value_feature_offsets[None, :].to(tl.int64) * prefill_stride_dim - ) - v_prefill = tl.load(prefill_ptr + prefill_v_offsets, mask=load_mask_prefill, other=0.0) - if HAS_DECODE: - load_mask_decode = is_decode[:, None] & dim_mask[None, :] - decode_rows = token_offsets - num_prefill_tokens - decode_v_offsets = ( - decode_rows[:, None].to(tl.int64) * decode_stride_token - + value_feature_offsets[None, :].to(tl.int64) * decode_stride_dim - ) - v_decode = tl.load(decode_ptr + decode_v_offsets, mask=load_mask_decode, other=0.0) - v_values = tl.where(is_prefill[:, None], v_prefill, v_decode) - else: - v_values = v_prefill - - output_offsets = ( - token_offsets[:, None].to(tl.int64) * (NUM_V_HEADS * HEAD_V_DIM) - + value_head_idx * HEAD_V_DIM - + dim_offsets[None, :] - ) - tl.store(v_ptr + output_offsets, v_values, mask=store_mask) - - a_offsets = token_offsets.to(tl.int64) * a_stride_token + value_head_idx * a_stride_head - b_offsets = token_offsets.to(tl.int64) * b_stride_token + value_head_idx * b_stride_head - a_values = tl.load(a_ptr + a_offsets, mask=token_mask, other=0.0).to(tl.float32) - b_values = tl.load(b_ptr + b_offsets, mask=token_mask, other=0.0).to(tl.float32) - A_log = tl.load(A_log_ptr + value_head_idx).to(tl.float32) - dt_bias = tl.load(dt_bias_ptr + value_head_idx).to(tl.float32) - - gate_input = a_values + dt_bias - softplus = tl.where( - gate_input <= softplus_threshold, - tl.log(1.0 + tl.exp(gate_input)), - gate_input, - ) - g_values = -tl.exp(A_log) * softplus - beta_values = tl.sigmoid(b_values) - gate_offsets = token_offsets.to(tl.int64) * NUM_V_HEADS + value_head_idx - tl.store(g_ptr + gate_offsets, g_values, mask=token_mask) - tl.store(beta_ptr + gate_offsets, beta_values, mask=token_mask) - - -def fused_gdn_post_conv( - prefill: torch.Tensor, - decode: torch.Tensor | None, - a: torch.Tensor, - b: torch.Tensor, - A_log: torch.Tensor, - dt_bias: torch.Tensor, - num_k_heads: int, + """Fused transpose-prefill + split-decode into contiguous q, k, v. + + Reads prefill from transposed layout [D, T_p] and decode from row-major + layout [T_d, D] with contiguous rows of arbitrary stride, writes both + into contiguous q/k/v outputs. + Grid: (num_seq_blocks_total, num_dim_blocks, 3) + program_id(2): 0=Q, 1=K, 2=V + """ + pid_seq = tl.program_id(0) + pid_dim = tl.program_id(1) + pid_out = tl.program_id(2) + + total_seq = num_prefill + num_decode + out_dim = tl.where(pid_out == 2, v_dim, tl.where(pid_out == 1, k_dim, q_dim)) + src_col_offset = tl.where(pid_out == 0, 0, tl.where(pid_out == 1, q_dim, q_dim + k_dim)) + + seq_offsets = pid_seq * BLOCK_SEQ + tl.arange(0, BLOCK_SEQ) + dim_offsets = pid_dim * BLOCK_DIM + tl.arange(0, BLOCK_DIM) + + seq_mask = seq_offsets < total_seq + dim_mask = dim_offsets < out_dim + mask = seq_mask[:, None] & dim_mask[None, :] + + # Determine if each row is prefill or decode + is_prefill = seq_offsets < num_prefill + is_decode = ~is_prefill & (seq_offsets < total_seq) + + # Prefill: read from prefill_t[D, T_p] transposed — index [col, row] + prefill_src_col = src_col_offset + dim_offsets + prefill_indices = prefill_src_col[None, :].to(tl.int64) * num_prefill + seq_offsets[:, None].to( + tl.int64 + ) + prefill_data = tl.load( + prefill_t_ptr + prefill_indices, mask=is_prefill[:, None] & dim_mask[None, :], other=0.0 + ) + + # Decode: read from decode_ptr[T_d, D] row-major + decode_row = seq_offsets - num_prefill + decode_indices = decode_row[:, None].to(tl.int64) * decode_stride_seq + ( + src_col_offset + dim_offsets[None, :] + ).to(tl.int64) + decode_data = tl.load( + decode_ptr + decode_indices, mask=is_decode[:, None] & dim_mask[None, :], other=0.0 + ) + + # Merge + data = tl.where(is_prefill[:, None], prefill_data, decode_data) + + # Write to output [total_seq, out_dim] + out_ptr = tl.where(pid_out == 0, q_ptr, tl.where(pid_out == 1, k_ptr, v_ptr)) + dst_indices = seq_offsets[:, None].to(tl.int64) * out_dim + dim_offsets[None, :] + tl.store(out_ptr + dst_indices, data, mask=mask) + + +def transpose_and_split_qkv( + prefill_t: torch.Tensor, + decode: torch.Tensor, + q_dim: int, + k_dim: int, + v_dim: int, + num_q_heads: int, head_k_dim: int, num_v_heads: int, head_v_dim: int, - l2_norm_eps: float = 1e-6, - softplus_threshold: float = 20.0, - beta_dtype: torch.dtype = torch.float32, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Fuse GDN post-conv split, Q/K normalization, and gate preparation. - - Args: - prefill: Prefill causal-conv output with shape ``[qkv_dim, num_prefill_tokens]``. - decode: Optional decode causal-conv output with shape ``[num_decode_tokens, qkv_dim]``. - a: Gate input with shape ``[num_tokens, num_v_heads]``. - b: Beta input with shape ``[num_tokens, num_v_heads]``. - A_log: Log decay with shape ``[num_v_heads]``. - dt_bias: Time-step bias with shape ``[num_v_heads]``. - num_k_heads: Number of local query/key heads. - head_k_dim: Query/key dimension per head. - num_v_heads: Number of local value heads. - head_v_dim: Value dimension per head. - l2_norm_eps: Epsilon added to the Q/K squared norm. - softplus_threshold: Linear fallback threshold for the decay gate softplus. - beta_dtype: Output dtype for beta. Target verification uses the input dtype for parity. - Returns: - Contiguous Q, K, V, log-space G, and beta tensors with shapes - ``[1, num_tokens, num_k_heads, head_k_dim]``, - ``[1, num_tokens, num_k_heads, head_k_dim]``, - ``[1, num_tokens, num_v_heads, head_v_dim]``, and twice - ``[1, num_tokens, num_v_heads]``. +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ - num_prefill_tokens = prefill.shape[1] - num_decode_tokens = 0 if decode is None else decode.shape[0] - num_tokens = num_prefill_tokens + num_decode_tokens + Fused transpose prefill [D, T_p] + split decode [T_d, D] into contiguous q, k, v. - q = torch.empty( - (1, num_tokens, num_k_heads, head_k_dim), dtype=prefill.dtype, device=prefill.device - ) - k = torch.empty_like(q) - v = torch.empty( - (1, num_tokens, num_v_heads, head_v_dim), dtype=prefill.dtype, device=prefill.device + The decode tensor may be a column-slice view of a wider tensor (contiguous + rows, arbitrary row stride). + + Replaces separate transpose_copy_back + split_qkv_contiguous for mixed batches. + """ + assert decode.stride(-1) == 1 + num_prefill = prefill_t.shape[1] + num_decode = decode.shape[0] + total_seq = num_prefill + num_decode + + q_flat = torch.empty(total_seq, q_dim, dtype=prefill_t.dtype, device=prefill_t.device) + k_flat = torch.empty(total_seq, k_dim, dtype=prefill_t.dtype, device=prefill_t.device) + v_flat = torch.empty(total_seq, v_dim, dtype=prefill_t.dtype, device=prefill_t.device) + + BLOCK_SEQ, BLOCK_DIM = 32, 128 + max_dim = max(q_dim, k_dim, v_dim) + grid = (triton.cdiv(total_seq, BLOCK_SEQ), triton.cdiv(max_dim, BLOCK_DIM), 3) + + _transpose_and_split_qkv_kernel[grid]( + prefill_t, + decode, + q_flat, + k_flat, + v_flat, + num_prefill, + num_decode, + decode.stride(0), + q_dim, + k_dim, + v_dim, + BLOCK_SEQ, + BLOCK_DIM, ) - g = torch.empty((1, num_tokens, num_v_heads), dtype=torch.float32, device=prefill.device) - beta = torch.empty((1, num_tokens, num_v_heads), dtype=beta_dtype, device=prefill.device) - if num_tokens == 0: - return q, k, v, g, beta - - has_decode = decode is not None - # Triton requires a tensor for every pointer argument. HAS_DECODE=False - # compile-time eliminates all decode address arithmetic and loads, so this - # pointer and its strides are never interpreted as a decode tensor. - decode_input = prefill if decode is None else decode - block_tokens = 16 - block_k = triton.next_power_of_2(head_k_dim) - block_v = triton.next_power_of_2(head_v_dim) - grid = (triton.cdiv(num_tokens, block_tokens), num_k_heads + num_v_heads) - _fused_gdn_post_conv_kernel[grid]( - prefill, - decode_input, - a, - b, - A_log, - dt_bias, - q, - k, - v, - g, - beta, - num_prefill_tokens, - num_decode_tokens, - prefill.stride(1), - prefill.stride(0), - decode_input.stride(0), - decode_input.stride(1), - a.stride(0), - a.stride(1), - b.stride(0), - b.stride(1), - l2_norm_eps, - softplus_threshold, - num_k_heads, - num_v_heads, - head_k_dim, - head_v_dim, - has_decode, - block_tokens, - block_k, - block_v, - num_warps=4, - num_stages=2, + + return ( + q_flat.view(1, total_seq, num_q_heads, head_k_dim), + k_flat.view(1, total_seq, num_q_heads, head_k_dim), + v_flat.view(1, total_seq, num_v_heads, head_v_dim), ) - return q, k, v, g, beta @triton.jit -def _pack_gdn_decode_qkv_kernel( - mixed_qkv_ptr, +def _split_qkv_contiguous_kernel( + src_ptr, q_ptr, k_ptr, v_ptr, - num_tokens, - stride_token, - stride_dim, - NUM_K_HEADS: tl.constexpr, - NUM_V_HEADS: tl.constexpr, - HEAD_K_DIM: tl.constexpr, - HEAD_V_DIM: tl.constexpr, - BLOCK_TOKENS: tl.constexpr, - BLOCK_K: tl.constexpr, - BLOCK_V: tl.constexpr, + seq_len, + src_stride_seq, + src_stride_dim, + q_dim: tl.constexpr, + k_dim: tl.constexpr, + v_dim: tl.constexpr, + BLOCK_SEQ: tl.constexpr, + BLOCK_DIM: tl.constexpr, ): - """Pack target-verification decode Q/K/V without normalization or gates.""" - token_offsets = tl.program_id(0) * BLOCK_TOKENS + tl.arange(0, BLOCK_TOKENS) - head_idx = tl.program_id(1) - token_mask = token_offsets < num_tokens - - if head_idx < NUM_K_HEADS: - dim_offsets = tl.arange(0, BLOCK_K) - dim_mask = dim_offsets < HEAD_K_DIM - mask = token_mask[:, None] & dim_mask[None, :] - q_feature_offsets = head_idx * HEAD_K_DIM + dim_offsets - q_offsets = ( - token_offsets[:, None].to(tl.int64) * stride_token - + q_feature_offsets[None, :].to(tl.int64) * stride_dim - ) - k_offsets = q_offsets + NUM_K_HEADS * HEAD_K_DIM * stride_dim - output_offsets = ( - token_offsets[:, None].to(tl.int64) * (NUM_K_HEADS * HEAD_K_DIM) - + q_feature_offsets[None, :] - ) - tl.store(q_ptr + output_offsets, tl.load(mixed_qkv_ptr + q_offsets, mask=mask), mask=mask) - tl.store(k_ptr + output_offsets, tl.load(mixed_qkv_ptr + k_offsets, mask=mask), mask=mask) - else: - value_head_idx = head_idx - NUM_K_HEADS - dim_offsets = tl.arange(0, BLOCK_V) - dim_mask = dim_offsets < HEAD_V_DIM - mask = token_mask[:, None] & dim_mask[None, :] - value_feature_offsets = ( - 2 * NUM_K_HEADS * HEAD_K_DIM + value_head_idx * HEAD_V_DIM + dim_offsets - ) - value_offsets = ( - token_offsets[:, None].to(tl.int64) * stride_token - + value_feature_offsets[None, :].to(tl.int64) * stride_dim - ) - output_offsets = ( - token_offsets[:, None].to(tl.int64) * (NUM_V_HEADS * HEAD_V_DIM) - + value_head_idx * HEAD_V_DIM - + dim_offsets[None, :] - ) - tl.store( - v_ptr + output_offsets, tl.load(mixed_qkv_ptr + value_offsets, mask=mask), mask=mask - ) - - -def pack_gdn_decode_qkv( + """Split mixed_qkv [T, qkv_dim] into 3 contiguous output tensors. + + Supports arbitrary source strides to handle both contiguous and + transposed inputs (e.g. from causal_conv1d_fn(...).transpose(0, 1)). + + Grid: (num_seq_blocks, num_dim_blocks_for_max_dim, 3) + program_id(2): 0=Q, 1=K, 2=V + """ + pid_seq = tl.program_id(0) + pid_dim = tl.program_id(1) + pid_out = tl.program_id(2) + + # Determine output dim and source column offset for this output + out_dim = tl.where(pid_out == 2, v_dim, tl.where(pid_out == 1, k_dim, q_dim)) + src_col_offset = tl.where(pid_out == 0, 0, tl.where(pid_out == 1, q_dim, q_dim + k_dim)) + + seq_offsets = pid_seq * BLOCK_SEQ + tl.arange(0, BLOCK_SEQ) + dim_offsets = pid_dim * BLOCK_DIM + tl.arange(0, BLOCK_DIM) + + seq_mask = seq_offsets < seq_len + dim_mask = dim_offsets < out_dim + mask = seq_mask[:, None] & dim_mask[None, :] + + # Read from src [T, qkv_dim] using actual strides + src_indices = ( + seq_offsets[:, None].to(tl.int64) * src_stride_seq + + (src_col_offset + dim_offsets[None, :]).to(tl.int64) * src_stride_dim + ) + data = tl.load(src_ptr + src_indices, mask=mask, other=0.0) + + # Write to output [T, out_dim] — contiguous, stride = (out_dim, 1) + out_ptr = tl.where(pid_out == 0, q_ptr, tl.where(pid_out == 1, k_ptr, v_ptr)) + dst_indices = seq_offsets[:, None].to(tl.int64) * out_dim + dim_offsets[None, :] + tl.store(out_ptr + dst_indices, data, mask=mask) + + +def split_qkv_contiguous( mixed_qkv: torch.Tensor, - num_k_heads: int, + q_dim: int, + k_dim: int, + v_dim: int, + num_q_heads: int, head_k_dim: int, num_v_heads: int, head_v_dim: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Pack target-verification decode Q/K/V while keeping them unnormalized. - - Args: - mixed_qkv: Decode causal-conv output with shape ``[num_tokens, qkv_dim]``. - num_k_heads: Number of local query/key heads. - head_k_dim: Query/key dimension per head. - num_v_heads: Number of local value heads. - head_v_dim: Value dimension per head. + """ + Split mixed_qkv [T, qkv_dim] into 3 contiguous tensors and reshape. Returns: - Contiguous Q, K, and V tensors with shapes - ``[1, num_tokens, num_k_heads, head_k_dim]``, - ``[1, num_tokens, num_k_heads, head_k_dim]``, and - ``[1, num_tokens, num_v_heads, head_v_dim]``. + q_out [1, T, num_q_heads, head_k_dim] — contiguous + k_out [1, T, num_q_heads, head_k_dim] — contiguous + v_out [1, T, num_v_heads, head_v_dim] — contiguous """ - num_tokens = mixed_qkv.shape[0] - q = torch.empty( - (1, num_tokens, num_k_heads, head_k_dim), - dtype=mixed_qkv.dtype, - device=mixed_qkv.device, + seq_len = mixed_qkv.shape[0] + src_stride_seq, src_stride_dim = mixed_qkv.stride() + + q_flat = torch.empty(seq_len, q_dim, dtype=mixed_qkv.dtype, device=mixed_qkv.device) + k_flat = torch.empty(seq_len, k_dim, dtype=mixed_qkv.dtype, device=mixed_qkv.device) + v_flat = torch.empty(seq_len, v_dim, dtype=mixed_qkv.dtype, device=mixed_qkv.device) + + BLOCK_SEQ = 32 + BLOCK_DIM = 128 + max_dim = max(q_dim, k_dim, v_dim) + grid = ( + triton.cdiv(seq_len, BLOCK_SEQ), + triton.cdiv(max_dim, BLOCK_DIM), + 3, ) - k = torch.empty_like(q) - v = torch.empty( - (1, num_tokens, num_v_heads, head_v_dim), - dtype=mixed_qkv.dtype, - device=mixed_qkv.device, - ) - if num_tokens == 0: - return q, k, v - - block_tokens = 16 - block_k = triton.next_power_of_2(head_k_dim) - block_v = triton.next_power_of_2(head_v_dim) - grid = (triton.cdiv(num_tokens, block_tokens), num_k_heads + num_v_heads) - _pack_gdn_decode_qkv_kernel[grid]( + + _split_qkv_contiguous_kernel[grid]( mixed_qkv, - q, - k, - v, - num_tokens, - mixed_qkv.stride(0), - mixed_qkv.stride(1), - num_k_heads, - num_v_heads, - head_k_dim, - head_v_dim, - block_tokens, - block_k, - block_v, - num_warps=4, - num_stages=2, + q_flat, + k_flat, + v_flat, + seq_len, + src_stride_seq, + src_stride_dim, + q_dim, + k_dim, + v_dim, + BLOCK_SEQ, + BLOCK_DIM, + ) + + return ( + q_flat.view(1, seq_len, num_q_heads, head_k_dim), + k_flat.view(1, seq_len, num_q_heads, head_k_dim), + v_flat.view(1, seq_len, num_v_heads, head_v_dim), ) - return q, k, v @triton.jit diff --git a/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py b/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py index 3d29e6b05cfc..52120a76bdf3 100644 --- a/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py +++ b/tensorrt_llm/_torch/modules/mamba/gdn_mixer.py @@ -14,10 +14,6 @@ from torch import nn from transformers import Qwen3NextConfig -from tensorrt_llm._torch.modules.fla.cached_replay import ( - CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, - fused_recurrent_gated_delta_rule_cached_replay_update, -) from tensorrt_llm._torch.modules.fla.fused_recurrent import fused_recurrent_gated_delta_rule_update from tensorrt_llm._torch.modules.fla.fused_sigmoid_gating_recurrent import ( _can_use_flashinfer_gdn_verify, @@ -25,22 +21,21 @@ fused_sigmoid_gating_delta_rule_update, ) from tensorrt_llm._utils import is_flashinfer_gdn_supported_arch -from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping from ...attention_backend import AttentionMetadata from ...distributed import AllReduceParams from ...model_config import ModelConfig from ...speculative import SpecMetadata -from ...utils import EventType, get_model_extra_attrs, is_gdn_replay_enabled, is_torch_compiling -from ..linear import FP8QDQLinearMethod, Linear, TensorParallelMode +from ...utils import EventType, get_model_extra_attrs, is_torch_compiling +from ..linear import Linear, TensorParallelMode from ..multi_stream_utils import maybe_execute_in_parallel from .causal_conv1d import causal_conv1d_fn, causal_conv1d_update from .causal_conv1d_triton import causal_conv1d_update as causal_conv1d_update_triton from .fuse_elementwise_ops import ( extract_transpose_prefill_slice, - fused_gdn_post_conv, - pack_gdn_decode_qkv, + split_qkv_contiguous, + transpose_and_split_qkv, ) from .layernorm_gated import RMSNorm as RMSNormGated from .layernorm_gated import rms_norm_gated_token_major @@ -167,6 +162,80 @@ def fused_gdn_gating( return g +@triton.jit +def fused_gdn_gating_with_sigmoid_kernel( + g, + beta_out, + A_log, + a, + dt_bias, + b, + stride_a_row, + stride_b_row, + NUM_HEADS: tl.constexpr, + sp_beta: tl.constexpr, + threshold: tl.constexpr, + BLK_HEADS: tl.constexpr, +): + """Fuse gdn_gating + sigmoid(b) into one kernel.""" + i_b, i_d = tl.program_id(0), tl.program_id(1) + head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS) + # a/b may be row-strided views sliced out of the packed ba projection; + # g/beta_out are always allocated packed. + off_a = i_b * stride_a_row + head_off + off_b = i_b * stride_b_row + head_off + off_out = i_b * NUM_HEADS + head_off + mask = head_off < NUM_HEADS + blk_A_log = tl.load(A_log + head_off, mask=mask) + blk_a = tl.load(a + off_a, mask=mask) + blk_bias = tl.load(dt_bias + head_off, mask=mask) + x = blk_a.to(tl.float32) + blk_bias.to(tl.float32) + softplus_x = tl.where( + sp_beta * x <= threshold, (1 / sp_beta) * tl.log(1 + tl.exp(sp_beta * x)), x + ) + blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x + tl.store(g + off_out, blk_g.to(g.dtype.element_ty), mask=mask) + # sigmoid(b) + blk_b = tl.load(b + off_b, mask=mask) + blk_beta = tl.sigmoid(blk_b.to(tl.float32)) + tl.store(beta_out + off_out, blk_beta.to(beta_out.dtype.element_ty), mask=mask) + + +def fused_gdn_gating_with_sigmoid( + A_log: torch.Tensor, + a: torch.Tensor, + dt_bias: torch.Tensor, + b: torch.Tensor, + sp_beta: float = 1.0, + threshold: float = 20.0, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fused GDN gating + sigmoid: compute g and beta in one kernel launch.""" + batch, num_heads = a.shape + grid = (batch, triton.cdiv(num_heads, 8)) + g = torch.empty(batch, num_heads, dtype=torch.float32, device=a.device) + # Allocate beta in fp32 since (1) the kernel already computes sigmoid in fp32 + # and was previously casting back to b.dtype only to be re-cast to fp32 by the + # FlashInfer GDN prefill wrapper, and (2) the Triton chunk_gated_delta_rule + # path also accepts fp32 beta. Eliminates a redundant cast in the FI hot path. + beta_out = torch.empty(batch, num_heads, dtype=torch.float32, device=b.device) + fused_gdn_gating_with_sigmoid_kernel[grid]( + g, + beta_out, + A_log, + a, + dt_bias, + b, + a.stride(0), + b.stride(0), + num_heads, + sp_beta, + threshold, + 8, + num_warps=1, + ) + return g, beta_out + + class Qwen3NextGatedDeltaNet(nn.Module): def __init__( self, @@ -178,17 +247,6 @@ def __init__( config = model_config.pretrained_config self.model_config = model_config self.pretrained_config = config - replay_enabled = is_gdn_replay_enabled() - if replay_enabled: - logger.info_once( - "Configured GDN MTP replay implementation: cached", - key="gdn_mtp_replay_cached", - ) - else: - logger.info_once( - "GDN MTP replay disabled; set TRTLLM_USE_GDN_REPLAY=1 to enable it", - key="gdn_mtp_replay_disabled", - ) # tensor parallel tp_size = model_config.mapping.tp_size @@ -332,19 +390,6 @@ def __init__( self.event_dict = {key: torch.cuda.Event() for key in [EventType.Main, EventType.Attention]} self.aux_stream = aux_stream - def cache_derived_state(self) -> None: - """Attach downstream static quantization state after loading weights.""" - self.norm.fp8_scale = None - if isinstance(self.out_proj.quant_method, FP8QDQLinearMethod): - scale = self.out_proj.quant_method.get_static_input_scale(self.out_proj) - # detach() strips the Parameter wrapper so nn.Module.__setattr__ - # does not register the derived scale into norm's state_dict; the - # detached view still shares storage with out_proj.input_scale. - self.norm.fp8_scale = scale.detach() if scale is not None else None - - def post_load_weights(self) -> None: - self.cache_derived_state() - def _compute_tokenwise_inputs(self, hidden_states: torch.Tensor): def _compute_projected_states_qkvz(): return self.in_proj_qkvz(hidden_states) @@ -388,78 +433,11 @@ def _postprocess_gdn_output( # gated norm reads it through its token stride instead of packing a # copy. attn_out = rms_norm_gated_token_major( - attn_out.reshape(-1, self.head_v_dim), - z, - self.norm.weight, - self.norm.eps, - fp8_scale=self.norm.fp8_scale, + attn_out.reshape(-1, self.head_v_dim), z, self.norm.weight, self.norm.eps ) attn_out = attn_out.view(-1, self.value_dim_per_tp) return self.out_proj(attn_out, all_reduce_params=all_reduce_params) - def _replay_verify_recurrent( - self, - query, - key, - value, - a, - b, - ssm_states, - state_indices_d, - num_decodes, - draft_token_num, - replay_metadata, - layer_cache, - replay_work_items, - replay_n_writes, - output_d=None, - packed_qkv=None, - use_all_layer_commit=False, - ): - """Run MTP target verification via the replay kernel. - - This avoids intermediate-state writes and accepted-state copies; commits - are deferred via the compact history cache. Cached replay reuses the - Mamba2 fields as old_x<->U, old_B<->normalized k, and - old_dt<->cumulative G. - """ - assert replay_metadata is not None, ( - "GDN replay enabled but replay metadata was not allocated." - ) - assert draft_token_num == replay_metadata.replay_step_width, ( - "GDN replay does not support dynamic draft length yet: " - f"{draft_token_num} != {replay_metadata.replay_step_width}" - ) - if draft_token_num > 8 or replay_metadata.replay_history_size > 16: - raise RuntimeError( - "GDN cached replay requires draft_token_num <= 8 and replay_history_size <= 16." - ) - return fused_recurrent_gated_delta_rule_cached_replay_update( - q=query, - k=key, - v=value, - g=a, - beta=b, - A_log=self.A_log, - dt_bias=self.dt_bias, - launch_with_pdl=True, - ssm_states=ssm_states, - state_indices=state_indices_d[:num_decodes], - old_u=layer_cache.old_x, - old_k=layer_cache.old_B, - old_G=layer_cache.old_dt, - old_beta=layer_cache.old_dA_cumsum, - cache_buf_idx=layer_cache.cache_buf_idx, - prev_num_accepted_tokens=layer_cache.prev_num_accepted_tokens, - history_size=replay_metadata.replay_history_size, - replay_work_items=replay_work_items, - n_writes=replay_n_writes, - use_qk_l2norm_in_kernel=True, - packed_qkv=packed_qkv, - use_all_layer_commit=use_all_layer_commit, - output=output_d, - ) - def forward_decode( self, conv_states, @@ -485,7 +463,7 @@ def forward_decode( assert a.shape[0] == num_decodes * draft_token_num assert b.shape[0] == num_decodes * draft_token_num assert intermediate_conv_states is not None - assert kwargs.get("use_replay", False) or intermediate_ssm_states is not None + assert intermediate_ssm_states is not None # Speculative verification path: # 1. run conv update with per-step intermediate cache writes @@ -505,8 +483,6 @@ def forward_decode( conv_state_indices=cache_indices[:num_decodes], intermediate_conv_window=intermediate_conv_states, intermediate_state_indices=intermediate_state_indices, - # PDL chain: conv1d -> replay verify kernel (replay only) - launch_dependent_kernels=kwargs.get("use_replay", False), ) mixed_qkv = mixed_qkv_processed.transpose(1, 2).reshape( num_decodes * draft_token_num, -1 @@ -535,39 +511,6 @@ def forward_decode( # intermediate states written to the batch-scoped [:num_decodes] # prefix consumed by update_mamba_states()); fall back to the # Triton recurrent kernel when unavailable. - if kwargs.get("use_replay", False): - output_d = None - if output is not None: - output_d = output.view( - num_decodes, - draft_token_num, - self.num_v_heads // self.attn_tp_size, - self.head_v_dim, - ) - return self._replay_verify_recurrent( - query, - key, - value, - a, - b, - ssm_states, - cache_indices, - num_decodes, - draft_token_num, - kwargs.get("replay_metadata"), - kwargs.get("layer_cache"), - kwargs.get("replay_work_items"), - kwargs.get("replay_n_writes"), - output_d, - packed_qkv=mixed_qkv, - use_all_layer_commit=kwargs.get("use_cached_replay_all_layer_commit", False), - ).view( - 1, - num_decodes * draft_token_num, - self.num_v_heads // self.attn_tp_size, - self.head_v_dim, - ) - if _can_use_flashinfer_gdn_verify( ssm_states, self.head_k_dim, self.head_v_dim, draft_token_num ): @@ -717,6 +660,8 @@ def forward_extend( seqlen_split_size = [num_prefill_tokens, num_decode_tokens] if num_decode_tokens > 0: mixed_qkv_p, mixed_qkv_d = torch.split(mixed_qkv, seqlen_split_size, dim=0) + a_p, a_d = torch.split(a, seqlen_split_size, dim=0) + b_p, b_d = torch.split(b, seqlen_split_size, dim=0) query_start_loc_p = query_start_loc[: num_prefill + 1] has_initial_states_p = has_initial_states[:num_prefill] @@ -738,15 +683,13 @@ def forward_extend( ) if is_target_verify: - a_d = a[num_prefill_tokens:] - b_d = b[num_prefill_tokens:] draft_token_num = spec_metadata.runtime_draft_len + 1 assert num_decodes > 0 assert mixed_qkv_d.shape[0] == num_decodes * draft_token_num assert a_d.shape[0] == num_decodes * draft_token_num assert b_d.shape[0] == num_decodes * draft_token_num assert intermediate_conv_states is not None - assert kwargs.get("use_replay", False) or intermediate_ssm_states is not None + assert intermediate_ssm_states is not None intermediate_state_indices = torch.arange( num_decodes, dtype=torch.int32, device=state_indices_d.device @@ -761,8 +704,6 @@ def forward_extend( conv_state_indices=state_indices_d, intermediate_conv_window=intermediate_conv_states, intermediate_state_indices=intermediate_state_indices, - # PDL chain: conv1d -> replay verify kernel (replay only) - launch_dependent_kernels=kwargs.get("use_replay", False), ) mixed_qkv_d = mixed_qkv_d.transpose(1, 2).reshape(num_decode_tokens, -1) else: @@ -774,41 +715,20 @@ def forward_extend( activation=self.activation, conv_state_indices=state_indices_d, ) - if is_target_verify: - if num_prefill_tokens > 0: - query_p, key_p, value_p, g_p, beta_p = fused_gdn_post_conv( - mixed_qkv_p_t, - None, - a[:num_prefill_tokens], - b[:num_prefill_tokens], - self.A_log, - self.dt_bias, - self.num_k_heads_per_tp, - self.head_k_dim, - self.num_v_heads_per_tp, - self.head_v_dim, - beta_dtype=b.dtype, - ) - query_d, key_d, value_d = pack_gdn_decode_qkv( - mixed_qkv_d, - self.num_k_heads_per_tp, - self.head_k_dim, - self.num_v_heads_per_tp, - self.head_v_dim, - ) - else: - query, key, value, g, beta = fused_gdn_post_conv( - mixed_qkv_p_t, - mixed_qkv_d, - a, - b, - self.A_log, - self.dt_bias, - self.num_k_heads_per_tp, - self.head_k_dim, - self.num_v_heads_per_tp, - self.head_v_dim, - ) + key_split_dim = self.key_dim // self.attn_tp_size + value_split_dim = self.value_dim // self.attn_tp_size + # Fused transpose + split for mixed prefill+decode batch + query, key, value = transpose_and_split_qkv( + mixed_qkv_p_t, + mixed_qkv_d, + key_split_dim, + key_split_dim, + value_split_dim, + self.num_k_heads_per_tp, + self.head_k_dim, + self.num_v_heads_per_tp, + self.head_v_dim, + ) else: mixed_qkv_t = extract_transpose_prefill_slice( mixed_qkv, @@ -826,13 +746,14 @@ def forward_extend( cache_indices=cache_indices, query_start_loc=query_start_loc, ) - query, key, value, g, beta = fused_gdn_post_conv( - mixed_qkv_t, - None, - a, - b, - self.A_log, - self.dt_bias, + key_split_dim = self.key_dim // self.attn_tp_size + value_split_dim = self.value_dim // self.attn_tp_size + # Fused split for pure prefill (data in [D,T] layout from conv1d) + query, key, value = split_qkv_contiguous( + mixed_qkv_t.transpose(0, 1), + key_split_dim, + key_split_dim, + value_split_dim, self.num_k_heads_per_tp, self.head_k_dim, self.num_v_heads_per_tp, @@ -842,6 +763,11 @@ def forward_extend( if is_target_verify and num_decode_tokens > 0: attn_out_prefill = None if num_prefill_tokens > 0: + query_p = query[:, :num_prefill_tokens, :, :] + key_p = key[:, :num_prefill_tokens, :, :] + value_p = value[:, :num_prefill_tokens, :, :] + beta_p = b_p.sigmoid().unsqueeze(0) + g_p = fused_gdn_gating(self.A_log, a_p, self.dt_bias).unsqueeze(0) recurrent_state_p = ssm_states[state_indices_p] output_p = output[:, :num_prefill_tokens, :, :] if output is not None else None attn_out_prefill, last_recurrent_state = chunk_gated_delta_rule( @@ -854,20 +780,20 @@ def forward_extend( output_final_state=True, cu_seqlens=query_start_loc_long[: num_prefill + 1], head_first=False, - use_qk_l2norm_in_kernel=False, + use_qk_l2norm_in_kernel=True, output=output_p, ) last_recurrent_state = last_recurrent_state.to(ssm_states.dtype, copy=False) ssm_states[state_indices_p] = last_recurrent_state draft_token_num = spec_metadata.runtime_draft_len + 1 - query_d = query_d.reshape( + query_d = query[:, num_prefill_tokens:, :, :].reshape( num_decodes, draft_token_num, self.num_k_heads // self.attn_tp_size, self.head_k_dim ) - key_d = key_d.reshape( + key_d = key[:, num_prefill_tokens:, :, :].reshape( num_decodes, draft_token_num, self.num_k_heads // self.attn_tp_size, self.head_k_dim ) - value_d = value_d.reshape( + value_d = value[:, num_prefill_tokens:, :, :].reshape( num_decodes, draft_token_num, self.num_v_heads // self.attn_tp_size, self.head_v_dim ) @@ -884,25 +810,7 @@ def forward_extend( self.head_v_dim, ) - if kwargs.get("use_replay", False): - attn_out_decode = self._replay_verify_recurrent( - query_d, - key_d, - value_d, - a_d, - b_d, - ssm_states, - state_indices_d, - num_decodes, - draft_token_num, - kwargs.get("replay_metadata"), - kwargs.get("layer_cache"), - kwargs.get("replay_work_items"), - kwargs.get("replay_n_writes"), - output_d, - use_all_layer_commit=kwargs.get("use_cached_replay_all_layer_commit", False), - ).reshape(1, num_decode_tokens, out_v_heads, self.head_v_dim) - elif _can_use_flashinfer_gdn_verify( + if _can_use_flashinfer_gdn_verify( ssm_states, self.head_k_dim, self.head_v_dim, draft_token_num ): # FI gathers the initial state from the pool via state_indices_d @@ -959,6 +867,11 @@ def forward_extend( return attn_out_decode return torch.cat((attn_out_prefill, attn_out_decode), dim=1) + g, beta = fused_gdn_gating_with_sigmoid(self.A_log, a, self.dt_bias, b) + + g = g.unsqueeze(0) + beta = beta.unsqueeze(0) + core_attn_out, _ = chunk_gated_delta_rule( q=query, k=key, @@ -973,7 +886,7 @@ def forward_extend( output_final_state=False, cu_seqlens=query_start_loc_long, head_first=False, - use_qk_l2norm_in_kernel=False, + use_qk_l2norm_in_kernel=True, output=output, ) @@ -1055,23 +968,7 @@ def forward_core( layer_cache.intermediate_conv_window if is_target_verify else None ) intermediate_ssm_states = layer_cache.intermediate_ssm if is_target_verify else None - use_replay = is_target_verify and getattr( - attn_metadata.kv_cache_manager, "use_replay_state_update", False - ) - replay_metadata = ( - attn_metadata.kv_cache_manager.get_replay_state_update_metadata() - if use_replay - else None - ) - use_cached_replay_all_layer_commit = ( - num_decodes >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE - and getattr( - attn_metadata.kv_cache_manager, - "use_gdn_cached_replay_all_layer_commit", - False, - ) - ) kwargs = { "mixed_qkv": mixed_qkv, "a": a, @@ -1087,16 +984,6 @@ def forward_core( "state_indices_d": state_indices_d, "num_prefill": num_prefills, "num_decodes": num_decodes, - "use_replay": use_replay, - "replay_metadata": replay_metadata, - "layer_cache": layer_cache, - "replay_work_items": mamba_metadata.replay_work_items[:num_decodes] - if use_replay and num_decodes >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE - else None, - "replay_n_writes": mamba_metadata.replay_n_writes - if use_replay and num_decodes >= CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE - else None, - "use_cached_replay_all_layer_commit": use_cached_replay_all_layer_commit, } if num_prefills > 0: attn_out = self.forward_extend( diff --git a/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py b/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py index 9c7ebc0181b0..1aaa276423c2 100644 --- a/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py +++ b/tensorrt_llm/_torch/modules/mamba/layernorm_gated.py @@ -19,7 +19,6 @@ import torch import triton import triton.language as tl -import triton.language.extra.libdevice as tldevice from ...._utils import get_sm_version from ...utils import Fp4QuantizedTensor @@ -44,7 +43,6 @@ def fused_gated_rmsnorm_quant_shape_ok(hidden_size: int, @triton.heuristics({"HAS_BIAS": lambda args: args["B"] is not None}) @triton.heuristics({"HAS_Z": lambda args: args["Z"] is not None}) -@triton.heuristics({"OUTPUT_FP8": lambda args: args["FP8_SCALE"] is not None}) @triton.jit def _layer_norm_fwd_1pass_kernel( X, # pointer to the input @@ -54,7 +52,6 @@ def _layer_norm_fwd_1pass_kernel( Z, # pointer to the other branch Mean, # pointer to the mean Rstd, # pointer to the 1/std - FP8_SCALE, # static scale for an optional FP8 output stride_x_row, # how much to increase the pointer when moving by 1 row stride_y_row, stride_z_row, @@ -66,7 +63,6 @@ def _layer_norm_fwd_1pass_kernel( HAS_Z: tl.constexpr, NORM_BEFORE_GATE: tl.constexpr, IS_RMS_NORM: tl.constexpr, - OUTPUT_FP8: tl.constexpr, ): # Map the program id to the row of X and Y it should compute. row = tl.program_id(0) @@ -109,14 +105,8 @@ def _layer_norm_fwd_1pass_kernel( if HAS_Z and NORM_BEFORE_GATE: z = tl.load(Z + cols, mask=mask).to(tl.float32) y *= z * tl.sigmoid(z) - if OUTPUT_FP8: - # Match the existing two-kernel path: RMSNorm first stores to the - # input dtype, then static quantization reloads and multiplies by the - # rounded reciprocal of its input scale. - y = y.to(X.dtype.element_ty).to(tl.float32) - y *= tldevice.rcp_rn(tl.load(FP8_SCALE).to(tl.float32)) # Write output - tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + cols, y, mask=mask) # Rows per program of the multi-row gated-RMSNorm kernel. At the GDN decode @@ -128,7 +118,6 @@ def _layer_norm_fwd_1pass_kernel( _MULTIROW_MAX_N = 256 -@triton.heuristics({"OUTPUT_FP8": lambda args: args["FP8_SCALE"] is not None}) @triton.jit def _rms_norm_gated_fwd_multirow_kernel( X, # pointer to the input @@ -136,7 +125,6 @@ def _rms_norm_gated_fwd_multirow_kernel( W, # pointer to the weights Z, # pointer to the gate branch Rstd, # pointer to the 1/std - FP8_SCALE, # static scale for an optional FP8 output stride_x_row, stride_y_row, stride_z_tok, @@ -145,7 +133,6 @@ def _rms_norm_gated_fwd_multirow_kernel( N: tl.constexpr, # row length; power of two, whole row per program ROWS: tl.constexpr, # rows per program HEADS_PER_TOK: tl.constexpr, - OUTPUT_FP8: tl.constexpr, ): """rmsnorm(x) * silu(z), several short rows per program. @@ -173,12 +160,6 @@ def _rms_norm_gated_fwd_multirow_kernel( cols[None, :]) z = tl.load(Z + z_off, mask=mask2d, other=0.0).to(tl.float32) y *= z * tl.sigmoid(z) - if OUTPUT_FP8: - # Match the existing two-kernel path: RMSNorm first stores to the - # input dtype, then static quantization reloads and multiplies by the - # rounded reciprocal of its input scale. - y = y.to(X.dtype.element_ty).to(tl.float32) - y *= tldevice.rcp_rn(tl.load(FP8_SCALE).to(tl.float32)) y_off = rows[:, None].to(tl.int64) * stride_y_row + cols[None, :] tl.store(Y + y_off, y.to(Y.dtype.element_ty), mask=mask2d) @@ -189,7 +170,7 @@ def _multirow_gated_rmsnorm_eligible(N, ngroups, bias, z, norm_before_gate, and ngroups == 1 and N <= _MULTIROW_MAX_N and (N & (N - 1)) == 0) -def rms_norm_gated_token_major(x, z, weight, eps, out=None, fp8_scale=None): +def rms_norm_gated_token_major(x, z, weight, eps, out=None): """rmsnorm(x) * silu(z) with z read in place from a 3D token-major view. x: [num_tokens * heads, N] with contiguous rows. z: [num_tokens, heads, N] @@ -197,10 +178,6 @@ def rms_norm_gated_token_major(x, z, weight, eps, out=None, fp8_scale=None): arbitrary (e.g. a column slice of a wider per-token projection). Falls back to the generic kernel on a packed copy of z when the shape is not eligible for the multi-row kernel. - - When fp8_scale (a scalar fp32 tensor holding the downstream static input - scale) is given, the output is quantized to float8_e4m3fn in the same - kernel. """ M, N = x.shape num_tokens, heads, n_z = z.shape @@ -219,12 +196,10 @@ def rms_norm_gated_token_major(x, z, weight, eps, out=None, fp8_scale=None): out=out, norm_before_gate=True, is_rms_norm=True, - fp8_scale=fp8_scale, ) return y if out is None: - out_dtype = torch.float8_e4m3fn if fp8_scale is not None else x.dtype - out = torch.empty_like(x, dtype=out_dtype) + out = torch.empty_like(x) rstd = torch.empty((M, ), dtype=torch.float32, device=x.device) grid = (triton.cdiv(M, _MULTIROW_ROWS), ) with torch.cuda.device(x.device.index): @@ -234,7 +209,6 @@ def rms_norm_gated_token_major(x, z, weight, eps, out=None, fp8_scale=None): weight, z, rstd, - fp8_scale, x.stride(0), out.stride(0), z.stride(0), @@ -258,7 +232,6 @@ def _layer_norm_fwd( group_size=None, norm_before_gate=True, is_rms_norm=False, - fp8_scale=None, ): M, N = x.shape if group_size is None: @@ -278,13 +251,8 @@ def _layer_norm_fwd( if out is not None: assert out.shape == x.shape else: - out_dtype = torch.float8_e4m3fn if fp8_scale is not None else x.dtype - out = torch.empty_like(x, dtype=out_dtype) + out = torch.empty_like(x) assert out.stride(-1) == 1 - if fp8_scale is not None: - assert fp8_scale.dtype == torch.float32 - assert fp8_scale.numel() == 1 - assert out.dtype == torch.float8_e4m3fn mean = (torch.empty((ngroups * M, ), dtype=torch.float32, device=x.device) if not is_rms_norm else None) rstd = torch.empty((ngroups * M, ), dtype=torch.float32, device=x.device) @@ -298,7 +266,6 @@ def _layer_norm_fwd( weight, z, rstd, - fp8_scale, x.stride(0), out.stride(0), z.stride(0), @@ -328,7 +295,6 @@ def _layer_norm_fwd( z, mean, rstd, - fp8_scale, x.stride(0), out.stride(0), z.stride(0) if z is not None else 0, @@ -371,14 +337,11 @@ def __init__( self.is_nvfp4 = is_nvfp4 # nvfp4_scale will be set externally if is_nvfp4 is True self.nvfp4_scale: torch.Tensor | None = None - # fp8_scale will be attached from the downstream static FP8 linear. - self.fp8_scale: torch.Tensor | None = None def forward( - self, - x: torch.Tensor, - z: torch.Tensor | None = None, - ) -> torch.Tensor | Fp4QuantizedTensor: + self, + x: torch.Tensor, + z: torch.Tensor | None = None) -> torch.Tensor | Fp4QuantizedTensor: """If z is not None, we do norm(x) * silu(z) if norm_before_gate, else norm(x * silu(z))""" x_shape_og = x.shape # reshape input data into 2D tensor @@ -395,12 +358,6 @@ def forward( if self.bias is not None: bias = self.bias.contiguous() - fp8_scale = self.fp8_scale - if fp8_scale is not None: - if self.is_nvfp4: - raise ValueError( - "FP8 and NVFP4 RMSNorm outputs are mutually exclusive") - # NVFP4 quantized path - uses optimized fused CUDA kernel # Fuses: SiLU gating + Group RMSNorm + FP4 quantization if self.is_nvfp4 and z is not None and not self.norm_before_gate and \ @@ -434,6 +391,5 @@ def forward( group_size=self.group_size, norm_before_gate=self.norm_before_gate, is_rms_norm=True, - fp8_scale=fp8_scale, ) return y.reshape(x_shape_og) diff --git a/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py b/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py index 5b97e32a6ee3..5cab7283f033 100644 --- a/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py +++ b/tensorrt_llm/_torch/modules/mamba/mamba2_metadata.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import contextlib import math from typing import Tuple @@ -215,23 +214,6 @@ def cu_seqlens_to_chunk_indices_offsets( class Mamba2Metadata: - # Warmup-only knob: when set via ``force_initial_states_for_warmup``, - # ``prepare()`` forces ``has_initial_states_cpu[:num_contexts]`` to True so - # the ``HAS_INITSTATES=True`` variants of the SSD Triton kernels compile - # during warmup. Class-scoped (not env-var) so it cannot leak into real - # inference from a stray shell export or a forked worker. - _warmup_force_initial_states: bool = False - - @classmethod - @contextlib.contextmanager - def force_initial_states_for_warmup(cls): - prev = cls._warmup_force_initial_states - cls._warmup_force_initial_states = True - try: - yield - finally: - cls._warmup_force_initial_states = prev - def __init__(self, max_batch_size: int, chunk_size: int): self.max_batch_size = max_batch_size self.chunk_size = chunk_size @@ -387,14 +369,9 @@ def prepare(self, attn_metadata: AttentionMetadata): self.state_indices_cpu[:batch_size], non_blocking=True) else: # indices is a Python sequence (e.g. List[int]); data - # already lives on host, CPU staging is fine. One bulk - # conversion instead of a per-element tensor write. - assert len(indices) == batch_size, ( - f"get_state_indices() returned {len(indices)} entries for " - f"a batch of {batch_size} requests.") - self.state_indices_cpu[:batch_size].copy_( - torch.as_tensor(indices, - dtype=self.state_indices_cpu.dtype)) + # already lives on host, CPU staging is fine. + for i, idx in enumerate(indices): + self.state_indices_cpu[i] = idx self.state_indices[:batch_size].copy_( self.state_indices_cpu[:batch_size], non_blocking=True) @@ -437,15 +414,6 @@ def prepare(self, attn_metadata: AttentionMetadata): device='cpu') self.has_initial_states_cpu[:num_contexts].copy_(initial_states_cpu) - # Warmup-only override: force HAS_INITSTATES=True path so the - # HAS_INITSTATES=True variants of _state_passing_fwd_kernel, - # _chunk_scan_fwd_kernel, and _chunk_state_varlen_kernel compile - # during warmup instead of the first real-request iter that hits - # chunked prefill with cached tokens. Gate is a class-scoped - # context manager (see ``force_initial_states_for_warmup``) so it - # cannot silently affect real inference. - if Mamba2Metadata._warmup_force_initial_states: - self.has_initial_states_cpu[:num_contexts].fill_(True) # Mirror CPU staging flags to the CUDA-side buffer asynchronously. self.has_initial_states[:num_contexts].copy_( self.has_initial_states_cpu[:num_contexts], non_blocking=True) diff --git a/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py b/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py index 688c48b147de..e807a904978f 100644 --- a/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py +++ b/tensorrt_llm/_torch/modules/mamba/mamba2_mixer.py @@ -430,47 +430,16 @@ def forward( f"{draft_token_num} must match fixed replay step " f"width {replay_step_width}.") - # Dynamic-tree verify uses per-request links; linear MTP skips it. - is_dyn_tree = getattr(spec_metadata, 'is_spec_dec_dynamic_tree', - False) - retrieve_next_token = retrieve_next_sibling = None - retrieve_parent_token = None - if is_dyn_tree: - if use_replay: - raise NotImplementedError( - "Dynamic-tree Mamba verify is not supported with " - "the replay SSM-cache path (TRTLLM_USE_MAMBA_REPLAY)." - ) - retrieve_next_token = spec_metadata.retrieve_next_token - retrieve_next_sibling = spec_metadata.retrieve_next_sibling - assert (retrieve_next_token is not None - and retrieve_next_sibling is not None), ( - "Dynamic-tree verify requires retrieve link " - "tensors on spec_metadata.") - retrieve_next_token = retrieve_next_token[:num_decodes] - retrieve_next_sibling = retrieve_next_sibling[:num_decodes] - # conv1d fills parent links used by tree-aware SSM restore. - retrieve_parent_token = torch.empty( - (num_decodes, draft_token_num), - dtype=torch.int32, - device=state_indices_d.device) - - # Prefer the cache_manager-owned arange; cached fallback storage - # can be recycled by CUDA graph warmup. - _km_isi = getattr(attn_metadata.kv_cache_manager, - 'intermediate_state_indices', None) - if _km_isi is not None: - intermediate_state_indices = _km_isi[:num_decodes] - else: - intermediate_state_indices = _cached_arange( - attn_metadata.kv_cache_manager.get_max_resource_count(), - state_indices_d.device)[:num_decodes] + intermediate_state_indices = _cached_arange( + attn_metadata.kv_cache_manager.get_max_resource_count(), + state_indices_d.device)[:num_decodes] - # Use reshape because dynamic-tree tokens may be non-contiguous. - xbc_d_reshaped = xbc_d.reshape(num_decodes, draft_token_num, - -1).transpose(1, 2) + # Reshape for batch processing + xbc_d_reshaped = xbc_d.view(num_decodes, draft_token_num, + -1).transpose(1, 2) def conv1d(): + # TODO:support tree structure [TRTLLM-10320] xbc_d_processed = causal_conv1d_update_triton( xbc_d_reshaped, conv_states, @@ -480,15 +449,11 @@ def conv1d(): conv_state_indices=state_indices_d[:num_decodes], intermediate_conv_window=intermediate_conv_states, intermediate_state_indices=intermediate_state_indices, - # None on linear MTP. - retrieve_next_token=retrieve_next_token, - retrieve_next_sibling=retrieve_next_sibling, - retrieve_parent_token=retrieve_parent_token, # PDL chain: conv1d → precompute → main (replay only) launch_dependent_kernels=use_replay, ) - return xbc_d_processed.transpose(1, 2).reshape( + return xbc_d_processed.transpose(1, 2).view( num_decode_tokens, -1) else: @@ -623,23 +588,13 @@ def convert_dt(): state_batch_indices=state_batch_indices, disable_state_update=True, intermediate_state_indices=intermediate_state_indices, - # None for linear MTP; tree parent map for dynamic tree. - retrieve_parent_token=retrieve_parent_token, ) else: # Triton kernel + flashinfer need contiguous for alignment. x_d_4d = x_d_4d.contiguous() B_d_4d = B_d_4d.contiguous() C_d_4d = C_d_4d.contiguous() - if is_dyn_tree: - # flashinfer SSU cannot restore tree-parent states. - ssu_func = selective_state_update_native - ssu_extra = dict( - retrieve_parent_token=retrieve_parent_token) - else: - ssu_func = self.selective_state_update_func - ssu_extra = {} - ssu_func( + self.selective_state_update_func( ssm_states, x_d_4d, dt_d_4d, @@ -656,7 +611,6 @@ def convert_dt(): intermediate_states_buffer=intermediate_ssm_states, cache_steps=draft_token_num, intermediate_state_indices=intermediate_state_indices, - **ssu_extra, **philox_kwargs, ) else: diff --git a/tensorrt_llm/_torch/modules/mhc/hyper_connection.py b/tensorrt_llm/_torch/modules/mhc/hyper_connection.py index 477ea75ddd66..73f2b59c6efe 100644 --- a/tensorrt_llm/_torch/modules/mhc/hyper_connection.py +++ b/tensorrt_llm/_torch/modules/mhc/hyper_connection.py @@ -276,13 +276,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: if not _cuda_available: raise RuntimeError("CUDA MHC kernels not available") dtype = x.dtype - # The CUDA head consumes a flat ``[M, mult, hidden]`` tensor. Preserve any - # leading dims (e.g. ``[batch, block, mult, hidden]`` from the DSpark draft - # block) by collapsing to a single token axis and restoring afterwards; - # the RMS-norm + weighted sum is independent per token, so this matches the - # reference ``hc_head`` which keeps the leading dims. - lead = x.shape[:-2] - x_bf16 = x.reshape(-1, self.mult, self.hidden_size).to(torch.bfloat16).contiguous() + x_bf16 = x.to(torch.bfloat16).contiguous() y = mhc_hc_head_cuda( x_bf16, self.fn, @@ -293,7 +287,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: norm_eps=self.norm_eps, eps=self.eps, ) - return y.reshape(*lead, self.hidden_size).to(dtype) + return y.to(dtype) def skip_forward(self, x: torch.Tensor) -> torch.Tensor: """Skip HCHead computation for pipeline parallelism on non-last ranks.""" diff --git a/tensorrt_llm/_torch/modules/mla.py b/tensorrt_llm/_torch/modules/mla.py index 63f9a1bd65d3..4bee770842f7 100644 --- a/tensorrt_llm/_torch/modules/mla.py +++ b/tensorrt_llm/_torch/modules/mla.py @@ -26,7 +26,6 @@ from tensorrt_llm._utils import get_sm_version, is_sm_100f, nvtx_range, nvtx_range_debug from tensorrt_llm.logger import logger from tensorrt_llm.mapping import Mapping -from tensorrt_llm.quantization.utils.fp4_utils import NVFP4_SF_VEC_SIZE from ..attention_backend import ( AttentionForwardArgs, @@ -48,13 +47,7 @@ from ..attention_backend.utils import create_attention from ..distributed import AllReduceParams from ..model_config import ModelConfig -from ..utils import ( - Fp4QuantizedTensor, - compute_swizzled_sf_shape, - is_torch_compiling, - maybe_compiled_cat, - maybe_compiled_copy_, -) +from ..utils import is_torch_compiling, maybe_compiled_cat, maybe_compiled_copy_ from .attention import ( _helix_cp_allgather_input, _helix_cp_output_projection, @@ -62,7 +55,7 @@ _helix_zero_kv_mask, extract_extra_attrs, ) -from .linear import Linear, TensorParallelMode, is_static_nvfp4_input_eligible +from .linear import Linear, TensorParallelMode from .multi_stream_utils import do_multi_stream, maybe_execute_in_parallel from .rms_norm import RMSNorm from .rotary_embedding import RotaryEmbedding @@ -78,55 +71,6 @@ def _is_env_truthy(name: str) -> bool: return os.environ.get(name, "").strip().lower() in ("1", "true", "on") -def _slice_hidden_states_to_num_tokens(hidden_states, num_tokens: int): - """Drop CUDA-graph padding by slicing hidden_states to num_tokens rows. - - For a plain tensor this is the usual row slice. For an Fp4QuantizedTensor - (produced when the previous layer's boundary fusion pre-quantized this - layer's input) the slice must act on the packed FP4 form: - - fp4_tensor: row-major [m, k//2], so [:num_tokens] is a plain row slice; - - scaling_factor: 1D swizzled buffer of padUp(m,128)*padUp(cols,4). NVFP4 - quant is per-row independent and the swizzle is laid out in independent - 128-row tiles, so the leading padUp(num_tokens,128)*padUp(cols,4) bytes - are exactly the scale factors for the first num_tokens rows (verified by - tests/unittest/_torch/modules/test_fp4_num_tokens_slice.py). - - unquantized_hidden_states (when present): plain row slice. - """ - if not isinstance(hidden_states, Fp4QuantizedTensor): - return hidden_states[:num_tokens, ...] - - fp4 = hidden_states.fp4_tensor - if fp4.shape[0] == num_tokens: - return hidden_states # no padding to strip - # The row-slice math below relies on the swizzled 128-row-tile layout; a - # non-swizzled (linear) scale buffer would be sliced incorrectly and then - # silently relabeled as swizzled. Reject it rather than corrupt the SF. - if not hidden_states.is_sf_swizzled: - raise ValueError( - "_slice_hidden_states_to_num_tokens only supports swizzled FP4 scaling factors" - ) - sf = hidden_states.scaling_factor - # fp4 packs 2 elements/byte, so the logical hidden dim is fp4_cols*2 and - # the scale-factor column count is that / NVFP4_SF_VEC_SIZE. The swizzled - # SF is laid out in independent 128-row tiles, so the leading num_tokens - # rows' scale factors occupy exactly the first padded_rows*padded_cols - # bytes. - sf_cols = fp4.shape[-1] * 2 // NVFP4_SF_VEC_SIZE - padded_rows, padded_cols = compute_swizzled_sf_shape(num_tokens, sf_cols) - sf_len = padded_rows * padded_cols - sliced_unquant = ( - hidden_states.unquantized_hidden_states[:num_tokens] - if hidden_states.unquantized_hidden_states is not None - else None - ) - return Fp4QuantizedTensor( - fp4_tensor=fp4[:num_tokens].contiguous(), - scaling_factor=sf.view(-1)[:sf_len].contiguous(), - is_sf_swizzled=True, - unquantized_hidden_states=sliced_unquant, - ) - - def _extract_mla_extra_attrs(layer_idx: str): metadata, mla_layer = extract_extra_attrs(layer_idx, "mla") assert isinstance(mla_layer, MLA), "MLA layer must be a subclass of MLA or an instance of MLA" @@ -169,28 +113,9 @@ def mla_custom_op_inplace( dsv4_output: Optional[torch.Tensor], dsv4_output_sf: Optional[torch.Tensor], enable_dsv4_epilogue_fusion: bool, - hidden_states_fp4: Optional[torch.Tensor] = None, - hidden_states_sf: Optional[torch.Tensor] = None, ) -> None: - # When hidden_states_fp4/_sf are provided, the previous layer's boundary - # fusion pre-quantized this layer's kv_a_proj NVFP4 input. Custom ops can't - # take dataclasses, so the Fp4QuantizedTensor is passed as explicit tensors - # and reconstructed here (mirrors mla_dsa_proj). unquantized_hidden_states carries - # the un-quantized view for the o_proj output sizing in create_output. metadata, mla_layer = _extract_mla_extra_attrs(layer_idx) - if hidden_states_fp4 is not None or hidden_states_sf is not None: - assert hidden_states_fp4 is not None and hidden_states_sf is not None, ( - "hidden_states_fp4 and hidden_states_sf must be passed together" - ) - hidden_states = Fp4QuantizedTensor( - fp4_tensor=hidden_states_fp4, - scaling_factor=hidden_states_sf, - unquantized_hidden_states=hidden_states, - ) if mla_layer.is_deepseek_v4: - # DeepSeek-V4 uses MQA mode and has no residual-less RMSNorm+quant - # fusion entry point, so it cannot be reached with a pre-quantized - # Fp4QuantizedTensor input; the call site passes plain hidden_states. if enable_dsv4_epilogue_fusion: if dsv4_output is None or dsv4_output_sf is None: raise RuntimeError( @@ -223,8 +148,6 @@ def mla_dsa_proj( hidden_states: torch.Tensor, position_ids: Optional[torch.Tensor], layer_idx: str, - hidden_states_fp4: Optional[torch.Tensor] = None, - hidden_states_sf: Optional[torch.Tensor] = None, ) -> List[torch.Tensor]: """Token-wise projections for DSA MLA (CUDA-graph-capturable). @@ -233,14 +156,6 @@ def mla_dsa_proj( update the indexer k cache — that happens in Op 2 (mla_dsa_attn_inplace) because the scatter kernel accesses batch-specific metadata. - When ``hidden_states_fp4`` / ``hidden_states_sf`` are provided, the - previous layer's boundary fusion already produced a fused NVFP4 view of - the post-RMSNorm hidden states (via fused_add_rmsnorm_fp4_quantize). The - op rebuilds an ``Fp4QuantizedTensor`` carrying the BF16 view in - ``hidden_states`` (for DSA's ``pre_indexer_proj``) plus the - pre-quantized FP4 form (consumed by ``kv_a_proj_with_mqa``). - Both must be passed together; passing either alone is rejected. - Returns [q, compressed_kv, k_pe, latent_cache] when the short-MHA path handles all tokens, or [q, compressed_kv, k_pe, latent_cache, q_fp8, k_fp8, k_scale, weights, q_scale] when the indexer runs. Under torch @@ -250,18 +165,7 @@ def mla_dsa_proj( path ignores it in forward_dsa_attn. """ metadata, mla_layer = _extract_mla_extra_attrs(layer_idx) - if hidden_states_fp4 is not None or hidden_states_sf is not None: - assert hidden_states_fp4 is not None and hidden_states_sf is not None, ( - "hidden_states_fp4 and hidden_states_sf must be passed together" - ) - hs = Fp4QuantizedTensor( - fp4_tensor=hidden_states_fp4, - scaling_factor=hidden_states_sf, - unquantized_hidden_states=hidden_states, - ) - else: - hs = hidden_states - return mla_layer.forward_dsa_proj(position_ids, hs, metadata) + return mla_layer.forward_dsa_proj(position_ids, hidden_states, metadata) @mla_dsa_proj.register_fake @@ -269,8 +173,6 @@ def _mla_dsa_proj_fake( hidden_states: torch.Tensor, position_ids: Optional[torch.Tensor], layer_idx: str, - hidden_states_fp4: Optional[torch.Tensor] = None, - hidden_states_sf: Optional[torch.Tensor] = None, ) -> List[torch.Tensor]: # Under torch compile _should_use_short_mha is False, so the result is # always 9 tensors (4 attention inputs + 5 indexer intermediates, with @@ -495,15 +397,6 @@ def __init__( if config is not None: if "mla_layers" not in config.extra_attrs: config.extra_attrs["mla_layers"] = {} - suffix = 0 - # ``layer_idx`` is local to an attention stack, while this registry is shared - # by target and draft modules in one-model speculative decoding. Keep the first - # registration under ``""`` and suffix later collisions as - # ``"_"`` so custom ops resolve the originating module without - # overwriting another stack's weak reference. - while self.layer_idx_str in config.extra_attrs["mla_layers"]: - self.layer_idx_str = str(layer_idx) + f"_{suffix}" - suffix += 1 config.extra_attrs["mla_layers"][self.layer_idx_str] = weakref.ref(self) self.register_to_config = True @@ -525,14 +418,6 @@ def __init__( "TRTLLM_DSV4_DISABLE_FMHA_EPILOGUE_FUSION" ) - # Fold of the residual-less q_a_layernorm -> q_b_proj NVFP4 input - # quantize into a single fused (rmsnorm + fp4_quantize) kernel. Self- - # gating on q_b_proj being static-NVFP4 (see _resolve_qa_fused_scale); - # resolved lazily on first forward because q_b_proj.input_scale is only - # finalized after weight loading. - # None = not yet resolved; False = disabled; tensor = q_b_proj input_scale. - self._qa_fused_scale = None - # tensor parallel if mapping_with_cp is not None: logger.warning_once( @@ -1139,17 +1024,6 @@ def _attn_forward_gen( return attn_output def create_output(self, hidden_states: torch.Tensor, num_contexts: int): - # Upstream POST_MoE/MLP fusion (or attention-DP no-fusion fold) may pass - # an Fp4QuantizedTensor here; unpack to the BF16 view for sizing. The - # producing fold must have requested return_norm_out so the BF16 view - # is present (folds feeding an MLA always do). - if isinstance(hidden_states, Fp4QuantizedTensor): - assert hidden_states.unquantized_hidden_states is not None, ( - "MLA.create_output received an Fp4QuantizedTensor without a " - "unquantized_hidden_states view; the producing fusion must use " - "return_norm_out=True" - ) - hidden_states = hidden_states.unquantized_hidden_states num_tokens = hidden_states.shape[0] if self.is_deepseek_v4: hidden_size = self.num_heads_tp_cp * self.v_head_dim @@ -1336,65 +1210,6 @@ def _deepseek_v4_o_proj( output = self.o_b_proj(o_lora) return output - def _resolve_qa_fused_scale(self): - """Lazily decide whether the residual-less q_a_layernorm -> q_b_proj - NVFP4 fusion can run, caching q_b_proj's static input_scale. - - Returns the input_scale tensor when eligible, else None. Eligible iff: - (1) model is non-lite (has a distinct q_a_layernorm / q_b_proj), - (2) q_b_proj is static-NVFP4 (calibrated input_scale, no - pre_quant_scale / AWQ, not forced-dynamic), - (3) q_a_layernorm is a plain (non-gemma, non-quantizing) RMSNorm. - """ - if self._qa_fused_scale is not None: - # Already resolved: tensor (enabled) or False (disabled). - return self._qa_fused_scale if self._qa_fused_scale is not False else None - eligible = False - if ( - not self.is_lite - and getattr(self, "q_a_layernorm", None) is not None - and getattr(self, "q_b_proj", None) is not None - ): - qb = self.q_b_proj - qa = self.q_a_layernorm - # q_b_proj must be static-NVFP4 (shared predicate) and q_a_layernorm - # a plain (non-gemma, non-quantizing) RMSNorm. - if is_static_nvfp4_input_eligible(qb) and not qa.use_gemma and not qa.is_nvfp4: - eligible = True - if eligible: - # Mark q_a_layernorm as an NVFP4 norm and attach q_b_proj's static - # input_scale, so RMSNorm.forward folds the residual-less - # q_a_layernorm + q_b_proj NVFP4 input-quant into one kernel (the - # size gate routes this N=q_lora_rank<2048 edge to reduce_fusion). - self.q_a_layernorm.is_nvfp4 = True - self.q_a_layernorm.nvfp4_scale = qb.input_scale - self._qa_fused_scale = qb.input_scale if eligible else False - return self._qa_fused_scale if eligible else None - - def _q_a_layernorm_maybe_fused(self, q: torch.Tensor, return_norm_out: bool = False): - """Apply q_a_layernorm, optionally folding the q_b_proj NVFP4 input - quantize into the same kernel. - - When fusion is disabled, returns the BF16/FP16 normed tensor (and, if - return_norm_out, the same tensor again so callers have a stable arity). - - When fusion is enabled, returns an Fp4QuantizedTensor (consumed by - q_b_proj without re-quantizing). If return_norm_out is set, also returns - the BF16 post-RMSNorm value, needed by the DSA indexer's - pre_indexer_proj which requires a float q.""" - scale = self._resolve_qa_fused_scale() - if scale is None: - normed = self.q_a_layernorm(q) - return (normed, normed) if return_norm_out else normed - # q is typically the leading q_lora_rank columns of the - # kv_a_proj_with_mqa output split, i.e. a column slice whose last dim is - # unit-stride but whose row pitch is the full projection width. The - # fused RMSNorm+NVFP4-quant kernel reads that strided layout directly, - # so no .contiguous() copy is needed. _resolve_qa_fused_scale has - # attached .nvfp4_scale to q_a_layernorm, so RMSNorm.forward folds the - # residual-less RMSNorm + NVFP4 quant automatically. - return self.q_a_layernorm(q, return_norm_out=return_norm_out) - def forward_impl( self, position_ids: Optional[torch.Tensor], @@ -1419,7 +1234,7 @@ def forward_impl( num_ctx_tokens = attn_metadata.num_ctx_tokens num_tokens = attn_metadata.num_tokens - hidden_states = _slice_hidden_states_to_num_tokens(hidden_states, num_tokens) + hidden_states = hidden_states[:num_tokens, ...] if position_ids is not None: position_ids = position_ids[..., :num_tokens] @@ -1435,7 +1250,7 @@ def forward_impl( ) q, compressed_kv = maybe_execute_in_parallel( - lambda: self._q_a_layernorm_maybe_fused(q), + lambda: self.q_a_layernorm(q), lambda: self.kv_a_layernorm(compressed_kv), self.ln_events[0], self.ln_events[1], @@ -1567,17 +1382,14 @@ def forward_dsa_proj( [self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], -1 ) - # When the q_a_layernorm -> q_b_proj NVFP4 fusion is active, q becomes an - # Fp4QuantizedTensor consumed by q_b_proj; qr (fed to the indexer) needs - # the BF16 post-RMSNorm value, so request return_norm_out here. - q_pair, compressed_kv = maybe_execute_in_parallel( - lambda: self._q_a_layernorm_maybe_fused(q, return_norm_out=True), + q, compressed_kv = maybe_execute_in_parallel( + lambda: self.q_a_layernorm(q), lambda: self.kv_a_layernorm(compressed_kv), self.ln_events[0], self.ln_events[1], self.aux_stream, ) - q, qr = q_pair + qr = q latent_cache = torch.concat([compressed_kv, k_pe], dim=-1) q = self.q_b_proj(q) @@ -3094,22 +2906,9 @@ def forward( else: attn_output = self.create_output(hidden_states, attn_metadata.num_contexts) if self.is_dsa: - # When the previous layer's boundary fusion pre-quantized - # this layer's kv_a_proj NVFP4 input, hidden_states is an - # Fp4QuantizedTensor with both BF16 + FP4 views. Custom ops - # can't take dataclasses, so pass tensors explicitly. - if isinstance(hidden_states, Fp4QuantizedTensor): - proj_outputs = torch.ops.trtllm.mla_dsa_proj( - hidden_states.unquantized_hidden_states, - position_ids, - self.layer_idx_str, - hidden_states.fp4_tensor, - hidden_states.scaling_factor, - ) - else: - proj_outputs = torch.ops.trtllm.mla_dsa_proj( - hidden_states, position_ids, self.layer_idx_str - ) + proj_outputs = torch.ops.trtllm.mla_dsa_proj( + hidden_states, position_ids, self.layer_idx_str + ) q, compressed_kv, k_pe, latent_cache = proj_outputs[:4] indexer_intermediates = proj_outputs[4:] torch.ops.trtllm.mla_dsa_attn_inplace( @@ -3123,35 +2922,16 @@ def forward( attn_output, ) else: - # Vanilla MLA (e.g. Kimi-K2.5): when the previous layer's - # boundary fusion pre-quantized this layer's input, - # hidden_states is an Fp4QuantizedTensor. Custom ops can't - # take dataclasses, so pass the BF16 + FP4 + SF views as - # explicit tensors. - if isinstance(hidden_states, Fp4QuantizedTensor): - torch.ops.trtllm.mla_custom_op_inplace( - hidden_states.unquantized_hidden_states, - position_ids, - self.layer_idx_str, - attn_output, - latent_cache_gen, - None, - None, - False, - hidden_states.fp4_tensor, - hidden_states.scaling_factor, - ) - else: - torch.ops.trtllm.mla_custom_op_inplace( - hidden_states, - position_ids, - self.layer_idx_str, - attn_output, - latent_cache_gen, - None, - None, - False, - ) + torch.ops.trtllm.mla_custom_op_inplace( + hidden_states, + position_ids, + self.layer_idx_str, + attn_output, + latent_cache_gen, + None, + None, + False, + ) else: enable_dsv4_epilogue_fusion = ( self.is_deepseek_v4 diff --git a/tensorrt_llm/_torch/modules/qk_norm_attention.py b/tensorrt_llm/_torch/modules/qk_norm_attention.py index 3d0dbec17c69..0ee43381dace 100644 --- a/tensorrt_llm/_torch/modules/qk_norm_attention.py +++ b/tensorrt_llm/_torch/modules/qk_norm_attention.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,17 +19,13 @@ import torch from transformers import PretrainedConfig -from tensorrt_llm.functional import RotaryScalingType from tensorrt_llm.mapping import Mapping from ..attention_backend.interface import PositionalEmbeddingParams from ..model_config import ModelConfig from ..modules.attention import Attention -from ..modules.fused_ops.fused_qk_norm_rope_gate import ( - fused_qkv_gemma_rmsnorm_rope_gate, fused_sigmoid_mul) from ..modules.multi_stream_utils import maybe_execute_in_parallel from ..modules.rms_norm import RMSNorm -from ..utils import is_torch_compiling # Move out from this class @@ -176,7 +172,6 @@ def __init__( # Gemma-style RMSNorm (scale by (1 + weight)) is supported by the fused # qk_norm_rope kernel via the use_gemma flag threaded through below. self.use_gemma_rms_norm = use_gemma_rms_norm - self._fuse_qk_norm_rope_gate = False # If fuse_qk_norm_rope is true, do not apply fused RoPE in attention OP, and self.rotary_emb # will be skipped in the overridden apply_rope. @@ -218,87 +213,6 @@ def __init__( self.aux_stream = torch.cuda.Stream() self.ln_events = [torch.cuda.Event(), torch.cuda.Event()] - def _get_qk_norm_rotary_dim(self) -> int: - partial_rotary_factor = getattr(self.pretrained_config, - "partial_rotary_factor", 1.0) - return int(self.head_dim * partial_rotary_factor) - - def _can_use_fused_qk_norm_rope_gate( - self, qkv: torch.Tensor, - position_ids: Optional[torch.Tensor]) -> bool: - if not self._fuse_qk_norm_rope_gate or is_torch_compiling(): - return False - if torch.version.hip is not None or qkv.device.type != "cuda": - return False - if qkv.dim() != 2 or qkv.stride(-1) != 1: - return False - if qkv.dtype not in (torch.bfloat16, torch.float16): - return False - if position_ids is None or position_ids.dim() > 3: - return False - if position_ids.dtype not in (torch.int32, torch.int64): - return False - use_mrope = position_ids.dim() == 3 - if use_mrope and not (getattr( - self.pos_embd_params, "mrope_interleaved", False) and getattr( - self.pos_embd_params, "mrope_section", None) is not None - and position_ids.shape[0] == 3): - return False - expected_positions = qkv.shape[0] * (3 if use_mrope else 1) - if position_ids.numel() != expected_positions: - return False - if not (self.attn_output_gate and self.fuse_qk_norm_rope - and self.is_qk_norm and self.use_gemma_rms_norm - and not self.skip_rope): - return False - if self.pos_embd_params is None or self.pos_embd_params.rope is None: - return False - if not self.pos_embd_params.is_neox or self.rotary_emb is None: - return False - rope = self.pos_embd_params.rope - if rope.scale_type not in (RotaryScalingType.none, - RotaryScalingType.mrope): - return False - if rope.scale != 1.0: - return False - rotary_dim = self._get_qk_norm_rotary_dim() - return (rotary_dim == rope.dim and rotary_dim > 0 - and rotary_dim <= self.head_dim and rotary_dim % 2 == 0) - - def preprocess_qkv(self, qkv, position_ids): - """Fuse gate split + gemma QK norm + RoPE into a single Triton kernel - when supported; otherwise fall back to the generic unfused path.""" - if not self._can_use_fused_qk_norm_rope_gate(qkv, position_ids): - return super().preprocess_qkv(qkv, position_ids) - use_mrope = position_ids.dim() == 3 - positions = position_ids.reshape( - 3, -1) if use_mrope else position_ids.reshape(-1) - qkv, gate = fused_qkv_gemma_rmsnorm_rope_gate( - qkv, - self.q_norm.weight, - self.k_norm.weight, - self.rotary_emb.rotary_cos_sin, - positions.contiguous(), - self.q_norm.variance_epsilon, - self.num_heads, - self.num_key_value_heads, - self.head_dim, - self._get_qk_norm_rotary_dim(), - tuple(self.pos_embd_params.mrope_section) if use_mrope else None, - ) - return qkv, None, None, gate - - def apply_output_gate(self, attention_output, gate): - if (self._fuse_qk_norm_rope_gate and not is_torch_compiling() - and torch.version.hip is None - and attention_output.device.type == "cuda" - and attention_output.dim() == 2 - and attention_output.stride(-1) == 1 and gate.dim() in (2, 3) - and gate.stride(-1) == 1 - and gate.numel() == attention_output.numel()): - return fused_sigmoid_mul(attention_output, gate, inplace=True) - return super().apply_output_gate(attention_output, gate) - def apply_qk_norm(self, q, k): def q_l2norm(): @@ -324,7 +238,9 @@ def apply_qk_norm_rope(self, qkv, position_ids): factor, low, high, attention_factor = compute_yarn_parameters( self.pretrained_config) - rotary_dim = self._get_qk_norm_rotary_dim() + partial_rotary_factor = self.pretrained_config.partial_rotary_factor if hasattr( + self.pretrained_config, "partial_rotary_factor") else 1.0 + rotary_dim = int(self.head_dim * partial_rotary_factor) # Interleaved mRoPE: position_ids is 3D [3, ...] (temporal/height/width) # and each rotary half-dim picks a section per diff --git a/tensorrt_llm/_torch/modules/rms_norm.py b/tensorrt_llm/_torch/modules/rms_norm.py index 0e65940959a6..8575f006be7b 100644 --- a/tensorrt_llm/_torch/modules/rms_norm.py +++ b/tensorrt_llm/_torch/modules/rms_norm.py @@ -25,17 +25,6 @@ from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE from ..utils import Fp4QuantizedTensor, is_nvfp4_marlin_enabled -# Hidden-dim bounds of the warp-specialized fused_add_rms_norm_quant kernel -# (cpp/tensorrt_llm/thop/fusedAddRMSNormQuant.cpp). -_WS_MIN_N = 2048 -_WS_MAX_N = 16384 -# Row-count crossover for the layer-boundary add+RMSNorm+quant edge: below this -# the one-CTA-per-row reduce_fusion kernel (fused_add_rmsnorm_fp4_quantize) is -# faster; at/above it the warp-specialized kernel's DMA-ahead pipeline wins. -# Measured at N=7168 on GB200 (ws first overtakes at M=4096, ~6% faster; -# M<=3072 favors reduce_fusion). -_WS_M_THRESHOLD = 4096 - class RMSNorm(nn.Module): @@ -86,36 +75,24 @@ def __init__( self.use_gemma = use_gemma self.use_cuda_tile = use_cuda_tile - # The fused NVFP4 epilogue (cvt_warp_fp16_to_fp4 in quantization.cuh) is - # guarded by __CUDA_ARCH__ >= 1000, so it only produces correct FP4 on - # SM 10.x (Blackwell); SM 12.x lacks the warp-cooperative SF path. On - # unsupported SMs (incl. SM 9.x / Hopper, where the epilogue would - # silently emit zeros) fall back to flashinfer/generic RMSNorm and let + # fused_add_rms_norm_quant only supports SM 9.x / 10.x because: + # - Device code is guarded by is_major_v<9> || is_major_v<10> + # (ws_layernorm.cuh:828, low_latency_layernorm.cuh:157). + # On unsupported SMs, fall back to flashinfer/generic RMSNorm and let # the downstream linear layer handle FP4 quantization. if self.is_nvfp4: sm_version = get_sm_version() - # Marlin is also incompatible with the fused NVFP4 path. - if not (100 <= sm_version < 120) or is_nvfp4_marlin_enabled(): + if not (90 <= sm_version < 120) or is_nvfp4_marlin_enabled(): self.is_nvfp4 = False return_hp_output = False self.return_hp_output = return_hp_output - # Static NVFP4 input scale of the Linear this norm feeds, enabling the - # fused (add +) RMSNorm + NVFP4-quantize path in forward. It cannot be - # set at construction time because the consuming Linear's calibrated - # input_scale only exists after weight loading; the owning model - # attaches it afterwards (DeepseekV3 post_load_weights / - # MLA._resolve_qa_fused_scale). None keeps the fusion disabled. - self.nvfp4_scale: Optional[torch.Tensor] = None - def forward( self, hidden_states: torch.Tensor, residual: Union[ Optional[torch.Tensor], _ArgumentNotSpecifiedSentinelType] = _ARGUMENT_NOT_SPECIFIED_SENTINEL, - *, - return_norm_out: bool = False, ) -> Union[torch.Tensor, Fp4QuantizedTensor, Tuple[Union[ torch.Tensor, Fp4QuantizedTensor], Optional[torch.Tensor]], Tuple[ Fp4QuantizedTensor, torch.Tensor, torch.Tensor]]: @@ -123,22 +100,63 @@ def forward( if not has_residual: residual = None - # Fused (add +) RMSNorm + NVFP4 input-quantize: when this norm feeds an - # NVFP4 Linear whose static input_scale is attached as self.nvfp4_scale, - # fold this norm and that Linear's input-quant into one kernel. With a - # residual this is the layer-boundary add+RMSNorm+quant; without, the - # residual-less variant (e.g. q_a_layernorm -> q_b_proj). The actual - # kernel is chosen by a size gate inside _fused_nvfp4_quant. The BF16 - # post-RMSNorm value is also produced when return_norm_out (stashed on - # the Fp4QuantizedTensor for DSA consumers / returned for residual-less - # callers) or when self.return_hp_output (appended as an extra output - # for MoE-gate consumers). When is_nvfp4 but no scale is attached yet - # (e.g. a layer's first input_layernorm whose consumer has no static - # scale), fall through to the plain norm below. - nvfp4_scale = self.nvfp4_scale if self.is_nvfp4 else None - if nvfp4_scale is not None and not self.use_gemma: - return self._fused_nvfp4_quant(hidden_states, residual, nvfp4_scale, - return_norm_out) + if self.is_nvfp4 and has_residual and not self.use_gemma: + nvfp4_scale = getattr(self, "nvfp4_scale", None) + if nvfp4_scale is None: + raise ValueError( + f"layeridx={getattr(self, 'layer_idx', None)} RMSNorm NVFP4 output requested " + "but no `nvfp4_scale` is attached; ") + + orig_shape = tuple(hidden_states.shape) + n = int(orig_shape[-1]) + hs_2d = hidden_states.reshape(-1, n).contiguous() + res_2d = residual.reshape(-1, n) + gamma = self.weight + + def _ensure_contiguous_with_dtype(t: torch.Tensor, key: str): + if t.dtype != hs_2d.dtype: + raise ValueError( + f"RMSNorm NVFP4 fused path: casting {key} from {t.dtype} to {hs_2d.dtype}." + ) + return t.contiguous() + + res_2d = _ensure_contiguous_with_dtype(res_2d, "residual") + gamma = _ensure_contiguous_with_dtype(gamma, "gamma") + + if hs_2d.device != res_2d.device or hs_2d.device != gamma.device: + raise RuntimeError( + "RMSNorm NVFP4 fused path requires all tensors on the same device. " + f"Got input={hs_2d.device}, residual={res_2d.device}, gamma={gamma.device}." + ) + + sf_scale = nvfp4_scale.contiguous() + + results = torch.ops.trtllm.fused_add_rms_norm_quant( + hs_2d, + res_2d, + gamma, + sf_scale, + True, + eps=self.variance_epsilon, + output_hp_norm=self.return_hp_output, + ) + normed_fp4_i32, residual_out_2d, sf_fused = results[:3] + normed_fp4_u8 = normed_fp4_i32.view(torch.uint8) + if len(orig_shape) != 2: + normed_fp4_u8 = normed_fp4_u8.reshape(*orig_shape[:-1], n // 2) + residual_out = residual_out_2d.reshape(orig_shape) + else: + residual_out = residual_out_2d + + hidden_states_fused = Fp4QuantizedTensor(normed_fp4_u8, sf_fused) + + outputs = [hidden_states_fused] + if has_residual: + outputs.append(residual_out) + if self.return_hp_output: + high_precision_normed_output = results[3].reshape(orig_shape) + outputs.append(high_precision_normed_output) + return outputs[0] if len(outputs) == 1 else tuple(outputs) if self.return_hp_output: raise ValueError( @@ -212,147 +230,6 @@ def forward( else: return hidden_states - def _ws_kernel_eligible(self, hidden_states: torch.Tensor, - residual: Optional[torch.Tensor]) -> bool: - """Whether the warp-specialized fused_add_rms_norm_quant kernel should - serve this call instead of the one-CTA-per-row reduce_fusion kernel. - - ws only wins, and is only legal, for the contiguous rank-2 residual-add - edge with a large enough row count and an in-range hidden dim. The - residual-less and row-strided (column-slice) edges have no ws equivalent - and always use reduce_fusion. See _WS_M_THRESHOLD / _WS_MIN_N.""" - if residual is None: - return False - n = hidden_states.shape[-1] - if not (_WS_MIN_N <= n <= _WS_MAX_N and n % 16 == 0): - return False - m = 1 - for d in hidden_states.shape[:-1]: - m *= d - if m < _WS_M_THRESHOLD: - return False - # ws requires contiguous rank-2 input + residual (it cannot read a - # row-strided column slice and issues padded vectorized stores). - return hidden_states.is_contiguous() and residual.is_contiguous() - - def _fused_nvfp4_quant( - self, - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - sf_scale: torch.Tensor, - return_norm_out: bool, - ): - """Fold this (add +) RMSNorm + the consuming NVFP4 Linear's input-quant - into one kernel. ``sf_scale`` is that Linear's static input_scale. - - Picks the kernel by problem size (see _ws_kernel_eligible): the - warp-specialized ``fused_add_rms_norm_quant`` for the large contiguous - residual edge, else the one-CTA-per-row ``fused_add_rmsnorm_fp4_quantize`` - / ``fused_rmsnorm_fp4_quantize`` (which also serve the residual-less and - row-strided column-slice edges ws cannot). - - With a residual: returns ``(Fp4QuantizedTensor, residual_out)``. Without: - returns an ``Fp4QuantizedTensor`` (the residual-less q_a edge). When - ``return_norm_out`` the BF16 post-RMSNorm view is stashed on the - Fp4QuantizedTensor's ``unquantized_hidden_states`` (and, residual-less, returned - alongside). When ``self.return_hp_output`` the BF16 normed value is - appended as a trailing output for MoE-gate consumers.""" - # Either flag means "also produce the BF16 normed value". - want_norm = return_norm_out or self.return_hp_output - - if self._ws_kernel_eligible(hidden_states, residual): - return self._fused_nvfp4_quant_ws(hidden_states, residual, sf_scale, - return_norm_out) - - if residual is not None: - results = torch.ops.trtllm.fused_add_rmsnorm_fp4_quantize( - hidden_states, - residual, - self.weight, - sf_scale, - float(self.variance_epsilon), - want_norm, - ) - if want_norm: - bf16_hs, act_fp4, act_sf, residual_out = results - else: - act_fp4, act_sf, residual_out = results - bf16_hs = None - fp4 = Fp4QuantizedTensor(act_fp4, - act_sf, - unquantized_hidden_states=bf16_hs) - outputs = [fp4, residual_out] - if self.return_hp_output: - outputs.append(bf16_hs) - return tuple(outputs) - - orig_shape = tuple(hidden_states.shape) - n = orig_shape[-1] - hs_2d = hidden_states.reshape(-1, n) - results = torch.ops.trtllm.fused_rmsnorm_fp4_quantize( - hs_2d, - self.weight, - sf_scale, - float(self.variance_epsilon), - want_norm, - ) - if want_norm: - norm_out, act_fp4, act_sf = results - norm_out = norm_out.reshape(orig_shape) - else: - act_fp4, act_sf = results - norm_out = None - if len(orig_shape) != 2: - act_fp4 = act_fp4.reshape(*orig_shape[:-1], n // 2) - fp4 = Fp4QuantizedTensor(act_fp4, - act_sf, - unquantized_hidden_states=norm_out) - return (fp4, norm_out) if return_norm_out else fp4 - - def _fused_nvfp4_quant_ws( - self, - hidden_states: torch.Tensor, - residual: torch.Tensor, - sf_scale: torch.Tensor, - return_norm_out: bool, - ): - """Warp-specialized fused add+RMSNorm+NVFP4-quant - (torch.ops.trtllm.fused_add_rms_norm_quant) for the large contiguous - residual edge. Returns the same shape as the reduce_fusion residual - branch of _fused_nvfp4_quant.""" - want_norm = return_norm_out or self.return_hp_output - orig_shape = tuple(hidden_states.shape) - n = int(orig_shape[-1]) - hs_2d = hidden_states.reshape(-1, n).contiguous() - res_2d = residual.reshape(-1, n).contiguous() - gamma = self.weight.contiguous() - - results = torch.ops.trtllm.fused_add_rms_norm_quant( - hs_2d, - res_2d, - gamma, - sf_scale.contiguous(), - True, - eps=self.variance_epsilon, - output_hp_norm=want_norm, - ) - normed_fp4_i32, residual_out_2d, sf_fused = results[:3] - normed_fp4_u8 = normed_fp4_i32.view(torch.uint8) - if len(orig_shape) != 2: - normed_fp4_u8 = normed_fp4_u8.reshape(*orig_shape[:-1], n // 2) - residual_out = residual_out_2d.reshape(orig_shape) - else: - residual_out = residual_out_2d - - bf16_hs = results[3].reshape(orig_shape) if want_norm else None - fp4 = Fp4QuantizedTensor(normed_fp4_u8, - sf_fused, - unquantized_hidden_states=bf16_hs) - outputs = [fp4, residual_out] - if self.return_hp_output: - outputs.append(bf16_hs) - return tuple(outputs) - def skip_forward( self, hidden_states: torch.Tensor, diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 0149ba7ae03a..ffd024eadf3a 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -46,7 +46,6 @@ from ..models.modeling_multimodal_mixin import MultimodalModelMixin from ..speculative import (get_num_extra_kv_tokens, get_num_spec_layers, get_spec_decoder, should_use_separate_draft_kv_cache) -from ..utils import is_gdn_replay_enabled from .config_utils import (extract_mamba_kv_cache_params, is_gemma4_hybrid, is_hybrid_linear, is_mla, is_nemotron_hybrid, is_qwen3_hybrid) @@ -582,10 +581,6 @@ def _create_dummy_encoder_inputs(self) -> List[MultimodalParams]: self._profiling_stage_data, dict) and not self._profiling_stage_data.get("enable_mm_reqs"): return [] - # No local multimodal encoder (disable_mm_encoder or MM E/P disagg): - # nothing to profile. - if getattr(self._model_engine.model, "mm_encoder", object()) is None: - return [] input_processor = self._model_engine.input_processor _, encoder_max_num_tokens = self._llm_args.get_encoder_runtime_sizes() # Modality-agnostic: the model declares each modality's per-item token @@ -633,21 +628,6 @@ def _encode_dummy_inputs(self): return self._model_engine.model.encode_multimodal_inputs( self._dummy_encoder_inputs) - def _reserve_multimodal_encoder_cache_memory( - self, - peak_memory: int, - ) -> int: - """Reserve encoder-cache capacity when the model implements that cache.""" - model = self._model_engine.model - if (not isinstance(model, MultimodalModelMixin) - or not model.supports_encoder_cache): - return peak_memory - - multimodal_config = model.model_config.multimodal_config - if multimodal_config is None: - return peak_memory - return peak_memory + multimodal_config.encoder_cache_max_bytes - def _get_token_num_for_estimation(self) -> int: """Compute KV cache capacity required for estimate_max_kv_cache_tokens to succeed.""" if 'cp_type' in self._mapping.cp_config: @@ -896,14 +876,6 @@ def configure_kv_cache_capacity(self, allocated_bytes = 0 activation_bytes = 0 - peak_memory_without_encoder_cache = peak_memory - peak_memory = self._reserve_multimodal_encoder_cache_memory(peak_memory) - if peak_memory != peak_memory_without_encoder_cache: - mem_gb = (peak_memory - peak_memory_without_encoder_cache) / GB - logger.info( - f"Reserving {mem_gb:.2f} GiB for the multimodal encoder cache while estimating KV cache " - "capacity.", ) - # calculate max memory from peak memory and free gpu memory fraction kv_cache_max_memory = self._cal_max_memory(peak_memory, total_gpu_memory, fraction, @@ -1988,47 +1960,6 @@ def _create_kv_cache_manager( spec_config=spec_config, quant_config=quant_config, ) - - # Replay state update for GDN MTP: mirrors the nemotron_hybrid gating - # above, minus the Mamba2-specific stochastic-rounding/Philox gate. - # The GDN replay kernel does a plain cast on checkpoint commit, so - # quantized SSM cache dtypes stay on the legacy path. - sm = get_sm_version() - use_replay = spec_config is not None and sm >= 80 - if spec_config is None: - logger.info( - "GDN replay kernel requires speculative decoding; using " - "non-replay path") - elif spec_config.tokens_per_gen_step > 8: - logger.info("GDN cached replay supports at most 8 tokens per " - "generation step; using non-replay path") - use_replay = False - - # Tree attention: replay assumes a linear token sequence. - if (spec_config is not None - and (getattr(spec_config, 'eagle_choices', None) is not None - or getattr(spec_config, 'use_dynamic_tree', False))): - logger.info("GDN replay kernel incompatible with tree attention; " - "using legacy MTP path") - use_replay = False - - if mamba_params.mamba_ssm_cache_dtype not in (torch.float32, - torch.bfloat16, - torch.float16): - logger.info( - "GDN replay kernel does not support quantized SSM cache " - f"dtype {mamba_params.mamba_ssm_cache_dtype}; using legacy " - "MTP path") - use_replay = False - - # Replay is opt-in because its end-to-end benefit is workload-dependent. - if not is_gdn_replay_enabled(): - logger.info("GDN replay kernel is disabled; set " - "TRTLLM_USE_GDN_REPLAY=1 to enable it") - use_replay = False - logger.info("GDN replay state update: " + - ("ENABLED" if use_replay else "DISABLED")) - kv_cache_manager = kv_cache_manager_cls( # mamba cache parameters mamba_params.state_size, @@ -2057,7 +1988,6 @@ def _create_kv_cache_manager( is_estimating_kv_cache=estimating_kv_cache, execution_stream=execution_stream, model_type="qwen3_next", - use_replay_state_update=use_replay, **manager_extra_kwargs, ) else: @@ -2110,38 +2040,16 @@ def _create_kv_cache_manager( return kv_cache_manager -def validate_kv_cache_compression_with_spec( - config: KvCacheCompressionConfig, - spec_config: Optional[SpeculativeConfig], - draft_kv_cache_manager: Optional[KVCacheManagerV2], -) -> None: - """Reject speculative setups the compression method cannot run with.""" - if (spec_config is None - or not config.kv_cache_compression_mode.is_eviction_method()): - return - # Evicting methods co-compact the draft KV, so the draft must be a - # standard paged cache in the same forward (one-model speculation). - mode = spec_config.spec_dec_mode - if not (mode.is_mtp_one_model() or mode.is_eagle3_one_model()): - raise ValueError( - f"KV-cache compression algorithm {config.algorithm!r} does not " - f"support speculative decoding mode {mode.name}: the draft KV " - "must be a standard paged cache compacted together with the " - "target (one-model MTP/EAGLE3).") - - def create_kv_cache_compression_manager( config: KvCacheCompressionConfig, kv_cache_manager: KVCacheManagerV2, - draft_kv_cache_manager: Optional[KVCacheManagerV2] = None, ) -> Optional[BaseKVCacheCompressionManager]: """Build the KV-cache compression manager for ``config.algorithm``, or return None if no algorithm matches. Called from ``create_py_executor`` and registered as a resource manager, like the KV cache manager itself. Concrete algorithms add a dispatch branch - here; the framework ships none. Speculative-decoding compatibility is - checked by the caller via ``validate_kv_cache_compression_with_spec``. + here; the framework ships none. """ logger.warning( "KV-cache compression algorithm '%s' is not registered; running without " @@ -2395,16 +2303,8 @@ def create_py_executor_instance( kv_cache_compression_config = getattr(llm_args, "kv_cache_compression_config", None) if kv_cache_compression_config is not None: - draft_kv_cache_manager = resources.get( - ResourceManagerType.DRAFT_KV_CACHE_MANAGER) - validate_kv_cache_compression_with_spec(kv_cache_compression_config, - spec_config, - draft_kv_cache_manager) compression_manager = create_kv_cache_compression_manager( - kv_cache_compression_config, - kv_cache_manager, - draft_kv_cache_manager=draft_kv_cache_manager, - ) + kv_cache_compression_config, kv_cache_manager) if compression_manager is not None: resources[ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER] = ( compression_manager) @@ -2416,16 +2316,17 @@ def create_py_executor_instance( if kv_cache_manager is not None: resource_manager.resource_managers.move_to_end( ResourceManagerType.KV_CACHE_MANAGER, last=True) + # Compression manager runs after the cache manager: reconciles history once it's resized. + if (ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER + in resource_manager.resource_managers): + resource_manager.resource_managers.move_to_end( + ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER, last=True) + cross_kv_cache_manager = resources.get( ResourceManagerType.CROSS_KV_CACHE_MANAGER) if cross_kv_cache_manager is not None: resource_manager.resource_managers.move_to_end( ResourceManagerType.CROSS_KV_CACHE_MANAGER, last=True) - # Compression is the final reconciler after every native KV manager. - if (ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER - in resource_manager.resource_managers): - resource_manager.resource_managers.move_to_end( - ResourceManagerType.KV_CACHE_COMPRESSION_MANAGER, last=True) # When scheduler_capacity == 1, attention dp dummy request will prevent the scheduling of DISAGG_GENERATION_INIT. # Enlarge scheduler capacity to avoid DISAGG_GENERATION_INIT stuck in the scheduler. diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index 601ab716dfcb..5f1e0cbc0a26 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -96,11 +96,6 @@ def is_mla(config): return False -def is_minimax_m3(sparse_attention_config): - """True when the sparse attention config selects the MiniMax-M3 algorithm.""" - return sparse_attention_config is not None and sparse_attention_config.algorithm == "minimax_m3" - - def is_qwen3_next(config): return hasattr( config, 'architectures' diff --git a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py index b8b67ed5ded8..e0bd91ac52a8 100644 --- a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py +++ b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py @@ -39,27 +39,6 @@ KeyType: TypeAlias = Tuple[int, int, bool, bool, bool] -def _save_spec_decode_capture_state( - attn_metadata: Any, enable_spec_decode: bool) -> Optional[torch.Tensor]: - if not enable_spec_decode or not hasattr(attn_metadata, 'kv_lens_cuda'): - return None - return attn_metadata.kv_lens_cuda[:attn_metadata.num_seqs].clone() - - -def _restore_spec_decode_capture_state( - attn_metadata: Any, saved_kv_lens_cuda: Optional[torch.Tensor]) -> None: - if saved_kv_lens_cuda is None: - return - # Speculative decoding updates kv_lens_cuda in-place during every forward. - # CUDA graph warmup reuses one dummy request for multiple eager forwards, so - # letting those updates accumulate would make later warmups/capture advertise - # more KV tokens than the dummy request actually allocated. Restore the - # single-step input state outside the graph after each forward instead. - batch_size = saved_kv_lens_cuda.shape[0] - attn_metadata.kv_lens_cuda[:batch_size].copy_(saved_kv_lens_cuda) - attn_metadata.on_update_kv_lens() - - @dataclass class CUDAGraphRunnerConfig: """Configuration for the CUDAGraphRunner, passed from the ModelEngine.""" @@ -121,7 +100,7 @@ class CUDAGraphRunner: and low-level execution (capturing, resource management, replaying) for multiple graphs, keyed by (batch size, draft_len, is_first_draft). """ - WARMUP_STEPS = 1 + WARMUP_STEPS = 2 def __init__(self, config: CUDAGraphRunnerConfig): self.config = config @@ -153,7 +132,6 @@ def __init__(self, config: CUDAGraphRunnerConfig): # tensor reallocation from invalidating addresses baked into existing # CUDA graphs. Use allow_capture() context manager during warmup. self._capture_allowed = False - self.is_warmup_only = False def _create_shared_static_tensors(self): """Allocates static tensors sized for the largest possible batch.""" @@ -343,7 +321,7 @@ def maybe_get_cuda_graph( key = self.get_graph_key(batch, new_tensors_device, spec_resource_manager, spec_metadata) - if key in self.graph_metadata: + if key in self.graphs: return self.graph_metadata[key][ "attn_metadata"], self.graph_metadata[key]["spec_metadata"], key @@ -400,8 +378,8 @@ def capture(self, forward_fn: Callable, initial_inputs: Dict[str, Any], enable_spec_decode: bool = False, - postprocess_fn: Optional[Callable] = None) -> Any: - """Warm up and/or capture the forward pass for a graph key.""" + postprocess_fn: Optional[Callable] = None): + """Captures the forward pass for a given batch size.""" batch_size = key[0] # [CUDA graph spec decode padding] # We pad input IDs/position IDs to the maximum draft length (token per request). @@ -429,12 +407,9 @@ def capture(self, capture_inputs = initial_inputs.copy() capture_inputs.update(sliced_static_tensors) - attn_metadata = capture_inputs["attn_metadata"] - saved_kv_lens_cuda = _save_spec_decode_capture_state( - attn_metadata, enable_spec_decode) self.graph_metadata[key] = { - "attn_metadata": attn_metadata, + "attn_metadata": initial_inputs["attn_metadata"], "spec_metadata": initial_inputs.get("spec_metadata", None), } @@ -447,38 +422,27 @@ def _setup_spec_decoding_and_forward(key: KeyType, forward_fn: Callable, capture_inputs['attn_metadata'].use_spec_decoding = True return forward_fn(capture_inputs) - output = None + # We have to do warm up runs to initialize PyTorch's + # internal states according to the docs: + # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics + # This also lets us initialize states in the attn_metadata. + graph = torch.cuda.CUDAGraph() with with_multi_stream(True), piecewise_cuda_graph(False): - # We have to do a warmup run to initialize PyTorch's internal - # states according to the docs: - # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics - # This also lets us initialize states in the attn_metadata and - # resize the shared attention workspace before any graph is captured. for _ in range(self.WARMUP_STEPS): - output = _setup_spec_decoding_and_forward( - key, forward_fn, capture_inputs) + _setup_spec_decoding_and_forward(key, forward_fn, + capture_inputs) if postprocess_fn is not None: postprocess_fn(capture_inputs) - _restore_spec_decode_capture_state(attn_metadata, - saved_kv_lens_cuda) - - if self.is_warmup_only: - return output - graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph, pool=self.memory_pool): output = _setup_spec_decoding_and_forward( key, forward_fn, capture_inputs) if postprocess_fn is not None: postprocess_fn(capture_inputs) - _restore_spec_decode_capture_state(attn_metadata, - saved_kv_lens_cuda) self.graphs[key] = graph - graph_output = make_weak_ref(output) - self.graph_outputs[key] = graph_output + self.graph_outputs[key] = make_weak_ref(output) self.memory_pool = graph.pool() - return graph_output def replay(self, key: KeyType, current_inputs: Dict[str, Any]) -> Optional[torch.Tensor]: @@ -740,7 +704,7 @@ class EncoderCUDAGraphRunner: Restricted to `TrtllmAttentionMetadata` — FlashInfer's per-batch planner state is not compatible with CUDA graph capture/replay. """ - WARMUP_STEPS = 1 + WARMUP_STEPS = 2 def __init__(self, config: EncoderCUDAGraphRunnerConfig): self.config = config @@ -766,7 +730,6 @@ def __init__(self, config: EncoderCUDAGraphRunnerConfig): self.cuda_graph_meta_buffers = get_memory_buffers() self._capture_allowed = False - self.is_warmup_only = False # CUDA graph H2D memcpy nodes require pinned host sources. In CC mode # prefer_pinned() is false: pageable host buffers are preferred, so the @@ -953,7 +916,7 @@ def maybe_get_cuda_graph( or not is_padding_successful: return None, None - if key in self.graph_metadata: + if key in self.graphs: return self.graph_metadata[key]["attn_metadata"], key # New key not yet captured. Only create metadata if capture is @@ -1017,8 +980,8 @@ def capture( key: EncoderKeyType, forward_fn: Callable[[Dict[str, Any]], Any], inputs: Dict[str, Any], - ) -> Any: - """Warm up and/or capture the forward pass for a graph key.""" + ) -> None: + """Capture a CUDA graph for the given key.""" _, padded_num_tokens, _ = key sliced_static_tensors = { @@ -1040,21 +1003,17 @@ def capture( attn_md = capture_inputs["attn_metadata"] - self.graph_metadata[key] = {"attn_metadata": attn_md} + self.graph_metadata[key] = { + "attn_metadata": attn_md, + } - output = None + graph = torch.cuda.CUDAGraph() + # Warmup runs required by CUDA graph semantics. See + # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics with with_multi_stream(True), piecewise_cuda_graph(False): - # Warmup runs required by CUDA graph semantics. See - # https://pytorch.org/docs/stable/notes/cuda.html#cuda-graph-semantics - # Warmups initialize PyTorch and attention metadata state, and - # resize the shared attention workspace before any graph is captured. for _ in range(self.WARMUP_STEPS): - output = forward_fn(capture_inputs) - - if self.is_warmup_only: - return output + forward_fn(capture_inputs) - graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph, pool=self.memory_pool): if self._capture_h2d_copy: # H2D copies for captured inside the graph: at replay @@ -1075,10 +1034,8 @@ def capture( "Encoder CUDA graph does not support nested tensor outputs. " "Disable encoder CUDA graphs for models with ragged outputs.") self.graphs[key] = graph - graph_output = make_weak_ref(output) - self.graph_outputs[key] = graph_output + self.graph_outputs[key] = make_weak_ref(output) self.memory_pool = graph.pool() - return graph_output def replay( self, diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py index 24808e8b55b5..5f91d0cde5a7 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache_manager_v2.py @@ -15,16 +15,14 @@ import hashlib import math import os -import sys from collections import OrderedDict, defaultdict -from dataclasses import dataclass, replace +from dataclasses import fields from typing import TYPE_CHECKING, Dict, Iterable, List, NamedTuple, Optional, Sequence, Tuple, Union import numpy as np import torch from strenum import StrEnum -from tensorrt_llm._torch.disaggregation.resource.page import MapperKind from tensorrt_llm._torch.distributed.communicator import Distributed, ReduceOp from tensorrt_llm._utils import ( TensorWrapper, @@ -40,41 +38,40 @@ from tensorrt_llm.llmapi.llm_args import KvCacheConfig from tensorrt_llm.runtime.kv_cache_hash import get_effective_kv_cache_event_hash_algo from tensorrt_llm.runtime.kv_cache_manager_v2 import ( - _KV_CACHE_ITERATION_STATS_DELTA_FIELDS, - BAD_PAGE_INDEX, - CACHE_LEVEL1, DEFAULT_BEAM_INDEX, - GPU_LEVEL, AttentionLayerConfig, - AttnLifeCycle, - BatchDesc, BufferConfig, - CacheLevel, CacheTierConfig, - CuError, - DataRole, DiskCacheTierConfig, GpuCacheTierConfig, HostCacheTierConfig, - KVCacheDesc, - KVCacheEventManager, KVCacheIterationStatsDelta, LayerId, - LifeCycleId, PageIndexMode, - PlannedDropHandle, - PoolGroupPeakBlockStats, ReuseScope, SwaScratchReuseConfig, TokenIdExt, _KVCache, - exact_div, - gen_multimodal_cache_key_tokens, - typed_range, ) from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheManager as KVCacheManagerPy from tensorrt_llm.runtime.kv_cache_manager_v2 import KVCacheManagerConfig as KVCacheManagerConfigPy -from tensorrt_llm.runtime.kv_cache_manager_v2 import OutOfMemoryError as KVCacheOutOfMemoryError +from tensorrt_llm.runtime.kv_cache_manager_v2._block_radix_tree import ( + gen_multimodal_cache_key_tokens, +) +from tensorrt_llm.runtime.kv_cache_manager_v2._common import ( + BAD_PAGE_INDEX, + CACHE_LEVEL1, + GPU_LEVEL, + CacheLevel, +) +from tensorrt_llm.runtime.kv_cache_manager_v2._config import DataRole +from tensorrt_llm.runtime.kv_cache_manager_v2._event_manager import KVCacheEventManager +from tensorrt_llm.runtime.kv_cache_manager_v2._exceptions import CuError +from tensorrt_llm.runtime.kv_cache_manager_v2._exceptions import ( + OutOfMemoryError as KVCacheOutOfMemoryError, +) +from tensorrt_llm.runtime.kv_cache_manager_v2._life_cycle_registry import AttnLifeCycle, LifeCycleId +from tensorrt_llm.runtime.kv_cache_manager_v2._utils import exact_div, typed_range from tensorrt_llm.sampling_params import SamplingParams from ..._utils import binding_to_torch_dtype, mpi_rank, nvtx_range, str_dtype_to_torch @@ -104,7 +101,9 @@ if TYPE_CHECKING: from tensorrt_llm._torch.attention_backend.interface import AttentionMetadata -KV_CACHE_ITERATION_STATS_DELTA_FIELDS = _KV_CACHE_ITERATION_STATS_DELTA_FIELDS +KV_CACHE_ITERATION_STATS_DELTA_FIELDS = tuple( + field.name for field in fields(KVCacheIterationStatsDelta) +) KV_CACHE_ITERATION_STATS_REUSE_FIELDS = ( "iter_reused_blocks", "iter_full_reused_blocks", @@ -126,8 +125,8 @@ class Role: # Sparse-attention per-layer index-K cache (MiniMax-M3 and similar # sparse-block-selection backends). Registered as a native V2 # BufferConfig on sparse layers via the extra_buffers_per_layer hook on - # _build_base_config, so allocation, free, slot reuse, and prefix reuse - # share the lifecycle of the main K/V buffers for the same layer. + # _build_cache_config, so allocation, free, slot reuse, and prefix + # reuse share the lifecycle of the main K/V buffers for the same layer. INDEX_KEY = DataRole("index_key") ALL = DataRole("all") @@ -135,90 +134,6 @@ class Role: class BlockReusePolicy(StrEnum): ALL_REUSABLE = "all_reusable" PER_REQUEST = "per_request" - PER_CONVERSATION = "per_conversation" - - -def _request_conversation_id(request: LlmRequest) -> Optional[str]: - if request.is_dummy_request: - return None - conversation_params = request.py_conversation_params - if conversation_params is None: - return None - conversation_id = conversation_params.conversation_id.strip() - return conversation_id or None - - -@dataclass(slots=True) -class _ConversationState: - current_request_id: Optional[int] = None - planned_drop_handle: Optional[PlannedDropHandle] = None - - -class ConversationManager: - """Track the current request and drop plan for each conversation.""" - - def __init__(self) -> None: - self._conversation_states: Dict[str, _ConversationState] = {} - - def save_drop_plan(self, request: LlmRequest, kv_cache: _KVCache) -> None: - """Save a completed context's drop plan and apply the preceding plan on success.""" - request_id = request.py_request_id - conversation_id = _request_conversation_id(request) - if conversation_id is None: - return - - state = self._conversation_states[conversation_id] - if state.current_request_id != request_id: - return - - drop_handle = kv_cache.plan_committed_block_drop() - if drop_handle is None: - logger.warning( - f"Committed blocks for request {request_id} in conversation " - f"{conversation_id} have been dropped." - ) - else: - previous_handle = state.planned_drop_handle - state.planned_drop_handle = drop_handle - if previous_handle is not None: - previous_handle.drop() - - self.finish_request(request) - - def prepare_request(self, request: LlmRequest) -> None: - """Register a context request unless its conversation has another active one.""" - conversation_id = _request_conversation_id(request) - if conversation_id is None: - return - request_id = request.py_request_id - state = self._conversation_states.setdefault(conversation_id, _ConversationState()) - current_request_id = state.current_request_id - if current_request_id is not None and current_request_id != request_id: - logger.warning( - f"Conversation {conversation_id} already has current request " - f"{current_request_id}. Request {request_id} will ignore " - "conversation params." - ) - return - - state.current_request_id = request_id - - def finish_request(self, request: LlmRequest) -> None: - """Clear a request as active while preserving any saved drop plan.""" - conversation_id = _request_conversation_id(request) - if conversation_id is None: - return - state = self._conversation_states.get(conversation_id) - if state is None or state.current_request_id != request.py_request_id: - return - - state.current_request_id = None - if state.planned_drop_handle is None: - self._conversation_states.pop(conversation_id) - - def clear(self) -> None: - """Clear state after reusable KV-cache blocks have been cleared.""" - self._conversation_states.clear() def _estimate_full_attn_size_per_token( @@ -231,46 +146,6 @@ def _estimate_full_attn_size_per_token( ) -def _compute_auto_host_tier_quota( - quota: int, - local_ranks: int, - mem_available: float, - memlock_limit: float, -) -> int: - """Compute the auto-provisioned host cache tier quota for a single rank. - - The host tier backs the MAX_UTILIZATION scheduler's suspend/resume path, - so it must be positive. It defaults to the device quota, capped by the - per-node available-memory budget shared with co-located ranks and by the - pinnable-memory (RLIMIT_MEMLOCK) limit. - - Args: - quota: Device (GPU) KV cache quota in bytes; must be positive. - local_ranks: Number of ranks co-located on this physical node. - mem_available: Available host memory in bytes, or ``float("inf")`` - if unknown. - memlock_limit: RLIMIT_MEMLOCK soft limit in bytes, or - ``float("inf")`` if unlimited or unknown. - - Returns: - Host cache tier quota in bytes (always positive). - """ - candidates = [quota] - if mem_available != float("inf"): - candidates.append(int(mem_available / local_ranks * 0.5)) - if memlock_limit != float("inf"): - candidates.append(int(memlock_limit * 0.8)) - host_quota = min(candidates) - if host_quota <= 0: - logger.warning( - f"KV cache manager v2 auto host tier sizing computed a " - f"non-positive quota ({host_quota}); falling back to the " - f"device quota {quota / (1 << 30):.2f}GiB" - ) - host_quota = quota - return host_quota - - def _estimate_swa_cache_size( layer_sizes: Sequence[int], attention_windows: Sequence[Optional[int]], @@ -786,8 +661,6 @@ def __init__( layer_mask=layer_mask, ) self.is_draft = is_draft - # Set True by a compression manager; generation-step resize then leaves history untouched. - self.kv_compression_manages_history: bool = False self.enable_swa_scratch_reuse = ( kv_cache_config.enable_swa_scratch_reuse and not self.is_draft ) @@ -928,8 +801,7 @@ def append_to_kv_heads_per_layer( self.is_vswa = len(set(self.max_attention_window_vec)) > 1 - max_util_for_resume = kv_cache_config.max_util_for_resume - quota = sys.maxsize + quota = float("inf") if ( kv_cache_config.max_gpu_total_bytes is not None and kv_cache_config.max_gpu_total_bytes > 0 @@ -940,7 +812,7 @@ def append_to_kv_heads_per_layer( quota_from_max_tokens = int( math.ceil( self._get_quota_from_max_tokens(kv_cache_config.max_tokens) - / max_util_for_resume + / kv_cache_config.max_util_for_resume ) ) quota = min(quota, quota_from_max_tokens) @@ -950,17 +822,17 @@ def append_to_kv_heads_per_layer( f"New quota is {quota / (1 << 30)}GiB" ) - assert quota < sys.maxsize, ( + assert quota != float("inf"), ( "Quota not set. Check kv_cache_config.max_tokens or kv_cache_config.max_gpu_total_bytes" ) - # Sync resumable token capacity across ranks so the scheduler produces - # identical batches. Normalize to tokens because cache costs vary - # across PP ranks, including fixed per-rank costs. + # Sync KV cache token capacity across ranks so all ranks allocate + # the same number of tokens and the scheduler produces identical + # batches. Normalize to token count before the allreduce because + # bytes_per_token varies across PP ranks (different local layers). if mapping.world_size > 1: dist = Distributed.get(mapping) - resumable_quota = int(quota * max_util_for_resume) - max_tokens = self._get_max_tokens_from_quota(resumable_quota) + max_tokens = self._get_max_tokens_from_quota(quota) max_tokens = dist.allreduce(max_tokens, op=ReduceOp.MIN) # inf max_tokens means all layers are SWA and every rank quota can # fit all SWA fixed cache. @@ -969,14 +841,11 @@ def append_to_kv_heads_per_layer( # token↔quota round-trip is not identity when SWA layers # dominate (full_attn_size_per_token==0), so clamp to guard # against a bogus inflation (nvbugs/6418103). - synced_quota = int( - math.ceil(self._get_quota_from_max_tokens(max_tokens) / max_util_for_resume) - ) - quota = min(quota, synced_quota) + quota = min(quota, self._get_quota_from_max_tokens(max_tokens)) logger.info(f"KV cache manager v2 device quota set to {quota / (1 << 30)}GiB") - cache_tiers: List[CacheTierConfig] = [GpuCacheTierConfig(quota=int(quota))] + cache_tiers: List[CacheTierConfig] = [GpuCacheTierConfig(quota=quota)] if kv_cache_config.host_cache_size is not None and kv_cache_config.host_cache_size >= 0: host_quota = kv_cache_config.host_cache_size else: @@ -991,14 +860,6 @@ def append_to_kv_heads_per_layer( # memory and pinnable memory limit to avoid allocation failures. import resource - # Rank-aware host tier sizing: divide the per-node memory budget - # by the number of ranks sharing this physical node. Each rank - # independently provisions a host tier; without this, N - # co-located ranks each reserving a device-quota-sized block can - # OOM the host (observed on GB300 NVL72 with 4 ranks/node and - # ~170GiB device quota each on a 975GiB node). - local_ranks = max(1, Distributed.get(mapping).local_world_size) - try: mem_available = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_AVPHYS_PAGES") except (ValueError, OSError): @@ -1008,16 +869,16 @@ def append_to_kv_heads_per_layer( memlock_limit = _soft if _soft != resource.RLIM_INFINITY else float("inf") except (ValueError, OSError): memlock_limit = float("inf") - host_quota = _compute_auto_host_tier_quota( - quota, local_ranks, mem_available, memlock_limit - ) - logger.info( - f"KV cache manager v2 auto host tier sizing: " - f"{local_ranks} co-located rank(s) on this node, " - f"available host memory {mem_available / (1 << 30):.2f}GiB" - ) + candidates = [quota] + if mem_available != float("inf"): + candidates.append(int(mem_available * 0.5)) + if memlock_limit != float("inf"): + candidates.append(int(memlock_limit * 0.8)) + host_quota = min(candidates) + if host_quota <= 0: + host_quota = quota if host_quota > 0: - cache_tiers.append(HostCacheTierConfig(quota=int(host_quota))) + cache_tiers.append(HostCacheTierConfig(quota=host_quota)) logger.info( f"KV cache manager v2 host cache quota set to {host_quota / (1 << 30):.2f}GiB" ) @@ -1025,21 +886,19 @@ def append_to_kv_heads_per_layer( if disk_cache_size is not None and disk_cache_size > 0: disk_cache_path = kv_cache_config.disk_cache_path assert disk_cache_path is not None - cache_tiers.append( - DiskCacheTierConfig(quota=int(disk_cache_size), path=disk_cache_path) - ) + cache_tiers.append(DiskCacheTierConfig(quota=disk_cache_size, path=disk_cache_path)) logger.info( f"KV cache manager v2 disk cache quota set to {disk_cache_size / (1 << 30):.2f}GiB at {disk_cache_path}" ) self.vocab_size = vocab_size - config = self._build_base_config( + config = self._build_cache_config( kv_cache_config, tokens_per_block=tokens_per_block, + vocab_size=vocab_size, cache_tiers=cache_tiers, ) - config = self._build_cache_config(config) self.kv_cache_manager_py_config = config @@ -1053,7 +912,12 @@ def append_to_kv_heads_per_layer( "Retrying without host cache tier." ) cache_tiers_gpu_only = [t for t in cache_tiers if isinstance(t, GpuCacheTierConfig)] - config = replace(config, cache_tiers=cache_tiers_gpu_only) + config = self._build_cache_config( + kv_cache_config, + tokens_per_block=tokens_per_block, + vocab_size=vocab_size, + cache_tiers=cache_tiers_gpu_only, + ) cache_tiers = cache_tiers_gpu_only self.kv_cache_manager_py_config = config self.impl = KVCacheManagerPy(config, event_manager=self.event_manager) @@ -1128,12 +992,6 @@ def append_to_kv_heads_per_layer( self.enable_block_reuse = kv_cache_config.enable_block_reuse self.enable_partial_reuse = kv_cache_config.enable_partial_reuse self.disk_prefetch_num_reqs = kv_cache_config.disk_prefetch_num_reqs - enable_conversation_manager = ( - self.enable_block_reuse - and self.block_reuse_policy == BlockReusePolicy.PER_CONVERSATION - and not self.is_draft - ) - self.conversation_manager = ConversationManager() if enable_conversation_manager else None # With pipeline parallelism, multiple microbatches can be in-flight # simultaneously, so we need slots for all concurrent sequences. @@ -1203,11 +1061,23 @@ def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: for layer_id in typed_range(LayerId(self.num_local_layers)): layer_group_id = self.impl.get_layer_group_id(layer_id) - key_base_addr = kv_cache_pool_pointers_list[layer_group_id][0] - offset = self._kv_pool_mapping_offset(layer_id, layer_group_id, key_base_addr) - - if self.dtype == DataType.NVFP4: + if self.dtype != DataType.NVFP4: + key_base_addr = kv_cache_pool_pointers_list[layer_group_id][0] + addr_offset = ( + self.impl.get_mem_pool_base_address( + layer_id, Role.KEY, PageIndexMode.SHARED + ) + - key_base_addr + ) + else: + key_base_addr = kv_cache_pool_pointers_list[layer_group_id][0] block_scale_base_addr = block_scale_pool_pointers_list[layer_group_id][0] + addr_offset = ( + self.impl.get_mem_pool_base_address( + layer_id, Role.KEY, PageIndexMode.SHARED + ) + - key_base_addr + ) block_scale_addr_offset = ( self.impl.get_mem_pool_base_address( layer_id, Role.KEY_BLOCK_SCALE, PageIndexMode.SHARED @@ -1220,6 +1090,14 @@ def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: * self.kv_factor * self.tokens_per_block, ) + offset = exact_div( + addr_offset, + self.get_layer_bytes_per_token(layer_id, Role.KEY) + * self.kv_factor + * self.tokens_per_block, + ) + + if self.dtype == DataType.NVFP4: assert block_scale_offset == offset, ( "Block scale offset and offset should be the same" ) @@ -1280,29 +1158,6 @@ def _prepare_page_table_tensor(self, index_mapper_capacity: int) -> None: if self.enable_swa_scratch_reuse: self._prepare_swa_scratch_copy_tensors(index_mapper_capacity) - def _kv_pool_mapping_offset( - self, layer_id: LayerId, layer_group_id: int, key_base_addr: int - ) -> int: - """Per-layer offset recorded in ``kv_cache_pool_mapping``. - - The default derives the layer's position within its pool from the K - base address, assuming every layer contributes exactly K(+V) to the - pool slot so the layer stride is uniform. Managers whose pool slots - may interleave extra per-layer buffers between layers (non-uniform - layer strides — e.g. MiniMax M3 when the index-K buffer coalesces - into the K/V pool) must override this with a positional formula. - """ - addr_offset = ( - self.impl.get_mem_pool_base_address(layer_id, Role.KEY, PageIndexMode.SHARED) - - key_base_addr - ) - return exact_div( - addr_offset, - self.get_layer_bytes_per_token(layer_id, Role.KEY) - * self.kv_factor - * self.tokens_per_block, - ) - def _get_runtime_cache_size_layer_components(self) -> tuple[List[int], List[Optional[int]]]: layer_sizes = [] attention_windows = [] @@ -1636,81 +1491,14 @@ def _copy_batch_block_offsets_per_layer( non_blocking=True, ) - def _build_base_config( + def _build_cache_config( self, kv_cache_config: KvCacheConfig, *, tokens_per_block: int, + vocab_size: Optional[int], cache_tiers: List[CacheTierConfig], ) -> KVCacheManagerConfigPy: - """Build the general cache configuration used by most models. - - Models that need a custom configuration should subclass - :class:`KVCacheManagerV2` and override :meth:`_build_cache_config` to - update the necessary fields. - """ - scratch_reuse_config = None - if self.enable_swa_scratch_reuse: - # Context requests allocate num_extra_kv_tokens for spec decoding. - # They should not count toward the scratch range. - scratch_reuse_config = SwaScratchReuseConfig(max_rewind_len=self.num_extra_kv_tokens) - - typical_step = None - constraints = [] - if kv_cache_config.pool_ratio is None: - typical_seq_len = ( - kv_cache_config.avg_seq_len - if kv_cache_config.avg_seq_len is not None - else self.max_seq_len - ) - if typical_seq_len > self.max_seq_len: - raise ValueError( - f"kv_cache_config.avg_seq_len ({typical_seq_len}) must be less than or " - f"equal to max_seq_len ({self.max_seq_len})" - ) - - # Model one context request and enough generation requests to fill - # max_batch_size without over-provisioning windowed cache pools. - context_capacity = ( - self.max_num_tokens if self.max_num_tokens is not None else typical_seq_len - ) + self.num_extra_kv_tokens - generation_history_length = max(0, typical_seq_len - self.max_draft_len - 1) - typical_step = BatchDesc( - [KVCacheDesc(capacity=context_capacity, history_length=0)] - + [ - KVCacheDesc( - capacity=typical_seq_len, - history_length=generation_history_length, - ) - ] - * (self.max_batch_size - 1) - ) - - # CUDA graph generation warmup uses one request at max_seq_len and - # enough minimal decode requests to fill max_batch_size. - min_decode_capacity = 1 + self.max_draft_len + self.num_extra_kv_tokens - constraints.append( - BatchDesc( - [KVCacheDesc(capacity=self.max_seq_len, history_length=self.max_seq_len - 1)] - + [KVCacheDesc(capacity=min_decode_capacity, history_length=0)] - * (self.max_batch_size - 1) - ) - ) - - # General and chunked-prefill warmup uses one fresh context request - # at the per-iteration token budget. - if self.max_num_tokens is not None: - constraints.append( - BatchDesc( - [ - KVCacheDesc( - capacity=self.max_num_tokens + self.num_extra_kv_tokens, - history_length=0, - ) - ] - ) - ) - buffer_type = [Role.KEY] if self.kv_cache_type != CacheTypeCpp.SELFKONLY: buffer_type.append(Role.VALUE) @@ -1723,6 +1511,12 @@ def _build_base_config( if self.kv_cache_type != CacheTypeCpp.SELFKONLY: buffer_type.append(Role.VALUE_BLOCK_SCALE) + scratch_reuse_config = None + if self.enable_swa_scratch_reuse: + # Context requests allocate num_extra_kv_tokens for spec decoding. + # They should not count toward the scratch range. + scratch_reuse_config = SwaScratchReuseConfig(max_rewind_len=self.num_extra_kv_tokens) + # Subclasses (e.g. MiniMax-M3 sparse cache) can register additional # per-layer BufferConfig entries — for example a sparse index-K # buffer — without overriding the K/V/NVFP4 scale wiring above. @@ -1763,10 +1557,8 @@ def _build_base_config( return KVCacheManagerConfigPy( tokens_per_block=tokens_per_block, + vocab_size=vocab_size, cache_tiers=cache_tiers, - layers=layer_configs, - typical_step=typical_step, - constraints=constraints, max_util_for_resume=kv_cache_config.max_util_for_resume, enable_partial_reuse=kv_cache_config.enable_partial_reuse, enable_stats=self.enable_stats, @@ -1776,12 +1568,9 @@ def _build_base_config( and self.block_reuse_policy != BlockReusePolicy.ALL_REUSABLE ), initial_pool_ratio=kv_cache_config.pool_ratio, + layers=layer_configs, ) - def _build_cache_config(self, config: KVCacheManagerConfigPy) -> KVCacheManagerConfigPy: - """Customize the general cache config for a specialized cache manager.""" - return config - def _extra_buffers_per_layer( self, *, tokens_per_block: int ) -> Optional[dict[int, List[BufferConfig]]]: @@ -1793,44 +1582,12 @@ def _extra_buffers_per_layer( a sparse index-K buffer for each sparse local layer. Each ``BufferConfig.size`` is interpreted as bytes per block (i.e., ``bytes_per_token * tokens_per_block``), matching the standard - buffers built in :meth:`_build_base_config`. The block storage + buffers built in :meth:`_build_cache_config`. The block storage groups buffers by lifecycle and size with an opaque role key, so new roles do not require C++ changes. """ return None - def get_disagg_role_mapper_kinds(self) -> dict[DataRole, MapperKind]: - """Map native cache roles to disaggregation mapper kinds. - - ``Role.ALL`` is the required fallback for roles without an explicit - entry. The default is the head-major (HND) ``INDEXED`` layout written - by the TRTLLM attention kernels — correct for V1 and standard V2 - managers. ``Role.INDEX_KEY`` defaults to ``REPLICATED``: every - index-key side cache shipped so far (DSA indexer-K on V1, MiniMax M3 - on V2) computes its projection replicated across TP ranks, so the - cache bytes are identical per rank; the entry is inert unless a - subclass actually registers ``INDEX_KEY`` buffers via - ``_extra_buffers_per_layer``. Model-specific managers may declare - logical layouts without requiring the shared extractor to inspect - private attributes or role names. MiniMax M3, for example, maps - ordinary K/V to ``NHD`` and keeps index-key ``REPLICATED``. - - This declaration does not influence storage pooling: V2 storage - coalesces buffers purely by ``(life_cycle, buffer size)``, so roles - with different transfer semantics may share a pool slot when their - per-block sizes coincide (e.g. MiniMax M3 at TP degrees where - K == V == INDEX_KEY bytes per block). The disagg page-table builder - splits each physical pool into one logical view per mapper kind, so - transfer correctness never depends on the coalescing outcome. - - Pool memory is layout-agnostic; this declaration describes what the - manager's paired attention backend actually writes. A static mapping - is valid only for a fixed manager/backend pair. A manager whose backend - selects the layout at runtime must derive the mapping from that - backend's configuration. - """ - return {Role.ALL: MapperKind.INDEXED, Role.INDEX_KEY: MapperKind.REPLICATED} - @property def blocks_in_primary_pool(self) -> int: """ @@ -2197,8 +1954,6 @@ def prepare_context(self, req: LlmRequest) -> bool: def _prepare_context_impl(self, req: LlmRequest) -> bool: if req.is_first_context_chunk: - if self.conversation_manager is not None: - self.conversation_manager.prepare_request(req) kv_cache = self.kv_cache_map.get(req.py_request_id) if kv_cache is None: all_tokens = req.get_tokens(DEFAULT_BEAM_INDEX) @@ -2483,47 +2238,14 @@ def _stats_life_cycle_window_size(self, life_cycle) -> Optional[int]: return None return self._stats_window_size(life_cycle.window_size) - def _get_storage_statistics(self, cache_level: CacheLevel): - return self.impl._storage.get_statistics(cache_level) - - def _stats_life_cycle_metadata(self) -> dict[int, tuple[int, Optional[int], str]]: - pool_groups_by_life_cycle = [ - self.impl._storage.get_pool_group_index(LifeCycleId(life_cycle_id)) - for life_cycle_id in range(len(self.impl.layer_grouping)) - ] - - metadata: dict[int, tuple[int, Optional[int], str]] = {} - for life_cycle_id, layer_ids in enumerate(self.impl.layer_grouping): - if not layer_ids: - continue - layer = self.kv_cache_manager_py_config.layers[int(layer_ids[0])] - is_attention = isinstance(layer, AttentionLayerConfig) - metadata[life_cycle_id] = ( - int(pool_groups_by_life_cycle[life_cycle_id]), - self._stats_window_size(layer.sliding_window_size) if is_attention else None, - "attention" if is_attention else "ssm", - ) - return metadata - def _storage_pool_groups_by_window(self) -> dict[int, set[int]]: pool_groups_by_window: dict[int, set[int]] = defaultdict(set) - for pool_group_id, window_size, _ in self._stats_life_cycle_metadata().values(): - if window_size is not None: - pool_groups_by_window[window_size].add(pool_group_id) - return pool_groups_by_window - - def _get_and_reset_iteration_peak_block_stats(self, cache_level: CacheLevel): - get_peak_stats = getattr(self.impl, "get_and_reset_iteration_peak_block_stats", None) - if get_peak_stats is not None: - return get_peak_stats(cache_level) - return [ - PoolGroupPeakBlockStats( - available=stats.available, - unavailable=stats.total - stats.available, - evictable=stats.evictable, + for life_cycle_id, life_cycle in self.impl._life_cycles.attention_life_cycles(): + pool_group_id = self.impl._storage.get_pool_group_index(life_cycle_id) + pool_groups_by_window[self._stats_window_size(life_cycle.window_size)].add( + int(pool_group_id) ) - for stats in self._get_storage_statistics(cache_level) - ] + return pool_groups_by_window @staticmethod def _windows_by_pool_group( @@ -2641,7 +2363,7 @@ def _build_iteration_stats( return stats def _collect_iteration_stats_deltas( - self, raw_iteration_stats, life_cycle_metadata + self, raw_iteration_stats, storage ) -> tuple[dict, dict, dict, dict]: reuse_deltas_by_window: dict[int, KVCacheIterationStatsDelta] = {} reuse_deltas_by_life_cycle: dict[int, KVCacheIterationStatsDelta] = {} @@ -2649,7 +2371,9 @@ def _collect_iteration_stats_deltas( pool_group_deltas: dict[int, KVCacheIterationStatsDelta] = {} for life_cycle_id, delta in raw_iteration_stats.items(): - pool_group_id, window_size, _ = life_cycle_metadata[int(life_cycle_id)] + life_cycle = self.impl._life_cycles.get_life_cycle(life_cycle_id) + pool_group_id = int(storage.get_pool_group_index(life_cycle_id)) + window_size = self._stats_life_cycle_window_size(life_cycle) pool_group_delta = self._filter_iteration_stats_delta( delta, KV_CACHE_ITERATION_STATS_POOL_GROUP_FIELDS @@ -2715,15 +2439,9 @@ def _build_pool_group_iteration_stats( secondary_peak_stats_by_level, pool_group_delta, ) -> KVCacheV2PoolGroupIterationStats: - primary_pool_group_stats = primary_stats[pool_group_id] - slot_size = ( - primary_pool_group_stats.slot_sizes - if hasattr(primary_pool_group_stats, "slot_sizes") - else primary_pool_group_stats.slot_size - ) return KVCacheV2PoolGroupIterationStats( pool_group_id=pool_group_id, - slot_size=tuple(slot_size), + slot_size=tuple(primary_stats[pool_group_id].slot_size), window_sizes=windows_by_pool_group.get(pool_group_id, ()), stats=self._build_iteration_stats( (pool_group_id,), @@ -2739,19 +2457,21 @@ def _build_pool_group_iteration_stats( def _build_life_cycle_iteration_stats( self, life_cycle_id: int, - life_cycle_metadata, + storage, primary_stats, secondary_stats_by_level, primary_peak_stats, secondary_peak_stats_by_level, reuse_delta, ) -> KVCacheV2LifeCycleIterationStats: - pool_group_id, window_size, kind = life_cycle_metadata[life_cycle_id] + typed_life_cycle_id = LifeCycleId(life_cycle_id) + life_cycle = self.impl._life_cycles.get_life_cycle(typed_life_cycle_id) + pool_group_id = int(storage.get_pool_group_index(typed_life_cycle_id)) return KVCacheV2LifeCycleIterationStats( life_cycle_id=life_cycle_id, pool_group_id=pool_group_id, - window_size=window_size, - kind=kind, + window_size=self._stats_life_cycle_window_size(life_cycle), + kind="attention" if isinstance(life_cycle, AttnLifeCycle) else "ssm", stats=self._build_iteration_stats( (), primary_stats, @@ -2765,7 +2485,7 @@ def _build_life_cycle_iteration_stats( def get_kv_cache_stats(self): kv_cache_stats = KvCacheStats() - pool_group_stats = self._get_storage_statistics(GPU_LEVEL) + pool_group_stats = self.impl._storage.get_statistics(GPU_LEVEL) max_num_blocks = sum(stat.total for stat in pool_group_stats) free_num_blocks = sum(stat.available for stat in pool_group_stats) committed_stats = self.impl.get_committed_stats() @@ -2807,29 +2527,29 @@ def get_iteration_stats(self): if not self.enable_stats: return None - life_cycle_metadata = self._stats_life_cycle_metadata() + storage = self.impl._storage pool_groups_by_window = self._storage_pool_groups_by_window() windows_by_pool_group = self._windows_by_pool_group(pool_groups_by_window) raw_iteration_stats = self.impl.get_and_reset_iteration_stats() - primary_peak_stats = self._get_and_reset_iteration_peak_block_stats(GPU_LEVEL) - num_cache_levels = len(self.impl.cache_tier_list) + primary_peak_stats = self.impl.get_and_reset_iteration_peak_block_stats(GPU_LEVEL) secondary_peak_stats_by_level = [ - self._get_and_reset_iteration_peak_block_stats(CacheLevel(level)) - for level in range(1, num_cache_levels) + self.impl.get_and_reset_iteration_peak_block_stats(CacheLevel(level)) + for level in range(1, int(storage.num_cache_levels)) ] ( reuse_deltas_by_window, reuse_deltas_by_life_cycle, pool_group_deltas_by_window, pool_group_deltas, - ) = self._collect_iteration_stats_deltas(raw_iteration_stats, life_cycle_metadata) + ) = self._collect_iteration_stats_deltas(raw_iteration_stats, storage) windows = set(pool_groups_by_window) windows.update(reuse_deltas_by_window) windows.update(pool_group_deltas_by_window) - primary_stats = self._get_storage_statistics(GPU_LEVEL) + primary_stats = storage.get_statistics(GPU_LEVEL) secondary_stats_by_level = [ - self._get_storage_statistics(CacheLevel(level)) for level in range(1, num_cache_levels) + storage.get_statistics(CacheLevel(level)) + for level in range(1, int(storage.num_cache_levels)) ] stats_by_window = { @@ -2864,7 +2584,7 @@ def get_iteration_stats(self): stats_by_life_cycle = { life_cycle_id: self._build_life_cycle_iteration_stats( life_cycle_id, - life_cycle_metadata, + storage, primary_stats, secondary_stats_by_level, primary_peak_stats, @@ -3070,8 +2790,6 @@ def release_index_slot(self, request_id: int) -> None: self._early_freed_index_requests.add(request_id) def free_resources(self, request: LlmRequest, pin_on_release: bool = False): - if self.conversation_manager is not None: - self.conversation_manager.finish_request(request) self._allocated_draft_lens.pop(request.py_request_id, None) kv_cache = self.kv_cache_map.pop(request.py_request_id, None) if kv_cache is None: @@ -3085,12 +2803,6 @@ def free_resources(self, request: LlmRequest, pin_on_release: bool = False): else: self.index_mapper.remove_sequence(request.py_request_id) - def get_layer_page_index_scale(self, layer_idx: int) -> int: - """Page-index scale of this layer's KV buffer. Layers in one pool can - have different scales (e.g. different head_dim), so per-layer callers - must not use the pool-level scale.""" - return int(self.impl.get_page_index_scale(self.layer_offsets[layer_idx], Role.KEY)) - def get_batch_cache_indices( self, request_ids: List[int], @@ -3099,16 +2811,13 @@ def get_batch_cache_indices( ) -> List[List[int]]: if layer_idx is None: pool_id = 0 - index_scale = None else: pool_id = self.layer_to_pool_mapping_dict[self.layer_offsets[layer_idx]] - index_scale = self.get_layer_page_index_scale(layer_idx) return self._get_batch_cache_indices_by_pool_id( request_ids, pool_id=pool_id, is_kv_aggregate=True, num_blocks_per_seq=num_blocks_per_seq, - index_scale=index_scale, ) def _get_batch_cache_indices_by_pool_id( @@ -3118,7 +2827,6 @@ def _get_batch_cache_indices_by_pool_id( pool_id: int = 0, is_kv_aggregate: bool = True, num_blocks_per_seq: Optional[Sequence[int]] = None, - index_scale: Optional[int] = None, ) -> List[List[int]]: if is_kv_aggregate: # Div by kv_factor to index kv cache with size @@ -3127,8 +2835,7 @@ def _get_batch_cache_indices_by_pool_id( else: div_factor = 1 - if index_scale is None: - index_scale = int(self.index_scales[pool_id]) + index_scale = int(self.index_scales[pool_id]) res = [] for req_idx, req_id in enumerate(request_ids): @@ -3171,14 +2878,10 @@ def get_batch_cache_indices_flat( """ if layer_idx is None: pool_id = 0 - scale = self._index_scale_ints[pool_id] else: pool_id = self.layer_to_pool_mapping_dict[self.layer_offsets[layer_idx]] - # Layers sharing a pool can still require different page-index - # scales (e.g. Gemma4 sliding/global head_dim). Use the per-layer - # scale so this flat block table matches get_batch_cache_indices() - # and never feeds out-of-range page ids to FlashInfer. - scale = self.get_layer_page_index_scale(layer_idx) + + scale = self._index_scale_ints[pool_id] div_factor = self.kv_factor out_tensor = torch.empty(sum(num_blocks), dtype=torch.int32, pin_memory=prefer_pinned()) @@ -3312,8 +3015,6 @@ def shutdown(self): kv_cache.close() self.kv_cache_map.clear() self.impl.shutdown() - if self.conversation_manager is not None: - self.conversation_manager.clear() def get_max_resource_count(self) -> int: # TODO: implement this @@ -3395,8 +3096,6 @@ def update_context_resources(self, scheduled_batch: ScheduledRequests): if should_commit: self.try_commit_blocks(req) if req.context_remaining_length == 0: - if self.conversation_manager is not None: - self.conversation_manager.save_drop_plan(req, kv_cache) # Scratch blocks are only for prefill chunks. Disable them at # the context/generation boundary so generation uses normal KV # pages before the first generation allocation. @@ -3430,15 +3129,12 @@ def update_resources( if req.state in (LlmRequestState.GENERATION_COMPLETE, LlmRequestState.CONTEXT_INIT) else kv_cache.capacity - req.py_rewind_len ) - history_length = ( - None if self.kv_compression_manages_history else req.max_beam_num_tokens - 1 - ) - success = kv_cache.resize(new_capacity, history_length) + success = kv_cache.resize(new_capacity, req.max_beam_num_tokens - 1) if not success: raise ValueError( f"Failed to resize KV cache for request {req.py_request_id} " f"to capacity {new_capacity} and history length " - f"{history_length} tokens at generation update" + f"{req.max_beam_num_tokens - 1} tokens at generation update" ) def copy_batch_block_offsets( @@ -3448,10 +3144,7 @@ def copy_batch_block_offsets( beam_width: int, num_contexts: int, num_seqs: int, - max_blocks: Optional[int] = None, ): - # max_blocks is accepted for signature parity with KVCacheManager; the - # device-side copy op here already scales with allocated blocks only. assert beam_width == 1, "beam_width must be 1 for KVCacheManagerV2" copy_idx = self.index_mapper.get_copy_index(request_ids, num_contexts, beam_width) @@ -3582,5 +3275,3 @@ def prefetch_for_context_tokens(self, requests: list) -> bool: def reset_reuse_state(self): self.impl.clear_reusable_blocks() - if self.conversation_manager is not None: - self.conversation_manager.clear() diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py b/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py index a686adc97528..087862b07d5f 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache_transceiver.py @@ -1,6 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - from abc import ABC, abstractmethod from os import getenv from typing import Any, Dict, List, Optional @@ -9,14 +6,12 @@ from tensorrt_llm import logger from tensorrt_llm._torch.distributed.communicator import Distributed from tensorrt_llm.bindings import WorldConfig -from tensorrt_llm.llmapi.llm_args import (_CACHE_TRANSCEIVER_BACKEND_ENV_VARS, - CacheTransceiverConfig) +from tensorrt_llm.llmapi.llm_args import CacheTransceiverConfig from tensorrt_llm.mapping import Mapping from .llm_request import LlmRequest from .mamba_cache_manager import (BaseMambaCacheManager, - CppMambaHybridCacheManager, - MixedMambaHybridCacheManager) + CppMambaHybridCacheManager) from .resource_manager import KVCacheManager CacheTransceiverCpp = tensorrt_llm.bindings.internal.batch_manager.CacheTransceiver @@ -24,80 +19,6 @@ CacheTransBufferManagerCpp = tensorrt_llm.bindings.internal.batch_manager.CacheTransBufferManager BackendTypeCpp = tensorrt_llm.bindings.executor.CacheTransceiverBackendType -_DISAGG_INFLIGHT_CANCEL_ENABLED_ENV = "TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL" -_NIXL_KVCACHE_BACKEND_ENV = "TRTLLM_NIXL_KVCACHE_BACKEND" -_DISABLE_KV_CACHE_TRANSFER_OVERLAP_ENV = "TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP" -_DISAGG_LAYERWISE_ENV = "TRTLLM_DISAGG_LAYERWISE" -_TRY_ZCOPY_FOR_KV_CACHE_TRANSFER_ENV = "TRTLLM_TRY_ZCOPY_FOR_KVCACHE_TRANSFER" -_SUPPORTED_INFLIGHT_CANCEL_NIXL_BACKEND = "UCX" -_disagg_inflight_cancel_enabled_cache: Optional[bool] = None - - -def is_disagg_inflight_cancel_enabled() -> bool: - """Return whether disaggregated in-flight KV transfer cancellation is enabled.""" - global _disagg_inflight_cancel_enabled_cache - if _disagg_inflight_cancel_enabled_cache is None: - _disagg_inflight_cancel_enabled_cache = (getenv( - _DISAGG_INFLIGHT_CANCEL_ENABLED_ENV, "0") == "1") - if _disagg_inflight_cancel_enabled_cache: - logger.warning( - f"{_DISAGG_INFLIGHT_CANCEL_ENABLED_ENV}=1: disagg KV " - "transfer in-flight cancellation was requested. It is active " - "only for cache transceivers that advertise support.") - return _disagg_inflight_cancel_enabled_cache - - -def _is_disagg_inflight_cancel_config_supported( - cache_transceiver_config: CacheTransceiverConfig) -> bool: - runtime = cache_transceiver_config.transceiver_runtime or "CPP" - nixl_backend = getenv(_NIXL_KVCACHE_BACKEND_ENV, - _SUPPORTED_INFLIGHT_CANCEL_NIXL_BACKEND) - return (runtime == "CPP" and cache_transceiver_config.backend == "NIXL" - and nixl_backend == _SUPPORTED_INFLIGHT_CANCEL_NIXL_BACKEND - and cache_transceiver_config.kv_transfer_timeout_ms is not None - and getenv(_DISABLE_KV_CACHE_TRANSFER_OVERLAP_ENV) != "1" - and getenv(_DISAGG_LAYERWISE_ENV) != "1" - and getenv(_TRY_ZCOPY_FOR_KV_CACHE_TRANSFER_ENV) != "1") - - -def _validate_disagg_inflight_cancel_config( - cache_transceiver_config: CacheTransceiverConfig) -> None: - if not is_disagg_inflight_cancel_enabled(): - return - - enabled_backend_env_vars = [ - env_name for env_name, _ in _CACHE_TRANSCEIVER_BACKEND_ENV_VARS - if getenv(env_name) == "1" - ] - if (cache_transceiver_config.backend == "DEFAULT" - and len(enabled_backend_env_vars) > 1): - raise ValueError( - f"{_DISAGG_INFLIGHT_CANCEL_ENABLED_ENV}=1 requires an " - "unambiguous cache transceiver backend, but multiple legacy " - f"backend selectors are enabled: {enabled_backend_env_vars}.") - - if _is_disagg_inflight_cancel_config_supported(cache_transceiver_config): - return - - runtime = cache_transceiver_config.transceiver_runtime or "CPP" - nixl_backend = getenv(_NIXL_KVCACHE_BACKEND_ENV, - _SUPPORTED_INFLIGHT_CANCEL_NIXL_BACKEND) - disable_overlap = getenv(_DISABLE_KV_CACHE_TRANSFER_OVERLAP_ENV) - layerwise = getenv(_DISAGG_LAYERWISE_ENV) - try_zcopy = getenv(_TRY_ZCOPY_FOR_KV_CACHE_TRANSFER_ENV) - raise ValueError( - f"{_DISAGG_INFLIGHT_CANCEL_ENABLED_ENV}=1 is experimental and " - "currently supported only with transceiver_runtime='CPP', " - "backend='NIXL', the UCX NIXL backend, a finite " - "kv_transfer_timeout_ms, asynchronous non-layer-wise transfer, and " - "zero-copy disabled; got " - f"transceiver_runtime={runtime!r}, " - f"backend={cache_transceiver_config.backend!r}, " - f"resolved_nixl_backend={nixl_backend!r}, " - f"kv_transfer_timeout_ms={cache_transceiver_config.kv_transfer_timeout_ms!r}, " - f"disable_transfer_overlap={disable_overlap!r}, " - f"layerwise={layerwise!r}, try_zcopy={try_zcopy!r}.") - def mapping_to_world_config(mapping: Mapping) -> WorldConfig: @@ -121,41 +42,33 @@ def create_kv_cache_transceiver( logger.info("cache_transceiver is disabled") return None - # "auto" is normally resolved against the model's preference at config - # load time (ModelLoader.load_config_and_apply_defaults); paths that skip - # that step (e.g. AutoDeploy) fall back to the C++ transceiver here. This - # must run before any consumer of transceiver_runtime below (e.g. the - # inflight-cancel validation, which treats non-CPP runtimes as - # unsupported). - if cache_transceiver_config.transceiver_runtime == "auto": - cache_transceiver_config.transceiver_runtime = None - - if (cache_transceiver_config.transceiver_runtime != "PYTHON" - and isinstance(mamba_cache_manager, MixedMambaHybridCacheManager)): - raise ValueError( - "MixedMambaHybridCacheManager requires the Python transceiver " - "runtime in disaggregated serving.") - - _validate_disagg_inflight_cancel_config(cache_transceiver_config) - if cache_transceiver_config.backend == "DEFAULT": # When cache_transceiver_config.backend is not set, fallback to env_vars settings # NIXL is the default backend for non hybrid models - backend, env_var = cache_transceiver_config._resolve_default_backend() - if env_var is not None: - logger.warning( - f"{env_var}=1 is set, but it's recommended to set cache_transceiver_config.backend in yaml config" - ) - cache_transceiver_config.backend = backend + cache_transceiver_config.backend = "NIXL" + # Ordered by priority + env_vars = [ + ("TRTLLM_USE_NIXL_KVCACHE", "NIXL"), + ("TRTLLM_USE_UCX_KVCACHE", "UCX"), + ("TRTLLM_USE_MOONCAKE_KVCACHE", "MOONCAKE"), + ("TRTLLM_USE_MPI_KVCACHE", "MPI"), + ] + for env_var, be_type in env_vars: + if getenv(env_var) == "1": + logger.warning( + f"{env_var}=1 is set, but it's recommended to set cache_transceiver_config.backend in yaml config" + ) + cache_transceiver_config.backend = be_type + break if cache_transceiver_config.backend == "MPI": logger.warning( "MPI CacheTransceiver is deprecated, UCX or NIXL is recommended") elif cache_transceiver_config.backend == "UCX": logger.info( - "Using UCX kv-cache transceiver. If your devices are not in the same domain, please consider setting " - "UCX_CUDA_IPC_ENABLE_MNNVL=n, UCX_RNDV_SCHEME=put_zcopy and/or unset UCX_NET_DEVICES upon server " - "hangs or lower-than-expected performance.") + f"Using UCX kv-cache transceiver. If your devices are not in the same domain, please consider setting " + f"UCX_CUDA_IPC_ENABLE_MNNVL=n, UCX_RNDV_SCHEME=put_zcopy and/or unset UCX_NET_DEVICES upon server " + f"hangs or lower-than-expected performance.") # Select transceiver implementation based on transceiver_runtime # transceiver_runtime == None or "CPP" -> use C++ transceiver (default) @@ -210,12 +123,6 @@ def check_gen_transfer_complete(self): def cancel_request(self, req: LlmRequest): raise NotImplementedError - def supports_inflight_request_cancellation(self) -> bool: - return False - - def has_poisoned_transfer_buffer(self) -> bool: - return False - @abstractmethod def prepare_context_requests(self, requests: List[LlmRequest]): """ @@ -237,10 +144,6 @@ def get_disaggregated_params(self) -> Dict[str, Any]: def commit_blocks_for_reuse(self, req: LlmRequest) -> None: """Commit received KV blocks to the radix tree for prefix reuse. No-op by default.""" - def get_data_transceiver_state(self) -> bytes: - """Get the serialized DataTransceiverState (CacheState + CommState).""" - return b"" - def shutdown(self): """Shut down the transceiver and release registered resources.""" @@ -254,7 +157,6 @@ def __init__(self, attention_type: AttentionTypeCpp, cache_transceiver_config: CacheTransceiverConfig, mamba_cache_manager: Optional[BaseMambaCacheManager] = None): - _validate_disagg_inflight_cancel_config(cache_transceiver_config) world_config = mapping_to_world_config(mapping) # Filter out mamba/recurrent state layers (kv_heads == 0) so that # CacheState::ModelConfig::mNbKvHeadsPerLayer only contains attention @@ -278,9 +180,6 @@ def __init__(self, self.kv_transfer_timeout_ms = cache_transceiver_config.kv_transfer_timeout_ms self.kv_transfer_sender_future_timeout_ms = cache_transceiver_config.kv_transfer_sender_future_timeout_ms self.kv_transfer_poll_interval_ms = cache_transceiver_config.kv_transfer_poll_interval_ms - self._supports_inflight_request_cancellation = ( - _is_disagg_inflight_cancel_config_supported( - cache_transceiver_config)) # Get RNN layer distribution if mamba_cache_manager is provided. rnn_layer_num_per_pp_rank = [] @@ -327,21 +226,10 @@ def check_gen_transfer_complete(self): def cancel_request(self, req: LlmRequest): return self.impl.cancel_request(req) - def supports_inflight_request_cancellation(self) -> bool: - return self._supports_inflight_request_cancellation - - def has_poisoned_transfer_buffer(self) -> bool: - if not is_disagg_inflight_cancel_enabled(): - return False - return self.impl.has_poisoned_transfer_buffer() - def prepare_context_requests(self, requests: List[LlmRequest]): # not implemented, an empty placeholder to allow being invoked unconditionally ... - def get_data_transceiver_state(self) -> bytes: - return self.impl.get_serialized_data_transceiver_state() - def get_disaggregated_params(self): # Cpp kv cache transceiver will set the disaggregated params to context response # Only new py cache transceiver will support gen-first disagg diff --git a/tensorrt_llm/_torch/pyexecutor/llm_request.py b/tensorrt_llm/_torch/pyexecutor/llm_request.py index 88513bc326d6..63bfb318e284 100644 --- a/tensorrt_llm/_torch/pyexecutor/llm_request.py +++ b/tensorrt_llm/_torch/pyexecutor/llm_request.py @@ -677,7 +677,6 @@ def __init__( self.py_lora_path: str | None = kwargs.pop("py_lora_path", None) # Multimodal data self.py_multimodal_data = kwargs.pop("py_multimodal_data", None) - self.py_mm_item_order = kwargs.pop("py_mm_item_order", None) encoder_input_tokens = kwargs.get("encoder_input_tokens") encoder_output_len = kwargs.get("encoder_output_len") return_encoder_output = bool(kwargs.get("return_encoder_output", False)) @@ -731,18 +730,12 @@ def __init__( self.py_batch_idx = None self.py_draft_pages_allocated = 0 self.py_rewind_len = 0 - # Tokens physically evicted by KV-cache compression; deducted in the engine. - self.py_num_compressed_tokens = 0 self.py_draft_tokens = [] if self.draft_tokens is None else self.draft_tokens self.py_last_context_chunk = (None, None) self.py_draft_logits = None self.py_target_probs = None self.py_last_draft_tokens = None self.py_num_accepted_draft_tokens = 0 - # One-model rejection: set per-iteration by _handle_dynamic_draft_len when - # this gen request produced 0 real draft tokens, so _prepare_tp_inputs - # one-hots its stale draft_probs slot. Consumed (and cleared) there. - self.py_needs_onehot_draft_probs = False self.py_num_accepted_draft_tokens_indices = [] self.py_rewind_draft_token_separate_adjustment = 0 self.py_per_pos_drafted = [0] * MAX_SPEC_DECODE_POSITIONS @@ -1185,7 +1178,6 @@ def executor_request_to_llm_request( arrival_time=getattr(executor_request, "py_arrival_time", None), py_multimodal_data=getattr(executor_request, "py_multimodal_data", None), - py_mm_item_order=getattr(executor_request, "py_mm_item_order", None), kv_cache_retention_config=executor_request.kv_cache_retention_config, agent_hierarchy=agent_hierarchy, logprobs_mode=getattr(executor_request, "py_logprobs_mode", @@ -1194,13 +1186,6 @@ def executor_request_to_llm_request( "py_logprobs_simple_format", False), ) - # Bad-words list for the TorchSampler path, kept in its native - # list[list[int]] form (single- and multi-token words). This is the - # TorchSampler's own input and is independent of any other sampler. - llm_request.py_bad_words = [ - list(word) for word in executor_request.bad_words - ] if executor_request.bad_words else None - llm_request.py_original_end_id = getattr(executor_request, "py_original_end_id", llm_request.py_end_id) diff --git a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py index 16fe22d1c777..4d8dc3b717c1 100644 --- a/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/mamba_cache_manager.py @@ -33,8 +33,7 @@ BaseResourceManager, CacheTypeCpp, DataType, KVCacheManager, PoolConfiguration, get_pp_layers) from tensorrt_llm._torch.pyexecutor.scheduler import ScheduledRequests -from tensorrt_llm._utils import (nvtx_range, prefer_pinned, - torch_dtype_to_binding) +from tensorrt_llm._utils import nvtx_range, prefer_pinned from tensorrt_llm.bindings.internal.batch_manager import ( LinearAttentionMetadata, LinearCacheType) from tensorrt_llm.llmapi.llm_args import KvCacheConfig @@ -784,20 +783,14 @@ def _drop(tensor): torch.cuda.empty_cache() @torch.compile(options={"max-autotune": True}) - def update_mamba_states( - self, - attn_metadata: "AttentionMetadata", - num_accepted_tokens: torch.Tensor, - state_indices: torch.Tensor, - accepted_leaf_positions: Optional[torch.Tensor] = None): + def update_mamba_states(self, attn_metadata: "AttentionMetadata", + num_accepted_tokens: torch.Tensor, + state_indices: torch.Tensor): batch_size = attn_metadata.num_seqs num_contexts = attn_metadata.num_contexts num_gens = batch_size - num_contexts num_accepted_draft_tokens = num_accepted_tokens[ num_contexts:num_contexts + num_gens] - 1 - # Dynamic tree passes tree-node leaf positions; linear MTP uses depth. - accepted_positions = (accepted_leaf_positions if accepted_leaf_positions - is not None else num_accepted_draft_tokens) state_indices_d = state_indices[num_contexts:num_contexts + num_gens] src_state_indices = self.intermediate_state_indices[:num_gens] @@ -834,7 +827,7 @@ def update_mamba_states( ssm_states = self.mamba_cache.temporal intermediate_ssm_cache = self.mamba_cache.intermediate_ssm accepted_ssm_state = intermediate_ssm_cache[:, src_state_indices, - accepted_positions] + num_accepted_draft_tokens] ssm_states[:, state_indices_d, :] = accepted_ssm_state # Conv: both paths save all intermediate conv windows, carry over the accepted one. @@ -842,7 +835,7 @@ def update_mamba_states( intermediate_conv_window_cache = self.mamba_cache.intermediate_conv_window accepted_conv_state = intermediate_conv_window_cache[:, src_state_indices, - accepted_positions] + num_accepted_draft_tokens] conv_states[:, state_indices_d, :] = accepted_conv_state @@ -986,18 +979,15 @@ def mamba_layer_cache( def shutdown(self): self._impl.shutdown() - def update_mamba_states( - self, - attn_metadata: "AttentionMetadata", - num_accepted_tokens: torch.Tensor, - state_indices: torch.Tensor, - accepted_leaf_positions: Optional[torch.Tensor] = None): + def update_mamba_states(self, attn_metadata: "AttentionMetadata", + num_accepted_tokens: torch.Tensor, + state_indices: torch.Tensor): # Non-speculative configs don't allocate intermediate state; the # promotion is a clean no-op. if not self._impl.is_speculative(): return self._impl.update_mamba_states(attn_metadata, num_accepted_tokens, - state_indices, accepted_leaf_positions) + state_indices) class MambaHybridCacheManager(BaseResourceManager, BaseMambaCacheManager): @@ -1145,6 +1135,26 @@ def update_resources(self, kv_cache_dtype_byte_size) +def calc_context_stop_positions(prompt_len: int, + tokens_per_block: int, + mamba_state_cache_interval: int, + save_last_snapshot: bool = False) -> list[int]: + """Compute token positions at which mamba state snapshots should be saved. + + Returns positions spaced by ``mamba_state_cache_interval`` plus the final + prompt length (and optionally the last block-aligned position). + """ + stop_positions = list( + range(mamba_state_cache_interval, prompt_len, + mamba_state_cache_interval)) + last_ckpt = prompt_len // tokens_per_block * tokens_per_block + if save_last_snapshot and (last_ckpt not in stop_positions): + stop_positions.append(last_ckpt) + if prompt_len not in stop_positions: + stop_positions.append(prompt_len) + return stop_positions + + @triton.jit def _promote_mamba_state_kernel( src_ptr, @@ -1316,9 +1326,6 @@ def __init__( # accessors (get_mamba_ssm_cache_dtype, use_replay_state_update) work # on ranks with no local mamba layers. self._use_replay_state_update = use_replay_state_update - self._use_gdn_cached_replay_all_layer_commit = ( - use_replay_state_update and model_type == "qwen3_next" - and self.local_num_mamba_layers > 0) self.replay_step_width: Optional[int] = ( spec_config.tokens_per_gen_step if spec_config is not None and use_replay_state_update else None) @@ -1355,13 +1362,6 @@ def __init__( is_estimating_kv_cache=is_estimating_kv_cache, is_draft=is_draft, ) - # PP ranks replay the same scheduling decisions, so a rank without - # local Mamba layers must still publish the configured boundaries. - self.kv_cache_config = kv_cache_config - self.linear_attention_metadata = LinearAttentionMetadata() - self.linear_attention_metadata.states_snapshot_interval = ( - kv_cache_config.mamba_state_cache_interval - if kv_cache_config.enable_block_reuse else 0) return # Derive ssm_state_shape and conv_state_shape from mamba params (same as MambaCacheManager) @@ -1448,21 +1448,6 @@ def __init__( LinearCacheType.RECURRENT_STATES. value if mamba_layer_mask[i] else max_seq_len) - recurrent_states_window = LinearCacheType.RECURRENT_STATES.value - local_windows = { - recurrent_states_window - if mamba_layer_mask[layer_idx] else max_seq_len - for layer_idx in self.pp_layers - } - kwargs["pool_configurations"] = [ - PoolConfiguration( - window_size=window_size, - head_dim=head_dim, - dtype=torch_dtype_to_binding(self.ssm_state_dtype) - if window_size == recurrent_states_window else dtype, - ) for window_size in sorted(local_windows) - ] - # Normalize num_kv_heads to a per-layer list and zero out mamba # layer positions: those layers carry SSM/conv state instead of KV # heads, so the parent KV cache should not allocate KV head storage @@ -1534,12 +1519,6 @@ def __init__( self._setup_states() self._setup_replay_buffers(spec_config) - if use_replay_state_update and model_type == "qwen3_next": - logger.info_once( - "Configured GDN cached replay commit mode: small-batch fused, " - "large-batch all-layer", - key="gdn_cached_replay_commit_mode_fused", - ) @staticmethod def get_cache_size_per_token( @@ -1613,46 +1592,6 @@ def get_cache_size_per_token( intercept = 0 return attention_slope + mamba_slope, intercept - @property - def use_gdn_cached_replay_all_layer_commit(self) -> bool: - return self._use_gdn_cached_replay_all_layer_commit - - def _commit_gdn_cached_replay_history_layers( - self, - attn_metadata: "AttentionMetadata", - num_decodes: int, - ) -> None: - """Synchronously advance every local GDN checkpoint in one launch.""" - from tensorrt_llm._torch.modules.fla.cached_replay import ( - CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE, - commit_gdn_cached_replay_history_layers) - - if (not self._use_gdn_cached_replay_all_layer_commit - or num_decodes < CACHED_REPLAY_PARTITION_MIN_BATCH_SIZE): - return - if (self.all_ssm_states is None or self.old_x is None - or self.old_B is None or self.old_dt is None - or self.replay_history_size is None): - raise RuntimeError( - "GDN cached replay all-layer commit requires replay state buffers." - ) - - mamba_metadata = attn_metadata.mamba_metadata - if mamba_metadata.replay_num_decodes != num_decodes: - raise RuntimeError( - "GDN replay metadata contains " - f"{mamba_metadata.replay_num_decodes} decode requests, " - f"but state update received {num_decodes}.") - commit_gdn_cached_replay_history_layers( - ssm_states=self.all_ssm_states, - old_u=self.old_x, - old_k=self.old_B, - old_G=self.old_dt, - replay_work_items=mamba_metadata.replay_work_items[:num_decodes], - n_writes=mamba_metadata.replay_n_writes, - history_size=self.replay_history_size, - ) - def shutdown(self): # Release tensor views into the pool before the pool memory is freed, # so their deleters don't see stale pointers. @@ -1800,12 +1739,10 @@ def is_speculative(self) -> bool: return self.spec_config is not None @nvtx_range("hybrid_update_mamba_states") - def update_mamba_states( - self, - attn_metadata: "AttentionMetadata", - num_accepted_tokens: torch.Tensor, - state_indices: Optional[torch.Tensor] = None, - accepted_leaf_positions: Optional[torch.Tensor] = None): + def update_mamba_states(self, + attn_metadata: "AttentionMetadata", + num_accepted_tokens: torch.Tensor, + state_indices: Optional[torch.Tensor] = None): if self.local_num_mamba_layers == 0: return batch_size = attn_metadata.num_seqs @@ -1814,10 +1751,6 @@ def update_mamba_states( num_accepted_draft_tokens = ( num_accepted_tokens[num_contexts:num_contexts + num_gens] - 1).to( torch.int32) - # Dynamic tree passes tree-node leaf positions; linear MTP uses depth. - accepted_positions = (accepted_leaf_positions.to(torch.int32) - if accepted_leaf_positions is not None else - num_accepted_draft_tokens) # Match the API of MambaCacheManager.update_mamba_states: callers # may pass per-request state slot indices explicitly (e.g. MTP via # attn_metadata.mamba_metadata.state_indices). Fall back to this @@ -1835,11 +1768,6 @@ def update_mamba_states( # writes through the view's real strides (~85% of HBM peak, one launch # per state). if self._use_replay_state_update: - # Every GDN layer has finished reading the old checkpoint and - # writing its candidate history. Advance all local checkpoints - # in one launch before PNAT and the active history buffer change. - self._commit_gdn_cached_replay_history_layers( - attn_metadata, num_gens) # SSM state is handled incrementally by the kernel. Mirror the # kernel's checkpoint predicate from the previous PNAT and fixed # replay step width: checkpoint steps flip buffers, while no-write @@ -1870,15 +1798,16 @@ def update_mamba_states( # Legacy: copy the accepted SSM state from the intermediate buffer. _promote_mamba_state_triton(self.all_ssm_states, self.intermediate_ssm_states, - src_state_indices, accepted_positions, + src_state_indices, + num_accepted_draft_tokens, state_indices_d) # Conv: both paths save all intermediate conv windows, carry over the # accepted one. _promote_mamba_state_triton(self.all_conv_states, self.intermediate_conv_states, - src_state_indices, accepted_positions, - state_indices_d) + src_state_indices, + num_accepted_draft_tokens, state_indices_d) @torch.inference_mode() def _refresh_dummy_request_mask(self, is_dummy: List[bool]) -> None: @@ -1909,10 +1838,7 @@ def get_num_available_tokens(self, # request, so no additional capping is required here. interval = (self.linear_attention_metadata.states_snapshot_interval if self.linear_attention_metadata is not None else 0) - # Attention-only PP ranks keep the interval so every rank publishes - # identical scheduling boundaries, but they have no recurrent-state - # pool whose capacity should constrain their attention KV cache. - if self.local_num_mamba_layers > 0 and interval and interval > 0: + if interval and interval > 0: stats = self.impl.get_kv_cache_stats() rs_free = stats.num_free_blocks_per_window_size.get( LinearCacheType.RECURRENT_STATES.value, 0) @@ -2042,18 +1968,15 @@ def _setup_state_indices(self, requests=None) -> None: self.cuda_state_indices.copy_(self._host_state_indices, non_blocking=True) - is_dummy = [req.is_dummy for req in requests] self._refresh_dummy_request_mask( - is_dummy if requests is - self.requests else [req.is_dummy for req in self.requests]) + [req.is_dummy for req in self.requests]) # Build request_id → pool block offset mapping so that # get_state_indices can return indices in arbitrary request order. - # Bulk tolist avoids a per-request tensor-index + .item() round-trip. - state_values = self._host_state_indices[:n].tolist() - for req, value, dummy in zip(requests, state_values, is_dummy): - self._request_id_to_state_index[req.py_request_id] = value - self._request_id_to_is_dummy[req.py_request_id] = dummy + for i, req in enumerate(requests): + self._request_id_to_state_index[ + req.py_request_id] = self._host_state_indices[i].item() + self._request_id_to_is_dummy[req.py_request_id] = req.is_dummy def get_state_indices(self, request_ids: Optional[List[int]] = None, @@ -2077,23 +2000,38 @@ def get_state_indices(self, return indices return self.cuda_state_indices - def prepare_expect_snapshot_points(self, - requests: List[LlmRequest]) -> None: - """Set reusable Mamba snapshot boundaries before scheduling.""" - if not self.kv_cache_config.enable_block_reuse: - for request in requests: - request.expect_snapshot_points = [] - return + def calc_next_context_chunk_size(self, request: LlmRequest) -> int: + """Compute the next prefill chunk size for a context request when block reuse is enabled. - interval = self.linear_attention_metadata.states_snapshot_interval - if interval is None or interval <= 0: - for request in requests: - request.expect_snapshot_points = [] - return + When kv_cache_config.enable_block_reuse is True, context prefill must stop exactly at + the positions returned by calc_context_stop_positions (mamba_state_cache_interval boundaries + and block boundaries). This returns the chunk_size to use for the next prefill step so + that the next stop position is not exceeded. + + Args: + request: Context request with prompt_len and context_current_position set. - for request in requests: - request.expect_snapshot_points = list( - range(interval, request.prompt_len + 1, interval)) + Returns: + Number of tokens to prefill in the next step (0 if context is already complete). + """ + prompt_len = request.prompt_len + current = request.context_current_position + if current >= prompt_len: + return 0 + if not self.kv_cache_config.enable_block_reuse: + assert current == 0, ( + "Expected context_current_position to be 0 when block reuse is " + f"disabled, but got {current}") + return prompt_len - current + step = self.linear_attention_metadata.states_snapshot_interval + stop_positions = calc_context_stop_positions(prompt_len, + self.tokens_per_block, + step) + stop_positions = sorted(set(stop_positions)) + for pos in stop_positions: + if pos > current: + return pos - current + return prompt_len - current def _setup_states(self) -> None: # Pool layout: {numLocalLayers, numBlocks, ssm_bytes + conv_bytes} (as uint8) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 50aa317e9e43..2ec62d5d3cae 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -22,7 +22,7 @@ prefer_pinned, release_gc, torch_dtype_to_str, trace_func) from tensorrt_llm.bindings.internal.runtime import TaskLayerModuleConfig -from tensorrt_llm.inputs.multimodal import (MultimodalInput, MultimodalParams, +from tensorrt_llm.inputs.multimodal import (MultimodalParams, MultimodalRuntimeData, _has_mm_payload_keys, check_mm_embed_cumsum_if_needed, @@ -59,7 +59,6 @@ from ..models.modeling_utils import DecoderModelForCausalLM from ..modules.fused_moe.moe_load_balancer import (MoeLoadBalancer, MoeLoadBalancerIterContext) -from ..modules.mamba.mamba2_metadata import Mamba2Metadata from ..peft.lora.cuda_graph_lora_manager import CudaGraphLoraManager from ..speculative import (SpecMetadata, get_draft_kv_cache_manager, get_num_extra_kv_tokens, get_spec_metadata, @@ -126,35 +125,29 @@ def _filter_piecewise_capture_num_tokens( ) -> Tuple[list[int], list[int]]: """Cap piecewise CUDA graph capture candidates at the engine's reachable `num_tokens` ceiling `max_batch_size * (max_seq_len - 1 - num_extra_decoding_steps)` - clamping user-requested sizes above it down to the ceiling. + and ensure the ceiling itself is captured. Each in-flight request must leave room for at least one decode token, so the ceiling is the largest forward-pass `num_tokens` the warmup - builder can construct. Candidates above the ceiling cannot be - recorded; clamping them down to the ceiling preserves the user's - intent (a requested 128 becomes 127 when only 127 is recordable) - without inventing capture sizes the user never asked - for. Appending sizes beyond the user's list is harmful: runtime - padding rounds iterations up to the nearest captured size, so a far - appended ceiling (e.g. 65536 over a list topping at 13914) would - make every iteration in the gap execute the full ceiling shape. - - Returns `(kept, unrecordable)` where `kept` is sorted ascending and - deduped, with above-ceiling candidates clamped to the ceiling. + builder can construct. Including it in the capture set closes the + runtime padding gap between the next-largest candidate and the ceiling + (otherwise ISLs in that gap have no graph >= them and fall back to + eager). + + Returns `(kept, unrecordable)` where `kept` is sorted ascending, + deduped, and contains the ceiling whenever it is positive. `unrecordable` is the sorted unique set of input entries above the - ceiling but within `max_num_tokens` (the clamped ones, reported so - the caller's warning fires). + ceiling but within `max_num_tokens`. """ max_capturable_num_tokens = max( 0, max_batch_size * (max_seq_len - 1 - num_extra_decoding_steps)) piecewise_capacity_limit = min(max_num_tokens, max_capturable_num_tokens) - if piecewise_capacity_limit > 0: - kept = sorted({ - min(i, piecewise_capacity_limit) - for i in candidate_num_tokens if 0 < i <= max_num_tokens - }) - else: - kept = [] + kept = sorted( + {i + for i in candidate_num_tokens if 0 < i <= piecewise_capacity_limit}) + if piecewise_capacity_limit > 0 and (not kept or kept[-1] + < piecewise_capacity_limit): + kept.append(piecewise_capacity_limit) unrecordable = sorted({ i for i in candidate_num_tokens @@ -163,25 +156,6 @@ def _filter_piecewise_capture_num_tokens( return kept, unrecordable -def _build_request_multimodal_input( - request: LlmRequest, cache_enabled: bool) -> Optional[MultimodalInput]: - # Skip building this input (and its `from_components` validation) when the cache is disabled. - if not cache_enabled or request.multimodal_hashes is None: - return None - # `multimodal_input` is consumed only by the encoder-cache key path - # (`MultimodalModelMixin._encoder_cache_keys`), which uses UUID-aware multimodal hashes - # internally. Although the `multimodal_uuids` are not exposed as an attribute, they remain in - # the backing C++ request for KV-cache block keys and cache events. - return MultimodalInput.from_components( - request.multimodal_hashes, - request.multimodal_positions, - request.multimodal_lengths, - mm_item_run_cu_offsets=request.multimodal_item_run_cu_offsets, - mm_run_positions=request.multimodal_run_positions, - mm_run_lengths=request.multimodal_run_lengths, - ) - - def _filter_cuda_graph_batch_sizes(cuda_graph_batch_sizes: list[int], max_batch_size: int, max_num_tokens: int, max_total_draft_tokens: int, @@ -387,11 +361,7 @@ def __init__( trust_remote_code=llm_args.trust_remote_code, **input_processor_kwargs) self.input_processor_with_hash = create_input_processor_with_hash( - self.input_processor, - encoder_cache_enabled=( - llm_args.multimodal_config is not None - and llm_args.multimodal_config.encoder_cache_max_bytes > 0), - ) + self.input_processor) if model is None: lora_config: Optional[ LoraConfig] = None if is_draft_model else llm_args.lora_config @@ -531,7 +501,7 @@ def __init__( f"{unrecordable}: exceeds reachable ceiling " f"max_batch_size*(max_seq_len-1-num_extra_decoding_steps)=" f"{max(0, self.batch_size * (self.max_seq_len - 1 - num_extra_decoding_steps))}. " - f"Clamping them to the ceiling; raise max_seq_len for larger graphs." + f"Capturing the ceiling itself; raise max_seq_len for larger graphs." ) try: @@ -621,13 +591,7 @@ def __init__( ) or self.model_is_wrapped self.max_total_draft_tokens = spec_config.tokens_per_gen_step - 1 self.max_draft_len = spec_config.max_draft_len - # Mutable per-iteration draft length (updated each iteration when - # dynamic draft length is enabled; otherwise stays fixed). Tree - # modes verify all tree nodes per step, which can be wider than the - # tree depth used by the drafter loop. - self.runtime_draft_len = (self.max_total_draft_tokens - if not spec_config.is_linear_tree else - self.max_draft_len) + self.runtime_draft_len = spec_config.max_draft_len else: self.without_logits = False @@ -688,18 +652,6 @@ def __init__( self.position_ids_cuda = torch.empty((self.max_num_tokens, ), dtype=torch.int, device='cuda') - # Steady-state generation-only prepare cache (non-speculative overlap - # decode). Holds the per-request lists that are invariant while the - # scheduled generation batch keeps the same composition, plus a pinned - # cached-token counter advanced by one per step (host-side bookkeeping - # only; the device position buffer is advanced in place and this - # buffer is never the source of an async H2D). Invalidated (set to - # None) by every full _prepare_tp_inputs pass. - self._steady_gen_cache: Optional[Dict[str, Any]] = None - self._steady_gen_positions_pinned = torch.empty( - (self.max_num_tokens, ), - dtype=torch.int, - pin_memory=prefer_pinned()) if self.use_mrope: self.mrope_position_ids_cuda = torch.empty( (3, 1, self.max_num_tokens), dtype=torch.int, device='cuda') @@ -813,12 +765,6 @@ def __init__( self._prepare_inputs_event: Optional[torch.cuda.Event] = None - # Cache for enc-dec cross-attention stable generation steps. - # Populated on the first CUDA-graph generation step; cleared whenever - # the batch composition changes (new encoder request arrives). - self._cross_attn_stable_cached_tokens: Optional[List[int]] = None - self._cross_attn_stable_request_ids: Optional[List[int]] = None - def register_forward_pass_callable(self, callable: Callable): self.forward_pass_callable = callable @@ -891,13 +837,6 @@ def use_mrope(self): logger.debug(f"Detected use_mrope: {use_mrope}") return use_mrope - @functools.cached_property - def _mm_encoder_cache_enabled(self) -> bool: - """Whether the multimodal encoder cache is active for this model.""" - multimodal_config = self.model.model_config.multimodal_config - return (multimodal_config is not None - and multimodal_config.encoder_cache_max_bytes > 0) - @property def is_warmup(self): return getattr(self, "_is_warmup", False) @@ -1163,44 +1102,15 @@ def warmup(self, resource_manager: ResourceManager) -> None: if not is_enc_dec and not self.mapping.has_cp_helix(): self._run_autotuner_warmup(resource_manager) log_mem_snapshot("warmup/after_autotuner") - # Pre-JIT Mamba SSD multi-seq + HAS_INITSTATES=True Triton kernels - # for Mamba hybrid models. Runs regardless of enable_autotuner, - # since MambaHybridCacheManager skips _general_warmup and the - # default autotuner shape is single-seq / no-initstates. Safe - # no-op for non-Mamba models. - self._run_mamba_hybrid_warmup(resource_manager) - log_mem_snapshot("warmup/after_mamba_hybrid") # Release the autotuner's exploration-mode intermediates. The # exploration leftovers are pure waste that hide tens of GiB from # non-torch allocators (cuBLAS handle workspace, UCX/NIXL, # NVSHMEM). gc.collect() torch.cuda.empty_cache() - # Warm up every graph shape before capturing any graph. Attention - # kernels can switch implementations at smaller batch sizes and require - # a larger workspace, so the first pass grows the workspace to its - # maximum size. The second pass runs the final per-shape warmup and - # captures without resizing the workspace. with self.cuda_graph_runner.allow_capture(): - self.cuda_graph_runner.is_warmup_only = True - try: - self._run_cuda_graph_warmup(resource_manager) - finally: - self.cuda_graph_runner.is_warmup_only = False - self.cuda_graph_runner.padding_dummy_requests = {} self._run_cuda_graph_warmup(resource_manager) log_mem_snapshot("warmup/after_cuda_graph_capture") - # Pre-compile DeepGEMM paged_mqa_logits_metadata for every 32-aligned - # batch bucket the runtime can produce (max_batch_size scaled by the - # MTP / DSL expansion factor when applicable). CUDA-graph warmup only - # exercises the batch sizes in cuda_graph_batch_sizes, which round - # up to a subset of buckets; any inference iter whose - # context_lens.size(0) lands on an uncovered bucket triggers an - # nvcc-driven JIT compile (~3s stall inside _prepare_inputs) on - # first touch. Pre-touching every bucket funnels that cost into - # warmup. No-op on non-DSA models. - self._warmup_dg_paged_mqa_logits_metadata() - log_mem_snapshot("warmup/after_dg_paged_mqa_logits_metadata") if can_run_general_warmup: # Pre-populate the memory pool with max-shape allocations to reduce # fragmentation at runtime. @@ -1209,102 +1119,6 @@ def warmup(self, resource_manager: ResourceManager) -> None: self._general_warmup(resource_manager, warmup_requests_configs) log_mem_snapshot("warmup/after_memory_pool_prepop") - def _warmup_dg_paged_mqa_logits_metadata(self) -> None: - """Pre-compile DeepGEMM's `get_paged_mqa_logits_metadata` helper for - every 32-aligned batch bucket the runtime can produce. - - DSA's `Indexer.prepare_scheduler_metadata` calls - `deep_gemm.get_paged_mqa_logits_metadata(context_lens, block_kv, - num_sms)` inside `_prepare_inputs` every iteration. The underlying - kernel is templated on `` - where `kAlignedBatchSize = align(context_lens.size(0), 32)` and - `split_kv` / `num_sms` are fixed for a given (block_kv, device). - deep_gemm's Python-side JIT compiles a fresh cubin (spawning - nvcc/cicc/ptxas, ~3s on GB300) the first time each `aligned_bs` - is requested. CUDA-graph warmup exercises only the batch sizes in - `cuda_graph_batch_sizes`, which round up to a subset of the 32- - aligned buckets; every uncovered bucket that the inference - workload later touches produces a 3s stall on that iteration. - Pre-touching every bucket here funnels those compiles into the - deterministic warmup phase. - - `context_lens.size(0)` is not always `num_generations`. For MTP - with `use_expanded_buffers_for_mtp=True` the expanded call passes - `num_generations * (1 + max_draft_tokens)`. For DSL expansion the - call passes `num_generations * dsl_expand_factor`, where - `dsl_expand_factor = next_n // eff` (`eff in kernel_atoms`, see - `_pick_dsl_expand` in `dsa.py`); its worst case is - `next_n = 1 + max_draft_tokens` when `eff == 1`. Reading the - current `dsl_expand_factor` off the metadata would under-estimate - the eventual max (it defaults to 1 before any prepare() has run, - and per-iter picks can differ across iters when CUDA graph is - off), so we use the static upper bound `1 + max_draft_tokens` - for both expansion paths. Bucket range is also scaled by - `max_beam_width` as a defense-in-depth ceiling for future beam - support (no-op today — DSA does not use beam). No-op on non-DSA - models. - - Best-effort: per-bucket JIT failures are logged and skipped so a - single broken bucket does not abort PyExecutor startup. - """ - attn_meta = getattr(self, "attn_metadata", None) - if attn_meta is None: - return - try: - from tensorrt_llm._torch.attention_backend.sparse.dsa import ( - _DG_SCHEDULE_BLOCK_KV, DSAtrtllmAttentionMetadata) - except ImportError: - return - if not isinstance(attn_meta, DSAtrtllmAttentionMetadata): - return - try: - from tensorrt_llm.deep_gemm import get_paged_mqa_logits_metadata - except ImportError: - logger.info( - "[DG warmup] deep_gemm.get_paged_mqa_logits_metadata not " - "available; skipping paged_mqa_logits_metadata prewarm.") - return - - num_sms = attn_meta.num_sms - max_bs = max(1, int(self.batch_size)) - beam_width = max(1, int(getattr(self, "max_beam_width", 1) or 1)) - # Static upper bound on the row-count multiplier applied to - # `context_lens`. Both MTP-expanded and DSL-expanded call sites - # are bounded above by `(1 + max_draft_tokens)`; see the - # docstring for why we don't read the runtime `dsl_expand_factor` - # here. - max_draft_tokens = int(getattr(attn_meta, "max_draft_tokens", 0) or 0) - expands_batch = (getattr(attn_meta, "use_expanded_buffers_for_mtp", - False) - or getattr(attn_meta, "expand_for_dsl", False)) - expand_factor = 1 + max_draft_tokens if expands_batch else 1 - max_aligned = ((max_bs * beam_width * expand_factor + 31) // 32) * 32 - buckets = list(range(32, max_aligned + 32, 32)) - logger.info(f"[DG warmup] Pre-compiling paged_mqa_logits_metadata for " - f"{len(buckets)} aligned batch buckets up to {max_aligned} " - f"(block_kv={_DG_SCHEDULE_BLOCK_KV}, num_sms={num_sms}, " - f"max_bs={max_bs}, beam_width={beam_width}, " - f"expand_factor={expand_factor})") - for aligned_bs in buckets: - # Kernel scans `context_lens` and prefix-sums schedules; a - # zero-filled 2D tensor of shape (aligned_bs, 1) is enough to - # trigger dispatch and compile — the metadata output is - # discarded. - dummy = torch.zeros(aligned_bs, 1, dtype=torch.int32, device="cuda") - try: - _ = get_paged_mqa_logits_metadata(dummy, _DG_SCHEDULE_BLOCK_KV, - num_sms) - except RuntimeError as e: - # Narrow to RuntimeError so signature drifts in - # get_paged_mqa_logits_metadata (TypeError / ValueError) - # surface loudly instead of silently degrading perf. - logger.warning( - f"[DG warmup] paged_mqa_logits_metadata prewarm failed " - f"for aligned_bs={aligned_bs} " - f"(block_kv={_DG_SCHEDULE_BLOCK_KV}, num_sms={num_sms}); " - f"skipping bucket. {type(e).__name__}: {e}") - torch.cuda.synchronize() - def _general_warmup(self, resource_manager: ResourceManager, warmup_requests_configs: List[Tuple[int, int]]): """ @@ -1522,145 +1336,6 @@ def _run_autotuner_warmup(self, resource_manager: ResourceManager): clear_memory_buffers() torch.cuda.empty_cache() - def _run_mamba_hybrid_warmup(self, resource_manager: ResourceManager): - """Pre-JIT the Mamba SSD multi-seq + HAS_INITSTATES=True Triton kernels. - - Mamba hybrid models (e.g. Nemotron 3 Super 120B, Nemotron-Nano-12B-v2) - skip ``_general_warmup`` because ``can_run_general_warmup`` is False - when the KV cache manager is a ``MambaHybridCacheManager``. The default - ``_run_autotuner_warmup`` then issues a single ``least_requests=True`` - prefill = 1 sequence with ``num_cached_tokens_per_seq = 0``, which only - compiles the ``num_seqs == 1`` / ``HAS_INITSTATES=False`` variants of - the SSD kernels. The first real serve iteration with chunked prefill - and multiple context requests then triggers autotune of the missing - variants mid-inference, producing a ~30 s stall / large P99 spike. - - This method runs two extra forward passes to compile those variants - during warmup: - - 1. ``least_requests=False`` — splits ``curr_max_num_tokens`` into many - short sequences, forcing the multi-seq path of - ``cu_seqlens_to_chunk_indices_offsets_triton`` and its - ``_cu_seqlens_triton_kernel``. - 2. ``least_requests=False`` inside - ``Mamba2Metadata.force_initial_states_for_warmup()`` — same as (1) - plus the ``HAS_INITSTATES=True`` variants of - ``_state_passing_fwd_kernel``, ``_chunk_scan_fwd_kernel``, and - ``_chunk_state_varlen_kernel``. - - Runs regardless of ``enable_autotuner``. Wraps in ``autotune()`` when - the autotuner is enabled so op-level (M,N,K) caches also get primed - for these shapes. Set ``TLLM_MAMBA_MULTISEQ_WARMUP=0`` to disable. - """ - if os.environ.get("TLLM_MAMBA_MULTISEQ_WARMUP", "1") != "1": - return - kv_cache_manager = resource_manager.get_resource_manager( - self.kv_cache_manager_key) - if kv_cache_manager is None or not isinstance(kv_cache_manager, - MambaHybridCacheManager): - return - - token_num_upper_bound = min(self.max_num_tokens, - self.batch_size * (self.max_seq_len - 1)) - curr_max_num_tokens = kv_cache_manager.get_num_available_tokens( - token_num_upper_bound=token_num_upper_bound, - max_num_draft_tokens=self.original_max_draft_len) - if curr_max_num_tokens < 4: - return - - # Cap the multi-seq warmup token count so we don't fill the KV cache - # to the brim. The autotuner warmup that ran just before this uses - # ``least_requests=True`` (few long sequences) which fits comfortably - # even when ``curr_max_num_tokens`` is close to the block ceiling. - # ``least_requests=False`` instead spreads the token budget across - # ``batch_size`` short sequences; when each sequence's length lands - # exactly on a block boundary AND the KV cache has ``num_extra_kv_tokens`` - # or ``num_extra_decoding_steps`` > 0 (e.g. spec decoding cases), - # ``add_token`` needs to allocate one extra block per sequence, which - # ``_create_warmup_request``'s ``blocks_to_use`` estimate doesn't - # account for. On a small KV pool (e.g. Qwen3.5 hybrid with DFlash spec - # decoding on a single H100: 259 blocks total, ``max_num_tokens=8192`` - # nearly saturates it), that extra per-sequence block overflows the - # pool and crashes with "Can't allocate new blocks for window size N". - # The point of this warmup is only to trigger ``num_seqs > 1`` + - # ``HAS_INITSTATES=True`` kernel variants — a modest token budget - # achieves that with plenty of block headroom. - WARMUP_TOKEN_CAP = 4096 - capped_num_tokens = min(curr_max_num_tokens, WARMUP_TOKEN_CAP) - - logger.info( - "Running Mamba hybrid warmup (multi-seq + HAS_INITSTATES=True)...") - - # (num_tokens, num_gen_requests, least_requests, force_initstates) - mamba_warmup_shapes = [ - (capped_num_tokens, 0, False, False), - (capped_num_tokens, 0, False, True), - ] - - autotuner_enabled = self.llm_args.enable_autotuner - cache_path = os.environ.get("TLLM_AUTOTUNER_CACHE_PATH", None) - autotune_ctx = (autotune(cache_path=cache_path) - if autotuner_enabled else contextlib.nullcontext()) - - with self.no_cuda_graph(), autotune_ctx: - for (num_tokens_i, num_gen_requests_i, least_req_i, - force_init_i) in mamba_warmup_shapes: - init_ctx = (Mamba2Metadata.force_initial_states_for_warmup() - if force_init_i else contextlib.nullcontext()) - try: - with init_ctx: - warmup_request = self._create_warmup_request( - resource_manager, - num_tokens_i, - num_gen_requests_i, - least_requests=least_req_i) - with self._release_batch_context( - warmup_request, resource_manager) as batch: - if batch is None and self.mapping.tp_size <= 1: - continue - self._assert_all_tp_ranks_have_warmup_batch( - batch, num_tokens_i) - if batch is None: - continue - spec_resource_manager = resource_manager.get_resource_manager( - ResourceManagerType.SPEC_RESOURCE_MANAGER) - if self.is_draft_model and isinstance( - spec_resource_manager, - Eagle3ResourceManager): - spec_resource_manager.is_first_draft = True - - self.forward(batch, - new_tensors_device=None, - resource_manager=resource_manager) - - if autotuner_enabled: - AutoTuner.get().cache_pp_recv() - AutoTuner.get().cache_pp_send() - AutoTuner.get().clean_pp_flag() - - torch.cuda.synchronize() - except (torch.OutOfMemoryError, RuntimeError) as e: - # Catch both OOM and RuntimeError. C++ KV cache block - # allocation ("Can't allocate new blocks for window size - # N") surfaces as RuntimeError, not torch.OutOfMemoryError. - # This warmup is a pure perf optimization: if a shape - # doesn't fit for any reason, log and skip; the model then - # JIT-compiles the missing kernel variants lazily on the - # first real request (i.e. the pre-fix behavior). - logger.warning(f"Mamba hybrid warmup skipped for shape " - f"num_tokens={num_tokens_i}, " - f"num_gen_requests={num_gen_requests_i}, " - f"force_initstates={force_init_i}: " - f"{type(e).__name__}: {e}") - # Mirror _general_warmup_impl: an OOM between dispatch() - # and combine() leaves MoE A2A state in ``dispatched``, - # tripping ``dispatch called twice`` on the next forward. - self._reset_moe_alltoall_state() - torch.cuda.empty_cache() - - clear_memory_buffers() - torch.cuda.empty_cache() - def _compute_dynamic_draft_len_mapping(self) -> Optional[Dict[int, int]]: """Compute graph_bs → draft_len mapping for dynamic draft length feature. @@ -1761,27 +1436,23 @@ def _get_graphs_to_capture( for draft_len in draft_lengths] def _run_cuda_graph_warmup(self, resource_manager: ResourceManager): - """Warm up or capture CUDA graphs for the configured graph shapes.""" + """Captures CUDA graphs for various batch sizes and draft lengths.""" if not (self.cuda_graph_runner.enabled or self._torch_compile_piecewise_cuda_graph): return self._capture_generation_cuda_graphs(resource_manager) - # Piecewise graphs have separate capture machinery and do not use the - # whole-model attention workspace. Capture them only on the second pass. - if not self.cuda_graph_runner.is_warmup_only: - self._capture_piecewise_cuda_graphs(resource_manager) + self._capture_piecewise_cuda_graphs(resource_manager) def _capture_generation_cuda_graphs(self, resource_manager: ResourceManager): - """Warm up or capture pure-generation CUDA graph shapes.""" + """Captures CUDA graphs for pure generation steps.""" if not self.cuda_graph_runner.enabled: return - operation = ("warmup" - if self.cuda_graph_runner.is_warmup_only else "capture") - logger.info(f"Running CUDA graph {operation} for " - f"{len(self._cuda_graph_batch_sizes)} batch sizes.") + logger.info( + f"Creating CUDA graph instances for {len(self._cuda_graph_batch_sizes)} batch sizes." + ) spec_resource_manager = resource_manager.get_resource_manager( ResourceManagerType.SPEC_RESOURCE_MANAGER) @@ -1789,7 +1460,7 @@ def _capture_generation_cuda_graphs(self, cuda_graph_batch_sizes = sorted(self._cuda_graph_batch_sizes, reverse=True) - # Determine which graph shapes to process. + # Determine which graphs to capture graphs_to_capture = self._get_graphs_to_capture(cuda_graph_batch_sizes, spec_resource_manager) graphs_to_capture = sorted(graphs_to_capture, reverse=True) @@ -1916,7 +1587,7 @@ def _run_capture_pass(force_non_greedy: bool, label: str) -> None: f"not enough KV cache space.") continue logger.info( - f"Run generation-only CUDA graph {operation} ({label}) " + f"Run generation-only CUDA graph warmup ({label}) " f"for batch size={bs}, draft_len={draft_len}, " f"max_seq_len={max_seq_len}") self.enable_spec_decode = draft_len > 0 or self.is_draft_model or ( @@ -2872,55 +2543,15 @@ def _get_all_rank_ctx_requests(self, num_ctx_requests: int): return list(self.dist.tp_allgather(num_ctx_requests)) return None - def _sync_group_all_greedy_sample(self, spec_metadata) -> None: - """All-gather the per-rank greedy flags and store the group AND. - - Why the sampling-path choice must be group-uniform under - ADP + LM-head TP is documented on the anchor, - ``SpecMetadata.group_all_greedy_sample``. Local contract: called once - per iteration, right after ``update_is_all_greedy_sample`` and BEFORE - the CUDA graph key is built. The gate is pure config (identical on - every rank), so ranks also agree on whether the exchange happens; the - gather spans the whole TP group, a superset of any LM-head-TP - subgroup. A dedicated host all-gather rather than a piggyback on the - ``all_rank_num_tokens`` exchange, which runs in ``_prepare_inputs`` -- - after the graph key, too late for the key to see the synced value. - """ - # enable_lm_head_tp_in_adp implies enable_attention_dp (asserted in - # Mapping.__init__), so ADP needs no separate check here. - if not (self.mapping.enable_lm_head_tp_in_adp - and spec_metadata.use_rejection_sampling): - return - local_flag = bool(spec_metadata.is_all_greedy_sample) - all_flags = self.dist.tp_allgather(local_flag) - spec_metadata.group_all_greedy_sample = all(all_flags) - # Also overwrite the live flag directly: this iteration's scan already - # ran (update_is_all_greedy_sample just returned) and the CUDA graph - # key reads the flag next -- the stored override only takes effect on - # the NEXT rescan (populate), which is after key selection. - spec_metadata.is_all_greedy_sample = spec_metadata.group_all_greedy_sample - def _set_spec_metadata_all_rank_num_tokens( - self, - spec_metadata: SpecMetadata, + self, spec_metadata: SpecMetadata, spec_all_rank_num_tokens: List[int], - all_rank_num_seqs: List[int], - all_rank_num_gens: Optional[List[int]] = None) -> None: + all_rank_num_seqs: List[int]) -> None: # Eagle3 / MTP-eagle one-model use subseq_all_rank_num_tokens for # draft loop iterations i>0 (per-sequence counts, since each # sequence contributes one token per iteration). spec_metadata.all_rank_num_tokens = spec_all_rank_num_tokens spec_metadata.all_rank_num_seqs = all_rank_num_seqs - # DSpark can draft only after the target processes the current bonus token, - # because it consumes captured target-layer hidden states for that token. - # Prefill computes hidden states for prompt tokens; the first generated token - # is sampled from the last prompt logits and has not itself passed through the - # target layers. Thus context requests seed the rolling window but do not run - # the draft. On mixed steps, num_seqs therefore over-counts the draft MoE - # workload; gen-only per-rank counts keep the FUSED_COMM (DeepGEMM MegaMoE) - # chunk loop identical across EP ranks. - if all_rank_num_gens is not None: - spec_metadata.all_rank_num_gens = all_rank_num_gens if (spec_metadata.spec_dec_mode.is_mtp_eagle_one_model() or spec_metadata.spec_dec_mode.is_eagle3_one_model()): spec_metadata.subseq_all_rank_num_tokens = all_rank_num_seqs @@ -3070,45 +2701,12 @@ def _prepare_enc_dec_cross_attn_inputs( skip_cross_kv_projection = True if attn_metadata.is_cuda_graph and attn_metadata.has_cross_sub_metadata: - # Fast path for stable CUDA-graph generation steps: the encoder - # KV lengths (kv_lens_cuda) and the frozen prompt lengths - # (prompt_lens_cuda) are identical across all generation steps - # for a fixed batch. Skip the expensive torch.tensor() allocations - # and H2D copies inside prepare() when nothing has changed. - is_stable_gen_step = ( - new_encoder_tokens == 0 # pure generation, no new cross-KV - and self._cross_attn_stable_cached_tokens - == encoder_num_cached_tokens_per_seq - and self._cross_attn_stable_request_ids - == attn_metadata.request_ids # same batch and row order + cross_attn_metadata = attn_metadata.update_cross_metadata( + encoder_seq_lens=encoder_seq_lens, + cross_kv_cache_manager=cross_kv_cache_manager, + encoder_num_cached_tokens_per_seq= + encoder_num_cached_tokens_per_seq, ) - if is_stable_gen_step: - cross_attn_metadata = attn_metadata.cross - # Only refresh the decoder-side Python references that the - # kernel reads; these are pointer-level updates with no alloc. - cross_attn_metadata._seq_lens = attn_metadata.seq_lens - cross_attn_metadata._seq_lens_cuda = attn_metadata.seq_lens_cuda - cross_attn_metadata.prompt_lens = attn_metadata.prompt_lens - cross_attn_metadata.request_ids = attn_metadata.request_ids - cross_attn_metadata.num_contexts = attn_metadata.num_contexts - else: - cross_attn_metadata = attn_metadata.update_cross_metadata( - encoder_seq_lens=encoder_seq_lens, - cross_kv_cache_manager=cross_kv_cache_manager, - encoder_num_cached_tokens_per_seq= - encoder_num_cached_tokens_per_seq, - ) - cross_attn_metadata.prepare() - if new_encoder_tokens == 0: - # Record this stable state for future fast-path use. - self._cross_attn_stable_cached_tokens = list( - encoder_num_cached_tokens_per_seq) - self._cross_attn_stable_request_ids = list( - attn_metadata.request_ids) - else: - # Batch changed (new encoder request); reset cache. - self._cross_attn_stable_cached_tokens = None - self._cross_attn_stable_request_ids = None else: cross_attn_metadata = attn_metadata.create_cross_metadata( cross_kv_cache_manager=cross_kv_cache_manager, @@ -3118,18 +2716,7 @@ def _prepare_enc_dec_cross_attn_inputs( ) if attn_metadata.is_cuda_graph: attn_metadata.cross = cross_attn_metadata - if new_encoder_tokens == 0: - self._cross_attn_stable_cached_tokens = list( - encoder_num_cached_tokens_per_seq) - self._cross_attn_stable_request_ids = list( - attn_metadata.request_ids) - else: - self._cross_attn_stable_cached_tokens = None - self._cross_attn_stable_request_ids = None - else: - self._cross_attn_stable_cached_tokens = None - self._cross_attn_stable_request_ids = None - cross_attn_metadata.prepare() + cross_attn_metadata.prepare() return { "encoder_hidden_states": packed_encoder_hidden_states, @@ -3293,14 +2880,12 @@ def _prepare_incremental_update_metadata( # Handle distributed spec metadata if enable_attention_dp: sequence_lengths = spec_metadata.seq_lens - all_rank_num_tokens = self.dist.tp_cp_allgather([ - spec_metadata.num_tokens, - len(sequence_lengths), attn_metadata.num_generations - ]) + all_rank_num_tokens = self.dist.tp_cp_allgather( + [spec_metadata.num_tokens, + len(sequence_lengths)]) self._set_spec_metadata_all_rank_num_tokens( spec_metadata, [item[0] for item in all_rank_num_tokens], - [item[1] for item in all_rank_num_tokens], - [item[2] for item in all_rank_num_tokens]) + [item[1] for item in all_rank_num_tokens]) # Set iteration states - batch dictionary updates self.iter_states.update({ @@ -3554,10 +3139,6 @@ def _apply_incremental_update_target( else: prompt_lengths[idx] = request.py_prompt_len - # Physical KV length for the kernels: subtract the tokens a - # KV-cache compression manager evicted (tracked on the request, - # 0 without compression). Position ids and the cached_tokens stat - # keep the logical count. if request.is_dummy: num_cached_tokens_per_seq[idx] = base_past_seen request.cached_tokens = base_past_seen @@ -3568,11 +3149,9 @@ def _apply_incremental_update_target( num_previous_batch] = request.py_batch_idx num_previous_batch += 1 - request.cached_tokens = (base_past_seen + - num_tokens_per_extend_request) - num_cached_tokens_per_seq[idx] = ( - base_past_seen + num_tokens_per_extend_request - - request.py_num_compressed_tokens) + num_cached_tokens_per_seq[ + idx] = base_past_seen + num_tokens_per_extend_request + request.cached_tokens = num_cached_tokens_per_seq[idx].item() request.py_batch_idx = request.py_seq_slot @@ -3660,143 +3239,6 @@ def _apply_incremental_update_target( return inputs, self.gather_ids_cuda[:num_generation_tokens] - def _can_use_steady_gen_fast_prepare( - self, scheduled_requests: ScheduledRequests, - new_tokens_device: Optional[torch.Tensor], - next_draft_tokens_device: Optional[torch.Tensor], - spec_metadata: Optional[SpecMetadata]) -> bool: - """Check whether the cached steady-state generation prepare applies. - - The cache is only recorded by a full _prepare_tp_inputs pass whose - batch consisted purely of non-dummy generation requests that all had - a previous overlap-scheduler tensor (see the recording site), so the - per-step check only needs to confirm the dynamic conditions: still a - generation-only batch with the exact same requests in the same order. - """ - cache = self._steady_gen_cache - if cache is None or self.is_warmup: - return False - if new_tokens_device is None or next_draft_tokens_device is not None \ - or spec_metadata is not None: - return False - if scheduled_requests.num_context_requests > 0: - return False - generation_requests = scheduled_requests.generation_requests - if len(generation_requests) != cache['num_requests']: - return False - return cache['request_ids'] == [ - request.py_request_id for request in generation_requests - ] - - @nvtx_range("_apply_steady_gen_fast_prepare") - def _apply_steady_gen_fast_prepare( - self, kv_cache_manager: Union[KVCacheManager, KVCacheManagerV2], - attn_metadata: AttentionMetadata, - new_tensors_device: SampleStateTensors, - resource_manager: Optional[ResourceManager]): - """Prepare inputs for an unchanged generation-only batch. - - Every request advanced by exactly one committed token since the last - prepare, so instead of re-walking the batch in Python this advances - the cached positions in place (device position buffer plus a pinned - host counter), reuses the seq-slot buffer already on device, and - refreshes only the per-step metadata. For mrope models (recorded only - for batches with no actual mrope work) the (3,1,N) broadcast buffer - the model reads is the one advanced. - """ - cache = self._steady_gen_cache - num_requests = cache['num_requests'] - - # Positions and cached-token counts are the same values in this - # regime; advance both by one. The device-side position buffer is - # advanced in place: it still holds the previous step's positions - # because only _prepare_tp_inputs writes it and the cache validity - # invariant guarantees the previous pass wrote these same rows. This - # avoids reusing a mutated pinned buffer as the source of an async - # H2D whose previous-step copy may still be pending under the overlap - # scheduler (the nvbug 6293536 hazard class; see - # KVCacheManager._stage_block_offsets_for_copy). The pinned buffer is - # host-side bookkeeping only. - use_mrope = cache['use_mrope'] - positions = self._steady_gen_positions_pinned[:num_requests] - positions.add_(1) - if use_mrope: - # Text-only batch on an mrope model: the recording pass broadcast - # the scalar positions onto all three axes of the (3,1,N) buffer, - # which is what the model (and any captured CUDA graph) reads, so - # advance it in place. position_ids_cuda is reseeded by the next - # full pass. - self.mrope_position_ids_cuda[:, :, :num_requests].add_(1) - else: - self.position_ids_cuda[:num_requests].add_(1) - num_cached_tokens_per_seq = positions.tolist() - - # Gather this step's input tokens from the previous iteration's device - # sample buffer; the seq-slot indices in previous_batch_indices_cuda - # are unchanged since the last full pass. - previous_slots = self.previous_batch_indices_cuda[:num_requests] - new_tokens = new_tensors_device.new_tokens[:1, previous_slots, :self. - max_beam_width] - self.input_ids_cuda[:num_requests * self.max_beam_width].copy_( - new_tokens.flatten(), non_blocking=True) - - if not attn_metadata.is_cuda_graph: - attn_metadata.seq_lens = cache['seq_lens_ones'] - attn_metadata.beam_width = 1 - attn_metadata.request_ids = cache['request_ids'] - attn_metadata.prompt_lens = cache['prompt_lens'] - attn_metadata.num_contexts = 0 - attn_metadata.num_chunked_ctx_requests = 0 - attn_metadata.kv_cache_params = KVCacheParams( - use_cache=True, - num_cached_tokens_per_seq=num_cached_tokens_per_seq, - num_extra_kv_tokens=get_num_extra_kv_tokens(None)) - attn_metadata.kv_cache_manager = kv_cache_manager - if hasattr(self.model.model_config.pretrained_config, 'chunk_size'): - attn_metadata.mamba_chunk_size = \ - self.model.model_config.pretrained_config.chunk_size - with nvtx_range("steady_gen_metadata_prepare"): - attn_metadata.prepare() - - attn_all_rank_num_tokens = self._get_all_rank_num_tokens(attn_metadata) - padded_num_tokens, can_run_piecewise_cuda_graph, attn_all_rank_num_tokens = \ - self._get_padding_params(num_requests, 0, attn_all_rank_num_tokens) - set_per_request_piecewise_cuda_graph_flag(can_run_piecewise_cuda_graph) - attn_metadata.padded_num_tokens = ( - padded_num_tokens if padded_num_tokens != num_requests else None) - virtual_num_tokens = num_requests - if attn_metadata.padded_num_tokens is not None: - self.input_ids_cuda[num_requests:padded_num_tokens].fill_(0) - # Zero-fill the padding tail of whichever position layout the - # model consumes, matching the full pass. - if use_mrope: - self.mrope_position_ids_cuda[:, :, num_requests: - padded_num_tokens].fill_(0) - else: - self.position_ids_cuda[num_requests:padded_num_tokens].fill_(0) - virtual_num_tokens = padded_num_tokens - - self.iter_states['num_ctx_requests'] = 0 - self.iter_states['num_ctx_tokens'] = 0 - self.iter_states['num_generation_tokens'] = num_requests - self.iter_states['cached_kv_tokens'] = sum(num_cached_tokens_per_seq) - - if use_mrope: - final_position_ids = \ - self.mrope_position_ids_cuda[:, :, :virtual_num_tokens] - else: - final_position_ids = \ - self.position_ids_cuda[:virtual_num_tokens].unsqueeze(0) - inputs = { - 'attn_metadata': attn_metadata, - 'input_ids': self.input_ids_cuda[:virtual_num_tokens], - 'position_ids': final_position_ids, - 'inputs_embeds': None, - 'multimodal_params': [], - 'resource_manager': resource_manager, - } - return inputs, None - def _prepare_tp_inputs( self, scheduled_requests: ScheduledRequests, @@ -3833,28 +3275,12 @@ def _prepare_tp_inputs( if self._can_use_incremental_update(scheduled_requests, new_tokens_device, next_draft_tokens_device): - # Spec engines never record the steady-gen cache, but invalidate - # defensively so the two fast paths can never interleave if the - # gates ever evolve. - self._steady_gen_cache = None return self._apply_incremental_update( scheduled_requests, kv_cache_manager, attn_metadata, spec_metadata, new_tensors_device, cache_indirection_buffer, num_accepted_tokens_device, req_id_to_old_request, resource_manager) - if self._can_use_steady_gen_fast_prepare(scheduled_requests, - new_tokens_device, - next_draft_tokens_device, - spec_metadata): - return self._apply_steady_gen_fast_prepare(kv_cache_manager, - attn_metadata, - new_tensors_device, - resource_manager) - # Any full pass invalidates the steady-state cache; it is re-recorded - # at the end of this pass when the batch qualifies. - self._steady_gen_cache = None - # Hoist self.use_mrope to a function-scope local so the per-request / # per-context-request mrope branches use LOAD_FAST instead of LOAD_ATTR. _use_mrope = self.use_mrope @@ -3870,10 +3296,6 @@ def _prepare_tp_inputs( draft_tokens = [] draft_lens = [] gen_request_seq_slots = [] # per generation request - # One-model rejection: slots of gen requests that produced 0 real draft - # tokens this step (marked in _handle_dynamic_draft_len); their stale - # draft_probs rows are one-hot'd after spec_metadata.prepare(). - padding_gen_slots = [] multimodal_params_list = [] mrope_position_ids = [ ] # (start_idx, end_idx, (3,1,L) mrope_pos_ids) per multimodal request @@ -3971,9 +3393,8 @@ def append_cross_attention_state(request: LlmRequest, py_request_id] = request.py_num_accepted_draft_tokens_indices prompt_lengths.append(len(prompt_tokens)) past_seen_token_num = begin_compute - num_cached_tokens_per_seq.append(past_seen_token_num - - request.py_num_compressed_tokens) - request.cached_tokens = past_seen_token_num + num_cached_tokens_per_seq.append(past_seen_token_num) + request.cached_tokens = num_cached_tokens_per_seq[-1] append_cross_attention_state( request, project_encoder_output=not request.py_skip_cross_kv_projection @@ -4000,11 +3421,8 @@ def append_cross_attention_state(request: LlmRequest, ) multimodal_params = MultimodalParams( - multimodal_input=_build_request_multimodal_input( - request, self._mm_encoder_cache_enabled), multimodal_data=request.py_multimodal_data, multimodal_runtime=py_multimodal_runtime, - mm_item_order=getattr(request, "py_mm_item_order", None), input_ids_start_offset=context_start_idx) # Transfer any cross-iter MM encoder prefetch event stamped on the request onto the # freshly-built MultimodalParams. The downstream consume site reads it from the wrapper, @@ -4123,10 +3541,6 @@ def append_cross_attention_state(request: LlmRequest, self.runtime_draft_len) runtime_draft_token_buffer_width = runtime_tokens_per_gen_step - 1 for request in extend_requests: - if getattr(request, "py_needs_onehot_draft_probs", False): - if request.py_seq_slot is not None: - padding_gen_slots.append(request.py_seq_slot) - request.py_needs_onehot_draft_probs = False # consume once request_ids.append(request.py_request_id) request_accepted_path[ request. @@ -4165,9 +3579,8 @@ def append_cross_attention_state(request: LlmRequest, list( range(past_seen_token_num, past_seen_token_num + 1 + num_draft_tokens))) - num_cached_tokens_per_seq.append( - past_seen_token_num - request.py_num_compressed_tokens) - request.cached_tokens = past_seen_token_num + num_cached_tokens_per_seq.append(past_seen_token_num) + request.cached_tokens = num_cached_tokens_per_seq[-1] # update batch index request.py_batch_idx = request.py_seq_slot else: @@ -4194,11 +3607,9 @@ def append_cross_attention_state(request: LlmRequest, previous_pos_indices.extend([previous_batch_idx] * runtime_tokens_per_gen_step) - num_cached_tokens_per_seq.append( - past_seen_token_num + runtime_tokens_per_gen_step - - request.py_num_compressed_tokens) - request.cached_tokens = (past_seen_token_num + - runtime_tokens_per_gen_step) + num_cached_tokens_per_seq.append(past_seen_token_num + + runtime_tokens_per_gen_step) + request.cached_tokens = num_cached_tokens_per_seq[-1] if self.enable_spec_decode and spec_config.spec_dec_mode.extend_ctx( self.attn_backend) and spec_config.is_linear_tree: prompt_lengths.append(runtime_tokens_per_gen_step) @@ -4255,8 +3666,7 @@ def append_cross_attention_state(request: LlmRequest, py_request_id] = request.py_num_accepted_draft_tokens_indices prompt_lengths.append(request.py_prompt_len) past_seen_token_num = begin_compute - num_cached_tokens_per_seq.append(past_seen_token_num - - request.py_num_compressed_tokens) + num_cached_tokens_per_seq.append(past_seen_token_num) append_cross_attention_state(request, project_encoder_output=False) # update batch index @@ -4333,8 +3743,7 @@ def append_cross_attention_state(request: LlmRequest, request.cached_tokens = past_seen_token_num for beam in range(beam_width): position_ids.append(position_id) - num_cached_tokens_per_seq.append( - past_seen_token_num - request.py_num_compressed_tokens) + num_cached_tokens_per_seq.append(past_seen_token_num) prompt_lengths.append(request.py_prompt_len) gather_ids.append(len(position_ids) - 1) @@ -4777,12 +4186,6 @@ def previous_seq_slots_device(): if hasattr(self.model.model_config.pretrained_config, 'chunk_size'): attn_metadata.mamba_chunk_size = self.model.model_config.pretrained_config.chunk_size - # Some sparse backends (RocketKV) clamp - # kv_cache_params.num_cached_tokens_per_seq in place during prepare(), - # and KVCacheParams holds the list by reference. Snapshot the true - # pre-prepare counts so the steady-gen recording below stores values - # that the per-step prepare() can re-clamp from scratch. - num_cached_tokens_snapshot = list(num_cached_tokens_per_seq) attn_metadata.prepare() cross_attention_inputs = (self._prepare_enc_dec_cross_attn_inputs( cross_encoder_hidden_states, @@ -4875,22 +4278,15 @@ def previous_seq_slots_device(): spec_metadata.populate_sampling_params_for_one_model( scheduled_requests.all_requests()) spec_metadata.prepare() - # One-model rejection: one-hot the stale draft_probs rows of gen - # requests that produced no draft tokens this step, so the (possibly - # captured) rejection kernel reads a legal placeholder distribution. - spec_metadata.write_padding_onehot_draft_probs( - padding_gen_slots, self.runtime_draft_len) inputs['spec_metadata'] = spec_metadata if self.enable_attention_dp: - all_rank_num_tokens = self.dist.tp_cp_allgather([ - spec_metadata.num_tokens, - len(sequence_lengths), spec_metadata.num_generations - ]) + all_rank_num_tokens = self.dist.tp_cp_allgather( + [spec_metadata.num_tokens, + len(sequence_lengths)]) self._set_spec_metadata_all_rank_num_tokens( spec_metadata, [item[0] for item in all_rank_num_tokens], - [item[1] for item in all_rank_num_tokens], - [item[2] for item in all_rank_num_tokens]) + [item[1] for item in all_rank_num_tokens]) if mm_token_indices is not None: self._ship_multimodal_indices( @@ -4913,50 +4309,6 @@ def previous_seq_slots_device(): self.previous_request_ids = all_gen_request_ids self.has_previous_device_draft = next_draft_tokens_device is not None - # Record the steady-state generation cache when this pass handled - # purely non-dummy generation requests that all carried a previous - # overlap-scheduler tensor (previous_batch_len == _n_gen implies - # every request took that branch and none appended input_ids). - # While the batch composition holds, the next passes only need to - # advance positions by one and refresh per-step metadata. - # MRoPE models are supported only for batches with no actual mrope - # work (text-only requests, empty mrope lists below): the full - # pass routes use_mrope models through the (3,1,N) - # mrope_position_ids_cuda layout even then (to keep torch.compile - # guards stable), with all three axes equal to the scalar - # positions, so the fast path advances that buffer in place and - # returns the same layout (see _apply_steady_gen_fast_prepare). - if (self.spec_config is None and not self.is_draft_model - and spec_metadata is None and new_tokens_device is not None - and self.guided_decoder is None - and not self.enable_attention_dp and not mrope_position_ids - and not mrope_delta_write_seq_slots - and not mrope_delta_read_seq_slots - and not self.use_beam_search and self.max_beam_width == 1 - and not is_enc_dec and not _has_cp_helix - and num_ctx_requests == 0 and not extend_requests - and not first_draft_requests and _n_gen > 0 - and previous_batch_len == _n_gen and num_tokens == 0 - and not _has_any_multimodal_request - and not multimodal_params_list and not lora_params - and attn_metadata.padded_num_tokens is None - and self._get_position_id_offset() == 0): - self._steady_gen_positions_pinned[:_n_gen].copy_( - torch.as_tensor(num_cached_tokens_snapshot, - dtype=torch.int)) - self._steady_gen_cache = { - 'num_requests': - _n_gen, - 'request_ids': - all_gen_request_ids, - 'prompt_lens': - prompt_lengths, - 'seq_lens_ones': - maybe_pin_memory(torch.ones(_n_gen, dtype=torch.int)), - 'use_mrope': - _use_mrope, - } - return inputs, self.gather_ids_cuda[:len( gather_ids)] if self.enable_spec_decode else None @@ -4999,10 +4351,7 @@ def _prepare_tp_inputs_no_cache( # Multimodal if request.py_multimodal_data is not None: multimodal_params = MultimodalParams( - multimodal_input=_build_request_multimodal_input( - request, self._mm_encoder_cache_enabled), multimodal_data=request.py_multimodal_data, - mm_item_order=getattr(request, "py_mm_item_order", None), input_ids_start_offset=context_start_idx) multimodal_params.to_device("multimodal_data", "cuda", @@ -5130,15 +4479,14 @@ def _prepare_tp_inputs_no_cache( if spec_metadata is not None: all_rank_num_tokens = self.dist.tp_cp_allgather([ attn_metadata.num_tokens, spec_metadata.num_tokens, - len(sequence_lengths), spec_metadata.num_generations + len(sequence_lengths) ]) attn_metadata.all_rank_num_tokens = [ item[0] for item in all_rank_num_tokens ] self._set_spec_metadata_all_rank_num_tokens( spec_metadata, [item[1] for item in all_rank_num_tokens], - [item[2] for item in all_rank_num_tokens], - [item[3] for item in all_rank_num_tokens]) + [item[2] for item in all_rank_num_tokens]) else: all_rank_num_tokens = self.dist.tp_cp_allgather( attn_metadata.num_tokens) @@ -5802,17 +5150,7 @@ def warmup_encoder(self) -> None: torch.cuda.empty_cache() self._run_autotuner_warmup_encoder() - # Warm up every encoder graph shape before capturing any graph. Some - # attention kernels switch implementations at smaller shapes and need - # a larger workspace, so the first pass grows the workspace to its - # maximum size. The second pass runs the final per-shape warmup and - # captures without resizing the workspace. with self.encoder_cuda_graph_runner.allow_capture(): - self.encoder_cuda_graph_runner.is_warmup_only = True - try: - self._run_cuda_graph_warmup_encoder() - finally: - self.encoder_cuda_graph_runner.is_warmup_only = False self._run_cuda_graph_warmup_encoder() # Pre-populate the memory pool with max-shape allocations to reduce @@ -5862,14 +5200,14 @@ def _run_autotuner_warmup_encoder(self) -> None: AutoTuner.get().print_profiling_cache() def _run_cuda_graph_warmup_encoder(self) -> None: - """Warm up or capture whole-model encode-only CUDA graphs.""" + """Captures whole-model CUDA graphs for the encode-only path.""" if not self.encoder_cuda_graph_runner.enabled: return self._capture_encoder_cuda_graphs() def _capture_encoder_cuda_graphs(self) -> None: - """Warm up or capture encoder CUDA graphs for all feasible keys. + """Capture whole-model encoder CUDA graphs for all feasible keys. Feasibility filter (also used in source): nt >= prev_sl + bs (enough tokens for this sl bucket) @@ -5885,9 +5223,8 @@ def _capture_encoder_cuda_graphs(self) -> None: num_tokens_list = sorted(self._cuda_graph_num_tokens) seq_lens_list = sorted(self._cuda_graph_seq_lens) - operation = "warmup" if runner.is_warmup_only else "capture" - num_processed = 0 - logger.info(f"Running encoder CUDA graph {operation} ...") + num_captured = 0 + logger.info("Capturing encoder CUDA graphs ...") for bs in batch_sizes: if bs > self.batch_size: continue @@ -5906,14 +5243,13 @@ def _capture_encoder_cuda_graphs(self) -> None: if inputs is None: continue - logger.info(f"Encoder CUDA graph {operation}: " + logger.info(f"Encoder CUDA graph capture: " f"bs={bs}, nt={nt}, sl={sl}") self.encoder_forward(inputs) torch.cuda.synchronize() - num_processed += 1 + num_captured += 1 - logger.info(f"Completed encoder CUDA graph {operation} for " - f"{num_processed} graph shape(s).") + logger.info(f"Captured {num_captured} encoder CUDA graph(s).") @torch.inference_mode() @with_model_extra_attrs(lambda self: self.model.extra_attrs) @@ -5966,9 +5302,7 @@ def encoder_forward(self, inputs: Dict[str, Any], return self._forward_step(model_inputs, **forward_kwargs) - needs_capture = self.encoder_cuda_graph_runner.needs_capture( - key) - if needs_capture: + if self.encoder_cuda_graph_runner.needs_capture(key): def forward_fn( capture_inputs: Dict[str, Any]) -> Dict[str, Any]: @@ -5978,20 +5312,16 @@ def forward_fn( return self._forward_step(capture_inputs, **forward_kwargs) - capture_outputs = self.encoder_cuda_graph_runner.capture( + self.encoder_cuda_graph_runner.capture( key, forward_fn, { **model_inputs, "_forward_kwargs": forward_kwargs }) - if self.encoder_cuda_graph_runner.is_warmup_only: - graph_outputs = capture_outputs - else: - with MoeLoadBalancerIterContext(moe_load_balancer): - graph_outputs = self.encoder_cuda_graph_runner.replay( - key, { - **model_inputs, "_forward_kwargs": - forward_kwargs - }) + with MoeLoadBalancerIterContext(moe_load_balancer): + graph_outputs = self.encoder_cuda_graph_runner.replay( + key, { + **model_inputs, "_forward_kwargs": forward_kwargs + }) # Return a clone to avoid sharing data_ptr with the static buffers. outputs = {} @@ -6116,7 +5446,6 @@ def forward(self, if spec_metadata is not None: spec_metadata.update_is_all_greedy_sample( padded_requests.all_requests()) - self._sync_group_all_greedy_sample(spec_metadata) maybe_attn_metadata, maybe_spec_metadata, key = self.cuda_graph_runner.maybe_get_cuda_graph( padded_requests, @@ -6166,8 +5495,7 @@ def forward(self, gather_ids=gather_ids, gather_context_logits=gather_context_logits) else: - needs_capture = self.cuda_graph_runner.needs_capture(key) - if needs_capture: + if self.cuda_graph_runner.needs_capture(key): def capture_forward_fn(inputs: Dict[str, Any]): with MoeLoadBalancerIterContext(moe_load_balancer): @@ -6179,16 +5507,13 @@ def capture_forward_fn(inputs: Dict[str, Any]): def capture_postprocess_fn(inputs: Dict[str, Any]): self._postprocess_inputs(inputs) - capture_outputs = self.cuda_graph_runner.capture( + self.cuda_graph_runner.capture( key, capture_forward_fn, inputs, enable_spec_decode=self.enable_spec_decode, postprocess_fn=capture_postprocess_fn) - if self.cuda_graph_runner.is_warmup_only: - outputs = capture_outputs - elif needs_capture: # Pre-replay: set DSA slot mappings for current batch's draft cache (fixes 2nd warmup) saved_draft = prepare_attn_metadata_for_draft_replay( attn_metadata, draft_kv_cache_manager) @@ -6454,7 +5779,7 @@ def _prepare_tp_inputs_encoder( encoder_attn_metadata.num_contexts = len(encoder_requests) encoder_attn_metadata.max_seq_len = self.max_seq_len encoder_attn_metadata.request_ids = request_ids - encoder_attn_metadata.prepare_encoder_only() + encoder_attn_metadata.prepare() encoder_input_ids_t = torch.tensor(encoder_input_ids, dtype=torch.int, diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index f2284e06bb46..7596c2a68291 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -13,17 +13,13 @@ from tensorrt_llm._torch.models.checkpoints.base_checkpoint_loader import ( AutoCheckpointMapper, BaseCheckpointLoader) from tensorrt_llm._torch.weight_sharing import ( - ArtifactIdentity, IdentityCheckPolicy, PostTransformFeature, - PostTransformProfile, PostTransformProfileRegistry, - PostTransformQualificationDecision, PostTransformTransferScope, - SourceIdentity, check_weight_sharing_compatibility) + IdentityCheckPolicy, SourceIdentity, check_weight_sharing_compatibility) from tensorrt_llm._utils import str_dtype_to_torch from tensorrt_llm.llmapi.llm_args import (DecodingBaseConfig, ExecutorMemoryType, ModelExpressConfig, SparseAttentionConfig, TorchLlmArgs) from tensorrt_llm.llmapi.llm_utils import (_resolve_kv_cache_manager_v2_auto, - _resolve_transceiver_runtime_auto, apply_model_defaults_to_llm_args) from tensorrt_llm.logger import logger from tensorrt_llm.lora_helper import LoraConfig @@ -35,8 +31,7 @@ from ..model_config import ModelConfig from ..models import AutoModelForCausalLM, LlamaForCausalLM from ..models.checkpoints.base_checkpoint_loader import BaseCheckpointLoader -from ..models.modeling_utils import (MODEL_CLASS_MAPPING, - DecoderModelForCausalLM, MetaInitMode, +from ..models.modeling_utils import (DecoderModelForCausalLM, MetaInitMode, timing) from ..modules.fused_moe.moe_load_balancer import ( MoeLoadBalancer, maybe_create_moe_load_balancer) @@ -71,7 +66,7 @@ def validate_and_set_mamba_ssm_cache_dtype( def validate_and_set_kv_cache_quant(model_config: ModelConfig, - pyt_kv_cache_dtype: str) -> None: + pyt_kv_cache_dtype: str) -> QuantAlgo: logger.info( f'Validating KV Cache config against kv_cache_dtype="{pyt_kv_cache_dtype}"' ) @@ -105,14 +100,6 @@ def validate_and_set_kv_cache_quant(model_config: ModelConfig, # Apply explicit override from kv_cache_config.dtype. model_config.quant_config.kv_cache_quant_algo = mapped_pyt_quant - # MIXED_PRECISION checkpoints carry per-layer QuantConfigs in - # quant_config_dict; modules built from them (e.g. attention) must agree - # with the global config on the KV element size, otherwise the KV pool is - # allocated with the overridden dtype while attention layers read/write - # with the checkpoint dtype -> out-of-bounds access. - if model_config.quant_config_dict is not None: - for layer_quant_config in model_config.quant_config_dict.values(): - layer_quant_config.kv_cache_quant_algo = mapped_pyt_quant def validate_encoder_decoder_kv_cache_config(model_config: ModelConfig, @@ -310,16 +297,9 @@ class ModelLoader: This class isolates model loading logic from the main execution engine. """ _MX_STAGED_RECEIVER_TRANSFORM_PROTOCOL_VERSION = 1 - _POST_TRANSFORM_PROFILE_REGISTRY = PostTransformProfileRegistry( - profiles=(PostTransformProfile( - profile_id="llama-for-causal-lm-target-v1", - root_model_class=LlamaForCausalLM, - architecture="LlamaForCausalLM", - model_type="llama", - speculative_mode=None, - protocol_version=_MX_STAGED_RECEIVER_TRANSFORM_PROTOCOL_VERSION, - transfer_scope=PostTransformTransferScope.TARGET_MODEL, - ), )) + _MX_STAGED_RECEIVER_ALLOWLIST = frozenset({ + (LlamaForCausalLM, _MX_STAGED_RECEIVER_TRANSFORM_PROTOCOL_VERSION) + }) def __init__(self, llm_args: TorchLlmArgs, @@ -366,9 +346,6 @@ def load_config_and_apply_defaults( checkpoint_loader: BaseCheckpointLoader) -> TorchLlmArgs: """Load model config and apply model-specific defaults to llm_args.""" if checkpoint_loader is None: - # No config to resolve a model class from; still resolve the - # "auto" sentinel so it never leaks past config loading. - _resolve_transceiver_runtime_auto(llm_args) return llm_args config_kwargs = { @@ -413,20 +390,6 @@ def load_config_and_apply_defaults( model_cls.__name__ if model_cls is not None else "unknown model") - # The transceiver preference follows the checkpoint's original - # architecture: _resolve_class may rewrite it to an execution class - # (e.g. MTPDraftModelForCausalLM), which must not drop the target - # model's preference. - preference_cls = model_cls - architectures = getattr(config.pretrained_config, 'architectures', None) - if architectures: - preference_cls = MODEL_CLASS_MAPPING.get(architectures[0], - model_cls) - - # Resolve "auto" sentinel values after model defaults are applied. - _resolve_transceiver_runtime_auto(llm_args, preference_cls, - config.pretrained_config) - return llm_args @staticmethod @@ -440,39 +403,6 @@ def _needs_source_identity(checkpoint_loader: BaseCheckpointLoader, """ return load_format == LoadFormat.GMS or checkpoint_loader.checkpoint_format == "MX" - @staticmethod - def _build_source_identity( - config: ModelConfig, - model: DecoderModelForCausalLM, - *, - checkpoint_dir: str, - model_name: str, - fallback_on_artifact_error: bool, - ) -> Optional[SourceIdentity]: - """Build the local identity without weakening artifact validation. - - Artifact construction remains fail-closed. MX may convert an artifact - error into an unavailable local identity so its compatibility gate - falls back to disk; GMS propagates the error because it has no fallback. - """ - try: - artifact_identity = ArtifactIdentity.from_checkpoint(checkpoint_dir) - except (OSError, RuntimeError, ValueError) as error: - if not fallback_on_artifact_error: - raise - logger.warning( - "Unable to build checkpoint artifact identity for MX checkpoint " - f"{checkpoint_dir}; falling back to regular checkpoint loading: {error}" - ) - return None - - return SourceIdentity.from_model_config( - config, - model, - artifact_identity=artifact_identity, - model_name=model_name, - ) - def load( self, checkpoint_dir: str, @@ -516,16 +446,12 @@ def load( # ground truth; building it here (post-construction, # pre-weight-load) gives producer and consumer a common, # comparable lifecycle point. - self._source_identity = self._build_source_identity( + self._source_identity = SourceIdentity.from_model_config( config, model, - checkpoint_dir=checkpoint_dir, model_name=str( getattr(self.llm_args, "model", None) or checkpoint_dir), - fallback_on_artifact_error=( - load_format != LoadFormat.GMS - and checkpoint_loader.checkpoint_format == "MX"), ) memo: dict[torch.Tensor, torch.Tensor] = {} @@ -604,7 +530,6 @@ def init_meta_tensor(t: torch.Tensor): loads_draft_weights = ( self.spec_config is not None and self.spec_config.spec_dec_mode.need_load_draft_weights()) - speculative_mode = self._speculative_mode_name(self.spec_config) # Set when either GMS RW or GMS RO branch has already run the # post_load_* hooks itself, so the shared post-load block below # must skip them. RW handles them inside `mem_pool_scope` so the @@ -629,15 +554,9 @@ def init_meta_tensor(t: torch.Tensor): # do not accept post-transform bytes for only the target # model. Enable this only after target and draft subgraphs # have an explicit mixed-layout policy. - qualification = self._qualify_post_transform_profile( - model, - speculative_mode=speculative_mode, - loads_draft_weights=loads_draft_weights) - load_weights_kwargs[ - "allow_post_transform_weights"] = qualification.qualified - if qualification.qualified: - load_weights_kwargs[ - "prepare_post_transform_receiver"] = self._setup_aliases + load_weights_kwargs["allow_post_transform_weights"] = ( + self._is_mx_staged_receiver_allowlisted(model) + and not loads_draft_weights) if hasattr(model, 'llm_checkpoint_dir'): weights = checkpoint_loader.load_weights( @@ -756,15 +675,10 @@ def init_meta_tensor_in_pool(t: torch.Tensor): "source_identity": self._source_identity, } if checkpoint_loader.checkpoint_format == "MX": - qualification = self._qualify_post_transform_profile( - model, - speculative_mode=speculative_mode, - loads_draft_weights=loads_draft_weights) load_weights_kwargs[ - "allow_post_transform_weights"] = qualification.qualified - if qualification.qualified: - load_weights_kwargs[ - "prepare_post_transform_receiver"] = self._setup_aliases + "allow_post_transform_weights"] = ( + self._is_mx_staged_receiver_allowlisted( + model) and not loads_draft_weights) weights = checkpoint_loader.load_weights( weight_source, **load_weights_kwargs) @@ -828,7 +742,6 @@ def init_meta_tensor_in_pool(t: torch.Tensor): checkpoint_loader, model, weights_preloaded=weights_preloaded, - speculative_mode=speculative_mode, loads_draft_weights=loads_draft_weights) if mx_staged_receiver_path: self._setup_aliases(model) @@ -854,9 +767,7 @@ def init_meta_tensor_in_pool(t: torch.Tensor): checkpoint_loader, model, checkpoint_dir=checkpoint_dir, - weights_preloaded=weights_preloaded, - speculative_mode=speculative_mode, - loads_draft_weights=loads_draft_weights) + weights_preloaded=weights_preloaded) # Pool closed. Commit the post-post_load layout. gms_backend.finalize_write(model) @@ -904,13 +815,10 @@ def init_meta_tensor_in_pool(t: torch.Tensor): gms_backend.materialize_module(model) self._walk_cache_state(model) - self._post_load_publish( - checkpoint_loader, - model, - checkpoint_dir=checkpoint_dir, - weights_preloaded=True, - speculative_mode=speculative_mode, - loads_draft_weights=loads_draft_weights) + self._post_load_publish(checkpoint_loader, + model, + checkpoint_dir=checkpoint_dir, + weights_preloaded=True) gms_post_load_handled = True logger.info("LoadFormat.GMS (RO): materialized weights") else: @@ -949,7 +857,6 @@ def init_meta_tensor_in_pool(t: torch.Tensor): checkpoint_loader, model, weights_preloaded=weights_preloaded, - speculative_mode=speculative_mode, loads_draft_weights=loads_draft_weights) if mx_staged_receiver_path: self._setup_aliases(model) @@ -960,9 +867,7 @@ def init_meta_tensor_in_pool(t: torch.Tensor): self._post_load_publish(checkpoint_loader, model, checkpoint_dir=checkpoint_dir, - weights_preloaded=weights_preloaded, - speculative_mode=speculative_mode, - loads_draft_weights=loads_draft_weights) + weights_preloaded=weights_preloaded) # TODO(GMS-MOE-LB): when the (MoE, GMS) combination is enabled, # `register_weight_slots_after_to_cuda` and `finalize_model` @@ -1022,12 +927,11 @@ def _should_run_mx_staged_receiver_path( model: DecoderModelForCausalLM, *, weights_preloaded: bool, - speculative_mode: Optional[str] = None, loads_draft_weights: bool = False) -> bool: """Whether an MX receiver can skip one-shot weight transforms. MXCheckpointLoader only accepts post-transform P2P bytes when this same - exact-profile check passes before transfer. It also refuses + allow-list check passes before transfer. It also refuses target-plus-draft mixed layouts until there is an explicit policy for that combination, so this post-load branch should never see an unsafe post-transform receiver in normal use. @@ -1041,93 +945,49 @@ def _should_run_mx_staged_receiver_path( ): return False - qualification = cls._qualify_post_transform_profile( - model, - speculative_mode=speculative_mode, - loads_draft_weights=loads_draft_weights) - profile = qualification.profile - if qualification.qualified and profile is not None: + if loads_draft_weights: + raise RuntimeError( + "MX receiver accepted post-transform weights while a separate " + "draft-model load is required. This is unsafe because the " + "target and draft subgraphs may need different post-load " + "transform handling. Disable post-transform MX for this load " + "or add an explicit mixed target/draft policy.") + + if cls._is_mx_staged_receiver_allowlisted(model): logger.info( - "MX receiver using staged post-load profile %s for %s " + "MX receiver using staged post-load path for %s " "(transform protocol v%d).", - profile.profile_id, type(model).__name__, cls._MX_STAGED_RECEIVER_TRANSFORM_PROTOCOL_VERSION, ) return True - unsupported_features = ",".join( - sorted(feature.value - for feature in qualification.unsupported_features)) - feature_detail = (f"; unsupported_features={unsupported_features}" - if unsupported_features else "") raise RuntimeError( f"MX receiver got post-transform weights for {type(model).__name__}, " - "but the load does not match a qualified staged post-load profile " - f"for protocol v{cls._MX_STAGED_RECEIVER_TRANSFORM_PROTOCOL_VERSION}: " - f"reason={qualification.reason.value}{feature_detail}. " + "but the model is not allow-listed for staged post-load transform " + f"protocol v{cls._MX_STAGED_RECEIVER_TRANSFORM_PROTOCOL_VERSION}. " "Refusing to run the full post-load path on already-transformed " "weights.") - @staticmethod - def _speculative_mode_name( - spec_config: Optional[DecodingBaseConfig]) -> Optional[str]: - if spec_config is None: - return None - spec_dec_mode = getattr(spec_config, "spec_dec_mode", None) - mode_name = getattr(spec_dec_mode, "name", None) - return mode_name.lower() if isinstance(mode_name, str) else "unknown" - @classmethod - def _qualify_post_transform_profile( - cls, model: DecoderModelForCausalLM, *, - speculative_mode: Optional[str], - loads_draft_weights: bool) -> PostTransformQualificationDecision: - pretrained_config = model.model_config.pretrained_config - architectures = getattr(pretrained_config, "architectures", None) - architecture = (architectures[0] - if isinstance(architectures, - (list, tuple)) and architectures - and isinstance(architectures[0], str) else None) - configured_model_type = getattr(pretrained_config, "model_type", None) - model_type = (configured_model_type if isinstance( - configured_model_type, str) else None) - enabled_features = set() - if loads_draft_weights: - enabled_features.add(PostTransformFeature.SEPARATE_DRAFT_MODEL) - return cls._POST_TRANSFORM_PROFILE_REGISTRY.qualify( - root_model_class=type(model), - architecture=architecture, - model_type=model_type, - speculative_mode=speculative_mode, - protocol_version=cls._MX_STAGED_RECEIVER_TRANSFORM_PROTOCOL_VERSION, - transfer_scope=PostTransformTransferScope.TARGET_MODEL, - enabled_features=frozenset(enabled_features), - ) + def _is_mx_staged_receiver_allowlisted( + cls, model: DecoderModelForCausalLM) -> bool: + for model_type, protocol_version in cls._MX_STAGED_RECEIVER_ALLOWLIST: + if (protocol_version + == cls._MX_STAGED_RECEIVER_TRANSFORM_PROTOCOL_VERSION + and isinstance(model, model_type)): + return True + return False def _post_load_publish(self, checkpoint_loader: BaseCheckpointLoader, model: DecoderModelForCausalLM, *, - checkpoint_dir: str, weights_preloaded: bool, - speculative_mode: Optional[str], - loads_draft_weights: bool) -> None: + checkpoint_dir: str, + weights_preloaded: bool) -> None: kwargs = { "checkpoint_dir": checkpoint_dir, "weights_preloaded": weights_preloaded, } if checkpoint_loader.checkpoint_format == "MX": - qualification = self._qualify_post_transform_profile( - model, - speculative_mode=speculative_mode, - loads_draft_weights=loads_draft_weights) - if not qualification.qualified: - if not weights_preloaded: - logger.info( - "Skipping MX post-transform publish for %s: " - "qualification reason=%s.", - type(model).__name__, - qualification.reason.value, - ) - return kwargs["source_identity"] = self._source_identity checkpoint_loader.post_load_publish(model, **kwargs) @@ -1317,8 +1177,6 @@ def _load_and_validate_config( force_dynamic_quantization=self.llm_args.force_dynamic_quantization, spec_config=self.spec_config, sparse_attention_config=self.sparse_attention_config, - kv_cache_compression_config=( - self.llm_args.kv_cache_compression_config), max_num_tokens=self.max_num_tokens, max_seq_len=self.max_seq_len, moe_max_num_tokens=self.llm_args.moe_config.max_num_tokens, @@ -1326,7 +1184,6 @@ def _load_and_validate_config( lora_config=self.lora_config, allreduce_strategy=self.llm_args.allreduce_strategy, mm_encoder_only=self.llm_args.mm_encoder_only, - disable_mm_encoder=self.llm_args.disable_mm_encoder, attn_backend=self.llm_args.attn_backend, moe_backend=self.llm_args.moe_config.backend, moe_disable_finalize_fusion=self.llm_args.moe_config. diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 34ebd340a5ee..bf20ed9e495e 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -69,8 +69,7 @@ from .hang_detector import HangDetector, propagate_hard_kill from .kv_cache_manager_v2 import KVCacheManagerV2 from .kv_cache_stats import append_kv_cache_iteration_stats -from .kv_cache_transceiver import (KvCacheTransceiver, - is_disagg_inflight_cancel_enabled) +from .kv_cache_transceiver import KvCacheTransceiver from .llm_request import (ATTENTION_DP_DUMMY_REQUEST_ID, MAX_SPEC_DECODE_POSITIONS, ExecutorRequest, LlmRequest, LlmRequestState, LlmResponse, @@ -765,10 +764,10 @@ def __init__( self.inflight_req_ids = ReqIdsSet() # Encoder-decoder models execute the encoder and decoder in separate - # iterations in both executor loops. PP usage is very rare for these - # models, so encoder PP send/recv support is not implemented in the - # PyTorch path for now. Reject pp_size > 1. - # TODO: Add support for pp + encoder models + # iterations. The encoder branch lives in ``_executor_loop`` only; + # ``_executor_loop_overlap`` has not been threaded yet. Reject + # pp_size > 1 for parity with the legacy TRT path (Encoder PP support + # is intentionally out of scope for this port). is_encoder_decoder = bool( getattr(getattr(self.model_engine.model, "model_config", None), "is_encoder_decoder", False)) @@ -778,6 +777,11 @@ def __init__( "pp_size > 1 is not supported for encoder-decoder models " "in the PyTorch flow; encoder send/recv hooks are out of " "scope. Set pp_size=1 to run T5/BART/mBART.") + if not self.disable_overlap_scheduler: + raise NotImplementedError( + "Overlap scheduler is not yet wired for encoder-decoder " + "models. Set disable_overlap_scheduler=True for " + "encoder-decoder runs.") if getattr(self.model_engine, "_torch_compile_piecewise_cuda_graph", False): raise NotImplementedError( @@ -822,9 +826,6 @@ def __init__( self.is_shutdown = False self._fatal_error: Optional[BaseException] = None self._error_budget = ErrorBudget() - self._disagg_timed_out_ctx_cancelled_ids: set[int] = set() - self._disagg_timed_out_gen_cancelled_ids: set[int] = set() - self._disagg_inflight_cancel_unsupported_logged = False self.max_batch_size = max_batch_size self.adp_ctx_waiting_iters_count = 0 self.adp_ctx_batching_wait_iters_count = 0 @@ -914,7 +915,6 @@ def on_detected(): # under steady state — see ping-pong comment in _profiler). self._latest_host_step_time_ms: Optional[float] = None self._latest_prev_device_step_time_ms: Optional[float] = None - self._emit_initial_stats() self.gather_all_responses = False self.kv_cache_transceiver = kv_cache_transceiver @@ -1077,14 +1077,7 @@ def _maybe_init_kv_connector_manager(self): self.kv_connector_manager.wait_for_initialization() def _end_transfer_and_maybe_terminate(self, request: LlmRequest): - transfer_failed = request.state == LlmRequestState.DISAGG_TRANS_ERROR if self.kv_cache_transceiver and request in self.active_requests: - if transfer_failed: - # End only the transfer that just became terminal. Keep the - # request active so the synchronized error path can emit an - # error response after every async transfer releases ownership. - self.async_transfer_manager.end_transfer(request) - return # Fast-transfer: KV transfer completed in the same iteration # before _handle_responses could run. Create the response now # while state is still TRANS_IN_PROGRESS (required by C++ @@ -1107,8 +1100,6 @@ def _end_transfer_and_maybe_terminate(self, request: LlmRequest): self._terminate_request(request) return if self.async_transfer_manager.end_transfer(request): - if transfer_failed: - return # Skip if the PP=1 early path already terminated this request; # under PP>1 that path is off, so terminate here on transfer-complete. if not self.force_terminate_ctx_for_partial_reuse: @@ -1276,21 +1267,6 @@ def start_worker(self): def _set_global_steady_clock_offset(self): assert self.global_rank >= 0, "rank should be >= 0" - # First calibration wins (mirrors the C++ guard in CacheTransceiver). - # PyExecutor is constructed twice per process (memory-profiling dry run, - # then the real executor), so this method runs twice. Recalibration is - # idempotent as long as the measurement below reads the raw - # steady_clock, but skipping it avoids a redundant barrier+allgather - # and protects the offset if the measurement path ever becomes - # offset-aware (which would make a second pass observe ~zero skew and - # wipe the correct value). - if LlmRequest.global_steady_clock_offset is not None: - logger.info( - f"global_steady_clock_offset already set " - f"({LlmRequest.global_steady_clock_offset}); skipping recalibration " - f"for rank {self.global_rank}") - return - # Sync all ranks self.dist.barrier() # Immediately take the local steady clock timestamp @@ -2028,9 +2004,6 @@ def _update_iter_stats( # Aggregate stats from all generation requests for req in scheduled_batch.generation_requests: - # exclude attention dp dummy / CUDA Graph padding requests from AL calculation - if getattr(req, 'is_dummy', False): - continue draft_len = getattr(req, 'num_draft_tokens', 0) py_draft_tokens = getattr(req, 'py_draft_tokens', None) py_num_accepted = getattr(req, 'py_num_accepted_draft_tokens', @@ -2564,10 +2537,6 @@ def _executor_loop_pp(self): # Retry until current rank can run first PP's schedule result. self._pp_retry_until_can_schedule(scheduled_batch) # Run scheduler locally because scheduler may change llm requests' state. - if hasattr(self.kv_cache_manager, - "prepare_expect_snapshot_points"): - self.kv_cache_manager.prepare_expect_snapshot_points( - self.active_requests) local_scheduler_output = self.scheduler.schedule_request( self.active_requests, self.inflight_req_ids) if self.kv_cache_transceiver: @@ -2602,9 +2571,6 @@ def _executor_loop_pp(self): f'{scheduled_batch.num_generation_requests} generation requests' ) - if scheduled_batch.encoder_requests: - self._run_encoder_step(scheduled_batch.encoder_requests) - can_queue, _ = self._can_queue(scheduled_batch) if not can_queue: self._revert_gen_alloc(scheduled_batch) @@ -3228,17 +3194,8 @@ def _handle_dynamic_draft_len(self, scheduled_batch.batch_size, self.model_engine.max_draft_len) # 2. Pad or truncate draft tokens to the resolved length DRAFT_BUFFER_PAD = 0 # Buffer sentinel, not PARD mask_token_id. - rejection_on = getattr(self.model_engine.spec_config, - "use_rejection_sampling", False) for request in scheduled_batch.generation_requests: current_num_draft_tokens = len(request.py_draft_tokens) - # One-model rejection: a gen request entering with 0 real draft - # tokens produced no draft-prob scatter for its slot last iter, - # so next iter's rejection kernel would read a stale draft_probs - # row. Mark it (pre-pad signal) so _prepare_tp_inputs writes a - # one-hot placeholder row after spec_metadata.prepare(). - request.py_needs_onehot_draft_probs = ( - rejection_on and current_num_draft_tokens == 0) if spec_dec_mode.is_pard(): # special case: PARD carries 2K-1 draft tokens per request runtime_draft_token_buffer_width = ( @@ -3277,7 +3234,6 @@ def _handle_dynamic_draft_len(self, if spec_config is not None and spec_config.is_linear_tree else self.model_engine.max_total_draft_tokens) - @nvtx_range("_can_queue") def _can_queue(self, scheduled_batch): # can_queue_this_rank is for case that the batch is not empty on this rank, but empty on other ranks @@ -3882,105 +3838,47 @@ def _check_benchmark_disagg_gate(self, scheduled_batch: ScheduledRequests, return can_forward, True return can_forward, False - @nvtx_range("_handle_disagg_cache_errors_synced") def _handle_disagg_cache_errors_synced(self): - """Rank-safe disagg cache error and poison handler. + """ADP-safe disagg cache error handler. - Called from the top of every executor iteration. Buffer poison is - reduced over the full executor world because one poisoned PP/DP rank - requires the whole distributed executor to stop. ADP TP ranks then - vote on failed request IDs and fail matching local replicas together; + Called from the top of every executor iteration. TP ranks vote on + failed request IDs and fail matching local replicas together; otherwise the downstream ``tp_gather`` in ``_enqueue_responses`` deadlocks or leaves peer replicas running. """ - if not self.kv_cache_transceiver: + if not (self.kv_cache_transceiver and self.enable_attention_dp + and self.dist.world_size != 1): return - if self._is_disagg_inflight_cancel_active(): - local_poisoned = self.kv_cache_transceiver.has_poisoned_transfer_buffer( - ) - if self.dist.world_size != 1: - any_poisoned = bool( - self.dist.allreduce(int(local_poisoned), op=ReduceOp.MAX)) - else: - any_poisoned = local_poisoned - if any_poisoned: - error_msg = ( - "Disagg KV cache transfer buffer is poisoned; process " - "restart is required") - self._fatal_error = RuntimeError(f"Fatal error: {error_msg}") - self.is_shutdown = True - self._handle_errors(error_msg, - requests=None, - charge_budget=False) - return - - if not (self.enable_attention_dp and self.dist.world_size != 1): - return + def request_vote_id(request: LlmRequest) -> int: + return (request.parent_request_id + if request.is_child else request.py_request_id) - local_error_requests = [ - request for request in self.active_requests - if request.state == LlmRequestState.DISAGG_TRANS_ERROR + local_error_requests = self._get_disagg_reqs_in_error_state() + local_error_ids = [ + request_vote_id(request) for request in local_error_requests ] - local_vote = { - "error_ids": [ - self._request_vote_id(request) - for request in local_error_requests - ], - "blocked_ids": [ - self._request_vote_id(request) - for request in local_error_requests - if self._is_disagg_error_cleanup_blocked(request) - ], - } - all_votes = self.dist.tp_allgather(local_vote) + all_error_ids = self.dist.tp_allgather(local_error_ids) voted_error_ids = { request_id - for rank_vote in all_votes - for request_id in rank_vote["error_ids"] - } - blocked_error_ids = { - request_id - for rank_vote in all_votes - for request_id in rank_vote["blocked_ids"] + for rank_error_ids in all_error_ids + for request_id in rank_error_ids } - ready_error_ids = voted_error_ids - blocked_error_ids - if not ready_error_ids: + if not voted_error_ids: return local_voted_error_requests = [ request for request in self.active_requests - if self._request_vote_id(request) in ready_error_ids + if request_vote_id(request) in voted_error_ids ] - logger.warning( - f"Disagg KV cache transfer error: rank={self.dist.rank} " - f"local_err_count={len(local_error_requests)}, " - f"voted_err_count={len(voted_error_ids)}, " - f"blocked_err_count={len(voted_error_ids & blocked_error_ids)}") + logger.warning(f"Disagg KV cache transfer error: rank={self.dist.rank} " + f"local_err_count={len(local_error_requests)}, " + f"voted_err_count={len(voted_error_ids)}") self._handle_errors( "Disagg KV cache transfer error", requests=local_voted_error_requests, charge_budget=False, ) - def _emit_initial_stats(self) -> None: - """Emit a startup stats snapshot so that cache_config_info is - immediately available to external metric scrapers (e.g. the - Kubernetes Inference Gateway EPP) before any inference request.""" - if not self.enable_iter_perf_stats: - return - stats = self._get_init_iter_stats(0, 0) - kv_cache_manager = self.resource_manager.resource_managers.get( - ResourceManagerType.KV_CACHE_MANAGER) - if kv_cache_manager is not None: - kv_stats = kv_cache_manager.get_kv_cache_stats() - kv_stats_to_save = KvCacheStats() - kv_stats_to_save.max_num_blocks = kv_stats.max_num_blocks - kv_stats_to_save.tokens_per_block = kv_stats.tokens_per_block - kv_stats_to_save.free_num_blocks = kv_stats.free_num_blocks - kv_stats_to_save.used_num_blocks = kv_stats.used_num_blocks - stats.kv_cache_stats = kv_stats_to_save - self._append_iter_stats(stats) - def _executor_loop(self): torch.cuda.set_device(self.device_id) # ensure the context is created, otherwise, some MPI calls will fail. @@ -4498,9 +4396,6 @@ def _executor_loop_overlap(self): if not self._is_kv_manager_v2: self._terminate_requests(scheduled_batch.paused_requests) - if scheduled_batch.encoder_requests: - self._run_encoder_step(scheduled_batch.encoder_requests) - gpu_forward_events_from_perf_pool = False can_queue, can_queue_this_rank = self._can_queue( scheduled_batch) @@ -5270,10 +5165,6 @@ def _waiting_requests(self, context_requests: list[LlmRequest], @nvtx_range("_schedule") def _schedule(self): - if hasattr(self.kv_cache_manager, "prepare_expect_snapshot_points"): - self.kv_cache_manager.prepare_expect_snapshot_points( - self.active_requests) - scheduler_output = self.scheduler.schedule_request( self.active_requests, self.inflight_req_ids) @@ -5463,143 +5354,6 @@ def _check_disagg_gen_transfer_status(self): # an empty ready set. self._check_disagg_gen_cache_transfer_status(0) - if self._is_disagg_inflight_cancel_active(): - self._cancel_timed_out_gen_transfers() - self._check_gen_cache_transfer_errors_consensus() - - return - - def _is_disagg_inflight_cancel_active(self) -> bool: - if not is_disagg_inflight_cancel_enabled(): - return False - - transceiver = getattr(self, "kv_cache_transceiver", None) - if transceiver is None: - return False - - supports = getattr(transceiver, - "supports_inflight_request_cancellation", None) - if callable(supports) and supports() is True: - return True - - if not getattr(self, "_disagg_inflight_cancel_unsupported_logged", - False): - logger.warning( - "TRTLLM_DISAGG_ENABLE_INFLIGHT_CANCEL=1 was requested, but " - f"{type(transceiver).__name__} does not advertise in-flight " - "request cancellation support. Cancellation and transfer-buffer " - "quarantine are currently scoped to the C++ NIXL transceiver " - "with the UCX plugin; using the existing timeout and " - "cancellation behavior for this transceiver.") - self._disagg_inflight_cancel_unsupported_logged = True - return False - - def _request_kv_transfer_cancellation(self, request: LlmRequest) -> bool: - """Best-effort cancellation that leaves ownership intact on errors.""" - try: - return self.kv_cache_transceiver.cancel_request(request) - except Exception as error: - logger.error(f"KV transfer cancellation failed for request " - f"{request.py_request_id}; will retry: {error}") - return False - - @nvtx_range("_cancel_timed_out_gen_transfers") - def _cancel_timed_out_gen_transfers(self) -> None: - """Request cancellation for timed-out generation transfers. - - This path is deliberately rank-synchronized. Under attention-DP, each - TP rank can own a different request subset, but later error responses - still pass through a TP collective. The timeout/cancel decision must - therefore be based on the TP-wide request-id union rather than a - rank-local timeout observation. - """ - timeout_ms = self.kv_cache_transceiver.kv_transfer_timeout_ms - if timeout_ms is None: - return - - requests_in_transfer = { - req.py_request_id: req - for req in self.active_requests - if req.is_disagg_generation_transmission_in_progress - } - current_time = time.monotonic() - for request in requests_in_transfer.values(): - if request.py_kv_transfer_start_time is None: - continue - elapsed_time = ((current_time - request.py_kv_transfer_start_time) * - 1000) - if (elapsed_time > timeout_ms - and not request.py_kv_transfer_timed_out): - logger.warning( - f"Requesting cancellation for generation request " - f"{request.py_request_id} due to KV cache transfer timeout") - request.py_kv_transfer_timed_out = True - - user_canceled_ids = set(self.canceled_req_ids) - local_timed_out_ids = sorted( - request_id for request_id, request in requests_in_transfer.items() - if request.py_kv_transfer_timed_out - and request_id not in user_canceled_ids - and request_id not in self._disagg_timed_out_gen_cancelled_ids) - - if self.dist.tp_size > 1: - any_timed_out = self.dist.tp_allreduce(int( - bool(local_timed_out_ids)), - op=ReduceOp.MAX) - else: - any_timed_out = int(bool(local_timed_out_ids)) - if not any_timed_out: - return - - if self.dist.tp_size > 1: - gathered_timed_out_ids = self.dist.tp_allgather(local_timed_out_ids) - timed_out_ids = sorted(set().union(*gathered_timed_out_ids)) - else: - timed_out_ids = local_timed_out_ids - - for request_id in timed_out_ids: - request = requests_in_transfer.get(request_id) - if request is None: - continue - - # A peer rank may have crossed the timeout first. Mirror the - # TP-wide decision locally so a failed cancel attempt keeps - # retrying even if this rank's wall clock had not yet expired. - request.py_kv_transfer_timed_out = True - - if request_id in self._disagg_timed_out_gen_cancelled_ids: - continue - - is_cancelled = self._request_kv_transfer_cancellation(request) - if is_cancelled: - self._disagg_timed_out_gen_cancelled_ids.add(request_id) - logger.warning( - f"Cancelled timed-out generation KV transfer for request " - f"{request.py_request_id}; waiting for C++ transfer " - "status to report final cleanup") - - @nvtx_range("_check_gen_cache_transfer_errors_consensus") - def _check_gen_cache_transfer_errors_consensus(self) -> None: - """Flush generation transfer errors through a TP-uniform path.""" - error_requests = [ - req for req in self._get_disagg_reqs_in_error_state() - if req.is_generation_only_request() - ] - local_needs_flush = bool(error_requests) - - if self.dist.tp_size > 1: - any_needs_flush = self.dist.tp_allreduce(int(local_needs_flush), - op=ReduceOp.MAX) - else: - any_needs_flush = int(local_needs_flush) - if not any_needs_flush: - return - - error_msg = "Error in kv cache transfer for generation requests" - self._handle_errors(error_msg, - requests=error_requests, - charge_budget=False) - @nvtx_range("_check_kv_transfer_timeout") def _check_kv_transfer_timeout(self): if not self.kv_cache_transceiver: @@ -5609,16 +5363,13 @@ def _check_kv_transfer_timeout(self): return def flag_if_kv_transfer_timed_out(req: LlmRequest, type: str) -> None: - current_time = time.monotonic() + current_time = time.time() if req.py_kv_transfer_start_time is None: return elapsed_time = (current_time - req.py_kv_transfer_start_time) * 1000 if elapsed_time > timeout_ms and not req.py_kv_transfer_timed_out: - verb = ("Requesting cancellation for" - if self._is_disagg_inflight_cancel_active() else - "Observed timeout on") logger.warning( - f"{verb} {type} request {req.py_request_id} due to KV " + f"Terminating {type} request {req.py_request_id} due to KV " f"cache transfer timeout: elapsed {elapsed_time:.0f}ms > " f"kv_transfer_timeout_ms={timeout_ms}ms") req.py_kv_transfer_timed_out = True @@ -6048,7 +5799,7 @@ def _recv_disagg_gen_cache(self, new_gen_reqs): if self.kv_cache_transceiver.kv_transfer_timeout_ms is not None: for req in new_gen_reqs: if req.state == LlmRequestState.DISAGG_GENERATION_TRANS_IN_PROGRESS: - req.py_kv_transfer_start_time = time.monotonic() + req.py_kv_transfer_start_time = time.time() self._check_disagg_gen_cache_transfer_status(0) @@ -6086,7 +5837,7 @@ def kv_connector_request_finished(req: LlmRequest): self.kv_cache_transceiver.respond_and_send_async(req) if self.kv_cache_transceiver.kv_transfer_timeout_ms is not None: - req.py_kv_transfer_start_time = time.monotonic() + req.py_kv_transfer_start_time = time.time() if self.kv_connector_manager: if not self.disable_overlap_scheduler: @@ -6101,28 +5852,10 @@ def kv_connector_request_finished(req: LlmRequest): if self.kv_cache_transceiver: self._check_disagg_ctx_cache_transfer_status(0) - @staticmethod - def _request_vote_id(request: LlmRequest) -> int: - return (request.parent_request_id - if request.is_child else request.py_request_id) - - def _is_disagg_error_cleanup_blocked(self, request: LlmRequest) -> bool: - request_id = self._request_vote_id(request) - if request_id in getattr(self, "canceled_req_ids", ()): - return True - - async_transfer_manager = getattr(self, "async_transfer_manager", None) - if (getattr(request, "is_context_only_request", False) is True - and async_transfer_manager is not None and request.py_request_id - in async_transfer_manager.requests_in_transfer()): - return True - return False - def _get_disagg_reqs_in_error_state(self): return [ req for req in self.active_requests if req.state == LlmRequestState.DISAGG_TRANS_ERROR - and not self._is_disagg_error_cleanup_blocked(req) ] def _check_cache_transfer_errors(self, error_msg_prefix: str): @@ -6135,10 +5868,10 @@ def _check_cache_transfer_errors(self, error_msg_prefix: str): return error_requests = self._get_disagg_reqs_in_error_state() if error_requests: - error_msg = f"Error in kv cache transfer for {error_msg_prefix}" - self._handle_errors(error_msg, - requests=error_requests, - charge_budget=False) + self._handle_errors( + f"Error in kv cache transfer for {error_msg_prefix}", + requests=error_requests, + charge_budget=False) @nvtx_range("_check_disagg_ctx_cache_transfer_status") def _check_disagg_ctx_cache_transfer_status(self, atLeastNum: int = 0): @@ -6167,27 +5900,15 @@ def _check_disagg_ctx_cache_transfer_status(self, atLeastNum: int = 0): for request_id in list(requests_in_transfer.keys()): request = requests_in_transfer[request_id] - if (not request.py_kv_transfer_timed_out - or request_id in completed_req_ids - or request_id in self._disagg_timed_out_ctx_cancelled_ids): - continue - - is_cancelled = self._request_kv_transfer_cancellation(request) - if not is_cancelled: - continue + if request.py_kv_transfer_timed_out and request_id not in completed_req_ids: + is_cancelled = self.kv_cache_transceiver.cancel_request(request) + # If cancel is successful, mark as complete so it can be cleaned up + # Otherwise, try at next iteration + if is_cancelled: + request.py_kv_transfer_start_time = None + request.state = LlmRequestState.DISAGG_CONTEXT_COMPLETE - if self._is_disagg_inflight_cancel_active(): - self._disagg_timed_out_ctx_cancelled_ids.add(request_id) - logger.warning(f"Cancelled timed-out context KV transfer for " - f"request {request.py_request_id}; waiting for " - "C++ transfer status to report final cleanup") - else: - # Preserve the legacy timeout behavior when in-flight - # cancellation is disabled: a queued transfer that can be - # cancelled is immediately released from the async manager. - request.py_kv_transfer_start_time = None - request.state = LlmRequestState.DISAGG_CONTEXT_COMPLETE - self._end_transfer_and_maybe_terminate(request) + self._end_transfer_and_maybe_terminate(request) self._check_cache_transfer_errors("context requests") @@ -6201,8 +5922,7 @@ def _check_disagg_gen_cache_transfer_status(self, atLeastNum: int = 0): req_id = req.py_request_id if not req.is_child else req.parent_request_id if req_id not in user_canceled_set: req.state = LlmRequestState.DISAGG_TRANS_ERROR - if not self._is_disagg_inflight_cancel_active(): - self._check_cache_transfer_errors("generation requests") + self._check_cache_transfer_errors("generation requests") def _maybe_prefetch_next_iter_mm_encoders( self, scheduled_batch: ScheduledRequests) -> None: @@ -6458,11 +6178,10 @@ def _handle_errors(self, is enqueued. Otherwise only the requests in *requests* are failed. When ``charge_budget`` is False, the error is treated as a - per-request failure unless the executor was already marked fatal by - the caller. The error budget is not consumed. Use this for - request-scoped errors (validation, KV-transfer timeout, - guided-decoder) that should not affect server health, and for the - cleanup phase of an already-classified fatal error. + per-request failure: only the specified requests are failed, the + error budget is not consumed, and shutdown is never triggered. + Use this for request-scoped errors (validation, KV-transfer + timeout, guided-decoder) that should not affect server health. .. note:: The ``charge_budget=False`` path reuses the full @@ -6486,17 +6205,15 @@ def _handle_errors(self, error_responses: Dict[int, LlmResponse] = {} error_msg = error_msg or "error" - budget_fatal = (self._error_budget.consume(error_msg) - if charge_budget else False) - is_fatal = self._fatal_error is not None or budget_fatal - if budget_fatal and self._error_budget.budget < 1e-9: + is_fatal = (self._error_budget.consume(error_msg) + if charge_budget else False) + if is_fatal and self._error_budget.budget < 1e-9: logger.error(f"Error budget exhausted " f"(budget={self._error_budget.budget:.3f}), " "treating as fatal") if is_fatal: - if self._fatal_error is None: - self._fatal_error = RuntimeError(f"Fatal error: {error_msg}") + self._fatal_error = RuntimeError(f"Fatal error: {error_msg}") self.is_shutdown = True logger.error( f"Fatal error detected, initiating shutdown: {error_msg}") @@ -6539,14 +6256,10 @@ def _handle_errors(self, client_id=getattr(item.request, 'client_id', None)))) - adp_collective_required = (self.enable_attention_dp - and self.dist.world_size != 1) - if waiting_responses or adp_collective_required: + if waiting_responses: self._enqueue_responses(waiting_responses) - if waiting_responses: - logger.info( - f"Drained {len(waiting_responses)} queued requests " - "on fatal error") + logger.info(f"Drained {len(waiting_responses)} queued requests " + "on fatal error") failed_requests = (list(self.active_requests) if requests is None else requests) @@ -6585,8 +6298,6 @@ def _terminate_request(self, request: LlmRequest): def _do_terminate_request(self, request: LlmRequest): self.resource_manager.free_resources(request) self._prefetched_request_ids.discard(request.py_request_id) - self._disagg_timed_out_ctx_cancelled_ids.discard(request.py_request_id) - self._disagg_timed_out_gen_cancelled_ids.discard(request.py_request_id) if self.gather_all_responses or self.dist.rank == 0: self.result_wait_queues.pop(request.py_request_id, None) @@ -6607,22 +6318,10 @@ def _try_cancel_request(self, request) -> bool: if self.kv_cache_transceiver is None: return True - async_transfer_manager = getattr(self, "async_transfer_manager", None) - if (getattr(request, "is_context_only_request", False) is True - and async_transfer_manager is not None and request.py_request_id - in async_transfer_manager.requests_in_transfer()): - if self._is_disagg_inflight_cancel_active(): - self._request_kv_transfer_cancellation(request) - return False - if not self._is_request_in_transmission(request): return True - if self._is_disagg_inflight_cancel_active(): - self._request_kv_transfer_cancellation(request) - return False - - return self._request_kv_transfer_cancellation(request) + return self.kv_cache_transceiver.cancel_request(request) @nvtx_range("_handle_canceled_requests") def _handle_canceled_requests(self): @@ -6645,7 +6344,6 @@ def _handle_canceled_requests(self): if is_cancelled: # Mark requests as finished, then, we reuse all existing code # to clean up the KV cache resources. - request.py_kv_transfer_timed_out = False request.finish_by_reason(FinishReason.CANCELLED) request.decoding_iter = request.py_decoding_iter else: @@ -6793,30 +6491,11 @@ def _handle_responses(self, emit_first_iter: bool = True): requests_to_terminate.append(request) continue - # Check if a generation request needs cleanup due to KV cache transfer timeout. + # Check if generation request needs cleanup due to KV cache transfer timeout if request.py_kv_transfer_timed_out: - if self._is_disagg_inflight_cancel_active(): - if (request.is_disagg_generation_transmission_in_progress - or request.state - == LlmRequestState.DISAGG_TRANS_ERROR): - new_active_requests.append(request) - else: - # The transfer completed after the deadline but before - # cancellation won the race. Fail the request without - # touching the now-quiesced transfer buffer. - timed_out_requests.append(request) - continue - - is_cancelled = self._request_kv_transfer_cancellation(request) + is_cancelled = self.kv_cache_transceiver.cancel_request(request) if is_cancelled: - # _handle_errors enters response collectives under ADP. - # Defer it until the rank-uniform vote below. timed_out_requests.append(request) - else: - # Legacy transceivers cannot cancel every in-flight - # transfer. Keep polling instead of dropping ownership of - # the request and its KV resources. - new_active_requests.append(request) continue if request.is_generation_only_request() and not request.is_finished: diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index 7920c8942e71..a4c2e48ddc53 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -40,7 +40,7 @@ from ._util import (KvCacheCreator, _adjust_torch_mem_fraction, create_py_executor_instance, instantiate_sampler, is_mla, validate_feature_combination) -from .config_utils import is_hybrid_linear, is_minimax_m3 +from .config_utils import is_hybrid_linear from .connectors.kv_cache_connector import KvCacheConnectorManager from .dwdp import DwdpManager from .guided_decoder import CapturableGuidedDecoder, GuidedDecoder @@ -48,15 +48,6 @@ from .model_loader import ModelLoader, _construct_checkpoint_loader from .py_executor import PyExecutor -_MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS = (90, 100, 103, 120, 121) -_MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS = (90, 100, 103, 120) -_MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS_STR = "/".join( - f"SM{sm_version}" - for sm_version in _MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS) -_MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS_STR = "/".join( - f"SM{sm_version}" - for sm_version in _MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS) - class _ExecutorMemoryMonitor: """Currently this focuses on tracking memory usage and related errors.""" @@ -420,13 +411,6 @@ def create_py_executor( if llm_args.attn_backend == "VANILLA": tokens_per_block = max_num_tokens - # The MSA kernels require a page size of 128; the Triton reference uses TRT-LLM's default - # of 32. - m3_sparse_config = llm_args.sparse_attention_config - if is_minimax_m3(m3_sparse_config): - tokens_per_block = 128 if m3_sparse_config.implementation == "msa" else 32 - kv_cache_config.tokens_per_block = tokens_per_block - if llm_args.attn_backend in ["FLASHINFER", "FLASHINFER_STAR_ATTENTION"]: # Workaround for flashinfer and star attention if kv_cache_config.enable_block_reuse: @@ -684,6 +668,10 @@ def drafting_loop_wrapper(model): max_num_tokens = model_engine.max_num_tokens sparse_attention_config = model_engine.sparse_attention_config + # Set default value for cache_transceiver_config.max_tokens_in_buffer + if cache_transceiver_config and cache_transceiver_config.max_tokens_in_buffer is None: + cache_transceiver_config.max_tokens_in_buffer = net_max_seq_len + config = model_engine.model.model_config.pretrained_config max_num_seq_slots = getattr(model_engine, "max_num_seq_slots", max_batch_size * getattr(mapping, "pp_size", 1)) @@ -716,11 +704,12 @@ def drafting_loop_wrapper(model): ) sm_version = get_sm_version() - if (kv_cache_config.enable_block_reuse and sm_version - not in _MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS): - logger.warning("KV cache reuse for MLA can only be enabled on " - f"{_MLA_KV_CACHE_REUSE_SUPPORTED_SM_VERSIONS_STR}, " - f"disable enable_block_reuse for SM{sm_version}") + if kv_cache_config.enable_block_reuse and sm_version not in [ + 90, 100, 103, 120 + ]: + logger.warning( + f"KV cache reuse for MLA can only be enabled on SM90/SM100/SM103/SM120, " + f"disable enable_block_reuse for SM{sm_version}") kv_cache_config.enable_block_reuse = False _set_model_engines_cache_reuse([model_engine, draft_model_engine], False) @@ -736,27 +725,15 @@ def drafting_loop_wrapper(model): kv_cache_config.enable_block_reuse = False _set_model_engines_cache_reuse([model_engine, draft_model_engine], False) - if (enable_chunked_context and sm_version - not in _MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS): - logger.warning("Chunked Prefill for MLA can only be enabled on " - f"{_MLA_CHUNKED_PREFILL_SUPPORTED_SM_VERSIONS_STR}, " - f"disable enable_chunked_context for SM{sm_version}") + if enable_chunked_context and sm_version not in [90, 100, 103, 120]: + logger.warning( + "Chunked Prefill for MLA can only be enabled on SM90/SM100/SM103/SM120, " + f"disable enable_chunked_context for SM{sm_version}") enable_chunked_context = False model_engine.attn_runtime_features.chunked_prefill = False if draft_model_engine is not None: draft_model_engine.attn_runtime_features.chunked_prefill = False - # Set default value for cache_transceiver_config.max_tokens_in_buffer. - # Placed after the FlashMLA tokens_per_block override and rounded up to a - # tokens_per_block multiple: CacheTransBufferManager requires - # max_tokens_in_buffer % tokens_per_block == 0 (cacheTransBuffer.cpp), - # and net_max_seq_len is in general not aligned (e.g. max_seq_len plus a - # non-power-of-two seq_len offset). - if cache_transceiver_config and cache_transceiver_config.max_tokens_in_buffer is None: - cache_transceiver_config.max_tokens_in_buffer = ( - (net_max_seq_len + tokens_per_block - 1) // tokens_per_block * - tokens_per_block) - if enable_chunked_context: chunk_unit_size = tokens_per_block max_attention_window = kv_cache_config.max_attention_window diff --git a/tensorrt_llm/_torch/pyexecutor/resource_manager.py b/tensorrt_llm/_torch/pyexecutor/resource_manager.py index e42c8e249dc6..e94d606e84f4 100644 --- a/tensorrt_llm/_torch/pyexecutor/resource_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/resource_manager.py @@ -19,8 +19,8 @@ from abc import ABC, abstractmethod from collections import OrderedDict, defaultdict, deque from dataclasses import dataclass -from typing import (TYPE_CHECKING, ClassVar, Dict, Iterable, List, Optional, - Sequence, Set, Tuple, Union) +from typing import (TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, + Set, Tuple, Union) import torch from mpi4py import MPI @@ -114,27 +114,6 @@ def _warn_if_unsupported_v1_kv_cache_event_hash_algo(hash_algo: str) -> None: "event hashes.") -def _merge_kv_cache_pool_pointers( - kv_cache_pool_pointers: torch.Tensor, - block_scale_pool_pointers: torch.Tensor, - layer_to_pool_mapping: torch.Tensor, - layer_pool_dtypes: Sequence["DataType"], -) -> torch.Tensor: - """Align compact scale rows with data rows in C++ physical-pool order.""" - dtype_by_physical_pool = dict( - zip(layer_to_pool_mapping[:, 0].tolist(), layer_pool_dtypes)) - fp4_pool_indices = [ - compact_pool_idx for compact_pool_idx, physical_pool_idx in enumerate( - sorted(dtype_by_physical_pool)) - if dtype_by_physical_pool[physical_pool_idx] == DataType.NVFP4 - ] - - aligned_scale_pool_pointers = torch.zeros_like(kv_cache_pool_pointers) - aligned_scale_pool_pointers[fp4_pool_indices] = block_scale_pool_pointers - return torch.stack([kv_cache_pool_pointers, aligned_scale_pool_pointers], - dim=-1) - - class BaseResourceManager(ABC): @abstractmethod @@ -307,9 +286,6 @@ def __init__( self.mapping = mapping self.dtype = dtype self.kv_cache_type = kv_cache_type - # Consumed by the disaggregation page-table builder to expose the DSA - # indexer K cache pool as a REPLICATED pool view. - self.enable_indexer_k_cache = enable_indexer_k_cache self.spec_config = spec_config self.pp_layers, self.num_layers = get_pp_layers( num_layers, @@ -645,31 +621,15 @@ def append_to_kv_heads_per_layer(num_kv_heads_per_layer: List[int], self.impl.allocate_pools(False) self.kv_cache_pool_pointers = self.impl.get_block_pool_pointers() - self.kv_cache_pool_mapping = self.impl.get_layer_to_pool_mapping() kv_cache_block_scale_pool_pointers = self.impl.get_block_scale_pool_pointers( ) if kv_cache_block_scale_pool_pointers.numel() > 0: - # C++ reports one effective configuration per actual window, - # including manager-level defaults when none were supplied. - dtype_by_window = { - config.window_size: config.dtype - for config in self.impl.pool_configurations - } - # Match the local layer order used by C++ to build the pointer - # mapping. The Python window helpers additionally account for - # global PP layer IDs and therefore do not describe these rows. - layer_pool_dtypes = [ - dtype_by_window[self.max_attention_window_vec[ - layer_offset % len(self.max_attention_window_vec)]] - for layer_offset in range(self.num_local_layers) - ] - self.kv_cache_pool_pointers = _merge_kv_cache_pool_pointers( - self.kv_cache_pool_pointers, - kv_cache_block_scale_pool_pointers, - self.kv_cache_pool_mapping, - layer_pool_dtypes, - ) + self.kv_cache_pool_pointers = torch.stack([ + self.kv_cache_pool_pointers, kv_cache_block_scale_pool_pointers + ], + dim=-1) + self.kv_cache_pool_mapping = self.impl.get_layer_to_pool_mapping() self.num_pools = self.impl.num_pools self.max_blocks_per_seq = self.impl.max_blocks_per_seq self.enable_block_reuse = kv_cache_config.enable_block_reuse @@ -1371,15 +1331,6 @@ def get_priority_by_block_id(self, window_size = self._resolve_window_size(window_size) return self.impl.get_priority_by_block_id(block_id, window_size) - def get_memory_pool_block_indices(self, block_ids: List[int], *, - window_size: int) -> List[int]: - # Translate logical block IDs to primary-pool slot indices. With host offload enabled, - # a block's ID and its current pool slot can diverge after an offload/onboard cycle. - # Every referenced block must be primary (asserted in C++): callers do primary-pool - # pointer arithmetic and the returned index carries no residency information. - return self.impl.get_memory_pool_block_indices(list(block_ids), - window_size) - def get_batch_cache_indices( self, request_ids: List[int], @@ -2176,13 +2127,9 @@ def _validate_and_adjust_attention_windows( def pin_blocks(self, request_id: int): self.impl.pin_blocks(request_id) - def copy_batch_block_offsets(self, - dst_tensor: torch.Tensor, - request_ids: List[int], - beam_width: int, - num_context: int, - num_seqs: int, - max_blocks: Optional[int] = None): + def copy_batch_block_offsets(self, dst_tensor: torch.Tensor, + request_ids: List[int], beam_width: int, + num_context: int, num_seqs: int): # Fill the persistent host buffer in place, exactly as before. CPU-side # consumers read self.host_kv_cache_block_offsets directly and depend on # its persistent, max_batch-sized layout: DSA sparse attention, the @@ -2248,39 +2195,23 @@ def copy_batch_block_offsets(self, # matching the already-safe kv_lens / block_ids_per_seq staging. The # persistent buffer above is untouched by this and stays valid for the # synchronous CPU readers. - host_block_offsets = self._stage_block_offsets_for_copy( - num_seqs, max_blocks) - width = host_block_offsets.shape[-1] + host_block_offsets = self._stage_block_offsets_for_copy(num_seqs) for pool_idx in range(self.num_pools): - dst_tensor[pool_idx, :num_seqs, :, :width].copy_( - host_block_offsets[pool_idx], non_blocking=True) + dst_tensor[pool_idx, :num_seqs].copy_(host_block_offsets[pool_idx], + non_blocking=True) - def _stage_block_offsets_for_copy( - self, - num_rows: int, - max_blocks: Optional[int] = None) -> torch.Tensor: + def _stage_block_offsets_for_copy(self, num_rows: int) -> torch.Tensor: """Snapshot the first ``num_rows`` rows of the persistent host block offset buffer into a fresh pinned buffer, to serve as the private source - of an asynchronous H2D copy (nvbug 6293536). - - ``max_blocks`` bounds the copied block width. The buffer is laid out - for max_seq_len (max_blocks_per_seq columns) but consumers only read - each sequence's allocated block prefix, so a caller that knows the - batch's maximum KV length can skip the unused tail — with a large - max_seq_len the tail dominates the copy cost.""" - if max_blocks is None: - width = self.max_blocks_per_seq - else: - width = min(max(max_blocks, 1), self.max_blocks_per_seq) + of an asynchronous H2D copy (nvbug 6293536).""" host_block_offsets = torch.empty(self.num_pools, num_rows, 2, - width, + self.max_blocks_per_seq, dtype=torch.int32, pin_memory=prefer_pinned(), device='cpu') - host_block_offsets.copy_( - self.host_kv_cache_block_offsets[:, :num_rows, :, :width]) + host_block_offsets.copy_(self.host_kv_cache_block_offsets[:, :num_rows]) return host_block_offsets def truncate_blocks(self, target_tokens: List[int], @@ -2424,34 +2355,14 @@ class BaseKVCacheCompressionManager(BaseResourceManager): base implementations below translate those callbacks into the lifecycle hooks. - Concrete compression methods subclass this directly. The hooks default to + Concrete compression methods subclass this directly. All 4 hooks default to no-op; subclasses override what they need. The manager never inherits from any cache manager because this layer decides *how* the physical KV is used, not *what* physical KV exists. Subclasses hold ``KVCacheManagerV2`` as a tool. - - A subclass compacts through the ``KVCacheManagerV2`` it holds and records - the evicted count on ``LlmRequest.py_num_compressed_tokens``; the model - engine subtracts that count when building ``num_cached_tokens_per_seq``. """ - adjusts_generation_kv_length: ClassVar[bool] = False - """Whether this manager can make target and logical KV lengths diverge.""" - - def __init__( - self, - kv_cache_manager: "KVCacheManagerV2", - draft_kv_cache_manager: Optional["KVCacheManagerV2"] = None, - ): - from .kv_cache_manager_v2 import KVCacheManagerV2 - - if not isinstance(kv_cache_manager, KVCacheManagerV2): - raise TypeError("KV-cache compression requires KVCacheManagerV2") - if draft_kv_cache_manager is not None and not isinstance( - draft_kv_cache_manager, KVCacheManagerV2): - raise TypeError( - "draft KV-cache compression requires KVCacheManagerV2") + def __init__(self, kv_cache_manager: "KVCacheManagerV2"): self.kv_cache_manager = kv_cache_manager - self.draft_kv_cache_manager = draft_kv_cache_manager # Compression evicts/rewrites stored keys and values, so a shared prefix # block is no longer safe to reuse (same constraint as RocketKVCacheManager). if kv_cache_manager.enable_block_reuse: @@ -2459,18 +2370,9 @@ def __init__( f"{type(self).__name__} changes stored keys and values and cannot " f"run with KV-cache block reuse. Set " f"KvCacheConfig.enable_block_reuse to False.") - kv_cache_manager.kv_compression_manages_history = self.adjusts_generation_kv_length - if draft_kv_cache_manager is not None: - # The draft cache is compacted together with the target. - draft_kv_cache_manager.kv_compression_manages_history = ( - self.adjusts_generation_kv_length) - - @property - def has_independent_draft_kv_cache(self) -> bool: - return self.draft_kv_cache_manager is not None # ================================================================== # - # KV-cache lifecycle hooks (5, in temporal order). # + # KV-cache lifecycle hooks (4, in temporal order). # # Subclasses override what they need; all default to no-op. # # ================================================================== # @@ -2481,23 +2383,20 @@ def on_request_init(self, request: "LlmRequest", **kwargs) -> None: scoring buffers). """ - def on_context_step_end(self, requests: List["LlmRequest"], - **kwargs) -> None: - """Fired once per iteration with the requests whose prefill finished - (their final chunk) this step. Batched like the generation hook so a - one-shot prefill-end eviction can process the cohort in one launch. - """ - - def on_generation_step_begin( + def on_context_step_end( self, - scheduled_batch: "ScheduledRequests", + request: "LlmRequest", + metadata: "AttentionMetadata", **kwargs, ) -> None: - """Fired once per generation step before this step's forward.""" + """Fired once per request, when its prefill finishes (its final + chunk). Override for a one-shot prefill-end eviction. + """ def on_generation_step_end( self, scheduled_batch: "ScheduledRequests", + attn_metadata: "AttentionMetadata", **kwargs, ) -> None: """Fired once per generation step, after every layer's forward @@ -2537,7 +2436,6 @@ def prepare_resources(self, scheduled_batch: "ScheduledRequests") -> None: for req in scheduled_batch.context_requests: if req.is_first_context_chunk: self.on_request_init(req) - self.on_generation_step_begin(scheduled_batch) def update_resources( self, @@ -2545,8 +2443,8 @@ def update_resources( attn_metadata: Optional["AttentionMetadata"] = None, kv_cache_dtype_byte_size: Optional[float] = None, ) -> None: - """Fire :meth:`on_context_step_end` with the requests whose final - prefill chunk ran this iteration, then :meth:`on_generation_step_end`. + """Fire :meth:`on_context_step_end` once per request, on the iteration its + final prefill chunk runs, then :meth:`on_generation_step_end` once. Uses the scheduler's ``context_requests_last_chunk`` split (computed at schedule time from ``is_last_context_chunk``) rather than tracking @@ -2557,10 +2455,9 @@ def update_resources( managers so PyExecutor passes ``attn_metadata`` / ``kv_cache_dtype_byte_size`` through transparently. """ - if scheduled_batch.context_requests_last_chunk: - self.on_context_step_end( - scheduled_batch.context_requests_last_chunk) - self.on_generation_step_end(scheduled_batch) + for req in scheduled_batch.context_requests_last_chunk: + self.on_context_step_end(req, attn_metadata) + self.on_generation_step_end(scheduled_batch, attn_metadata) def free_resources(self, request: "LlmRequest") -> None: """Fire :meth:`on_request_finish`.""" @@ -2597,9 +2494,9 @@ def update_resources( attn_metadata: Optional["AttentionMetadata"] = None, kv_cache_dtype_byte_size: Optional[float] = None, ): - for resource_type, resource_manager in self.resource_managers.items(): + for _, resource_manager in self.resource_managers.items(): if hasattr(resource_manager, "update_resources"): - if resource_type == ResourceManagerType.KV_CACHE_MANAGER: + if isinstance(resource_manager, KVCacheManager): resource_manager.update_resources(scheduled_batch, attn_metadata, kv_cache_dtype_byte_size) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py b/tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py index 42670b965a0f..902472d8f562 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/ops/flashinfer.py @@ -15,80 +15,87 @@ """FlashInfer-accelerated sampling kernels. These ops depend on flashinfer; the import is guarded so the module stays -importable without it (sampling_utils imports it unconditionally, and the -vanilla/TRTLLM sampler paths must keep working without flashinfer). Without -flashinfer, calling any op raises an ImportError with installation guidance. -Components that will invoke these ops are expected to fail fast at startup -instead of relying on that call-time error: TorchSampler enforces flashinfer -availability in its constructor. - -Randomness can be supplied either way (flashinfer accepts both in one -signature; explicit ``seed``/``offset`` take precedence over ``generator``): - -- ``generator``: stateful host-side ``torch.Generator``, for eager paths. -- ``seed``/``offset``: stateless device tensors, required under CUDA graph - capture (a ``torch.Generator`` advances host-side at launch time, so its - state would be frozen into the graph and every replay would reuse the same - random values). - -Every op is ``@_compiler_disable``d: nothing inside flashinfer is opaque to -Dynamo (its kernels sit behind a ``functools.cache``-d lazy JIT bootstrap and -its own custom-op registration is a no-op), so tracing in turns each -untraceable builtin of the bootstrap into a warn-once plus a permanent -per-call graph break. Disabling keeps one clean graph break per op and -preserves the bootstrap's cache fast path. +importable without it. """ -from typing import Any, Callable, Optional, TypeVar, Union, cast +from typing import Optional import torch from tensorrt_llm._torch.flashinfer_utils import IS_FLASHINFER_AVAILABLE, get_env_enable_pdl -_OpT = TypeVar("_OpT", bound=Callable[..., Any]) +if IS_FLASHINFER_AVAILABLE: + import flashinfer.sampling -def _compiler_disable(fn: _OpT) -> _OpT: - """``torch.compiler.disable``, typed: the torch stub is untyped and would - fail mypy strict (untyped-decorator) if applied directly.""" - return cast(_OpT, torch.compiler.disable(fn)) +def top_k_top_p_sampling_from_logits_op( + logits: torch.Tensor, + top_k: torch.Tensor, + top_p: torch.Tensor, + seed: Optional[int] = None, + offset: Optional[int] = None, +) -> torch.Tensor: + tokens: torch.Tensor = flashinfer.sampling.top_k_top_p_sampling_from_logits( + logits, top_k, top_p, seed=seed, offset=offset + ) + return tokens -if IS_FLASHINFER_AVAILABLE: - import flashinfer.sampling -else: +def sampling_from_probs_op( + probs: torch.Tensor, + seed: Optional[torch.Tensor] = None, + offset: Optional[torch.Tensor] = None, +) -> torch.Tensor: + tokens: torch.Tensor = flashinfer.sampling.sampling_from_probs( + probs, deterministic=True, seed=seed, offset=offset + ) + return tokens - class _FlashInferUnavailable: - """Placeholder that raises on first use instead of a bare NameError.""" - def __getattr__(self, name: str) -> Any: - raise ImportError( - "flashinfer is required for the FlashInfer sampling ops but is " - "not installed; please install the version pinned in " - "requirements.txt." - ) +def softmax_op( + logits: torch.Tensor, + temperature: Optional[torch.Tensor], +) -> torch.Tensor: + probs: torch.Tensor = flashinfer.sampling.softmax( + logits, temperature, enable_pdl=get_env_enable_pdl() + ) + return probs - flashinfer = _FlashInferUnavailable() # type: ignore[assignment] -SeedOrTensor = Union[int, torch.Tensor] +def top_k_mask_logits_op( + logits: torch.Tensor, + top_k: torch.Tensor, +) -> torch.Tensor: + masked: torch.Tensor = flashinfer.sampling.top_k_mask_logits(logits, top_k) + return masked -@_compiler_disable -def top_k_top_p_sampling_from_logits_op( +def top_p_renorm_probs_op( + probs: torch.Tensor, + top_p: torch.Tensor, +) -> torch.Tensor: + renormed: torch.Tensor = flashinfer.sampling.top_p_renorm_probs(probs, top_p) + return renormed + + +def sampling_from_probs_generator_op( + probs: torch.Tensor, + generator: Optional[torch.Generator], + check_nan: bool = False, +) -> torch.Tensor: + tokens: torch.Tensor = flashinfer.sampling.sampling_from_probs( + probs, deterministic=True, generator=generator, check_nan=check_nan + ) + return tokens + + +def top_k_top_p_sampling_from_logits_with_generator_op( logits: torch.Tensor, top_k: torch.Tensor, top_p: torch.Tensor, - *, - generator: Optional[torch.Generator] = None, - seed: Optional[SeedOrTensor] = None, - offset: Optional[SeedOrTensor] = None, + generator: Optional[torch.Generator], check_nan: bool = False, ) -> torch.Tensor: - """Fused top-k + top-p sampling from pre-softmax logits. - - Randomness: pass ``generator`` (eager) or ``seed``/``offset`` (CUDA graph); - see module docstring for the full contract. - """ tokens: torch.Tensor = flashinfer.sampling.top_k_top_p_sampling_from_logits( logits, top_k=top_k, @@ -97,127 +104,42 @@ def top_k_top_p_sampling_from_logits_op( deterministic=True, check_nan=check_nan, generator=generator, - seed=seed, - offset=offset, ) return tokens -@_compiler_disable -def sampling_from_probs_op( - probs: torch.Tensor, - *, - generator: Optional[torch.Generator] = None, - seed: Optional[SeedOrTensor] = None, - offset: Optional[SeedOrTensor] = None, - check_nan: bool = False, -) -> torch.Tensor: - """Categorical sampling from probabilities. - - Randomness: pass ``generator`` (eager) or ``seed``/``offset`` (CUDA graph); - see module docstring for the full contract. - """ - tokens: torch.Tensor = flashinfer.sampling.sampling_from_probs( - probs, - deterministic=True, - check_nan=check_nan, - generator=generator, - seed=seed, - offset=offset, - ) - return tokens - - -@_compiler_disable -def top_k_sampling_from_probs_op( +def top_k_sampling_from_probs_generator_op( probs: torch.Tensor, top_k: torch.Tensor, - *, - generator: Optional[torch.Generator] = None, - seed: Optional[SeedOrTensor] = None, - offset: Optional[SeedOrTensor] = None, + generator: Optional[torch.Generator], check_nan: bool = False, ) -> torch.Tensor: - """Top-k filtered sampling from probabilities. - - Randomness: pass ``generator`` (eager) or ``seed``/``offset`` (CUDA graph); - see module docstring for the full contract. - """ tokens: torch.Tensor = flashinfer.sampling.top_k_sampling_from_probs( probs, top_k=top_k, deterministic=True, check_nan=check_nan, generator=generator, - seed=seed, - offset=offset, ) return tokens -@_compiler_disable -def top_p_sampling_from_probs_op( +def top_p_sampling_from_probs_generator_op( probs: torch.Tensor, top_p: torch.Tensor, - *, - generator: Optional[torch.Generator] = None, - seed: Optional[SeedOrTensor] = None, - offset: Optional[SeedOrTensor] = None, + generator: Optional[torch.Generator], check_nan: bool = False, ) -> torch.Tensor: - """Top-p filtered sampling from probabilities. - - Randomness: pass ``generator`` (eager) or ``seed``/``offset`` (CUDA graph); - see module docstring for the full contract. - """ tokens: torch.Tensor = flashinfer.sampling.top_p_sampling_from_probs( probs, top_p=top_p, deterministic=True, check_nan=check_nan, generator=generator, - seed=seed, - offset=offset, ) return tokens -# The three ops below wrap the mask -> softmax -> renorm pipeline stages 1:1. -# The wrappers exist so callers stay importable without flashinfer installed -# (the flashinfer import above is guarded); softmax_op additionally centralizes -# the PDL env decision. - - -@_compiler_disable -def softmax_op( - logits: torch.Tensor, - temperature: Optional[torch.Tensor], -) -> torch.Tensor: - probs: torch.Tensor = flashinfer.sampling.softmax( - logits, temperature, enable_pdl=get_env_enable_pdl() - ) - return probs - - -@_compiler_disable -def top_k_mask_logits_op( - logits: torch.Tensor, - top_k: torch.Tensor, -) -> torch.Tensor: - masked: torch.Tensor = flashinfer.sampling.top_k_mask_logits(logits, top_k) - return masked - - -@_compiler_disable -def top_p_renorm_probs_op( - probs: torch.Tensor, - top_p: torch.Tensor, -) -> torch.Tensor: - renormed: torch.Tensor = flashinfer.sampling.top_p_renorm_probs(probs, top_p) - return renormed - - -@_compiler_disable def compute_probs_from_logits_op( logits: torch.Tensor, temperatures: torch.Tensor, diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py b/tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py index 8cfcca1e13b0..ffb962a60fc6 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/ops/vanilla.py @@ -26,7 +26,7 @@ from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.bindings.executor import FinishReason -BEAM_SEARCH_PAD_TOKEN = -1 +_BEAM_SEARCH_PAD_TOKEN = -1 @dataclass(kw_only=True) @@ -50,27 +50,66 @@ class BeamSearchMetadata(StrategyMetadata): beam_idx_arange: torch.Tensor -def top_k_top_p_sampling_batch( +def top_k_sampling_batch( logits: torch.Tensor, *, + top_k: int, temperature: float, - top_k: Optional[int] = None, - top_p: float = 1.0, generator: Optional[torch.Generator] = None, ) -> tuple[torch.Tensor, torch.Tensor]: - """Temperature + optional top-k / top-p filtering + multinomial sampling. + return top_k_top_p_sampling_batch( + logits, + top_k=top_k, + temperature=temperature, + generator=generator, + top_p=1, + ) - ``top_k=None`` (or ``vocab_size``) disables top-k filtering; ``top_p=1`` - disables top-p filtering. With both disabled this is plain temperature - sampling. - """ + +def top_p_sampling_batch( + logits: torch.Tensor, + *, + top_p: float, + temperature: float, + generator: Optional[torch.Generator] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + return top_k_top_p_sampling_batch( + logits, + top_p=top_p, + top_k=logits.size(1), + temperature=temperature, + generator=generator, + ) + + +def temperature_sampling_batch( + logits: torch.Tensor, + *, + temperature: float, + generator: Optional[torch.Generator] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + return top_k_top_p_sampling_batch( + logits, + top_p=1, + top_k=logits.size(1), + temperature=temperature, + generator=generator, + ) + + +def top_k_top_p_sampling_batch( + logits: torch.Tensor, + *, + top_k: int, + top_p: float, + temperature: float, + generator: Optional[torch.Generator] = None, +) -> tuple[torch.Tensor, torch.Tensor]: logits_dim = logits.dim() assert logits_dim == 2, "logits should be 2D: [batch_size, vocab_size]" assert temperature > 0, "non-greedy sampling requires valid temperature" logits = logits / max(temperature, 1e-5) batch_size, vocab_size = logits.size() - if top_k is None: - top_k = vocab_size assert top_k > 1, "non-greedy sampling requires valid top_k" need_top_k = top_k < vocab_size @@ -235,7 +274,7 @@ def beam_search_sampling_batch( next_tokens = next_tokens % vocab_size ended_predecessor_mask = torch.gather(dim=1, index=predecessor_beam, input=finished_beams_mask) - next_tokens = torch.where(ended_predecessor_mask, BEAM_SEARCH_PAD_TOKEN, next_tokens) + next_tokens = torch.where(ended_predecessor_mask, _BEAM_SEARCH_PAD_TOKEN, next_tokens) old_cum_log_probs = beam_search_args.cum_log_probs[beam_search_args.seq_slots].view(-1) beam_search_args.new_log_probs[beam_search_args.seq_slots, :beam_width_out] = ( @@ -283,19 +322,24 @@ def sample_rejected( return cast(int, new_token.item()) -# Rows whose temperature is at or below this threshold are treated as greedy. -# Contract with the spec-decoding metadata layer: greedy requests are -# normalized to a sentinel temperature strictly below this threshold (see -# DISABLE_TEMP_VAL in speculative/interface.py, which derives from it). -GREEDY_TEMPERATURE_THRESHOLD = 1e-4 +_GREEDY_TEMPERATURE_THRESHOLD = 1e-4 + + +def greedy( + logits: torch.Tensor, + *, + return_probs: bool = True, +) -> tuple[torch.Tensor, Optional[torch.Tensor]]: + """Greedy decoding; returns exact one-hot when return_probs=True.""" + return greedy_search_sampling_batch(logits, return_probs=return_probs) -def safely_apply_temperature_inplace( +def _safely_apply_temperature_inplace( logits_inout: torch.Tensor, temp: torch.Tensor ) -> torch.Tensor: """Divide logits by per-row temperature in place, guarding the greedy sentinel. - Greedy requests carry a temperature of 0 / <= ``GREEDY_TEMPERATURE_THRESHOLD``. + Greedy requests carry a temperature of 0 / <= ``_GREEDY_TEMPERATURE_THRESHOLD``. Dividing by it would blow logits up to inf/nan and corrupt downstream sampling (argmax / softmax / multinomial). Those rows are clamped to a temperature of 1.0 so the division is numerically safe; callers are expected to overwrite the greedy @@ -305,11 +349,11 @@ def safely_apply_temperature_inplace( ``logits_inout`` is modified in place (``div_``) and also returned for convenience; ``temp`` is left untouched. """ - safe_temp = torch.where(temp <= GREEDY_TEMPERATURE_THRESHOLD, torch.ones_like(temp), temp) + safe_temp = torch.where(temp <= _GREEDY_TEMPERATURE_THRESHOLD, torch.ones_like(temp), temp) return logits_inout.div_(safe_temp.unsqueeze(dim=1)) -class Fusions: +class _Fusions: @staticmethod @torch.compile(dynamic=None, fullgraph=True) def _gather_scatter_impl( @@ -331,7 +375,7 @@ def gather_scatter( torch._dynamo.mark_dynamic(dst_index_cuda, 0) torch._dynamo.mark_dynamic(src_cuda, 0) torch._dynamo.mark_dynamic(src_index_cuda, 0) - Fusions._gather_scatter_impl(dst_cuda, dst_index_cuda, src_cuda, src_index_cuda) + _Fusions._gather_scatter_impl(dst_cuda, dst_index_cuda, src_cuda, src_index_cuda) @staticmethod @torch.compile(dynamic=None, fullgraph=True) @@ -349,7 +393,7 @@ def determine_sampled_rank( ) -> torch.Tensor: torch._dynamo.mark_dynamic(group_logprobs_cuda, 0) torch._dynamo.mark_dynamic(sampled_logprobs_cuda, 0) - return Fusions._determine_sampled_rank_impl(group_logprobs_cuda, sampled_logprobs_cuda) + return _Fusions._determine_sampled_rank_impl(group_logprobs_cuda, sampled_logprobs_cuda) @staticmethod @torch.compile( @@ -372,106 +416,4 @@ def _gather_log_softmax_impl( def gather_log_softmax(inputs_cuda: torch.Tensor, indices_cuda: torch.Tensor) -> torch.Tensor: torch._dynamo.mark_dynamic(inputs_cuda, 0) torch._dynamo.mark_dynamic(indices_cuda, 0) - return Fusions._gather_log_softmax_impl(inputs_cuda, indices_cuda) - - # --- Top-P Decay ops --------------------------------------------------- - # Host-launch-bound per-step ops (a few dozen elements per row), fused with - # Inductor to keep the launch count low. mode="max-autotune-no-cudagraphs": - # cudagraphs is unsafe here (the update mutates persistent per-slot state - # in place and the gather's output is consumed outside the compiled region; - # cudagraph static output buffers get overwritten by subsequent replays). - # mark_dynamic on the batch-varying dims avoids recompilation as the batch - # composition changes. Compilation is lazy: the first decay-active request - # pays it (roughly a second); non-decay workloads never trigger it. See - # TorchSampler.TopPDecayStore for the feature-level semantics. - - @staticmethod - @torch.compile(mode="max-autotune-no-cudagraphs") - def _top_p_decay_update_impl( - runtime_top_p: torch.Tensor, - initial_top_p: torch.Tensor, - top_p_decay: torch.Tensor, - top_p_min: torch.Tensor, - reset_ids: torch.Tensor, - is_decay_slot: torch.Tensor, - step_tokens: torch.Tensor, - sampled_slots: torch.Tensor, - ) -> None: - active = is_decay_slot[sampled_slots] - current = runtime_top_p[sampled_slots] - updated = torch.where( - step_tokens[sampled_slots] == reset_ids[sampled_slots], - initial_top_p[sampled_slots], - torch.maximum(current * top_p_decay[sampled_slots], top_p_min[sampled_slots]), - ) - runtime_top_p[sampled_slots] = torch.where(active, updated, current) - - @staticmethod - def top_p_decay_update( - *, - runtime_top_p: torch.Tensor, - initial_top_p: torch.Tensor, - top_p_decay: torch.Tensor, - top_p_min: torch.Tensor, - reset_ids: torch.Tensor, - is_decay_slot: torch.Tensor, - step_tokens: torch.Tensor, - sampled_slots: torch.Tensor, - ) -> None: - """Fused in-place update of ``runtime_top_p`` for the sampled decay slots. - - Applies the Top-P Decay recurrence (see ``TorchSampler.TopPDecayStore`` - for the feature-level semantics) to every sampled row whose slot is - decay-active per ``is_decay_slot``. - - All per-slot tensors are 1-D of length ``max_num_sequences``; - ``step_tokens`` is a slot-indexed 1-D (possibly strided) int32 view of - the new-tokens buffer for a fixed step/beam - (``new_tokens[step, :, beam]``); ``sampled_slots`` is 1-D of length - ``num_sampled`` (this iteration's rows). ``runtime_top_p`` is mutated - in place; nothing is returned. - """ - torch._dynamo.mark_dynamic(sampled_slots, 0) - Fusions._top_p_decay_update_impl( - runtime_top_p, - initial_top_p, - top_p_decay, - top_p_min, - reset_ids, - is_decay_slot, - step_tokens, - sampled_slots, - ) - - @staticmethod - @torch.compile(mode="max-autotune-no-cudagraphs") - def _top_p_decay_gather_impl( - runtime_top_p: torch.Tensor, - is_decay_slot: torch.Tensor, - static_top_p: torch.Tensor, - slots: torch.Tensor, - ) -> torch.Tensor: - return torch.where(is_decay_slot[slots], runtime_top_p[slots], static_top_p) - - @staticmethod - def top_p_decay_gather( - *, - runtime_top_p: torch.Tensor, - is_decay_slot: torch.Tensor, - static_top_p: torch.Tensor, - slots: torch.Tensor, - ) -> torch.Tensor: - """Fused pre-sample per-row top-p gather for decay-active rows. - - Returns a new per-row tensor:: - - row_top_p[i] = runtime_top_p[slots[i]] if is_decay_slot[slots[i]] - = static_top_p[i] otherwise - - ``runtime_top_p`` / ``is_decay_slot`` are per-slot arrays; - ``static_top_p`` and ``slots`` are per-row (length = the group's - per-step row count). - """ - torch._dynamo.mark_dynamic(slots, 0) - torch._dynamo.mark_dynamic(static_top_p, 0) - return Fusions._top_p_decay_gather_impl(runtime_top_p, is_decay_slot, static_top_p, slots) + return _Fusions._gather_log_softmax_impl(inputs_cuda, indices_cuda) diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py index 041cb31f97be..6c0f1140f971 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampler.py @@ -93,17 +93,15 @@ GREEDY, BeamSearchMetadata, FlashInferGroupedStrategySampler, - Fusions, GenericStrategyKeyType, Strategy, StrategyMetadata, - TopPDecayMetadata, UtilsSamplingParams, + _Fusions, get_rejected_indices, resolve_sampling_strategy, sample, sample_rejected, - top_p_decay_active, ) if sys.version_info[:2] >= (3, 12): @@ -477,17 +475,9 @@ def _get_max_beam_width(request: LlmRequest) -> int: def _request_get_sampling_params(request: LlmRequest) -> UtilsSamplingParams: sampling_config = request.sampling_config - # These sampling fields live on the C++ SamplingConfig as optional> - # (a shape designed for the batched TRT-LLM sampler); the torch sampler consumes - # them per request, so we unwrap the singleton lists into scalars here. When the - # TRT-LLM sampler is removed, this SamplingConfig-based plumbing should be removed - # too in favor of reading the values directly from the per-request params. temperature = _unwrap_singleton(cast(Optional[list[float]], sampling_config.temperature)) top_p = _unwrap_singleton(cast(Optional[list[float]], sampling_config.top_p)) top_k = _unwrap_singleton(cast(Optional[list[int]], sampling_config.top_k)) - top_p_decay = _unwrap_singleton(cast(Optional[list[float]], sampling_config.top_p_decay)) - top_p_min = _unwrap_singleton(cast(Optional[list[float]], sampling_config.top_p_min)) - top_p_reset_ids = _unwrap_singleton(cast(Optional[list[int]], sampling_config.top_p_reset_ids)) beam_width_out = _get_beam_width_out(request) beam_width_in = _get_beam_width_in(request) use_beam_search = _get_max_beam_width(request) > 1 @@ -499,9 +489,6 @@ def _request_get_sampling_params(request: LlmRequest) -> UtilsSamplingParams: beam_width_in=beam_width_in, beam_width_out=beam_width_out, use_beam_search=use_beam_search, - top_p_decay=top_p_decay, - top_p_min=top_p_min, - top_p_reset_ids=top_p_reset_ids, ) @@ -2260,79 +2247,6 @@ class LogProbsStore: """Shape: batch_size, max_tokens, max_topk_logprobs Usage: Stores the values of the topk logprobs""" - @dataclass(kw_only=True) - class TopPDecayStore: - """Per-slot runtime state for Top-P Decay -- the single source of truth - for the feature's semantics on the torch path. - - Semantics (matching the legacy C++ ``computeToppDecay`` kernel): after - every sampled token of a decay-active request:: - - runtime_top_p = initial_top_p if token == reset_id - = max(runtime_top_p * top_p_decay, top_p_min) otherwise - - A negative ``reset_ids`` sentinel (-1, "reset disabled") never matches, - since sampled token ids are non-negative. Decay is active iff - ``top_p_decay`` is set and < 1.0 (``SamplingParams. - params_imply_top_p_decay_active``); an active decay forces a - top-p-capable strategy even for an otherwise implicitly-greedy request - (initial top-p defaults to 1.0), while explicit greedy controls win. - Beam search and speculative draft tokens are rejected at admission - (``_validate_top_p_decay_request``); parameter ranges are enforced by - ``SamplingParams._validate`` / the executor::SamplingConfig constructor. - - Lifecycle per slot (each tensor has shape ``(max_num_sequences,)``): - - 1. Admission (``_setup_top_p_decay_for_new_requests``): membership is - cleared then re-set for the newly-admitted slots -- both the host-side - ``TorchSampler._top_p_decay_slots`` set (an O(1) hot-path early-out) - and its device mirror ``is_top_p_decay_slot_cuda`` (the gate the - fused ops use, so the hot path needs no host-side filtering) -- - and the per-slot buffers are initialized. This clear-then-set also - covers slot reuse: stale entries from a prior occupant are never - consumed. - 2. Pre-sample (``_build_top_p_decay_metadata`` -> ``TopPDecayMetadata`` - -> ``TopPDecayMixin``): the per-row top-p fed to top_p / - top_k_top_p sampling is overridden with the decayed runtime value - for decay-active rows (fused gather, ``top_p_decay_gather``). - 3. Post-sample (``_update_top_p_decay_after_sample``): the recurrence - above is applied in place for the sampled decay-active slots (fused - update, ``top_p_decay_update``). - 4. Finish (``_retire_top_p_decay_slot``): the slot leaves the - membership set so the early-outs re-arm; the device buffers need no - cleanup (a freed slot is never sampled, reuse re-initializes it). - """ - - runtime_top_p_decay_cuda: torch.Tensor - """The current (decaying) top-p per slot; mutated post-sample each step.""" - initial_top_p_decay_cuda: torch.Tensor - """The initial top-p per slot; used to reset on a reset-id match.""" - top_p_decay_cuda: torch.Tensor - """Per-slot multiplicative decay factor.""" - top_p_decay_min_cuda: torch.Tensor - """Per-slot lower bound for the decayed top-p.""" - top_p_decay_reset_ids_cuda: torch.Tensor - """Per-slot reset token id (< 0 never matches a sampled token).""" - is_top_p_decay_slot_cuda: torch.Tensor - """Per-slot bool gate (device mirror of ``_top_p_decay_slots``). Lets the - fused post-sample update op filter decay-active slots on the GPU, - avoiding a host-side ``.tolist()`` / set intersection each step.""" - - @classmethod - def create(cls, max_num_sequences: int) -> "TorchSampler.TopPDecayStore": - n = (max_num_sequences,) - return cls( - runtime_top_p_decay_cuda=torch.empty(n, dtype=torch.float32, device="cuda"), - initial_top_p_decay_cuda=torch.empty(n, dtype=torch.float32, device="cuda"), - top_p_decay_cuda=torch.empty(n, dtype=torch.float32, device="cuda"), - top_p_decay_min_cuda=torch.empty(n, dtype=torch.float32, device="cuda"), - top_p_decay_reset_ids_cuda=torch.empty(n, dtype=torch.int, device="cuda"), - # The gate buffer IS the gate, so (unlike the others) it must start - # False: a slot that was never admitted as decay-active must read - # False. - is_top_p_decay_slot_cuda=torch.zeros(n, dtype=torch.bool, device="cuda"), - ) - @dataclass(kw_only=True) class Store: new_tokens: torch.Tensor @@ -2344,8 +2258,6 @@ class Store: """Holds data related to beam search.""" log_probs_store: "TorchSampler.LogProbsStore" """Holds data related to log-probs handling.""" - top_p_decay_store: "TorchSampler.TopPDecayStore" - """Holds per-slot runtime state for Top-P Decay.""" def _create_store(self) -> Store: # Tensors necessary for all sampling methods @@ -2396,15 +2308,10 @@ def _create_store(self) -> Store: seq_offsets=seq_offsets, beam_idx_arange=beam_idx_arange, ) - # Per-slot Top-P Decay runtime state (FlashInfer path). Allocated for all - # sampler instances; only slots in self._top_p_decay_slots are ever read. - top_p_decay_store = self.TopPDecayStore.create(self.max_num_sequences) - return self.Store( new_tokens=new_tokens, log_probs_store=log_probs_store, beam_search_store=beam_search_store, - top_p_decay_store=top_p_decay_store, ) @dataclass(frozen=True, kw_only=True) @@ -2422,9 +2329,6 @@ def __init__(self, args: Args): self.max_seq_len = args.max_seq_len self.max_tokens = args.max_total_draft_tokens + 1 self.max_beam_width = args.max_beam_width - # Snapshot of `not self._use_beam_search` so the update_requests - # fast-path avoids a property call per iteration. - self._batch_fastpath_eligible: bool = self.max_beam_width == 1 # The current maximum number of topk logprobs which can be stored in the sampler's store self.max_topk_logprobs = MAX_TOP_LOGPROBS # The maximum number of topk logprobs for the current batch of requests @@ -2432,13 +2336,6 @@ def __init__(self, args: Args): if args.max_total_draft_tokens > 0 and args.max_beam_width > 1: raise ValueError("TorchSampler does not support beam search with speculative decoding") self.max_num_sequences = args.max_num_sequences - # With the overlap scheduler, sample_async for step i runs before - # update_requests for step i-1, so the host-side token lists lag the - # device state. Track, per seq slot, how many sampled steps have not - # been folded back into the request yet; bad-words handling uses this - # to decide whether the newest token must be read device-side. - self._track_pending_steps = not args.disable_overlap_scheduler - self._pending_steps = [0] * self.max_num_sequences self.NEW_TOKENS_SHAPE = (self.max_tokens, self.max_num_sequences, self.max_beam_width) self.CACHE_INDIRECTION_SHAPE = ( self.max_num_sequences, @@ -2457,10 +2354,6 @@ def __init__(self, args: Args): "in requirements.txt." ) self._grouped_sampler_cls = FlashInferGroupedStrategySampler - # Slots with an active top-p-decay request. Sole gate for reading/updating - # the per-slot TopPDecayStore buffers; discarded on slot reuse so stale - # buffer entries are never consumed. - self._top_p_decay_slots: set[int] = set() # AutoDeploy build creates the sampler in inference mode, # which would disallow in-place mutating of new_tokens. @@ -2513,22 +2406,6 @@ def __init__(self, args: Args): None ] * self.max_num_sequences - @staticmethod - def _is_draft_batch(requests: list[LlmRequest]) -> bool: - """Whether this batch belongs to the draft model. - - Batches are homogeneous by construction: ModelDrafter builds all-draft - batches for its sample_async/update_requests calls on this shared - sampler, and PyExecutor's batches are all-target. The pending-steps - accounting relies on this to skip draft batches wholesale; assert it so - a mixed batch fails loudly instead of silently corrupting the counters. - """ - is_draft: bool = requests[0].py_is_draft - assert all(r.py_is_draft == is_draft for r in requests), ( - "sampler batch must be homogeneous (all-draft or all-target)" - ) - return is_draft - def get_generator(self, device: torch.device) -> torch.Generator: """Get a deterministic generator for the specified device. @@ -2561,48 +2438,6 @@ def get_spec_tree_manager( def _use_beam_search(self) -> bool: return self.max_beam_width > 1 - def _validate_top_p_decay_request(self, request: LlmRequest) -> None: - """Reject unsupported combinations for a top-p-decay-active request. - - Top-p decay is supported only for single-token decode steps without beam - search. Called from validate_request (request admission), so a violating - request is failed individually instead of aborting the whole batch. - """ - params = _request_get_sampling_params(request) - # NB: value ranges need no re-check here. Every request enters through - # the executor::SamplingConfig constructor, which hard-validates - # top_p_decay in (0, 1], top_p_min in (0, 1] and top_p_reset_ids >= 0 - # (samplingConfig.cpp check* helpers) for all frontends. A reset id - # >= vocab_size is not checked anywhere but is semantically inert: it - # can never match a sampled token, i.e. it behaves as "reset disabled". - if not top_p_decay_active(params): - return - if params.use_beam_search: - raise ValueError("top_p_decay is not supported with beam search.") - # A non-zero draft length means the request carries speculative draft - # tokens and produces multiple tokens per step (req_num_steps = - # 1 + draft_token_length). One-model speculation (vanilla MTP, one-model - # Eagle3 / MTP-Eagle, SA, draft-target-one-model) uses its own - # SpecSamplerBase-derived sampler and never reaches TorchSampler; the - # drafter-based modes that DO flow draft tokens through TorchSampler - # (two-model draft-target, NGram, user-provided, two-model Eagle3 / - # MTP-Eagle) are what can make this length non-zero. top-p decay does not - # support these multi-token steps. - # NB: at admission time the draft tokens of drafter-based modes are - # usually not attached yet, so this check is best-effort. Two-model - # speculation (the only source of such requests in TorchSampler) is - # slated for removal, so in practice no speculative request reaches the - # decay path; a debug assert in _build_top_p_decay_metadata guards the - # invariant at sample time. - if get_draft_token_length(request) > 0: - raise ValueError( - "top_p_decay is not supported for requests carrying speculative " - "draft tokens (req_num_steps > 1). This covers the drafter-based " - "modes routed through TorchSampler (two-model draft-target, NGram, " - "user-provided, two-model Eagle3 / MTP-Eagle); one-model " - "speculation uses its own sampler and is unaffected." - ) - def _can_use_fast_greedy_path(self, requests: list[LlmRequest]) -> bool: """ Check if we can use the fast argmax path for greedy sampling. @@ -2709,6 +2544,31 @@ def _handle_finish_reasons_impl( return True return False + def _handle_finish_reasons( + self, + request: LlmRequest, + finish_reasons: torch.Tensor, + finish_reasons_list: list[list[list[int]]], + ) -> bool: + """Check if all beams of a request have finished and set the request state accordingly + + Args: + request: LlmRequest. The request to check. + finish_reasons: torch.Tensor. Shape: (max_tokens, max_batch_size, max_beam_width) + The finish reasons for each beam. + finish_reasons_list: list[list[list[int]]]. The finish reasons for each beam. + Returns: + True if all beams have finished, False otherwise. + """ + assert request.py_seq_slot is not None + beam_width = request.py_beam_width + return self._handle_finish_reasons_impl( + request, + beam_width, + finish_reasons[DEFAULT_STEP_IDX, request.py_seq_slot], + finish_reasons_list[request.py_seq_slot][DEFAULT_STEP_IDX], + ) + def _handle_first_finish_reasons( self, request: LlmRequest, @@ -3063,10 +2923,6 @@ def _collect_new_requests_for_setup( @override def validate_request(self, request: LlmRequest) -> None: - # Reject unsupported top-p-decay combinations at admission, so only the - # offending request fails (raising later, inside setup_sampler_step or - # sampling, would abort the whole executor step). - self._validate_top_p_decay_request(request) if self._use_beam_search: if request.py_return_log_probs: if request.py_num_logprobs > 1: @@ -3141,10 +2997,6 @@ def setup_sampler_step(self, scheduled_requests: ScheduledRequests) -> None: all_sampling_requests=new_requests + scheduled_requests.generation_requests, ) - self._setup_top_p_decay_for_new_requests( - new_requests, new_seq_slots_cuda_long=seq_slots_tensor_cuda_long - ) - if self._use_beam_search: beam_search_store = self.store.beam_search_store assert beam_search_store is not None @@ -3155,106 +3007,6 @@ def setup_sampler_step(self, scheduled_requests: ScheduledRequests) -> None: max_prompt_len=max_prompt_len, ) - def _setup_top_p_decay_for_new_requests( - self, - new_requests: list[LlmRequest], - *, - new_seq_slots_cuda_long: torch.Tensor, - ) -> None: - """Refresh top-p-decay membership and per-slot buffers for admitted requests - (lifecycle step 1, see ``TopPDecayStore``). - - Drops stale membership from prior occupants of the newly-admitted slots - (host set and device gate), then re-admits the decay-active requests and - initializes their per-slot store entries. Unsupported decay combinations - were already rejected per-request in validate_request at admission. - """ - # Clear the device decay gate for every newly-admitted slot (covers slot - # reuse: a slot previously decay-active but reused by a non-decay request - # must read False). Decay-active slots are then set True below. - decay_gate = self.store.top_p_decay_store.is_top_p_decay_slot_cuda - decay_gate.index_fill_(0, new_seq_slots_cuda_long, False) - - decay_seq_slots: list[int] = [] - initial_top_p: list[float] = [] - top_p_decay: list[float] = [] - top_p_min: list[float] = [] - top_p_reset_ids: list[int] = [] - for request in new_requests: - slot = request.py_seq_slot - assert slot is not None - self._top_p_decay_slots.discard(slot) - sampling_params = _request_get_sampling_params(request) - if not top_p_decay_active(sampling_params): - continue - self._top_p_decay_slots.add(slot) - decay_seq_slots.append(slot) - # Initial runtime top-p defaults to 1.0 when top_p is unset. - initial_top_p.append( - sampling_params.top_p if sampling_params.top_p is not None else 1.0 - ) - # decay is guaranteed non-None and < 1.0 here (top_p_decay_active); - # min/reset fall back to the C++ runtime defaults when unset. - assert sampling_params.top_p_decay is not None - top_p_decay.append(sampling_params.top_p_decay) - top_p_min.append( - sampling_params.top_p_min if sampling_params.top_p_min is not None else 1e-6 - ) - top_p_reset_ids.append( - sampling_params.top_p_reset_ids - if sampling_params.top_p_reset_ids is not None - else -1 - ) - - if decay_seq_slots: - self._update_top_p_decay_store_for_new_requests( - decay_seq_slots=decay_seq_slots, - initial_top_p=initial_top_p, - top_p_decay=top_p_decay, - top_p_min=top_p_min, - top_p_reset_ids=top_p_reset_ids, - ) - - def _update_top_p_decay_store_for_new_requests( - self, - *, - decay_seq_slots: list[int], - initial_top_p: list[float], - top_p_decay: list[float], - top_p_min: list[float], - top_p_reset_ids: list[int], - ) -> None: - """Initialize per-slot Top-P Decay buffers for newly admitted decay requests. - - runtime_top_p and initial_top_p both start at the effective initial top-p; - the runtime value is decayed post-sample each step. - """ - store = self.store.top_p_decay_store - device = store.runtime_top_p_decay_cuda.device - slots_cuda = torch.tensor( - decay_seq_slots, device="cpu", dtype=torch.int64, pin_memory=prefer_pinned() - ).to(device, non_blocking=True) - floats_host = torch.tensor( - [initial_top_p, top_p_decay, top_p_min], - device="cpu", - dtype=torch.float32, - pin_memory=prefer_pinned(), - ) - floats_cuda = floats_host.to(device, non_blocking=True) - initial_cuda = floats_cuda[0] - reset_ids_cuda = torch.tensor( - top_p_reset_ids, device="cpu", dtype=torch.int32, pin_memory=prefer_pinned() - ).to(device, non_blocking=True) - - store.runtime_top_p_decay_cuda.index_copy_(0, slots_cuda, initial_cuda) - store.initial_top_p_decay_cuda.index_copy_(0, slots_cuda, initial_cuda) - store.top_p_decay_cuda.index_copy_(0, slots_cuda, floats_cuda[1]) - store.top_p_decay_min_cuda.index_copy_(0, slots_cuda, floats_cuda[2]) - store.top_p_decay_reset_ids_cuda.index_copy_(0, slots_cuda, reset_ids_cuda) - # Enable the device gate for these decay-active slots (cleared for all new - # slots in setup_sampler_step just before this call). - store.is_top_p_decay_slot_cuda.index_fill_(0, slots_cuda, True) - @staticmethod def _prepare_beam_search( beam_search_store: BeamSearchStore, @@ -3447,10 +3199,8 @@ def _get_logprobs_from_request( pin_memory=pin_memory, dtype=torch.int32, ) - # NB: forward slicing, because [:, :-0, :] would yield an empty view - # instead of the full history when preallocate_extra_steps == 0. - logprobs_tensor = logprobs_tensor_full[:, :num_generated_tokens, :] - logprobs_indices_tensor = logprobs_indices_tensor_full[:, :num_generated_tokens, :] + logprobs_tensor = logprobs_tensor_full[:, :-preallocate_extra_steps, :] + logprobs_indices_tensor = logprobs_indices_tensor_full[:, :-preallocate_extra_steps, :] if logprobs_tensor.numel() > 0: logprobs_list = request.py_result.log_probs assert logprobs_list is not None @@ -3740,7 +3490,6 @@ def _add_metadata_to_grouped_requests( *, seq_slots_cuda: torch.Tensor, seq_lens_cuda: torch.Tensor, - req_num_steps: torch.Tensor, ) -> dict[RequestGroupKey[GenericStrategyKeyType], RequestGroupValueWithMetadata]: grouped_requests_with_metadata: dict[ RequestGroupKey[GenericStrategyKeyType], RequestGroupValueWithMetadata @@ -3750,7 +3499,6 @@ def _add_metadata_to_grouped_requests( num_requests = len(requests) for key, value in grouped_requests.items(): metadata_type = get_metadata_type_for_group_fn(key.strategy_key) - metadata: StrategyMetadata | None if metadata_type is BeamSearchMetadata: assert beam_search_store is not None assert seq_lens is not None, "seq_lens is required for beam search" @@ -3779,13 +3527,6 @@ def _add_metadata_to_grouped_requests( seq_offsets=beam_search_store.seq_offsets, beam_idx_arange=beam_search_store.beam_idx_arange, ) - elif metadata_type is TopPDecayMetadata: - metadata = self._build_top_p_decay_metadata( - group_req_indices=value.indices, - req_num_steps=req_num_steps, - seq_slots=seq_slots, - seq_slots_cuda=seq_slots_cuda, - ) elif metadata_type is None: metadata = None else: @@ -3906,12 +3647,6 @@ def update_requests( if not state.requests: return - if self._track_pending_steps and not self._is_draft_batch(state.requests): - for req in state.requests: - slot = req.py_seq_slot - if slot is not None and self._pending_steps[slot] > 0: - self._pending_steps[slot] -= 1 - assert state.host is not None new_tokens = state.host.new_tokens finish_reasons = state.host.finish_reasons_list() @@ -3939,51 +3674,8 @@ def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: else: return None - # Fast-path (batched pybind): when the batch is greedy with no beam - # search, no logprobs, no draft tokens, no stop-words, and no - # speculative tree, collapse per-request pybind chatter into one - # batched add_new_tokens_to_requests call. Single-pass eligibility - # check with early-break; falls through when any invariant breaks. - if ( - self._batch_fastpath_eligible - and logprobs_state_list is None - and self.get_spec_tree_manager(resource_manager) is None - ): - alive_reqs: list[LlmRequest] = [] - tokens_flat: list[int] = [] - fastpath_ok = True - new_tokens_step0 = new_tokens_list[0] - for req in state.requests: - if req.state == LlmRequestState.GENERATION_COMPLETE: - continue - if get_draft_token_length(req) != 0 or req.py_stop_words_list: - fastpath_ok = False - break - assert req.py_seq_slot is not None - alive_reqs.append(req) - tokens_flat.append(new_tokens_step0[req.py_seq_slot][DEFAULT_BEAM_IDX]) - if fastpath_ok and alive_reqs: - add_new_tokens_to_requests(alive_reqs, tokens_flat, DEFAULT_BEAM_IDX) - _valid_finish_reasons = { - FinishReason.END_ID, - FinishReason.LENGTH, - FinishReason.STOP_WORDS, - } - for req in alive_reqs: - assert req.py_seq_slot is not None - reason_val = finish_reasons[req.py_seq_slot][0][DEFAULT_BEAM_IDX] - if reason_val != 0: - reason = FinishReason(reason_val) - if reason in _valid_finish_reasons: - req.finish_by(reason, DEFAULT_BEAM_IDX) - req.py_num_accepted_draft_tokens = 0 - req.py_rewind_len = 0 - req.py_decoding_iter += 1 - return - for req_idx, req in enumerate(state.requests): if req.state == LlmRequestState.GENERATION_COMPLETE: - self._retire_top_p_decay_slot(req) continue if req.py_beam_width > 1: @@ -4050,20 +3742,6 @@ def _maybe_build_beam_history(req_idx: int) -> BeamHistory | None: self._finish_reasons_handler.store.num_accepted_draft_tokens_host[ req.py_seq_slot ] = req.py_num_accepted_draft_tokens - if req.state == LlmRequestState.GENERATION_COMPLETE: - self._retire_top_p_decay_slot(req) - - def _retire_top_p_decay_slot(self, req: LlmRequest) -> None: - """Retire a finished request's slot from the top-p-decay membership set - (lifecycle step 4, see ``TopPDecayStore``), so the O(1) hot-path - early-outs re-arm once decay traffic drains. - - Callers ensure ``req`` has finished. Requests that finish outside the - sampler (e.g. cancellation) are covered by the slot-reuse cleanup at - admission instead. - """ - if self._top_p_decay_slots and req.py_seq_slot is not None: - self._top_p_decay_slots.discard(req.py_seq_slot) def _return_log_probs(self, requests: list[LlmRequest]) -> bool: return any(req.py_return_log_probs for req in requests) @@ -4103,16 +3781,6 @@ def sample_async( self.setup_sampler_step(scheduled_requests) new_tokens = self.store.new_tokens - if self._track_pending_steps: - # A context request claims a (possibly reused) slot: clear any - # counter leaked by a prior occupant that never got its final - # update_requests. Must happen before _process_requests, which - # reads the counters for bad-words staleness. - for r in scheduled_requests.context_requests: - if not r.py_is_draft: - assert r.py_seq_slot is not None - self._pending_steps[r.py_seq_slot] = 0 - # seq_slots_cuda / seq_lens_cuda are cast once inside # _process_requests and shared with the beam-search metadata builder. ( @@ -4129,11 +3797,6 @@ def sample_async( num_context_logits_prefix_sum, ) - if self._track_pending_steps and requests and not self._is_draft_batch(requests): - for r in requests: - assert r.py_seq_slot is not None - self._pending_steps[r.py_seq_slot] += 1 - finish_reasons_host: torch.Tensor | None = None first_finish_reasons_host: torch.Tensor | None = None beam_history_builders: list[BeamHistoryBuilder | None] | None = None @@ -4331,65 +3994,6 @@ def provision_bias_index() -> int: # sharing). logits[logits_bias_mask_cuda] += biases_tensor_cuda - def _build_top_p_decay_metadata( - self, - *, - group_req_indices: torch.Tensor, - req_num_steps: torch.Tensor, - seq_slots: torch.Tensor, - seq_slots_cuda: torch.Tensor, - ) -> Optional[TopPDecayMetadata]: - """Build the Top-P Decay metadata for a top_p / top_k_top_p group. - - Lifecycle step 2, see ``TopPDecayStore``. Returns None when no request - currently uses decay. The metadata's ``slots`` tensor is aligned to the - group's per-STEP row order (matching group_strategies_per_step); - non-decay rows (possibly multi-step draft rows) are gated out on-device - by ``is_decay_slot``, so decay presence in the group is not checked - host-side: a group without decay rows samples every row with its static - top-p -- same result as returning None. - """ - if not self._top_p_decay_slots: - return None - store = self.store.top_p_decay_store - # Fast path (steady-state decoding): if every row in the group is - # single-token, the per-STEP row order equals the per-request order, and - # (group_req_indices being sorted ascending) a contiguous group's slots - # are just a slice of seq_slots_cuda -- no host layout build and no H2D - # copy. - first_req = int(group_req_indices[0].item()) - last_req = int(group_req_indices[-1].item()) - group_steps = req_num_steps[group_req_indices] - if last_req - first_req + 1 == group_req_indices.size(0) and ( - group_steps.max().item() == 1 - ): - per_step_slots_cuda = seq_slots_cuda[first_req : last_req + 1] - else: - # Build the per-STEP slot layout (each request contributes - # req_num_steps rows). - group_seq_slots = seq_slots[group_req_indices] - if __debug__: - # Internal invariant (stripped under python -O): a decay-active - # row is always single-token -- the only source of multi-step - # rows in TorchSampler is two-model speculation (slated for - # removal), and decay + draft tokens is rejected per-request at - # admission in validate_request. - decay_row_steps = group_steps[ - torch.isin(group_seq_slots, torch.tensor(list(self._top_p_decay_slots))) - ] - assert decay_row_steps.numel() == 0 or decay_row_steps.max().item() == 1, ( - "top_p_decay row with req_num_steps != 1; decay + draft tokens " - "should have been rejected at admission" - ) - per_step_slots_cuda = torch.repeat_interleave(group_seq_slots.long(), group_steps).to( - seq_slots_cuda.device, non_blocking=True - ) - return TopPDecayMetadata( - slots=per_step_slots_cuda, - runtime_top_p=store.runtime_top_p_decay_cuda, - is_decay_slot=store.is_top_p_decay_slot_cuda, - ) - @nvtx_range("sample_batched_by_strategy") @torch.inference_mode() def _sample_batched_by_strategy( @@ -4426,7 +4030,6 @@ def _sample_batched_by_strategy( get_metadata_type_for_group_fn=self._grouped_sampler_cls.get_metadata_type_for_group, seq_slots_cuda=seq_slots_cuda, seq_lens_cuda=seq_lens_cuda, - req_num_steps=req_num_steps, ) generator_cuda = self.get_generator(cuda_device) @@ -4614,7 +4217,7 @@ def _sample_batched_by_strategy( logit_indices_for_raw_logprobs_cuda += batch_next_tokens_offset_start # NB: Copy could be avoided by storing logit indices (and temperature) instead (cf. comment on # processed logprobs above). - Fusions.gather_scatter( + _Fusions.gather_scatter( batch_logits_for_logprobs_cuda, logit_indices_for_raw_logprobs_cuda, group_logits_cuda, @@ -4660,7 +4263,6 @@ def _unbatch_sampling_results( new_tokens_cuda: torch.Tensor, req_num_generated_tokens: torch.Tensor, seq_slots: torch.Tensor, - seq_slots_cuda: torch.Tensor, ) -> torch.Tensor: batch_req_indices = batched_sampling_result.batch_req_indices batch_next_tokens_cuda_int = batched_sampling_result.batch_next_tokens_cuda_int @@ -4694,224 +4296,52 @@ def _dims_canonically_ordered(t: torch.Tensor) -> bool: new_tokens_cuda.view(-1, *new_tokens_cuda.shape[2:]).scatter_( 0, batch_dest_indices_1d_cuda, batch_next_tokens_cuda_int ) - # Post-sample: decay the runtime top-p for any decay-active slots that were - # sampled this iteration (must run after tokens land in new_tokens_cuda). - # batch_req_indices is a permutation of all sampled requests, so the set of - # sampled slots is exactly seq_slots (the kernel updates each slot - # independently; order is irrelevant) -- pass the resident device copy - # instead of gathering seq_slots[batch_req_indices] on host and copying it. - self._update_top_p_decay_after_sample( - new_tokens_cuda=new_tokens_cuda, - sampled_slots_cuda=seq_slots_cuda, - ) return self._copy_to_host(new_tokens_cuda) - def _update_top_p_decay_after_sample( - self, - *, - new_tokens_cuda: torch.Tensor, - sampled_slots_cuda: torch.Tensor, - ) -> None: - """Apply the post-sample decay recurrence for sampled decay-active slots. - - See ``TopPDecayStore`` for the feature-level semantics (lifecycle - step 3). Restricting to the sampled slots avoids reading stale - new_tokens_cuda for slots that were not scheduled this iteration. - """ - # Host-side O(1) early-out: skip the kernel launch when no request uses - # decay; otherwise a single fused (torch.compile) op gates on - # is_decay_slot on-device and gathers the sampled token in place (decay is - # single-token-only, so the token is at local step 0, beam 0). - if not self._top_p_decay_slots: - return - store = self.store.top_p_decay_store - Fusions.top_p_decay_update( - runtime_top_p=store.runtime_top_p_decay_cuda, - initial_top_p=store.initial_top_p_decay_cuda, - top_p_decay=store.top_p_decay_cuda, - top_p_min=store.top_p_decay_min_cuda, - reset_ids=store.top_p_decay_reset_ids_cuda, - is_decay_slot=store.is_top_p_decay_slot_cuda, - step_tokens=new_tokens_cuda[DEFAULT_STEP_IDX, :, DEFAULT_BEAM_IDX], - sampled_slots=sampled_slots_cuda, - ) - @staticmethod @torch.inference_mode() def _apply_min_length_penalty( logits: torch.Tensor, requests: list[LlmRequest], - num_steps_tensor: torch.Tensor, - num_beams_tensor: torch.Tensor, - ) -> None: - """Apply min_length_penalty to logits, mutating ``logits`` in place. + num_steps: list[int], + num_beams: list[int], + ) -> torch.Tensor: + """Inplace apply min_length_penalty to logits. Args: logits: The logits to apply min length penalty to requests: The requests to apply min length penalty to - num_steps_tensor: The number of steps per request (host tensor) - num_beams_tensor: The number of beams per request (host tensor) + num_steps: The number of steps per request + + Returns: + The logits with min length penalty applied """ if not any( r.py_min_length and (r.max_beam_num_tokens - r.py_orig_prompt_len) < r.py_min_length[0] for r in requests ): - return - - # Deferred host conversion: only needed on the (rare) penalty path. - num_steps = num_steps_tensor.tolist() - num_beams = num_beams_tensor.tolist() - - rows: list[int] = [] - cols: list[int] = [] - current_offset = 0 - for index, r in enumerate(requests): - # Advance the offset before any guard below can skip the request: - # every request occupies its logits rows, penalized or not. - req_offset = current_offset - current_offset += num_steps[index] * num_beams[index] - - if not r.py_min_length: - continue - # Use the original end_id (before ignore_eos override) - # so we suppress the real EOS token, not token -1. - end_id = getattr(r, "py_original_end_id", r.py_end_id) - if end_id is None or end_id <= -1: - continue - - for beam_idx in range(num_beams[index]): - for step in range(num_steps[index]): - if (r.get_num_tokens(beam_idx) - r.py_orig_prompt_len) + step < r.py_min_length[ - 0 - ]: - rows.append(req_offset + num_steps[index] * beam_idx + step) - cols.append(end_id) - else: - break - - if rows: - neg_inf = torch.full((), float("-inf"), dtype=logits.dtype, device=logits.device) - row_idx = torch.tensor(rows, dtype=torch.long, pin_memory=prefer_pinned()).to( - logits.device, non_blocking=True - ) - col_idx = torch.tensor(cols, dtype=torch.long, pin_memory=prefer_pinned()).to( - logits.device, non_blocking=True - ) - logits.index_put_((row_idx, col_idx), neg_inf, accumulate=False) - - @staticmethod - @torch.inference_mode() - def _apply_bad_words( - logits: torch.Tensor, - requests: list[LlmRequest], - num_steps: list[int], - num_beams: list[int], - *, - new_tokens_cuda: torch.Tensor | None = None, - stale_by_one: list[bool] | None = None, - ) -> None: - """Inplace ban "bad words" by masking their final token's logit to -inf. - - A single-token word is banned unconditionally; a multi-token word - ``[t0, ..., t_{k-1}]`` bans its final token ``t_{k-1}`` only when the - ``k-1`` most recently generated tokens exactly match the prefix - ``[t0, ..., t_{k-2}]``. - - With the overlap scheduler, ``sample_async`` for step ``i`` runs before - ``update_requests`` for step ``i-1``, so ``r.get_tokens()`` is missing - exactly the previous step's token; that token still lives device-side in - ``new_tokens_cuda``. For requests flagged in ``stale_by_one``, the - prefix match is therefore split: the first ``k-2`` prefix tokens are - matched on the host against the (complete up to there) host context, and - the final prefix token is compared on the GPU against - ``new_tokens_cuda[0, seq_slot, 0]``, without any device-to-host - synchronization. This path only supports ``num_steps == 1`` and - ``num_beams == 1`` (no speculation, no beam search). + return logits - Args: - logits: Flattened ``[total_rows, vocab]`` logits; rows are packed per - request as ``num_steps * num_beams`` consecutive entries, in - beam-major / step-minor order (same layout as - ``_apply_min_length_penalty``). Modified in-place. - requests: The requests, aligned with the packed logits rows. - num_steps: Number of steps per request. - num_beams: Number of beams per request. - new_tokens_cuda: Device buffer holding the previous step's sampled - tokens, shape ``[max_tokens, max_num_sequences, max_beam_width]``. - Required when any entry of ``stale_by_one`` is True. - stale_by_one: Per-request flag; True when the host token list lags - the device state by exactly one token (overlap scheduler). - """ rows: list[int] = [] cols: list[int] = [] - # Overlap path: bans whose last prefix token must be compared on-device. - cond_rows: list[int] = [] - cond_cols: list[int] = [] - cond_slots: list[int] = [] - cond_expected: list[int] = [] current_offset = 0 for index, r in enumerate(requests): - request_offset = current_offset - # Advance to the next request's rows before any early continue. + if r.py_min_length: + # Use the original end_id (before ignore_eos override) + # so we suppress the real EOS token, not token -1. + end_id = getattr(r, "py_original_end_id", r.py_end_id) + if end_id is not None and end_id > -1: + for beam_idx in range(num_beams[index]): + for step in range(num_steps[index]): + if ( + r.get_num_tokens(beam_idx) - r.py_orig_prompt_len + ) + step < r.py_min_length[0]: + rows.append(current_offset + num_steps[index] * beam_idx + step) + cols.append(end_id) + else: + break current_offset += num_steps[index] * num_beams[index] - bad_words = getattr(r, "py_bad_words", None) - if not bad_words: - continue - - if stale_by_one is not None and stale_by_one[index]: - assert num_steps[index] == 1 and num_beams[index] == 1, ( - "stale-host bad-words path only supports a single step and beam" - ) - assert r.py_seq_slot is not None - # Host context is missing the previous step's token; the true - # sequence is context + [new_tokens_cuda[0, seq_slot, 0]]. - context = r.get_tokens(0) - for word in bad_words: - k = len(word) - if k == 0: - continue - if k == 1: - # Single-token word: banned unconditionally. - rows.append(request_offset) - cols.append(word[0]) - continue - # True sequence length is len(context) + 1; need >= k - 1. - if len(context) < k - 2: - continue - # Host part: all prefix tokens except the newest one. - if k > 2 and context[-(k - 2) :] != word[: k - 2]: - continue - # Device part: the previous step's token must equal the - # last prefix token; resolved on the GPU below. - cond_rows.append(request_offset) - cond_cols.append(word[-1]) - cond_slots.append(r.py_seq_slot) - cond_expected.append(word[k - 2]) - continue - - for beam_idx in range(num_beams[index]): - # Full token sequence for this beam (prompt + generated), so a - # bad-word prefix ending inside the prompt is also matched. - context = r.get_tokens(beam_idx) - for word in bad_words: - k = len(word) - if k == 0: - continue - if k == 1: - # Single-token word: banned unconditionally. - col = word[0] - elif len(context) >= k - 1 and context[-(k - 1) :] == word[:-1]: - # Multi-token word: ban the final token only when the - # generated suffix matches the word prefix. - col = word[-1] - else: - continue - # Apply to every step row of this beam. - for step in range(num_steps[index]): - rows.append(request_offset + num_steps[index] * beam_idx + step) - cols.append(col) - if rows: neg_inf = torch.full((), float("-inf"), dtype=logits.dtype, device=logits.device) row_idx = torch.tensor(rows, dtype=torch.long, pin_memory=prefer_pinned()).to( @@ -4922,32 +4352,7 @@ def _apply_bad_words( ) logits.index_put_((row_idx, col_idx), neg_inf, accumulate=False) - if cond_rows: - assert new_tokens_cuda is not None - device = logits.device - cond_row_idx = torch.tensor(cond_rows, dtype=torch.long, pin_memory=prefer_pinned()).to( - device, non_blocking=True - ) - cond_col_idx = torch.tensor(cond_cols, dtype=torch.long, pin_memory=prefer_pinned()).to( - device, non_blocking=True - ) - slot_idx = torch.tensor(cond_slots, dtype=torch.long, pin_memory=prefer_pinned()).to( - device, non_blocking=True - ) - expected = torch.tensor( - cond_expected, dtype=new_tokens_cuda.dtype, pin_memory=prefer_pinned() - ).to(device, non_blocking=True) - # Previous step's token per request (single step, single beam). - prev_tokens = new_tokens_cuda[0].index_select(0, slot_idx)[:, 0] - # -inf where the device-side prefix token matches, 0 otherwise. - # Additive update keeps the op shape-static (boolean-mask indexing - # would force a device-to-host sync). - penalty = torch.where( - prev_tokens == expected, - torch.full((), float("-inf"), dtype=logits.dtype, device=device), - torch.zeros((), dtype=logits.dtype, device=device), - ) - logits.index_put_((cond_row_idx, cond_col_idx), penalty, accumulate=True) + return logits @staticmethod def _select_generated_logits( @@ -5061,6 +4466,37 @@ def _select_generated_logits( return sampling_requests, sampling_requests_metadata, logits_cuda + @staticmethod + def _longest_stop_word_len(requests: Iterable[LlmRequest]) -> int: + max_stop_word_len = 0 + for req in requests: + assert req.py_stop_words_list is not None + _, cumsum = req.py_stop_words_list + if -1 in cumsum: + cumsum = cumsum[: cumsum.index(-1)] + request_max_stop_word_len = np.max(np.diff(cumsum, prepend=0), initial=0).item() + max_stop_word_len = max(max_stop_word_len, request_max_stop_word_len) + return max_stop_word_len + + @staticmethod + def _requests_with_stop_words(requests: list[LlmRequest]) -> list[LlmRequest]: + return [ + r + for r in requests + if (r.py_stop_words_list is not None and len(r.py_stop_words_list[0]) > 0) + ] + + def _request_indices_with_stop_words(self, requests: list[LlmRequest]) -> torch.Tensor: + return torch.tensor( + [ + ridx + for ridx, r in enumerate(requests) + if (r.py_stop_words_list is not None and len(r.py_stop_words_list[0]) > 0) + ], + dtype=torch.int32, + pin_memory=prefer_pinned(), + ).to(device="cuda", non_blocking=True) + @nvtx_range("_process_logprobs") def _process_logprobs( self, @@ -5127,7 +4563,7 @@ def _process_logprobs( ) # (batch_size, vocab_size) - group_logprobs_cuda = Fusions.gather_log_softmax( + group_logprobs_cuda = _Fusions.gather_log_softmax( batched_sampling_result.batch_logits_for_logprobs_cuda, group_logits_indices_cuda ) @@ -5167,7 +4603,7 @@ def _process_logprobs( # NB: Computation of sampled rank could be lowered into FlashInferGroupedStrategySampler, s.t., e.g., for # greedy sampling, logits management and log_softmax could be completely skipped (sampled rank # computation is trivial in this case). - sampled_rank_cuda = Fusions.determine_sampled_rank( + sampled_rank_cuda = _Fusions.determine_sampled_rank( group_logprobs_cuda, sampled_vals_cuda ) @@ -5254,44 +4690,13 @@ def _process_requests( logits_cuda, sampling_requests, sampling_requests_metadata.req_num_steps ) - self._apply_min_length_penalty( + logits_cuda = self._apply_min_length_penalty( logits_cuda, sampling_requests, - sampling_requests_metadata.req_num_steps, - sampling_requests_metadata.req_num_beams, + sampling_requests_metadata.req_num_steps.tolist(), + sampling_requests_metadata.req_num_beams.tolist(), ) - if any(getattr(r, "py_bad_words", None) for r in sampling_requests): - stale_by_one: list[bool] | None = None - if self._track_pending_steps and not self._is_draft_batch(sampling_requests): - pending = [ - self._pending_steps[r.py_seq_slot] if r.py_seq_slot is not None else 0 - for r in sampling_requests - ] - if any(pending): - if self.max_tokens == 1 and self.max_beam_width == 1 and max(pending) == 1: - stale_by_one = [p > 0 for p in pending] - else: - # Speculative decoding / beam search under overlap: - # the missing host tokens cannot be reconstructed with - # the single-token device-side check; multi-token bad - # words may be applied one step late. - logger.warning_once( - "bad_words with the overlap scheduler and speculative " - "decoding or beam search: multi-token bad words are " - "matched against a host token history that lags the " - "device state and may be enforced inexactly.", - key="bad_words_stale_overlap", - ) - self._apply_bad_words( - logits_cuda, - sampling_requests, - sampling_requests_metadata.req_num_steps.tolist(), - sampling_requests_metadata.req_num_beams.tolist(), - new_tokens_cuda=new_tokens_cuda, - stale_by_one=stale_by_one, - ) - # Fast path for greedy sampling if self._can_use_fast_greedy_path(sampling_requests): # Compute destination indices on CPU (same pattern as _unbatch_sampling_results) @@ -5370,7 +4775,6 @@ def _process_requests( new_tokens_cuda=new_tokens_cuda, req_num_generated_tokens=sampling_requests_metadata.req_num_generated_tokens, seq_slots=seq_slots_host, - seq_slots_cuda=seq_slots_cuda, ) # NB: update_requests syncs w/ device computation and async D2H copies diff --git a/tensorrt_llm/_torch/pyexecutor/sampler/sampling_utils.py b/tensorrt_llm/_torch/pyexecutor/sampler/sampling_utils.py index 20bc59316844..0c4d59b0ecc4 100644 --- a/tensorrt_llm/_torch/pyexecutor/sampler/sampling_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/sampler/sampling_utils.py @@ -31,50 +31,55 @@ # These op wrappers are safe to import without flashinfer installed; they are # only called on the flashinfer sampler / speculative-worker paths. from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( - sampling_from_probs_op, - softmax_op, - top_k_mask_logits_op, - top_k_sampling_from_probs_op, - top_k_top_p_sampling_from_logits_op, - top_p_renorm_probs_op, - top_p_sampling_from_probs_op, + sampling_from_probs_generator_op as sampling_from_probs_generator_op, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import softmax_op as softmax_op +from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( + top_k_mask_logits_op as top_k_mask_logits_op, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( + top_k_sampling_from_probs_generator_op as top_k_sampling_from_probs_generator_op, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( + top_k_top_p_sampling_from_logits_with_generator_op as top_k_top_p_sampling_from_logits_with_generator_op, # noqa: E501 +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( + top_p_renorm_probs_op as top_p_renorm_probs_op, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.flashinfer import ( + top_p_sampling_from_probs_generator_op as top_p_sampling_from_probs_generator_op, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( + BeamSearchMetadata as BeamSearchMetadata, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import StrategyMetadata as StrategyMetadata +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import _Fusions as _Fusions +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( + beam_search_sampling_batch as beam_search_sampling_batch, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( + get_rejected_indices as get_rejected_indices, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import greedy as _torch_greedy +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( + greedy_search_sampling_batch as greedy_search_sampling_batch, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import sample_rejected as sample_rejected +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( + temperature_sampling_batch as temperature_sampling_batch, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( + top_k_sampling_batch as top_k_sampling_batch, +) +from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( + top_k_top_p_sampling_batch as top_k_top_p_sampling_batch, ) from tensorrt_llm._torch.pyexecutor.sampler.ops.vanilla import ( - GREEDY_TEMPERATURE_THRESHOLD, - BeamSearchMetadata, - Fusions, - StrategyMetadata, - beam_search_sampling_batch, - get_rejected_indices, - greedy_search_sampling_batch, - sample_rejected, - top_k_top_p_sampling_batch, + top_p_sampling_batch as top_p_sampling_batch, ) from tensorrt_llm._utils import prefer_pinned from tensorrt_llm.sampling_params import SamplingParams -# Ops imported above are re-exported for dependent modules (sampler, drafting -# loops, tests). mypy runs in strict mode (no implicit re-export), so they must -# be listed here. -__all__ = [ - "GREEDY_TEMPERATURE_THRESHOLD", - "BeamSearchMetadata", - "Fusions", - "StrategyMetadata", - "beam_search_sampling_batch", - "get_rejected_indices", - "greedy_search_sampling_batch", - "sample_rejected", - "sampling_from_probs_op", - "softmax_op", - "top_k_mask_logits_op", - "top_k_sampling_from_probs_op", - "top_k_top_p_sampling_batch", - "top_k_top_p_sampling_from_logits_op", - "top_p_renorm_probs_op", - "top_p_sampling_from_probs_op", -] - if sys.version_info[:2] >= (3, 12): from typing import override else: @@ -91,8 +96,7 @@ Strategy: TypeAlias = TopK | TopP | Greedy | TopKTopP | TemperatureOnly | BeamSearch -# Re-exported from the beam-search op implementation (single source of truth). -BEAM_SEARCH_PAD_TOKEN = vanilla.BEAM_SEARCH_PAD_TOKEN +BEAM_SEARCH_PAD_TOKEN = -1 @dataclass(frozen=True, kw_only=True) @@ -106,10 +110,6 @@ class UtilsSamplingParams: use_beam_search: Whether to use beam search. beam_width_in: The beam_width of a request before the sampling step. beam_width_out: The beam_width of a request after the sampling step. - top_p_decay: Per-step multiplicative decay applied to the runtime top-p. - top_p_min: Lower bound for the decayed runtime top-p. - top_p_reset_ids: Token id which, when sampled, resets the runtime top-p to - its initial value. A value < 0 never matches a token. """ temperature: Optional[float] @@ -118,38 +118,6 @@ class UtilsSamplingParams: use_beam_search: Optional[bool] beam_width_in: Optional[int] = None beam_width_out: Optional[int] = None - top_p_decay: Optional[float] = None - top_p_min: Optional[float] = None - top_p_reset_ids: Optional[int] = None - - -@dataclass(kw_only=True) -class TopPDecayMetadata(StrategyMetadata): - """Per-group runtime top-p override for Top-P Decay (attached to top_p / - top_k_top_p groups via the ``StrategyMetadata`` mechanism). - - ``slots`` maps each per-step group row to its sequence slot; the decayed - per-row top-p is gathered on-device from the per-slot ``runtime_top_p`` - store, gated by ``is_decay_slot`` (non-decay rows keep their static top-p). - Consumed by the TopP*/TopKTopP* strategy impls in ``sample()``. See - ``TorchSampler.TopPDecayStore`` for the feature-level semantics. - """ - - slots: torch.Tensor - """Per-step group rows' sequence slots (int64, device).""" - runtime_top_p: torch.Tensor - """Per-slot runtime (decayed) top-p store (float32, device).""" - is_decay_slot: torch.Tensor - """Per-slot decay-active gate (bool, device).""" - - -def top_p_decay_active(params: UtilsSamplingParams) -> bool: - """Whether dynamic top-p decay is active for a request. - - Delegates to the single-source predicate on SamplingParams; note that - ``top_p_min`` / ``top_p_reset_ids`` alone do not activate dynamic behavior. - """ - return SamplingParams.params_imply_top_p_decay_active(params.top_p_decay) def resolve_sampling_strategy(params: UtilsSamplingParams, *, vocab_size: int) -> Strategy: @@ -160,15 +128,11 @@ def resolve_sampling_strategy(params: UtilsSamplingParams, *, vocab_size: int) - top_p = params.top_p top_k = params.top_k - # The greedy verdict (including the top-p-decay override of the implicit - # all-unset greedy default, and explicit greedy controls winning over decay) - # is single-sourced in SamplingParams.params_imply_greedy_decoding. if SamplingParams.params_imply_greedy_decoding( temperature=temperature, top_p=top_p, top_k=top_k, use_beam_search=use_beam_search, - top_p_decay=params.top_p_decay, ): return GREEDY @@ -192,10 +156,7 @@ def resolve_sampling_strategy(params: UtilsSamplingParams, *, vocab_size: int) - assert top_k > 1, "non-greedy sampling requires valid top_k" need_top_k = top_k < vocab_size assert top_p > 0, "non-greedy sampling requires valid top_p" - # A decay-active request must go through a top-p-capable path even when its - # initial top_p is 1.0, so the runtime top-p (sourced per-row at sample time) - # can shrink the nucleus on later steps. - need_top_p = top_p < 1 or top_p_decay_active(params) + need_top_p = top_p < 1 if need_top_p: if need_top_k: @@ -218,14 +179,14 @@ def sample( # 'cast' needed b/c of https://github.com/python/mypy/issues/19081 match strategy: case ("top_k", top_k, temperature): - tokens, softmax = top_k_top_p_sampling_batch( + tokens, softmax = top_k_sampling_batch( logits, top_k=cast(int, top_k), temperature=cast(float, temperature), generator=generator, ) case ("top_p", top_p, temperature): - tokens, softmax = top_k_top_p_sampling_batch( + tokens, softmax = top_p_sampling_batch( logits, top_p=cast(float, top_p), generator=generator, @@ -240,7 +201,7 @@ def sample( generator=generator, ) case ("temperature", temperature): - tokens, softmax = top_k_top_p_sampling_batch( + tokens, softmax = temperature_sampling_batch( logits, temperature=cast(float, temperature), generator=generator, @@ -294,19 +255,8 @@ def sample( ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: pass - # TODO: Revisit this after determining performance impact - # - # NB: NaN logits can lead to crashes, see - # https://github.com/flashinfer-ai/flashinfer/issues/1575 - # @staticmethod def _flashinfer_check_nans(inputs: torch.Tensor) -> bool: - # Deliberately returns False to keep FlashInfer's own 'check_nan' path - # disabled: that path is a host-side `if torch.any(torch.isnan(...))`, - # which forces a device sync on every call. The explicit async - # device-side assert below provides the same protection without - # stalling the pipeline. - # https://github.com/pytorch/pytorch/issues/36853 torch._assert_async(~torch.any(torch.isnan(inputs))) return False @@ -346,8 +296,8 @@ def _sample_from_probs( probs: torch.Tensor, generator: Optional[torch.Generator], ) -> torch.Tensor: - return sampling_from_probs_op( - probs, generator=generator, check_nan=cls._flashinfer_check_nans(probs) + return sampling_from_probs_generator_op( + probs, generator, check_nan=cls._flashinfer_check_nans(probs) ) def _sample_greedy_with_probs( @@ -389,34 +339,6 @@ def _sample_with_probs( new_tokens = cls._sample_from_probs(probs, generator=generator) return new_tokens, probs - class TopPDecayMixin: - """Mixed into the TopP*/TopKTopP* impls (the owners of a per-row - ``_top_p`` tensor) to consume ``TopPDecayMetadata``.""" - - _top_p: torch.Tensor - - def _maybe_apply_top_p_decay(self, group_metadata: Optional[StrategyMetadata]) -> None: - """Override the per-row static top-p with the decayed runtime top-p. - - Only decay-active rows (per the on-device ``is_decay_slot`` gate) are - overridden, so a group mixing top-p-decay and plain top-p requests - keeps each row's correct value. The overridden ``self._top_p`` tensor - then feeds both sampling and ``top_p_renorm_probs_op`` (so processed - logprobs match). Fused via torch.compile (gather + gate + select). - """ - if not isinstance(group_metadata, TopPDecayMetadata): - return - assert self._top_p.shape == group_metadata.slots.shape, ( - self._top_p.shape, - group_metadata.slots.shape, - ) - self._top_p = Fusions.top_p_decay_gather( - runtime_top_p=group_metadata.runtime_top_p, - is_decay_slot=group_metadata.is_decay_slot, - static_top_p=self._top_p, - slots=group_metadata.slots, - ) - class StrategyImplWithProbs(StrategyImpl): @override @classmethod @@ -445,7 +367,7 @@ def sample( ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: return self._sample_greedy_with_probs(logits, group_logit_indices=group_logit_indices) - class TopKTopPWithProbs(TopPDecayMixin, StrategyImplWithProbs): + class TopKTopPWithProbs(StrategyImplWithProbs): def __init__(self, top_k: torch.Tensor, top_p: torch.Tensor, temperature: torch.Tensor): self._top_k = top_k self._top_p = top_p @@ -471,7 +393,6 @@ def sample( generator: Optional[torch.Generator] = None, group_metadata: Optional[StrategyMetadata] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: - self._maybe_apply_top_p_decay(group_metadata) return self._sample_with_probs( logits, group_logit_indices=group_logit_indices, @@ -514,7 +435,7 @@ def sample( generator=generator, ) - class TopPWithProbs(TopPDecayMixin, StrategyImplWithProbs): + class TopPWithProbs(StrategyImplWithProbs): def __init__(self, top_p: torch.Tensor, temperature: torch.Tensor): self._top_p = top_p self._temperature = temperature @@ -538,7 +459,6 @@ def sample( generator: Optional[torch.Generator] = None, group_metadata: Optional[StrategyMetadata] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: - self._maybe_apply_top_p_decay(group_metadata) return self._sample_with_probs( logits, group_logit_indices=group_logit_indices, @@ -607,7 +527,7 @@ def sample( logits = logits[group_logit_indices] return torch.argmax(logits, dim=-1), None - class TopKTopPSampleOnly(TopPDecayMixin, StrategyImplSampleOnly): + class TopKTopPSampleOnly(StrategyImplSampleOnly): def __init__(self, top_k: torch.Tensor, top_p: torch.Tensor, temperature: torch.Tensor): self._top_k = top_k self._top_p = top_p @@ -633,15 +553,14 @@ def sample( generator: Optional[torch.Generator] = None, group_metadata: Optional[StrategyMetadata] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: - self._maybe_apply_top_p_decay(group_metadata) logits = self._prepare_logits_with_temperature( logits, group_logit_indices, self._temperature ) - return top_k_top_p_sampling_from_logits_op( + return top_k_top_p_sampling_from_logits_with_generator_op( logits, self._top_k, self._top_p, - generator=generator, + generator, check_nan=self._flashinfer_check_nans(logits), ), None @@ -672,14 +591,11 @@ def sample( probs = self._prepare_probs_with_temperature( logits, group_logit_indices, self._temperature ) - return top_k_sampling_from_probs_op( - probs, - self._top_k, - generator=generator, - check_nan=self._flashinfer_check_nans(probs), + return top_k_sampling_from_probs_generator_op( + probs, self._top_k, generator, check_nan=self._flashinfer_check_nans(probs) ), None - class TopPSampleOnly(TopPDecayMixin, StrategyImplSampleOnly): + class TopPSampleOnly(StrategyImplSampleOnly): def __init__(self, top_p: torch.Tensor, temperature: torch.Tensor): self._top_p = top_p self._temperature = temperature @@ -703,15 +619,11 @@ def sample( generator: Optional[torch.Generator] = None, group_metadata: Optional[StrategyMetadata] = None, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: - self._maybe_apply_top_p_decay(group_metadata) probs = self._prepare_probs_with_temperature( logits, group_logit_indices, self._temperature ) - return top_p_sampling_from_probs_op( - probs, - self._top_p, - generator=generator, - check_nan=self._flashinfer_check_nans(probs), + return top_p_sampling_from_probs_generator_op( + probs, self._top_p, generator, check_nan=self._flashinfer_check_nans(probs) ), None class TemperatureOnlySampleOnly(StrategyImplSampleOnly): @@ -830,8 +742,6 @@ def get_metadata_type_for_group( match strategy_key: case ("beam_search", _, _): return BeamSearchMetadata - case "top_p" | "top_k_top_p": - return TopPDecayMetadata case _: return None @@ -903,6 +813,9 @@ def sample_grouped_strategies( return next_tokens, softmax, strategy_impl.get_temperature() +# Re-export the torch greedy op (used by drafting_loops and speculative/interface). +greedy = _torch_greedy + # --------------------------------------------------------------------------- # Spec-decoding interface: compute_probs_from_logits (per-request tensor params) # --------------------------------------------------------------------------- @@ -912,13 +825,14 @@ def sanitize_top_k(top_k: torch.Tensor, vocab_size: int) -> torch.Tensor: """Map ``top_k`` into a backend-safe range before top-k filtering. Per ``SamplingParams``, ``top_k == 0`` means "all logits" (top-k disabled), - but the flashinfer top-k kernels (``top_k_mask_logits``) break on a literal - 0 — they mask the entire row (all-zero probs). Map any non-positive value - (and any oversized disable sentinel such as ``INT32_MAX``) to - ``vocab_size`` (== keep all tokens), leaving genuine top_k values - untouched. + but the flashinfer (``top_k_mask_logits``) and PyTorch-native top-k paths + break on a literal 0 — they mask the entire row (all-zero probs) or gather + out of bounds. Mirror the C++ op (``dynamicTreeKernels.cu``): map any + non-positive value (and any oversized disable sentinel such as + ``INT32_MAX``) to ``vocab_size`` (== keep all tokens), leaving genuine + top_k values untouched. """ - return top_k.clamp(max=vocab_size).masked_fill_(top_k <= 0, vocab_size) + return torch.where(top_k > 0, top_k, torch.full_like(top_k, vocab_size)).clamp(max=vocab_size) @torch.compile(options={"max-autotune": True}) @@ -945,23 +859,26 @@ def sampling_batch_spec_dec_one_model( temperatures: torch.Tensor, top_k: torch.Tensor, top_p: torch.Tensor, - seed: Optional[torch.Tensor] = None, - offset: Optional[torch.Tensor] = None, + seed: Optional[int] = None, + offset: Optional[int] = None, ) -> torch.Tensor: """CUDA-graph compatible sampling; supports mixed sampling params. Returns sampled tokens.""" top_k = sanitize_top_k(top_k, logits.shape[-1]) - # Greedy rows (temperature <= threshold) reduce to top_k=1 sampling: with the - # divisor clamped to 1.0 by safely_apply_temperature_inplace (order-preserving - # for those rows), flashinfer deterministically returns the max-probability - # token, i.e. the argmax of the original logits. All ops remain branch-free - # (no data-dependent control flow), so this stays CUDA-graph safe. - is_greedy = temperatures <= vanilla.GREEDY_TEMPERATURE_THRESHOLD - top_k = torch.where(is_greedy, torch.ones_like(top_k), top_k) - top_p = torch.where(is_greedy, torch.ones_like(top_p), top_p) - logits = vanilla.safely_apply_temperature_inplace(logits, temperatures) - return flashinfer.top_k_top_p_sampling_from_logits_op( + # Greedy rows (temperature <= threshold) must return the argmax token, not a + # sample from the temperature-scaled distribution. Capture the argmax from the + # *original* logits up front; _safely_apply_temperature_inplace then guards the division + # against the greedy sentinel, and torch.where restores the greedy rows below. + # All ops are branch-free (no data-dependent control flow), so this stays + # CUDA-graph safe. + is_greedy = temperatures <= vanilla._GREEDY_TEMPERATURE_THRESHOLD + greedy_tokens = logits.argmax(dim=-1) + logits = vanilla._safely_apply_temperature_inplace(logits, temperatures) + sampled = flashinfer.top_k_top_p_sampling_from_logits_op( logits, top_k, top_p, seed=seed, offset=offset ) + # argmax yields int64; cast so torch.where preserves the sampler's dtype + # (flashinfer returns int32) instead of promoting the result to int64. + return torch.where(is_greedy, greedy_tokens.to(sampled.dtype), sampled) @torch.compile(options={"max-autotune": True}) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py index 21fd55971bd6..acea0242edcb 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py @@ -201,7 +201,6 @@ def create( dist=dist, max_sessions=attention_dp_config.kv_cache_routing_max_sessions, fair_share_multiplier=attention_dp_config.kv_cache_routing_fair_share_multiplier, - new_conv_placement=attention_dp_config.kv_cache_routing_new_conv_placement, ) if ( @@ -798,11 +797,10 @@ def _sort_key(req_item): class ConversationAwareADPRouter(ADPRouter): """Pins each conversation to a single attention-DP rank: the first request - of a conversation is placed by ``new_conv_placement`` (round-robin by - default, or least-queued), and every later request with the same + of a conversation is round-robined, and every later request with the same ``conversation_id`` returns to that rank, keeping the conversation's - KV-cache prefix on one rank. Requests without a ``conversation_id`` fall - back to the same ``new_conv_placement`` policy. + KV-cache prefix on one rank. Falls back to load-balanced round-robin when no + ``conversation_id`` is present. """ # Default LRU cap on the conversation->rank map (entries are ~tens of @@ -814,15 +812,11 @@ def __init__( dist: "Distributed", max_sessions: int = DEFAULT_MAX_SESSIONS, fair_share_multiplier: float = 2.0, - new_conv_placement: str = "round_robin", ): super().__init__(dist) self._conv_to_rank: "OrderedDict[str, int]" = OrderedDict() self._max_sessions = max(1, int(max_sessions)) self._fair_share_multiplier = max(1.0, float(fair_share_multiplier)) - self._new_conv_placement = ( - "least_queued" if new_conv_placement == "least_queued" else "round_robin" - ) self._round_robin_cursor = 0 def create_rank_state( @@ -929,13 +923,8 @@ def _next_rr(soft_cap: int) -> int: if rank is None: # First turn of a new conversation, sticky-overflow, or no - # conversation_id -> spread under the soft cap: round-robin - # (count-uniform) or least-queued (steers away from ranks kept - # busy by heavy pinned conversations). - if self._new_conv_placement == "least_queued": - rank = _least_loaded(expected_num_active_requests) - else: - rank = _next_rr(expected_num_active_requests) + # conversation_id -> round-robin spread under the soft cap. + rank = _next_rr(expected_num_active_requests) if conv_id is not None and conv_id not in self._conv_to_rank: # Bind this new conversation to its first-turn rank. self._record_target_rank(conv_id, rank) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py index caa8e3cb3de1..22d7ac2a8821 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler.py @@ -116,22 +116,6 @@ def _get_lora_task_id(req: LlmRequest): return (1, lora_id) -def _get_forced_context_chunk_size(req: LlmRequest) -> int: - next_point = min( - (point for point in req.expect_snapshot_points if point > req.context_current_position), - default=None, - ) - if next_point is None: - return req.context_remaining_length - next_position = min(next_point, req.prompt_len) - return max(0, next_position - req.context_current_position) - - -def _is_forced_context_chunk_boundary(req: LlmRequest, chunk_size: int) -> bool: - next_position = req.context_current_position + chunk_size - return next_position >= req.prompt_len or next_position in req.expect_snapshot_points - - class ScheduledRequests: """Scheduled requests separated into disjoint sets. @@ -680,7 +664,6 @@ def schedule( all_context_requests_fit = False if ctx_chunk_config and ctx_chunk_config.chunking_policy == ChunkingPolicy.FORCE_CHUNK: - # Run snapshot-boundary selection even when the full contexts fit. all_context_requests_fit = False # 3. Apply Chunking Strategy if needed @@ -903,9 +886,8 @@ def _chunk_fcfs( def _chunk_forced(self, requests: RequestList, capacity: Optional[int], unit_size: int): """Mirrors the kFORCE_CHUNK specialization of setCtxRequestsChunkSize (microBatchScheduler.cpp). - Requests advance to their next expected snapshot point. With no - remaining snapshot point, they consume the full remaining context. - Capacity-limited chunks are rounded down to a unit_size multiple. + Every request is assigned exactly min(context_remaining_length, unit_size) tokens. + Requests that would exceed the capacity budget are zeroed out. This policy is designed for linear attention / Mamba2 state caching, which doesn't support estimating reusable tokens, so we don't subtract them from the budget. @@ -917,14 +899,9 @@ def _chunk_forced(self, requests: RequestList, capacity: Optional[int], unit_siz ) total_tokens = 0 for req in requests: - assert isinstance(req.expect_snapshot_points, list) - chunk_size = _get_forced_context_chunk_size(req) - if self.max_context_length is not None and chunk_size > self.max_context_length: - chunk_size = (self.max_context_length // unit_size) * unit_size - if capacity is not None and total_tokens + chunk_size > capacity: - remaining_capacity = max(0, capacity - total_tokens) - chunk_size = (min(chunk_size, remaining_capacity) // unit_size) * unit_size - req.context_chunk_size = int(chunk_size) + req.context_chunk_size = min(req.context_remaining_length, unit_size) + if capacity is not None and total_tokens + req.context_chunk_size > capacity: + req.context_chunk_size = 0 total_tokens += req.context_chunk_size assert capacity is None or total_tokens <= capacity diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py index 958afc59ac04..2accbcc152b3 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/scheduler_v2.py @@ -17,7 +17,7 @@ import os from typing import Optional -from tensorrt_llm.llmapi.llm_args import CapacitySchedulerPolicy, ContextChunkingPolicy +from tensorrt_llm.llmapi.llm_args import CapacitySchedulerPolicy from tensorrt_llm.logger import logger from ..llm_request import LlmRequest, LlmRequestState, get_draft_token_length @@ -25,9 +25,7 @@ RequestList, RequestScheduler, SchedulerOutput, - _get_forced_context_chunk_size, _get_lora_task_id, - _is_forced_context_chunk_boundary, drop_decoder_context_requests_waiting_for_encoder_output, ) @@ -177,9 +175,8 @@ def __init__( self.policy = scheduler_policy self.peft_cache_manager = peft_cache_manager - # Chunking config. + # Chunking config — only FCFS supported self.chunking_enabled = False - self.chunking_policy: Optional[ContextChunkingPolicy] = None self.chunk_unit_size = 0 self.max_context_length = max_num_tokens self.tokens_per_block = kv_cache_manager.tokens_per_block @@ -197,7 +194,6 @@ def __init__( ) if ctx_chunk_config is not None: self.chunking_enabled = True - self.chunking_policy = ctx_chunk_config[0] self.chunk_unit_size = ctx_chunk_config[1] # State value caches for fast comparison. @@ -578,13 +574,10 @@ def _try_schedule_context_chunked( """ remaining_budget = budget.remaining_tokens pre_prepare_context_remaining = req.context_remaining_length - force_chunk = self.chunking_policy == ContextChunkingPolicy.FORCE_CHUNK - if remaining_budget is not None: - no_budget = remaining_budget <= 0 - fcfs_under_min = not force_chunk and remaining_budget < self.chunk_unit_size - if no_budget or fcfs_under_min: - return ScheduleAction.SKIP, 0, False + # Min budget check — need at least one chunk unit + if remaining_budget is not None and remaining_budget < self.chunk_unit_size: + return ScheduleAction.SKIP, 0, False # Prepare context (create _KVCache, block reuse, resume — no resize) if not self.kv_cache_manager.prepare_context(req): @@ -594,36 +587,22 @@ def _try_schedule_context_chunked( # Calculate chunk size from remaining budget # (context_remaining_length is now correct after block reuse) context_remaining = req.context_remaining_length - if force_chunk: - # Snapshot boundaries can be shorter than chunk_unit_size. - assert isinstance(req.expect_snapshot_points, list) - # With no remaining state to snapshot, avoid artificial boundaries. - chunk_size = _get_forced_context_chunk_size(req) - budget_context_remaining = context_remaining - else: - budget_context_remaining = ( - context_remaining - if self.enable_prefix_aware_scheduling - else pre_prepare_context_remaining - ) - chunk_size = ( - min(remaining_budget, budget_context_remaining) - if remaining_budget is not None - else budget_context_remaining - ) + budget_context_remaining = ( + context_remaining + if self.enable_prefix_aware_scheduling + else pre_prepare_context_remaining + ) + chunk_size = ( + min(remaining_budget, budget_context_remaining) + if remaining_budget is not None + else budget_context_remaining + ) if self.max_context_length is not None: chunk_size = min(chunk_size, self.max_context_length) - chunk_size = min( - chunk_size, remaining_budget if remaining_budget is not None else chunk_size - ) - - # Round down to chunk_unit_size boundary only when not hitting the end - # or a checkpoint. - if chunk_size < budget_context_remaining and not ( - force_chunk and _is_forced_context_chunk_boundary(req, chunk_size) - ): + # Round down to chunk_unit_size boundary (unless last chunk). + if chunk_size < budget_context_remaining: chunk_size = (chunk_size // self.chunk_unit_size) * self.chunk_unit_size if chunk_size <= 0: diff --git a/tensorrt_llm/_torch/speculative/dflash.py b/tensorrt_llm/_torch/speculative/dflash.py index 585b796a6dfb..fece18aacc27 100644 --- a/tensorrt_llm/_torch/speculative/dflash.py +++ b/tensorrt_llm/_torch/speculative/dflash.py @@ -102,10 +102,10 @@ def prepare(self): worker._ctx_len[slot] = 0 worker._free_slots.append(slot) - # Route unknown request IDs (cuda graph padding or warmup dummies) - # to dummy slot to avoid corrupting real request's context + # Default to slot 0 for unknown request IDs (e.g. during warmup + # where synthetic requests may not have assigned slots). mapping = torch.tensor( - [worker._req_to_slot.get(rid, worker._dummy_slot) for rid in self.request_ids], + [worker._req_to_slot.get(rid, 0) for rid in self.request_ids], dtype=torch.long, device="cpu", pin_memory=prefer_pinned(), @@ -172,25 +172,14 @@ def __init__( # graph compatible. self._ctx_buf_inited = False self._ctx_len = None - # Snapshot for rolling back in-place _ctx_len updates when a forward - # fails (or after warmup). See _ensure_spec_dec_state_restored. - self._saved_ctx_len = None - self._ctx_len_restore_pending = False - # Deferred kv_lens_cuda rewind state (see _prepare_kv_for_draft_forward, - # _apply_kv_rewind_after_draft, _ensure_spec_dec_state_restored). - self._kv_rewind_pending = False - self._kv_rewind_amount = None - self._kv_rewind_nc = None - self._kv_rewind_bs = None self._batch_to_slot = None self._max_ctx = 0 - self._ctx_k_buf = None # [max_batch+1, L, max_ctx+block, nkv, hd] + self._ctx_k_buf = None # [max_batch, L, max_ctx+block, nkv, hd] self._ctx_v_buf = None # Slot management (Python, updated in prepare() and eager mode) self._req_to_slot = {} # request_id -> slot index self._free_slots = deque() # available slot indices - self._dummy_slot = None # for cudagraph padding or warmup dummy requests logger.info( f"DFlashWorker initialized with use_separate_draft_kv_cache={use_separate_draft_kv_cache}" @@ -232,13 +221,7 @@ def _lazy_init_ctx_buffers(self, draft_model, spec_metadata, attn_metadata): dtype = draft_model.fc.weight.dtype if hasattr(draft_model, "fc") else torch.bfloat16 - # Reserve slot index max_batch as a scratch slot for padding/unknown - # dummies; real requests only draw slots 0..max_batch-1, so dummy - # writes land here and can't corrupt a real request's context. - self._dummy_slot = max_batch - num_slots = max_batch + 1 - - self._ctx_len = torch.zeros(num_slots, dtype=torch.long, device="cuda") + self._ctx_len = torch.zeros(max_batch, dtype=torch.long, device="cuda") self._batch_to_slot = torch.zeros(max_batch, dtype=torch.long, device="cuda") self._free_slots = deque(range(max_batch)) @@ -257,7 +240,7 @@ def _lazy_init_ctx_buffers(self, draft_model, spec_metadata, attn_metadata): L = draft_model._num_attn_layers nkv = draft_model._num_kv_heads hd = draft_model._head_dim - kv_shape = (num_slots, L, self._max_ctx + self._resolved_block_size, nkv, hd) + kv_shape = (max_batch, L, self._max_ctx + self._resolved_block_size, nkv, hd) self._ctx_k_buf = torch.zeros(kv_shape, dtype=dtype, device="cuda") self._ctx_v_buf = torch.zeros(kv_shape, dtype=dtype, device="cuda") self._ctx_buf_inited = True @@ -291,20 +274,16 @@ def _prepare_kv_for_draft_forward( if batch_size > num_contexts: attn_metadata.kv_lens_cuda[num_contexts:batch_size] += 1 - self._kv_rewind_pending = True attn_metadata.update_for_spec_dec() def _apply_kv_rewind_after_draft(self, attn_metadata, spec_metadata): """Apply the deferred kv_lens rewind after the draft forward.""" - self._kv_rewind_pending = False is_warmup = spec_metadata.is_cuda_graph and not torch.cuda.is_current_stream_capturing() if is_warmup: - # kv_lens_cuda was saved by prepare_for_spec_dec in this mode and - # is restored wholesale, so no rewind is needed. return - if self._kv_rewind_amount is not None and hasattr(attn_metadata, "kv_lens_cuda"): + if hasattr(self, "_kv_rewind_amount") and hasattr(attn_metadata, "kv_lens_cuda"): nc = self._kv_rewind_nc bs = self._kv_rewind_bs attn_metadata.kv_lens_cuda[nc:bs] -= self._kv_rewind_amount @@ -387,28 +366,7 @@ def _store_prefill_context( self._ctx_v_buf[slot, :, cur:end] = chunk_v.permute(1, 0, 2, 3) offset += slen - def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): - # Restore first (in warmup mode kv_lens_cuda was saved and comes back - # wholesale), then apply any pending rewind for the other modes so a - # failed draft forward does not leave kv_lens_cuda incremented. - super()._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) - if ( - getattr(self, "_kv_rewind_pending", False) - and attn_metadata is not None - and spec_metadata is not None - ): - self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) - if ( - getattr(self, "_ctx_len_restore_pending", False) - and self._ctx_len is not None - and self._saved_ctx_len is not None - ): - # A failed forward must not keep this iteration's in-place - # _ctx_len updates: roll back to the pre-forward snapshot. - self._ctx_len.copy_(self._saved_ctx_len) - self._ctx_len_restore_pending = False - - def _forward_impl( + def forward( self, input_ids, position_ids, @@ -441,23 +399,24 @@ def _forward_impl( self._lazy_init_ctx_buffers(draft_model, spec_metadata, attn_metadata) spec_metadata._dflash_worker = self - # Save context lengths so both warmup and a failed forward can roll - # back the in-place _ctx_len updates made during drafting. + # Save context lengths before warmup to prevent accumulation is_warmup = spec_metadata.is_cuda_graph and not torch.cuda.is_current_stream_capturing() - if not torch.cuda.is_current_stream_capturing(): - # Never allocate the snapshot while capturing a CUDA graph: the - # clone would live in the graph memory pool, and its replay-time - # writes could alias blocks reused by later captures. Rollback is - # only meaningful for eager/warmup forwards anyway; a failure - # during capture aborts the graph itself, and captured ops do not - # mutate _ctx_len until replay. - self._saved_ctx_len = self._ctx_len.clone() - self._ctx_len_restore_pending = True + if is_warmup: + saved_ctx_len = self._ctx_len.clone() self._execute_guided_decoder_if_present(logits) - accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( - logits, attn_metadata, spec_metadata + # Target now emits K+1 logits per gen request and the previous step + # stored K draft tokens per gen request (no filler padding). + if num_gens > 0: + draft_tokens = spec_metadata.draft_tokens.reshape(num_gens, K) + else: + draft_tokens = spec_metadata.draft_tokens.reshape(0, K) + + logits_for_accept = logits + + accepted_tokens, num_accepted_tokens = self._sample_and_accept_draft_tokens_base( + logits_for_accept, draft_tokens, num_contexts, batch_size, spec_metadata ) # Update GDN/Mamba recurrent states to the accepted token's state. @@ -492,10 +451,7 @@ def _forward_impl( # Rebuild batch_to_slot after prefill assigns new slots if self._ctx_buf_inited and spec_metadata.request_ids: num_seqs = len(spec_metadata.request_ids) - mapping = [ - self._req_to_slot.get(rid, self._dummy_slot) - for rid in spec_metadata.request_ids - ] + mapping = [self._req_to_slot.get(rid, 0) for rid in spec_metadata.request_ids] self._batch_to_slot[:num_seqs].copy_( torch.tensor(mapping, dtype=torch.long, device="cuda") ) @@ -541,22 +497,17 @@ def _forward_impl( vocab_size = gen_logits.shape[-1] gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - gen_draft_tokens = self.sample_draft_tokens( - gen_logits, - spec_metadata, - batch_size, - num_contexts=num_contexts, - ) + d2t = getattr(draft_model.model, "d2t", None) + gen_draft_tokens = torch.argmax(gen_logits, dim=-1, keepdim=False).long() + + if d2t is not None: + gen_draft_tokens = d2t[gen_draft_tokens] + gen_draft_tokens + + gen_draft_tokens = gen_draft_tokens.type(torch.int32) else: gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) - if num_contexts > 0 and num_gens > 0: ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) @@ -576,10 +527,9 @@ def _forward_impl( num_accepted_tokens, ) - # Restore context lengths after warmup; real runs keep the updates. + # Restore context lengths after warmup if is_warmup: - self._ctx_len.copy_(self._saved_ctx_len) - self._ctx_len_restore_pending = False + self._ctx_len.copy_(saved_ctx_len) return { "logits": raw_logits, diff --git a/tensorrt_llm/_torch/speculative/draft_target.py b/tensorrt_llm/_torch/speculative/draft_target.py index 6edeac334fdd..fe6dc7d1e4c7 100644 --- a/tensorrt_llm/_torch/speculative/draft_target.py +++ b/tensorrt_llm/_torch/speculative/draft_target.py @@ -153,7 +153,7 @@ def _update_kv_for_chained_draft_step( attn_metadata.update_for_spec_dec() - def _forward_impl( + def forward( self, input_ids, position_ids, @@ -257,14 +257,10 @@ def _forward_impl( hidden_states[gather_ids], draft_model.lm_head, attn_metadata, True ) if self.guided_decoder is not None: - self.guided_decoder.execute_draft_batch(logits, self._d2t, draft_step=i) + d2t = getattr(draft_model.model, "d2t", None) + self.guided_decoder.execute_draft_batch(logits, d2t, draft_step=i) - new_draft_token = self.sample_draft_tokens( - logits, - spec_metadata, - batch_size, - draft_step=i, - ) + new_draft_token = self.draft_decoder(logits, draft_model) next_draft_tokens.append(new_draft_token) # Update inputs and metadata for next draft step @@ -318,6 +314,36 @@ def _forward_impl( "next_new_tokens": next_new_tokens, } + def sample_and_accept_draft_tokens( + self, + logits: torch.Tensor, + attn_metadata: AttentionMetadata, + spec_metadata: DraftTargetOneModelSpecMetadata, + ): + batch_size = attn_metadata.num_seqs + num_contexts = attn_metadata.num_contexts + num_gens = batch_size - num_contexts + runtime_draft_len = spec_metadata.runtime_draft_len + + if spec_metadata.draft_tokens is None: + draft_tokens = torch.zeros( + (num_gens, runtime_draft_len), dtype=torch.int, device=logits.device + ) + else: + draft_tokens = spec_metadata.draft_tokens.reshape(num_gens, runtime_draft_len) + + return self._sample_and_accept_draft_tokens_base( + logits, draft_tokens, num_contexts, batch_size, spec_metadata + ) + + def draft_decoder( + self, + logits: torch.Tensor, + draft_model: nn.Module, + ): + d2t = getattr(draft_model.model, "d2t", None) + return self._draft_sampler_greedy(logits, d2t) + def prepare_1st_drafter_inputs( self, input_ids: torch.LongTensor, diff --git a/tensorrt_llm/_torch/speculative/drafting_loops.py b/tensorrt_llm/_torch/speculative/drafting_loops.py index e2be40bfce5e..231f7f21c147 100644 --- a/tensorrt_llm/_torch/speculative/drafting_loops.py +++ b/tensorrt_llm/_torch/speculative/drafting_loops.py @@ -15,8 +15,7 @@ import torch from tensorrt_llm._torch.attention_backend.interface import AttentionMetadata -from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import \ - greedy_search_sampling_batch +from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import greedy from tensorrt_llm._torch.speculative.eagle3 import Eagle3SpecMetadata from tensorrt_llm._torch.speculative.interface import SpecMetadata from tensorrt_llm._torch.speculative.spec_tree_manager import SpecTreeManager @@ -151,7 +150,7 @@ def forward(self, input_ids: torch.Tensor, position_ids: torch.Tensor, def sample(self, logits: torch.Tensor) -> torch.Tensor: # TODO: inject the sampler here so we can support non-greedy - tokens, _ = greedy_search_sampling_batch(logits, return_probs=False) + tokens, _ = greedy(logits, return_probs=False) if hasattr(self.draft_model.model, "d2t"): d2t = self.draft_model.model.d2t.data return tokens + d2t[tokens] diff --git a/tensorrt_llm/_torch/speculative/dspark.py b/tensorrt_llm/_torch/speculative/dspark.py deleted file mode 100644 index 7f43f103ec4e..000000000000 --- a/tensorrt_llm/_torch/speculative/dspark.py +++ /dev/null @@ -1,613 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DSpark worker / metadata mirror the DFlash plumbing (capture target-layer -# hidden states, accept the previous block with standard verification, draft a -# new block in one backbone forward), adapted to DSpark's draft model which -# produces the whole block (and its confidence-truncated length) inside a single -# ``DSparkDraftModel.forward`` rather than via mask-token cross-attention. - -from collections import deque -from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional - -import torch - -from tensorrt_llm._utils import prefer_pinned -from tensorrt_llm.logger import logger -from tensorrt_llm.mapping import Mapping - -from ..pyexecutor.llm_request import ATTENTION_DP_DUMMY_REQUEST_ID -from .interface import SpecMetadata, SpecWorkerBase - -if TYPE_CHECKING: - from ...llmapi.llm_args import DSparkDecodingConfig - - -@dataclass -class DSparkSpecMetadata(SpecMetadata): - """Metadata for DSpark speculative decoding. - - Captures hidden states from the target model's ``layers_to_capture`` during - the target forward pass. DSpark captures the *mean over the multi-head - (mHC) residual streams* at each captured layer (handled by the target-side - capture hook), concatenated across layers, and feeds them to the draft - model's ``main_proj`` + ``main_norm`` (inside ``DSparkDraftModel.forward``) - as the captured-context attention input (``main_x``). - - Mirrors :class:`DFlashSpecMetadata`; the only DSpark-specific detail is that - the per-layer captured width is the model hidden size (post hc-mean), so the - buffer is ``[max_num_tokens, hidden_size * num_capture_layers]``. - """ - - batch_indices_cuda: Optional[torch.Tensor] = None - - # Hidden state capture fields - layers_to_capture: Optional[List[int]] = None - hidden_size: int = 0 - max_num_tokens: int = 0 - dtype: torch.dtype = torch.bfloat16 - captured_hidden_states: Optional[torch.Tensor] = None - - def __post_init__(self): - self.batch_indices_cuda = torch.empty( - [self.max_num_requests], - dtype=torch.int, - device="cuda", - ) - - self.is_spec_dec_tree = False - self.is_spec_dec_dynamic_tree = False - - # Set up hidden state capture buffer - if self.layers_to_capture is not None and len(self.layers_to_capture) > 0: - self.layers_to_capture = sorted(list(self.layers_to_capture)) - self.num_capture_layers = len(self.layers_to_capture) - # O(1) lookups for is_layer_capture() and maybe_capture_hidden_states() - self._capture_layer_set = frozenset(self.layers_to_capture) - self._layer_to_idx = {lid: i for i, lid in enumerate(self.layers_to_capture)} - self.captured_hidden_states = torch.empty( - (self.max_num_tokens, self.hidden_size * self.num_capture_layers), - dtype=self.dtype, - device="cuda", - ) - logger.info( - f"DSpark: capturing hidden states from layers {self.layers_to_capture}, " - f"buffer shape {self.captured_hidden_states.shape}" - ) - else: - self.num_capture_layers = 0 - self._capture_layer_set = frozenset() - self._layer_to_idx = {} - - def prepare(self): - assert self.request_ids is not None - num_seqs = len(self.request_ids) - batch_indices = torch.arange( - num_seqs, dtype=torch.int, device="cpu", pin_memory=prefer_pinned() - ) - self.batch_indices_cuda[:num_seqs].copy_(batch_indices, non_blocking=True) - - # CUDA-graph-safe path: maintain the request->slot mapping on the host - # (outside the captured region) and mirror it into ``_batch_to_slot`` so the - # captured gen forward can index the rolling windows by tensor. Mirrors - # ``DFlashSpecMetadata.prepare`` (dflash.py:96-113). - worker = getattr(self, "_dspark_worker", None) - if worker is not None and worker._win_inited: - current = set(self.request_ids) - for rid in list(worker._req_to_slot.keys()): - if rid not in current: - slot = worker._req_to_slot.pop(rid) - worker._ctx_len[slot] = 0 - worker._kv_windows[slot].zero_() - worker._free_slots.append(slot) - # Assign a persistent rolling-window slot to every real generation - # request that never ran a context/seed forward on this worker. In - # disaggregated serving the prompt is prefilled (and the window - # seeded) on the *context* server, so ``_seed_context_windows`` never - # runs on the generation server and ``_req_to_slot`` stays empty; - # without this, all concurrent gen requests fall through to the shared - # scratch row below and corrupt each other's draft window at batch - # size > 1 (GitHub #16767). Context-prefix entries are left to - # ``_seed_context_windows``; the ADP-idle (id 0) and CUDA-graph - # padding dummies are kept on the scratch row. - num_contexts = max(0, len(self.request_ids) - self.num_generations) - for rid in self.request_ids[num_contexts:]: - if ( - rid != ATTENTION_DP_DUMMY_REQUEST_ID - and rid < worker._graph_dummy_id_floor - and rid not in worker._req_to_slot - ): - worker._assign_slot(rid, reset=False) - # Unknown request IDs (synthetic warmup / CUDA-graph padding, ADP idle - # requests, or disagg seed forwards without a real id) map to the - # dedicated throwaway scratch row so they cannot overwrite a live - # request's rolling window (they previously aliased to slot 0). - scratch = worker._scratch_slot - mapping = torch.tensor( - [worker._req_to_slot.get(rid, scratch) for rid in self.request_ids], - dtype=torch.long, - device="cpu", - pin_memory=prefer_pinned(), - ) - worker._batch_to_slot[:num_seqs].copy_(mapping, non_blocking=True) - - def is_layer_capture(self, layer_id: int) -> bool: - return layer_id in self._capture_layer_set - - def maybe_capture_hidden_states( - self, layer_id: int, hidden_states: torch.Tensor, residual: Optional[torch.Tensor] = None - ) -> None: - """Capture hidden states from a target model layer into the buffer. - - DeepSeek-V4 keeps the multi-head (mHC) residual stream flattened as - ``[num_tokens, hc_mult * hidden]``; DSpark captures the *mean over the hc - streams* (reference ``h.mean(dim=2)`` with ``h`` shaped - ``[*, hc_mult, hidden]``). We reduce here so the V4 decoder layer's - existing capture call is unchanged. A ``[num_tokens, hidden]`` input - (already reduced / non-mHC) is stored as-is. - """ - if self.captured_hidden_states is None: - return - i = self._layer_to_idx.get(layer_id) - if i is not None: - num_tokens = hidden_states.shape[0] - to_save = hidden_states + residual if residual is not None else hidden_states - # mHC residual -> mean over the hc_mult streams. - if to_save.shape[-1] != self.hidden_size: - hc_mult = to_save.shape[-1] // self.hidden_size - to_save = to_save.reshape(num_tokens, hc_mult, self.hidden_size).mean(dim=1) - self.captured_hidden_states[ - :num_tokens, i * self.hidden_size : (i + 1) * self.hidden_size - ].copy_(to_save, non_blocking=True) - - def get_hidden_states(self, num_tokens: int) -> Optional[torch.Tensor]: - """Get captured hidden states (all layers concatenated).""" - if self.captured_hidden_states is None: - return None - return self.captured_hidden_states[ - :num_tokens, : self.hidden_size * self.num_capture_layers - ] - - -class DSparkWorker(SpecWorkerBase): - """Worker for DSpark speculative decoding. - - DSpark drafts a whole block of ``block_size`` tokens in one backbone forward - (``DSparkDraftModel.forward``): it projects the captured target-layer hidden - states (``main_proj`` + ``main_norm``) into the draft's captured-context - attention, runs the ``num_stages`` DSpark blocks over a rolling captured - window, refines the per-position logits with the Markov head, and predicts a - per-position acceptance confidence used to truncate the proposed prefix. - - Unlike DFlash, the draft does NOT use the paged KV cache or mask-token - cross-attention: its attention K/V come from the worker-owned rolling window - of projected captured context (one ``main_kv`` per decode step, per stage). - Acceptance of the previous block goes through the unified - :meth:`SpecWorkerBase.sample_and_accept_draft_tokens` (strict target-verify, - or rejection sampling for a non-greedy batch), so greedy parity with no-spec - is preserved regardless of draft quality. - - The rolling window is kept consistent across the whole decode: it is seeded - from the prompt's captured context at prefill and back-filled with the - intermediate accepted tokens of a multi-accept step (both via - ``DSparkDraftModel.write_context_windows``), in addition to the per-step bonus - write done by the generation path. These affect draft acceptance rate only, - not correctness, which the standard target verify guarantees. - - Reference: DeepSeek DeepSpec (https://github.com/deepseek-ai/DeepSpec). - """ - - def __init__( - self, - spec_config: "DSparkDecodingConfig", - mapping: Mapping, - use_separate_draft_kv_cache: bool = False, - ): - super().__init__(use_separate_draft_kv_cache) - self.spec_config = spec_config - self.mapping = mapping - - # Per-slot rolling captured-context KV windows, built lazily on the - # first forward (fixed-size for slot-indexed reads/writes). - self._win_inited = False - self._kv_windows: Optional[torch.Tensor] = None # [max_batch, num_stages, win, hd] - self._ctx_len: Optional[torch.Tensor] = None # [max_batch] abs decode position - self._win = 0 - - # Slot management. ``_req_to_slot`` (python dict) + ``_free_slots`` are the - # source of truth, updated in prepare()/forward(); ``_batch_to_slot`` is the - # CUDA mirror (request-order -> slot) read by the CUDA-graph-safe batched - # gen path (set on the host in prepare(), so the captured forward indexes - # the rolling windows through a tensor instead of a python dict lookup). - self._req_to_slot = {} # request_id -> slot index - self._free_slots = deque() # available slot indices - self._batch_to_slot: Optional[torch.Tensor] = None # [max_batch] long, cuda - # Index of the throwaway "scratch" window row that absorbs padded / - # unknown request IDs (set in ``_lazy_init`` to ``max_batch``); it is - # never handed out through ``_free_slots``. - self._scratch_slot = 0 - - # The generation draft path is the batched, host-sync-free - # ``_draft_gen_block_batched`` + ``DSparkDraftModel.forward_batched`` + - # ``dspark_attention_forward_batched``: it is correct in eager mode AND safe - # to capture into the target's CUDA graph (DSpark is a one-engine drafter — - # its worker forward runs inside that graph, so the draft path MUST be - # capture-safe whenever ``cuda_graph_config`` is set). - - logger.info( - f"DSparkWorker initialized with " - f"use_separate_draft_kv_cache={use_separate_draft_kv_cache}" - ) - - @property - def max_draft_len(self) -> int: - return self.spec_config.max_draft_len - - def _lazy_init(self, draft_model, spec_metadata) -> None: - block_size = int(draft_model.block_size) - if block_size != self.max_draft_len: - raise ValueError( - "DSpark draft model block_size must equal worker max_draft_len; " - f"got block_size={block_size} and max_draft_len={self.max_draft_len}" - ) - - if self._win_inited: - return - max_batch = spec_metadata.max_num_requests - num_stages = draft_model.num_stages - self._win = int(draft_model._attn_params["window_size"]) - head_dim = int(draft_model._attn_params["head_dim"]) - - # Real requests occupy slots ``[0, max_batch)``; one extra "scratch" row - # at index ``max_batch`` absorbs padded / unknown request IDs (CUDA-graph - # padding, ADP idle requests, or disagg seed forwards that arrive without - # a real request id) so they can never overwrite a live request's rolling - # window. Previously such IDs aliased to slot 0 and corrupted whichever - # real request occupied it. The scratch row is never handed out through - # ``_free_slots`` and its contents are throwaway. - self._scratch_slot = max_batch - num_rows = max_batch + 1 - - # CUDA-graph padding requests carry ids in - # ``[CUDA_GRAPH_DUMMY_REQUEST_ID - runtime_draft_len, CUDA_GRAPH_DUMMY_REQUEST_ID]``, - # while real request ids start at ``max_batch_size`` and grow, so a simple - # floor cleanly separates them. Together with ``ATTENTION_DP_DUMMY_REQUEST_ID`` - # (0) these dummies must route to the scratch row (see ``prepare()``) and - # never consume a real slot. Imported lazily to break the - # dspark -> cuda_graph_runner -> speculative.utils -> dspark import cycle. - from ..pyexecutor.cuda_graph_runner import CUDA_GRAPH_DUMMY_REQUEST_ID - - self._graph_dummy_id_floor = CUDA_GRAPH_DUMMY_REQUEST_ID - self.max_draft_len - - self._kv_windows = torch.zeros( - (num_rows, num_stages, self._win, head_dim), - dtype=torch.bfloat16, - device="cuda", - ) - self._ctx_len = torch.zeros(num_rows, dtype=torch.long, device="cuda") - self._batch_to_slot = torch.zeros(max_batch, dtype=torch.long, device="cuda") - self._free_slots = deque(range(max_batch)) - self._req_to_slot = {} - self._win_inited = True - logger.info( - f"DSpark: allocated rolling KV windows " - f"[{num_rows}, {num_stages}, {self._win}, {head_dim}] " - f"({max_batch} request slots + 1 scratch row)" - ) - - def _assign_slot(self, req_id: int, reset: bool) -> int: - """Get (or refresh) the slot for a request; reset clears its window.""" - if reset and req_id in self._req_to_slot: - old = self._req_to_slot.pop(req_id) - self._ctx_len[old] = 0 - self._kv_windows[old].zero_() - self._free_slots.append(old) - if req_id not in self._req_to_slot: - if not self._free_slots: - raise RuntimeError( - "DSpark has no free rolling-window slots for request " - f"{req_id}; increase max_num_requests" - ) - slot = self._free_slots.popleft() - self._req_to_slot[req_id] = slot - self._ctx_len[slot] = 0 - self._kv_windows[slot].zero_() - return self._req_to_slot[req_id] - - def _seed_context_windows( - self, - draft_model, - spec_metadata: "DSparkSpecMetadata", - attn_metadata, - position_ids: torch.Tensor, - total_target_tokens: int, - ) -> None: - """Seed context chunks using their absolute positions. - - A request can arrive in multiple prefill chunks. Only its first chunk - starts at position zero and resets the persistent rolling window; - continuation chunks append to the same request slot. - """ - captured = spec_metadata.get_hidden_states(total_target_tokens) - flat_position_ids = position_ids.reshape(-1) - context_offset = 0 - for i in range(attn_metadata.num_contexts): - chunk_len = int(attn_metadata._seq_lens[i]) - chunk_positions = flat_position_ids[context_offset : context_offset + chunk_len].long() - if chunk_len == 0: - context_offset += chunk_len - continue - - req_id = spec_metadata.request_ids[i] - first_position = int(chunk_positions[0].item()) - slot = self._assign_slot(req_id, reset=first_position == 0) - self._ctx_len[slot] = chunk_positions[-1] + 1 - - if captured is not None: - keep = min(self._win, chunk_len) - hidden = captured[context_offset + chunk_len - keep : context_offset + chunk_len] - # A prompt token at absolute position p is stored in frame p+1, - # matching the generation path's start_pos convention. - window_positions = chunk_positions[-keep:] + 1 - draft_model.write_context_windows(hidden, window_positions, self._kv_windows[slot]) - context_offset += chunk_len - - def _draft_gen_block_batched( - self, - draft_model, - spec_metadata: "DSparkSpecMetadata", - attn_metadata, - accepted_tokens: torch.Tensor, - num_accepted_tokens: torch.Tensor, - num_contexts: int, - batch_size: int, - total_target_tokens: int, - all_rank_num_tokens: Optional[List[int]] = None, - ) -> torch.Tensor: - """CUDA-graph-safe batched gen draft (all gen requests in one forward). - - Free of host syncs and data-dependent shapes: per-request quantities - (``nacc``, the bonus, ``main_hidden``, ``start_pos``, the multi-accept - back-fill) are gathered as tensors, slots come from the host-built - ``_batch_to_slot`` mirror, and the backbone runs once via - ``DSparkDraftModel.forward_batched``. Returns the per-position corrected - block logits ``[num_gens, K, vocab]`` (or ``None`` when there is nothing to - draft); the worker feeds them to ``SpecWorkerBase.sample_draft_tokens``. - Confidence truncation stays disabled — the full block is proposed. - """ - num_gens = batch_size - num_contexts - K = self.max_draft_len - Kp1 = K + 1 - device = accepted_tokens.device - - if num_gens == 0: - return None - captured = spec_metadata.get_hidden_states(total_target_tokens) - if captured is None: - return None - - # gen-only graph batches have num_ctx_tokens == 0; mixed eager batches put - # the gen tokens after the context tokens. - gen_start = attn_metadata.num_ctx_tokens - slots = self._batch_to_slot[num_contexts:batch_size] # [G] - nacc = num_accepted_tokens[num_contexts:batch_size].long() # [G] - gidx = nacc - 1 # [G] index of the bonus within each verified prefix - - # Bonus token = last accepted token of the verified prefix. - bonus = ( - accepted_tokens[num_contexts:batch_size].gather(1, gidx.unsqueeze(1)).squeeze(1).long() - ) # [G] - - # Captured target hidden at the bonus position within each request's Kp1 - # processed tokens. - arange_g = torch.arange(num_gens, device=device) - base = gen_start + arange_g * Kp1 # [G] - main_hidden = captured[base + gidx] # [G, ncap*hidden] - - # Fixed-size ([G, K]) masked back-fill of the intermediate accepted tokens - # (everything but the bonus) into the rolling window — same frames as the - # eager path (old+1 .. old+nacc-1), with j >= nacc-1 masked out. - old = self._ctx_len[slots] # [G] pre-increment decode position - j = torch.arange(K, device=device) # [K] - interim_valid = j.unsqueeze(0) < (nacc.unsqueeze(1) - 1) # [G, K] - interim_pos = old.unsqueeze(1) + 1 + j.unsqueeze(0) # [G, K] - interim_base = (base.unsqueeze(1) + j.unsqueeze(0)).clamp( - min=0, max=captured.shape[0] - 1 - ) # [G, K] (clamped; invalid entries are masked out anyway) - interim_hidden = captured[interim_base] # [G, K, ncap*hidden] - draft_model.write_context_windows_batched( - interim_hidden, interim_pos, slots, interim_valid, self._kv_windows - ) - - # Advance the decode position by the accepted count; start_pos (= post- - # increment ctx_len) matches the eager path's frame value. - start_pos = old + nacc # [G] - self._ctx_len[slots] = start_pos - - # Surface the per-position corrected block logits ([num_gens, K, vocab]) - # and let SpecWorkerBase.sample_draft_tokens do the (greedy or rejection) - # sampling + TP gather + draft_probs scatter, rather than argmaxing here. - _toks, _num_proposed, block_logits = draft_model.forward_batched( - main_hidden, - bonus, - start_pos, - kv_windows=self._kv_windows, - slots=slots, - temperature=0.0, - confidence_threshold=0.0, - return_logits=True, - all_rank_num_tokens=all_rank_num_tokens, - ) - return block_logits - - def _forward_impl( - self, - input_ids, - position_ids, - hidden_states, - logits, - attn_metadata, - spec_metadata, - draft_model, - resource_manager=None, - ): - batch_size = attn_metadata.num_seqs - num_contexts = attn_metadata.num_contexts - num_gens = batch_size - num_contexts - raw_logits = logits - K = self.max_draft_len - - self._lazy_init(draft_model, spec_metadata) - # Backref so DSparkSpecMetadata.prepare() can maintain the host slot map - # and mirror it into _batch_to_slot for the CUDA-graph-safe gen path. - spec_metadata._dspark_worker = self - self._execute_guided_decoder_if_present(logits) - - # Target-verify acceptance via the unified SpecWorkerBase entry: it - # reshapes the stored draft tokens (default (num_gens, runtime_draft_len) - # hook), then routes to strict or rejection sampling. Greedy parity with - # the previous hand-rolled path is preserved (rejection only engages for a - # non-greedy batch with valid draft_probs). - accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( - logits, attn_metadata, spec_metadata - ) - - total_target_tokens = input_ids.shape[0] - - # CUDA-graph warmup guard: the warmup forwards (is_cuda_graph set, stream - # NOT yet capturing) run synthetic gen batches that would otherwise advance - # the persistent rolling-window state. Snapshot and restore it so warmup is - # side-effect-free. (During the capture pass itself the stream IS capturing, - # so we skip the save/restore and let the ops be recorded; real requests - # reset their slot's window+ctx_len at prefill, wiping any capture-time - # mutation.) - is_warmup = ( - getattr(spec_metadata, "is_cuda_graph", False) - and not torch.cuda.is_current_stream_capturing() - ) - if is_warmup: - saved_ctx_len = self._ctx_len.clone() - saved_windows = self._kv_windows.clone() - - # Assign / reset window slots for context (prefill) requests and seed each - # request's rolling KV window from its prompt's captured context, so the - # first generation step drafts against real context instead of an all-zero - # window (acceptance-rate only; verified decoding keeps output correct). - if num_contexts > 0: - self._seed_context_windows( - draft_model, - spec_metadata, - attn_metadata, - position_ids, - total_target_tokens, - ) - - # FUSED_COMM MoE backends (DeepGEMM MegaMoE) synchronize EP ranks with an - # in-kernel phase-flip NVLink barrier that flips on every kernel call, so - # every rank must invoke the draft MoE the same number of times and with - # the same globally-gathered per-rank token list, or the barrier desyncs - # (hang / "unspecified launch failure"). The draft runs over generation - # requests only, each expanded to ``block`` positions, so the per-rank - # draft-MoE token count is ``num_gens * block``. ``all_rank_num_gens`` is - # gathered at metadata-prep time (model_engine, outside any CUDA-graph - # capture region); it is None for non-ADP / single-rank runs, where the - # local ``[num_tokens]`` fallback in ``_forward_stage`` is correct. - block = int(draft_model.block_size) - all_rank_num_gens = getattr(spec_metadata, "all_rank_num_gens", None) - # A rank with zero local gen requests still has to cross the draft MoE's - # cross-rank barrier, but DeepseekV4MoE's router / shared-expert dense - # GEMMs reject a 0-row input (cuBLAS CUBLAS_STATUS_INVALID_VALUE), so such - # a rank runs a single 1-row dummy through the MoE (like ADP padding). - # Encode that as ``1`` in the globally-shared per-rank token list so every - # rank agrees on the FUSED_COMM chunk count and per-rank slice. - all_rank_draft_tokens = ( - [max(1, int(g) * block) for g in all_rank_num_gens] - if all_rank_num_gens is not None - else None - ) - global_has_gen = ( - max(all_rank_num_gens) > 0 if all_rank_num_gens is not None else num_gens > 0 - ) - - if num_gens > 0: - # The batched gen-block draft returns the per-position corrected block - # logits [num_gens, K, vocab] and is CUDA-graph-safe. - gen_logits = self._draft_gen_block_batched( - draft_model, - spec_metadata, - attn_metadata, - accepted_tokens, - num_accepted_tokens, - num_contexts, - batch_size, - total_target_tokens, - all_rank_num_tokens=all_rank_draft_tokens, - ) - if gen_logits is not None: - # SpecWorkerBase samples the draft tokens (greedy argmax, or - # rejection sampling for a non-greedy batch), performs the TP - # gather, and scatters the proposal distribution into draft_probs. - gen_draft_tokens = self.sample_draft_tokens( - gen_logits, spec_metadata, batch_size, num_contexts=num_contexts - ) - # The context one-hot must match the width the gen scatter just - # published to draft_probs, NOT gen_logits.shape[-1]: under TP the - # draft logits are vocab-sharded and sample_draft_tokens gathers - # them to full vocab before scattering, so the pre-gather shard - # width would leave stale columns and corrupt rejection. - gen_vocab = spec_metadata.draft_probs_last_dim - else: - gen_draft_tokens = torch.zeros((num_gens, K), dtype=torch.int32, device="cuda") - gen_vocab = None - else: - # No local generation requests: if any peer EP rank has some, we must - # still cross the draft MoE's cross-rank barrier the same number of - # times (zero-token) so a FUSED_COMM phase-flip barrier stays lockstep. - if global_has_gen: - draft_model.run_moe_lockstep_noop(all_rank_draft_tokens, accepted_tokens.device) - gen_draft_tokens = torch.empty((0, K), dtype=torch.int32, device="cuda") - gen_vocab = None - - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a legal one-hot so they are - # a valid distribution when they become gen requests next iteration. - self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) - - if num_contexts > 0: - ctx_draft_tokens = torch.zeros((num_contexts, K), dtype=torch.int32, device="cuda") - next_draft_tokens = torch.cat([ctx_draft_tokens, gen_draft_tokens], dim=0) - else: - next_draft_tokens = gen_draft_tokens - - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) - - if is_warmup: - self._ctx_len.copy_(saved_ctx_len) - self._kv_windows.copy_(saved_windows) - - return { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - } diff --git a/tensorrt_llm/_torch/speculative/eagle3.py b/tensorrt_llm/_torch/speculative/eagle3.py index a1f4f82869bd..3978040724b1 100644 --- a/tensorrt_llm/_torch/speculative/eagle3.py +++ b/tensorrt_llm/_torch/speculative/eagle3.py @@ -10,6 +10,7 @@ from tensorrt_llm.mapping import Mapping from ..attention_backend import AttentionMetadata +from ..distributed.ops import allgather from ..model_config import ModelConfig from ..pyexecutor.llm_request import LlmRequest from ..pyexecutor.mamba_cache_manager import MambaHybridCacheManager @@ -398,10 +399,6 @@ class Eagle3OneModelSpecMetadata(SpecMetadata): # prepare() before self.num_tokens is decremented to the attention-DP subseq # shape; maybe_capture_hidden_states must bound by this, not self.num_tokens. num_capture_tokens: int = 0 - # Per-generation tree links for Mamba verify in dynamic-tree one-model paths. - retrieve_next_token: Optional[torch.Tensor] = None - retrieve_next_sibling: Optional[torch.Tensor] = None - retrieve_parent_token: Optional[torch.Tensor] = None def __post_init__(self): if self.layers_to_capture is None: @@ -447,10 +444,9 @@ def __post_init__(self): self.hidden_size * len(self.layers_to_capture)), dtype=self.dtype, device='cuda') - batch_indices_cuda = getattr(self.spec_resource_manager, - "batch_indices_cuda", None) - if batch_indices_cuda is not None: - self.batch_indices_cuda = batch_indices_cuda + if (self.spec_resource_manager is not None + and self.spec_resource_manager.batch_indices_cuda is not None): + self.batch_indices_cuda = self.spec_resource_manager.batch_indices_cuda assert self.batch_indices_cuda.shape[0] >= self.max_num_requests, ( f"batch_indices_cuda shape mismatch: " f"{type(self.spec_resource_manager).__name__} has " @@ -535,23 +531,6 @@ def prepare(self): if gen_request_ids: sa_manager.prepare(gen_request_ids, self.runtime_draft_len) - self.retrieve_next_token = None - self.retrieve_next_sibling = None - self.retrieve_parent_token = None - spec_tree_manager = getattr(self.spec_resource_manager, - 'spec_tree_manager', None) - if self.use_dynamic_tree and spec_tree_manager is not None: - num_gens = self.num_generations - if num_gens > 0: - num_contexts = num_seqs - num_gens - slot_storage = spec_tree_manager.slot_storage - gen_slot_ids = slot_storage.all_ids_buf[ - num_contexts:num_contexts + num_gens] - next_token, next_sibling = slot_storage.next_links_from_slots( - gen_slot_ids, num_gens) - self.retrieve_next_token = next_token - self.retrieve_next_sibling = next_sibling - def maybe_capture_hidden_states( self, layer_id: int, @@ -646,14 +625,6 @@ def __init__(self, # MTP Eagle: lazily-resolved flag for Mamba hybrid cache support self._is_mamba_hybrid_cache = None - # Worker-side saved spec-dec params; initialized so that a failure - # inside _prepare_attn_metadata_for_spec_dec (e.g. an OOM in the - # clone calls) cannot turn the cleanup restore into AttributeError. - self._saved_packed_mask = None - self._saved_position_offsets = None - self._saved_position_offsets_cpp = None - self._saved_generation_lengths = None - @property def max_draft_len(self) -> int: return self.spec_config.max_draft_len @@ -709,15 +680,15 @@ def _restore_attn_metadata_from_spec_dec(self, attn_metadata): # Skip torch.compile for now since current Torch is not compatible with Triton 3.4 # @torch.compile(options={"max-autotune": True}) - def _forward_impl(self, - input_ids, - position_ids, - hidden_states, - logits, - attn_metadata, - spec_metadata, - draft_model, - resource_manager=None): + def forward(self, + input_ids, + position_ids, + hidden_states, + logits, + attn_metadata, + spec_metadata, + draft_model, + resource_manager=None): runtime_draft_len = spec_metadata.runtime_draft_len # skip the draft forward if the runtime draft length is 0 @@ -876,32 +847,11 @@ def _forward_linear_draft_loop(self, inputs, attn_metadata, spec_metadata, # ADP+LM-head-TP padding to ``max_num_requests`` so every TP # rank produces logits of the same shape. # Eagle3: logits_processor of the EAGLE draft model. - # - # The LM-head-TP fast path (group-stacked rows, vocab-sharded - # weight) is only usable when the consumer is an argmax: the - # distributed greedy sampler recovers the global argmax from - # vocab-shard maxima without materializing full distributions. - # Advanced (rejection) sampling needs each request's full-vocab - # distribution, so a batch headed for the advanced path - # bypasses LM-head-TP and computes full-vocab logits locally -- - # under ADP the lm_head weight is replicated (the sliced-shard - # trick is a runtime optimization), exactly like the target - # head. With rejection off, non-greedy batches keep the - # LM-head-TP argmax path unconditionally, where the per-rank - # greedy flag never enters control flow. This branch is safe - # to take group-uniformly because is_all_greedy_sample is - # group-synchronized whenever rejection+ADP+LM-head-TP are - # combined -- see SpecMetadata.group_all_greedy_sample (anchor - # for the group-sync semantics). - advanced_draft_sampling = ( - spec_metadata.wants_advanced_draft_sampling) - # enable_lm_head_tp_in_adp implies enable_attention_dp - # (asserted in Mapping.__init__); no separate ADP check. - lm_head_tp_in_adp_configured = ( + use_lm_head_tp_in_adp = ( self.is_mtp_eagle and self.model_config is not None - and self.model_config.mapping.enable_lm_head_tp_in_adp) - use_lm_head_tp_in_adp = (lm_head_tp_in_adp_configured - and not advanced_draft_sampling) + and self.model_config.mapping.enable_attention_dp + and getattr(self.model_config.mapping, + 'enable_lm_head_tp_in_adp', False)) if self.is_mtp_eagle: if use_lm_head_tp_in_adp: hidden_states_gathered = hidden_states[gather_ids] @@ -927,15 +877,6 @@ def _forward_linear_draft_loop(self, inputs, attn_metadata, spec_metadata, logits = draft_model.mtp_layers[0].shared_head( padded_hidden_states, draft_model.lm_head, attn_metadata, True) - elif lm_head_tp_in_adp_configured: - # Advanced-sampling bypass: the model's shared_head - # would re-apply the LM-head-TP stacked/sharded path - # from config on its own, so call lm_head directly. - # Under ADP the LMHead weight is replicated and - # is_spec_decoding_head defaults to False, so this is - # a plain local full-vocab GEMM over this rank's own - # rows -- the same computation the target head runs. - logits = draft_model.lm_head(hidden_states[gather_ids]) else: logits = draft_model.mtp_layers[0].shared_head( hidden_states[gather_ids], draft_model.lm_head, @@ -950,34 +891,27 @@ def _forward_linear_draft_loop(self, inputs, attn_metadata, spec_metadata, self.guided_decoder.execute_draft_batch(logits, draft_step=i) else: + d2t = getattr(draft_model.model, "d2t", None) self.guided_decoder.execute_draft_batch(logits, - self._d2t, + d2t, draft_step=i) - # ADP+LM-head-TP logits are the LM-head-TP group's row-stacked - # batch (each rank's rows padded to max_num_requests, then - # all-gathered along dim 0) with the vocab sharded across the - # group. Rows [:token_count] would be group rank 0's requests, - # not this rank's, and a per-rank argmax would return a - # shard-local index -- so keep the full stacked logits and let - # greedy_sample_draft_with_tp_gather combine the group's vocab - # shards and slice this rank's own row segment; only then trim - # the max_num_requests padding down to token_count. - mapping_lm_head_tp = None - if use_lm_head_tp_in_adp: - # The MTP head built this per-forward mapping when producing - # the vocab-sharded logits; the sampler needs it to gather. - mapping_lm_head_tp = getattr( - draft_model.mtp_layers[0].shared_head, - "mapping_lm_head_tp", None) - new_draft_token = self.sample_draft_tokens( - logits, - spec_metadata, - batch_size, - draft_step=i, - mapping_lm_head_tp=mapping_lm_head_tp) + # When ADP+LM-head-TP pads logits to max_num_requests, the + # padded rows are zero-filled placeholders only required so + # every TP rank produces logits of identical shape for the + # LM-head-TP all-gather. Drop them *before* sampling: the + # per-request sampling params (temperatures/top_k/top_p) are + # sized to token_count (== batch_size), so the padded logits + # would otherwise fail to broadcast in apply_temperature. This + # also keeps next_draft_tokens and the draft_probs buffer + # token_count-sized without a post-hoc trim. if use_lm_head_tp_in_adp: - new_draft_token = new_draft_token[:token_count] + logits = logits[:token_count] + new_draft_token = self.draft_decoder(logits, + draft_model, + spec_metadata, + batch_size, + draft_step=i) next_draft_tokens.append(new_draft_token) # Update hidden states for the next iteration. @@ -1043,6 +977,20 @@ def _forward_linear_draft_loop(self, inputs, attn_metadata, spec_metadata, gen_draft_tokens) next_draft_tokens[num_contexts:] = gen_draft_tokens + # Probs were already scattered into the slot-indexed buffer by + # _draft_sampler_advanced_for_rejection on each draft step (non-greedy + # batches only). All-greedy batches skip storage — rejection sampling + # will be bypassed by _can_use_rejection_sampling. Finalize the validity + # flag and d2t for next-iter target-side verification. + if spec_metadata.use_rejection_sampling: + if not spec_metadata.is_all_greedy_sample: + d2t_param = getattr(getattr(draft_model, 'model', None), "d2t", + None) + spec_metadata.d2t = d2t_param.data if d2t_param is not None else None + spec_metadata.draft_probs_valid = True + else: + spec_metadata.draft_probs_valid = False + return next_draft_tokens def _get_step_all_rank_num_tokens(self, spec_metadata, step_idx: int): @@ -1095,6 +1043,63 @@ def _prepare_flash_mla_generation_layout(self, attn_metadata, num_contexts, attn_metadata.block_ids_per_seq[:batch_size, :].copy_( reorder_block_ids_per_seq, non_blocking=True) + @torch.compile(options={"max-autotune": True}) + def _get_local_max_and_combined(self, logits, mapping_lm_tp=None): + local_max_values, local_argmax = torch.max(logits, dim=-1, keepdim=True) + vocab_per_rank = logits.shape[-1] + mapping_lm_tp = mapping_lm_tp if mapping_lm_tp is not None else \ + self.model_config.mapping + max_index_per_rank = local_argmax.type( + torch.int32) + (mapping_lm_tp.tp_rank * vocab_per_rank) + max_index_per_rank_float = max_index_per_rank.float() + local_max_values_float32 = local_max_values.float() + combined = torch.stack( + [max_index_per_rank_float, local_max_values_float32], + dim=-1).flatten(-2) + return combined + + @torch.compile(options={"max-autotune": True}) + def _get_draft_tokens_from_gathered(self, gathered): + gathered_indices_float = gathered[..., 0::2] + gathered_values_float = gathered[..., 1::2] + max_indices = torch.argmax(gathered_values_float, dim=-1, keepdim=True) + draft_tokens = torch.gather(gathered_indices_float, -1, + max_indices).squeeze(-1).type(torch.int32) + return draft_tokens + + def draft_sampler( + self, + logits: torch.Tensor, + mapping_lm_head_tp=None, + ): + """TP-aware greedy draft token sampler (MTP Eagle path). + + Falls back to simple argmax when no tensor parallelism is active or + when only attention DP is enabled without LM-head TP. + """ + if (self.model_config is not None + and hasattr(self.model_config, 'mapping') + and self.model_config.mapping.tp_size > 1 + and not self.model_config.mapping.enable_attention_dp): + combined = self._get_local_max_and_combined(logits) + gathered = allgather(combined, self.model_config.mapping, dim=-1) + return self._get_draft_tokens_from_gathered(gathered) + elif (self.model_config is not None + and hasattr(self.model_config, 'mapping') + and self.model_config.mapping.tp_size > 1 + and self.model_config.mapping.enable_lm_head_tp_in_adp): + combined = self._get_local_max_and_combined(logits, + mapping_lm_head_tp) + gathered = allgather(combined, mapping_lm_head_tp, dim=-1) + batch_size = logits.shape[0] + local_batch_size = batch_size // mapping_lm_head_tp.tp_size + gathered = gathered.view(mapping_lm_head_tp.tp_size, + local_batch_size, -1) + sliced_gathered = gathered[mapping_lm_head_tp.tp_rank] + return self._get_draft_tokens_from_gathered(sliced_gathered) + else: + return self._draft_sampler_greedy(logits) + @torch.compile(options={"max-autotune": True}) def _topk_kernel(self, gen_logprobs, num_gens, mtp_num_modules, spec_metadata): @@ -1205,6 +1210,78 @@ def sample_and_accept_draft_tokens( return self._accept_draft_tokens(logits, draft_tokens, num_contexts, batch_size, spec_metadata) + def draft_decoder( + self, + logits: torch.Tensor, + draft_model: nn.Module, + spec_metadata: Eagle3OneModelSpecMetadata, + batch_size: int, + draft_step: Optional[int] = None, + ): + ''' + Sample draft tokens using the target's per-request sampling params + (temperature/top_k/top_p). + + When rejection sampling is enabled and draft_step is provided, take the + single-pass path that also scatters the draft prob distribution into the + slot-indexed buffer (avoids a redundant softmax later). + + Args: + logits: [batch_size, vocab_size] - Draft model logits. + draft_model: The draft model. + spec_metadata: Carries per-request sampling param tensors. + batch_size: Active requests, used to slice per-request tensors. + draft_step: Current draft step index (0..max_draft_len-1). Required + for the rejection-sampling code path so probs are written to + the correct slice of spec_metadata.draft_probs. + ''' + + d2t = getattr(getattr(draft_model, 'model', None), "d2t", None) + # All-greedy fast path must stay TP-aware. When the draft LM head is + # tensor-parallel (tp_size>1 without attention DP, or LM-head-TP in + # ADP), the draft logits are sharded along the vocab dim. A plain + # per-rank argmax then picks a different token on each rank, which + # desyncs the speculative-decoding control flow across ranks and + # deadlocks the next collective (observed as a generation hang on + # MTP-Eagle + TP). draft_sampler() all-gathers the sharded logits + # before argmax (and falls back to a plain argmax when no TP gather is + # needed). Eagle3 (non-MTP) keeps its d2t-aware argmax. + if spec_metadata.is_all_greedy_sample: + # Only plain tensor parallelism (tp_size>1 without attention DP) + # shards the draft logits over the vocab dim and thus needs + # draft_sampler()'s all-gather argmax. The LM-head-TP-in-ADP case + # already produces full-vocab logits per rank (gathered upstream), + # and the no-TP / Eagle3 cases need nothing, so they take the plain + # d2t-aware argmax. (Routing ADP/LM-head-TP through draft_sampler + # without its mapping_lm_head_tp arg hits the None-mapping branch + # and crashes with 'NoneType has no attribute tp_group'.) + if (self.is_mtp_eagle and self.model_config is not None + and hasattr(self.model_config, 'mapping') + and self.model_config.mapping.tp_size > 1 + and not self.model_config.mapping.enable_attention_dp): + return self.draft_sampler(logits) + return self._draft_sampler_greedy(logits, d2t) + # Non-greedy (advanced) draft sampling has the same TP hazard as the + # greedy path: when the draft LM head is plain tensor-parallel + # (tp_size>1 without attention DP), each rank only holds a vocab shard + # of the draft logits. Random per-rank sampling then draws different + # tokens on different ranks, desyncing the spec-decode control flow and + # deadlocking the next collective. All-gather the shards into the full + # vocab first so every rank samples from the same distribution with the + # shared seed. (Greedy uses draft_sampler()'s lighter max+index gather; + # random sampling needs the full distribution. The LM-head-TP-in-ADP + # case is handled upstream and must not be gathered again here.) + if (self.is_mtp_eagle and self.model_config is not None + and hasattr(self.model_config, 'mapping') + and self.model_config.mapping.tp_size > 1 + and not self.model_config.mapping.enable_attention_dp): + logits = allgather(logits, self.model_config.mapping, dim=-1) + if spec_metadata.use_rejection_sampling and draft_step is not None: + return self._draft_sampler_advanced_for_rejection( + logits, spec_metadata, batch_size, d2t, draft_step) + return self._draft_sampler_advanced(logits, spec_metadata, batch_size, + d2t) + def prepare_1st_drafter_inputs( self, input_ids: torch.LongTensor, @@ -1267,12 +1344,10 @@ class MTPEagleWorker(Eagle3OneModelWorker): def __init__(self, spec_config, model_config: Optional[ModelConfig] = None, - use_separate_draft_kv_cache: bool = False, - *, - mapping: Optional[Mapping] = None): + use_separate_draft_kv_cache: bool = False): super().__init__( spec_config, - mapping=mapping, + mapping=None, model_config=model_config, use_separate_draft_kv_cache=use_separate_draft_kv_cache) # Preserved for callers/tests that still expect this attribute. diff --git a/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py b/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py index b988e6366c4a..7d9e836f5b9a 100644 --- a/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py +++ b/tensorrt_llm/_torch/speculative/eagle3_dynamic_tree.py @@ -375,7 +375,7 @@ def _ensure_spec_tree_manager(self, resource_manager): ) @nvtx_range("eagle3_dyn.forward") - def _forward_impl( + def forward( self, input_ids, position_ids, @@ -387,11 +387,11 @@ def _forward_impl( resource_manager=None, ): """Override to add accepted_draft_tokens_indices to output.""" - # Initialize spec_tree_manager before super()._forward_impl() which calls + # Initialize spec_tree_manager before super().forward() which calls # _forward_draft_loop needing spec_tree_manager. if resource_manager is not None: self._ensure_spec_tree_manager(resource_manager) - output = super()._forward_impl( + output = super().forward( input_ids, position_ids, hidden_states, @@ -585,6 +585,7 @@ def _forward_draft_loop( ): """Dynamic tree draft loop with growing context.""" spec_tree_manager = self.spec_tree_manager + self._d2t = getattr(draft_model.model, "d2t", None) assert batch_size <= self._max_batch_size, ( f"batch_size {batch_size} exceeds pre-allocated max_batch_size {self._max_batch_size}" @@ -635,7 +636,9 @@ def _forward_draft_loop( hidden_states[gather_ids], draft_model.lm_head, attn_metadata, True ) - new_draft_tokens, new_draft_scores = self.sample(logits, self.K) + new_draft_tokens, new_draft_scores = self.sample( + logits, self.K, draft_model=draft_model + ) previous_draft_scores = self.update_draft_tokens_and_scores( cur_draft_idx=0, @@ -695,7 +698,9 @@ def _forward_draft_loop( selected_hs, draft_model.lm_head, attn_metadata, True ) - new_draft_tokens, new_draft_scores = self.sample(logits, self.K) + new_draft_tokens, new_draft_scores = self.sample( + logits, self.K, draft_model=draft_model + ) # Reshape for update: [batch_size, K, K] new_draft_tokens = new_draft_tokens.reshape(batch_size, self.K, self.K) @@ -1010,12 +1015,14 @@ def _finalize_dynamic_tree_verify_outputs( ).to(torch.int32) @nvtx_range("eagle3_dyn.sample") - def sample(self, logits: torch.Tensor, max_top_k: int) -> tuple[torch.Tensor, torch.Tensor]: + def sample( + self, logits: torch.Tensor, max_top_k: int, draft_model=None + ) -> tuple[torch.Tensor, torch.Tensor]: """TopK sampling with softmax for dynamic tree.""" topk_indices, topk_values = _sample_softmax_topk(logits, max_top_k) - # Apply draft-to-target vocab mapping when draft/target vocabs differ. - if self._d2t is not None: - d2t = self._d2t.data + # Apply draft-to-target vocab mapping if the draft model has it + if draft_model is not None and hasattr(draft_model.model, "d2t"): + d2t = draft_model.model.d2t.data topk_indices = topk_indices + d2t[topk_indices] return topk_indices, topk_values diff --git a/tensorrt_llm/_torch/speculative/interface.py b/tensorrt_llm/_torch/speculative/interface.py index 50e23d087ca7..6244873efc43 100644 --- a/tensorrt_llm/_torch/speculative/interface.py +++ b/tensorrt_llm/_torch/speculative/interface.py @@ -28,7 +28,6 @@ from tensorrt_llm.logger import logger from ..._utils import get_sm_version, prefer_pinned -from ..attention_backend.interface import AttentionMetadata from ..attention_backend.trtllm import (AttentionBackend, TrtllmAttention, TrtllmAttentionMetadata) from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE @@ -43,8 +42,7 @@ import flashinfer from ..pyexecutor.sampler.sampling_utils import ( - compute_probs_from_logits, greedy_search_sampling_batch, - sampling_batch_spec_dec_one_model, + compute_probs_from_logits, greedy, sampling_batch_spec_dec_one_model, sampling_batch_spec_dec_one_model_for_rejection) @@ -113,10 +111,6 @@ def should_use_separate_draft_kv_cache(spec_config) -> bool: return False if not spec_config.spec_dec_mode.use_one_engine(): return False - # DSpark owns a dedicated rolling-window cache in DSparkWorker. Its draft - # model does not read the paged draft KV cache managed by attention metadata. - if spec_config.spec_dec_mode.is_dspark(): - return False return spec_config._allow_separate_draft_kv_cache @@ -282,7 +276,6 @@ class SpeculativeDecodingMode(IntEnum): SAVE_HIDDEN_STATES = auto() PARD = auto() DFLASH = auto() - DSPARK = auto() NONE = auto() AUTO = auto() @@ -317,11 +310,8 @@ def is_pard(self): def is_dflash(self): return self == SpeculativeDecodingMode.DFLASH - def is_dspark(self): - return self == SpeculativeDecodingMode.DSPARK - def is_parallel_draft(self): - return self.is_pard() or self.is_dflash() or self.is_dspark() + return self.is_pard() or self.is_dflash() def is_ngram(self): return self == SpeculativeDecodingMode.NGRAM @@ -486,11 +476,6 @@ class SpecMetadata: # The number of sequences for speculative model/layer of different rank all_rank_num_seqs: Optional[List[int]] = None - # The number of generation requests for the speculative model/layer of each - # rank (num_seqs - num_contexts). Used by external drafters (e.g. DSpark) - # whose draft forward processes only generation requests and must size a - # FUSED_COMM MoE (DeepGEMM MegaMoE) chunk loop identically across EP ranks. - all_rank_num_gens: Optional[List[int]] = None # The number of extra kv tokens # Some speculative decoding methods need to use different kv lengths for the # draft/target layers. But KVCacheManager can only support kv caches with the @@ -526,16 +511,6 @@ class SpecMetadata: # Defaults to True so non-one-engine paths (where populate is a no-op) # never accidentally select the advanced graph variant. is_all_greedy_sample: bool = True - # Group-synchronized override for ``is_all_greedy_sample`` (AND over the - # TP group's local flags; None = no group sync configured, use the local - # value). Under ADP + LM-head TP with rejection sampling, the greedy-vs- - # advanced choice gates group collectives, so all ranks must take the same - # path even though their batches (and thus local flags) differ. Set by - # ``_sync_group_all_greedy_sample`` before the CUDA graph key is built and - # re-applied by ``_scan_one_model_sampling`` on every rescan. AND is safe: - # a greedy rank pulled onto the advanced path still samples greedily via - # its sentinel params. - group_all_greedy_sample: Optional[bool] = None # Whether to use rejection sampling for one-model speculative decoding. use_rejection_sampling: bool = False # Sampling parameters for non-greedy sampling (per-request) @@ -546,6 +521,7 @@ class SpecMetadata: skip_temperature: bool = False skip_top_k: bool = False skip_top_p: bool = False + has_greedy_requests: bool = False # Pre-computed top_k_max scalar (CPU-side) to avoid CUDA-graph-incompatible # dynamic boolean tensor indexing inside verify_dynamic_tree_rejection_from_logits_out. top_k_max: int = 0 @@ -563,120 +539,68 @@ class SpecMetadata: # Slot-indexed buffers (draft_probs) must span this full range. # 0 falls back to max_num_requests. num_seq_slots: int = 0 - # Draft-model vocab size. full_draft_probs is allocated only when it differs - # from vocab_size; 0 (unknown) or a value equal to vocab_size means shared - # vocab and skips the buffer. - draft_vocab_size: int = 0 # Draft probabilities buffer for rejection sampling, indexed by py_seq_slot # so per-request data is stable across iterations regardless of batch # composition shifts (chunking ctx, gen completion, new ctx joining). # Shape: [num_seq_slots, max_draft_len, vocab_size]. draft_probs: Optional[torch.Tensor] = None draft_probs_vocab_size: int = 0 - # Scratch row index that dummy/padding requests (py_seq_slot is None) route - # to, captured when draft_probs is allocated so it tracks the buffer's real - # size. It must NOT be re-derived from max_num_requests at use time: - # create_cuda_graph_metadata shrinks max_num_requests to the graph bucket - # size while sharing the full-size buffer, so a bucket-relative index would - # collide with a real request's slot row and overwrite its draft probs. - dummy_slot_row: int = 0 + # Whether draft_probs contains valid data. + draft_probs_valid: bool = False # Last dimension size of the draft logits/probs stored in draft_probs. draft_probs_last_dim: int = 0 # Per-request slot ids (py_seq_slot) for the current batch, in batch order. # Used to scatter draft probs by slot at write time and gather them by slot # at the next iter's verify. Shape: [max_num_requests], dtype=long. batch_slot_ids: Optional[torch.Tensor] = None - # Draft probs expanded to the target vocab size. Zero-filled once at - # prepare(); each rejection iter overwrites only the d2t-selected positions - # (or [:draft_vocab] when there is no d2t). - # Shape: [max_num_requests, max_draft_len, vocab_size]. + # Draft-to-target vocab offset tensor. + d2t: Optional[torch.Tensor] = None + # Pre-allocated scratch for draft probs expanded to the target vocab size. + # Filled with zeros once at prepare(); each rejection iter only overwrites + # the positions selected by d2t (or [:draft_vocab] when there is no d2t), + # so the zeros outside those positions persist across iterations and we + # avoid a per-iter 64 MB zero-fill on the (max_num_requests, max_draft_len, + # vocab_size) tensor. Shape: [max_num_requests, max_draft_len, vocab_size]. full_draft_probs: Optional[torch.Tensor] = None - # Cached d2t-projected target vocab indices, computed once on first use. - # Shape: [draft_vocab_size], dtype long. + # Cached d2t-projected target vocab indices, computed once on first use + # (d2t is a model-static tensor). Replaces the per-iter + # arange + (source + d2t) % vocab_size kernel sequence inside the d2t + # padding step. Shape: [draft_vocab_size], dtype long. d2t_target_indices: Optional[torch.Tensor] = None def __post_init__(self): pass - def prepare_rejection_sampling_buffers(self): + def prepare(self): """ - Allocate the slot-indexed buffers used by one-model rejection sampling. - - Idempotent and gated on ``use_rejection_sampling``. + Hook to be called before the forward step of the model. """ - if not self.use_rejection_sampling: - return - - # Slot-indexed buffers span the full SeqSlotManager pool: py_seq_slot - # can range over [0, num_seq_slots), which under DeepSeek-V4 overlap - # exceeds max_num_requests. Fall back to max_num_requests when the pool - # size is unknown (0). One extra scratch row at index ``slot_capacity`` - # absorbs CUDA-graph dummy/padding requests (``py_seq_slot is None``). - slot_capacity = self.num_seq_slots or self.max_num_requests - num_slot_rows = slot_capacity + 1 - - if self.draft_probs is None and self.vocab_size > 0: - # [slot, draft_step, vocab]: scatter/gather by stable slot id. + if (self.use_rejection_sampling and self.draft_probs is None + and self.vocab_size > 0): + # 3D [slot, draft_step, vocab] so we can scatter/gather by slot id + # and avoid the brittle "batch position == buffer position" mapping. + slot_capacity = self.num_seq_slots or self.max_num_requests self.draft_probs = torch.empty( - (num_slot_rows, self.max_draft_len, self.vocab_size), + (slot_capacity, self.max_draft_len, self.vocab_size), dtype=torch.float32, device='cuda') self.draft_probs_vocab_size = self.vocab_size - # Dummy requests route to the extra scratch row (the buffer's last - # row at index ``slot_capacity``). Capture it here against the real - # allocation size so it stays correct on graph copies that shrink - # max_num_requests and under overlap where slot_capacity exceeds it. - self.dummy_slot_row = slot_capacity - if self.batch_slot_ids is None and self.max_num_requests > 0: + if (self.use_rejection_sampling and self.batch_slot_ids is None + and self.max_num_requests > 0): self.batch_slot_ids = torch.empty((self.max_num_requests, ), dtype=torch.long, device='cuda') - # full_draft_probs (d2t-expanded) is read only when draft and target - # vocabularies differ; skip it otherwise. Zero-filled once. - if (self.full_draft_probs is None and self.vocab_size > 0 - and self.draft_vocab_size not in (0, self.vocab_size)): + if (self.use_rejection_sampling and self.full_draft_probs is None + and self.vocab_size > 0): + # Zero-fill once. Subsequent iters only overwrite the d2t-mapped + # positions (constant across iters since d2t is model-static), so + # untouched positions stay 0 forever — saves the per-iter 64 MB + # zero-fill in _sample_and_accept_draft_tokens_rejection. self.full_draft_probs = torch.zeros( - (num_slot_rows, self.max_draft_len, self.vocab_size), + (self.max_num_requests, self.max_draft_len, self.vocab_size), dtype=torch.float32, device='cuda') - def write_padding_onehot_draft_probs(self, padding_slot_ids, draft_len): - """Write a one-hot draft-prob row (prob 1.0 at draft-vocab token id 0, - the placeholder token) into each padding gen request's stable slot row. - - Padding requests are gen requests that entered this iteration with 0 real - draft tokens (e.g. a runtime_draft_len K->0->K dynamic-draft-len toggle). - Their slot's draft_probs row was never scattered by the draft sampler, so - the next iteration's (possibly CUDA-graph-captured) rejection kernel would - read a stale/uninitialized distribution. Writing a legal one-hot row makes - acceptance reject the placeholder and resample from the target (equivalent - to strict acceptance) for those rows. Written eagerly before graph replay - into the stable draft_probs buffer, so the replayed kernel reads it. - - Idempotent w.r.t. context->gen transitions whose row was already one-hot'd - by write_context_onehot_draft_probs. The width matches the value already - published in draft_probs_last_dim (what acceptance reads), so it is NOT - overwritten here. No-op unless rejection is enabled and slots exist. - Static shapes -> CUDA-graph safe. - """ - if (not padding_slot_ids - or not getattr(self, "use_rejection_sampling", False) - or self.draft_probs is None): - return - onehot_vocab = (self.draft_probs_last_dim if self.draft_probs_last_dim - > 0 else self.draft_probs_vocab_size) - slots = torch.tensor(padding_slot_ids, - dtype=torch.long, - device=self.draft_probs.device) - self.draft_probs[slots, :draft_len, :onehot_vocab] = 0.0 - self.draft_probs[slots, :draft_len, 0] = 1.0 - - def prepare(self): - """ - Hook to be called before the forward step of the model. - """ - self.prepare_rejection_sampling_buffers() - def create_cuda_graph_metadata(self, max_batch_size: int): """ Creates metadata for CUDA graph execution. @@ -710,23 +634,17 @@ def _scan_one_model_sampling( ) -> tuple[list[tuple[float, int, float, int]], list[int]]: """Single source of truth for one-engine sampling-param detection. - Scans the batch's sampling configs and sets skip_*/is_all_greedy_sample - (honoring the warmup capture override). Returns + Scans the batch's sampling configs and sets skip_*/has_greedy_requests/ + is_all_greedy_sample (honoring the warmup capture override). Returns ``(per_request_normalized, per_request_slot_ids)`` for buffer population. Does NOT allocate or fill GPU buffers, so it is safe to call before the CUDA graph key is built. """ from tensorrt_llm._torch.pyexecutor.llm_request import LlmRequestState - from tensorrt_llm._torch.pyexecutor.sampler.sampling_utils import \ - GREEDY_TEMPERATURE_THRESHOLD from tensorrt_llm.sampling_params import SamplingParams - # Sentinel temperature for greedy / temperature-disabled rows. Must stay - # strictly below GREEDY_TEMPERATURE_THRESHOLD so the sampling kernels - # recognize these rows as greedy; small enough that even if a row were - # (incorrectly) sampled, softmax(logits / val) is effectively one-hot, - # and non-zero to avoid division by 0. - DISABLE_TEMP_VAL = GREEDY_TEMPERATURE_THRESHOLD / 10 + # Need to use a very small value for temperature when disabled to avoid division by 0 + DISABLE_TEMP_VAL = 1e-5 # Very large values disable topk. DISABLE_TOPK_VAL = torch.iinfo(torch.int32).max DISABLE_TOPP_VAL = 1.0 @@ -775,7 +693,7 @@ def _normalize_request_sampling_params( temperature_enabled = False top_k_enabled = False top_p_enabled = False - has_non_greedy_requests = False + has_greedy_requests = False per_request_slot_ids: list[int] = [] for request in requests: @@ -804,32 +722,32 @@ def _normalize_request_sampling_params( temperature_enabled |= use_temperature top_k_enabled |= use_top_k top_p_enabled |= use_top_p - has_non_greedy_requests |= not is_greedy + has_greedy_requests |= is_greedy per_request_normalized.append( (temp_val, tk_val, tp_val, num_tokens)) - # py_seq_slot is a stable per-request id used to scatter/gather draft - # probs across iterations. Dummy/padding requests (py_seq_slot is - # None) route to the scratch row captured at allocation time (the - # buffer's real last row), not max_num_requests, which a graph copy - # shrinks to the bucket size and would alias a real request's slot. + # py_seq_slot is a stable per-request id used to scatter / gather + # draft probs across iterations. Dummies / unallocated slots fall + # back to 0 (any valid index is fine — the data at that slot will + # be overwritten on the next real iteration before being read). per_request_slot_ids.append( - request.py_seq_slot if request. - py_seq_slot is not None else self.dummy_slot_row) + request.py_seq_slot if request.py_seq_slot is not None else 0) self.skip_temperature = not temperature_enabled self.skip_top_k = not top_k_enabled self.skip_top_p = not top_p_enabled + self.has_greedy_requests = has_greedy_requests # Used in the CUDA graph key to pick the argmax / advanced variant. - # All-greedy iff EVERY request is greedy. Derived from per-request - # greediness, not from the skip_* filter flags (a non-greedy request may - # enable no filter, e.g. temperature=1.0 with top_k/top_p unset). - self.is_all_greedy_sample = not has_non_greedy_requests - - # Warmup-time override: force the advanced-sampling path so the CUDA - # graph for the (is_all_greedy_sample=False) key gets captured. Dummy - # warmup requests carry no sampling params, so substitute synthetic - # non-greedy scalars to populate the GPU buffers. + self.is_all_greedy_sample = (self.skip_temperature and self.skip_top_k + and self.skip_top_p) + + # Warmup-time override (set via runtime attribute by the model engine): + # force the advanced-sampling code path so the CUDA graph for the + # (is_all_greedy_sample=False) key gets captured. Dummy warmup requests + # carry no sampling params, so the natural detection above always + # returns True; this branch substitutes synthetic non-greedy scalars + # into the per-request data and lets Phase 2 run normally to populate + # the GPU buffers used by the captured kernels. if getattr(self, '_force_non_greedy_for_capture', False): self.skip_temperature = False self.skip_top_k = False @@ -840,36 +758,20 @@ def _normalize_request_sampling_params( for (_, _, _, num_tokens) in per_request_normalized ] - # Apply the group-synchronized override last (semantics: see the - # ``group_all_greedy_sample`` field comment). Local contract: the - # synced value already incorporates any capture override, and rescans - # (e.g. populate after the graph key) must converge to it rather than - # resurrect the local value. - if self.group_all_greedy_sample is not None: - self.is_all_greedy_sample = self.group_all_greedy_sample - return per_request_normalized, per_request_slot_ids - @property - def wants_advanced_draft_sampling(self) -> bool: - """Whether the current batch takes the advanced (rejection) draft - path: rejection sampling enabled AND not an all-greedy batch. - - Single source of truth for the greedy-vs-advanced decision: the - sampler branch (``sample_draft_tokens``) and the worker's LM-head-TP - bypass (``_forward_linear_draft_loop``) must agree exactly -- a - divergence feeds the wrong logits layout to the sampler -- so both - read this property instead of re-deriving the predicate. - """ - return self.use_rejection_sampling and not self.is_all_greedy_sample - def update_is_all_greedy_sample(self, requests: list["LlmRequest"]) -> None: """Refresh ``is_all_greedy_sample`` for the *current* batch. Must be called BEFORE the CUDA graph key is built (the key includes ``is_all_greedy_sample`` to choose the argmax vs advanced-sampling graph - variant), so the selected graph stays consistent with the buffers - ``populate_sampling_params_for_one_model`` fills later. + variant). ``populate_sampling_params_for_one_model`` runs later, inside + ``_prepare_inputs``, and re-derives the same flag while filling the GPU + sampling buffers. Computing the flag here first keeps the selected graph + consistent with the buffers ``populate`` fills; otherwise the key would + use the previous iteration's stale value and could replay the advanced + graph against unpopulated (greedy) buffers, which can hang/corrupt the + run (notably for MTP with num_nextn>=2). """ if not self.spec_dec_mode.use_one_engine(): return @@ -887,11 +789,6 @@ def populate_sampling_params_for_one_model( if not self.spec_dec_mode.use_one_engine(): return - # Allocate the rejection buffers before copying py_seq_slot values into - # batch_slot_ids below; this runs earlier than prepare() in the - # model-engine flow. No-op unless use_rejection_sampling is set. - self.prepare_rejection_sampling_buffers() - if self.temperatures is None: # Ensures determinism across ranks. torch.manual_seed(0) @@ -1020,9 +917,6 @@ def __init__(self, use_separate_draft_kv_cache: bool = False): self.seed: Optional[torch.Tensor] = None self.offset: Optional[torch.Tensor] = None self.use_separate_draft_kv_cache = use_separate_draft_kv_cache - # Static draft->target vocab offset map, cached once the draft model is - # loaded (see set_draft_model). None when draft and target share a vocab. - self._d2t: Optional[torch.Tensor] = None # Lazily-initialized state for the fractional synthetic acceptance # rate. The pool is a fixed-seed, rank-independent table of uniform # [0, 1) values; the counter is a device-side int64 advanced in-place @@ -1031,54 +925,6 @@ def __init__(self, use_separate_draft_kv_cache: bool = False): self._force_accept_rng_pool: Optional[torch.Tensor] = None self._force_accept_rng_counter: Optional[torch.Tensor] = None - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - if "forward" in cls.__dict__: - raise TypeError( - f"{cls.__name__} must not override SpecWorkerBase.forward; " - f"implement _forward_impl instead. SpecWorkerBase.forward " - f"guarantees spec-dec attn-metadata cleanup when a forward " - f"fails (https://nvbugs/6442074).") - - def forward(self, *args, **kwargs): - """Run _forward_impl with guaranteed spec-dec metadata cleanup. - - Tolerated forward failures (e.g. an OOM during the max-shape general - warmup, or an error-budget-tolerated serving exception) must not leak - the attn-metadata state saved by prepare_for_spec_dec: a stale save - fails every subsequent forward at the pairing assert. - https://nvbugs/6442074 - """ - attn_metadata = kwargs.get("attn_metadata") - spec_metadata = kwargs.get("spec_metadata") - if attn_metadata is None or spec_metadata is None: - for a in args: - if attn_metadata is None and isinstance(a, AttentionMetadata): - attn_metadata = a - elif spec_metadata is None and isinstance(a, SpecMetadata): - spec_metadata = a - try: - return self._forward_impl(*args, **kwargs) - finally: - self._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) - - @abstractmethod - def _forward_impl(self, *args, **kwargs): - """Worker-specific forward logic, called by SpecWorkerBase.forward.""" - - def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): - """Restore attn-metadata spec-dec state if a failure skipped it. - - No-op on the success path: workers restore at their preferred point - and this sees no saved state. Subclasses with extra transient state - (e.g. the deferred kv_lens rewind in PARD/DFlash) extend this. - """ - if attn_metadata is not None and attn_metadata.has_spec_dec_saved_state: - logger.warning( - "Spec-dec worker forward failed between prepare_for_spec_dec " - "and restore_from_spec_dec; restoring attn metadata state.") - self._restore_attn_metadata_from_spec_dec(attn_metadata) - @property @abstractmethod def max_draft_len(self) -> int: @@ -1179,13 +1025,6 @@ def set_guided_decoder(self, self.guided_decoder = guided_decoder return True - def set_draft_model(self, draft_model) -> None: - """Cache the static draft->target vocab offset map (``d2t``) once the - draft model is loaded. ``d2t`` is a model-static parameter present only - when the draft and target vocabularies differ; stays None otherwise. - """ - self._d2t = getattr(getattr(draft_model, "model", None), "d2t", None) - def _prepare_attn_metadata_for_spec_dec(self, attn_metadata): """ Prepare attention metadata before speculative decoding draft token generation. @@ -1409,223 +1248,26 @@ def _accept_draft_tokens(self, logits, draft_tokens, num_contexts, stored_vocab = (spec_metadata.draft_probs_last_dim if spec_metadata.draft_probs_last_dim > 0 else spec_metadata.draft_probs_vocab_size) - # Fail closed: run the rejection kernel only when every buffer is - # present and correctly shaped; otherwise fall back to strict - # acceptance. - if self._rejection_buffers_valid(draft_tokens, draft_len, - stored_vocab, num_contexts, - batch_size, logits, spec_metadata): - # Gather the gen subset's slot rows, filled at the previous draft - # step indexed by py_seq_slot. - gen_slot_ids = spec_metadata.batch_slot_ids[ - num_contexts:batch_size] - draft_probs = spec_metadata.draft_probs[ - gen_slot_ids, :draft_len, :stored_vocab] - return self._sample_and_accept_draft_tokens_rejection( - logits, draft_tokens, draft_probs, num_contexts, batch_size, - spec_metadata) + # Gather the slot rows for the gen subset. The buffer was filled + # at the previous draft step indexed by py_seq_slot, so each gen + # request reads back exactly its own probs, regardless of batch + # composition changes since then. + gen_slot_ids = spec_metadata.batch_slot_ids[num_contexts:batch_size] + draft_probs = spec_metadata.draft_probs[ + gen_slot_ids, :draft_len, :stored_vocab] + return self._sample_and_accept_draft_tokens_rejection( + logits, draft_tokens, draft_probs, num_contexts, batch_size, + spec_metadata) return self._sample_and_accept_draft_tokens_base( logits, draft_tokens, num_contexts, batch_size, spec_metadata) - def _draft_logits_are_sharded(self, logits, spec_metadata): - """Whether the draft logits are vocab-sharded and need a TP gather. - - Sharded when tp_size>1 and the logits' last dim is narrower than the - DRAFT head's own full vocab -- either plain TP (no attention DP) or the - ADP + LM-head-TP mode, both of which produce vocab-sharded draft logits. - Replicated full-vocab logits (borrowed/gathered LM head, plain attention - DP, or a single rank) are not sharded. - - The reference width is the draft head's own full vocab (``draft_vocab_size``, - falling back to ``vocab_size`` when unknown/shared), NOT the target - ``vocab_size``: an Eagle3 reduced-vocab draft head produces full, - replicated ``[tokens, draft_vocab_size]`` logits that are narrower than the - target vocab; comparing against the target vocab would misclassify those as - sharded and gather identical copies (overflowing the d2t table). - """ - mapping = self.mapping - if mapping is None or getattr(mapping, "tp_size", 1) <= 1: - return False - # Under attention DP every rank is data-parallel -- it owns a distinct - # set of requests -- so the draft logits must NOT be cross-rank gathered, - # whether they are replicated full-vocab (plain ADP) or vocab-sharded - # (ADP + LM-head TP). A per-rank argmax on the rank's own logits is the - # correct proposal (verified later); a gather would splice in a - # mismatched collective across ranks that hold different token counts, - # desyncing them into a hang / DeepEP launch failure. Only plain TP - # (tp>1 without ADP), where all ranks share the same tokens sharded over - # the vocab dim, needs the gather. - if getattr(mapping, "enable_attention_dp", False): - return False - draft_full_vocab = (getattr(spec_metadata, "draft_vocab_size", 0) - or getattr(spec_metadata, "vocab_size", 0) or 0) - return bool(draft_full_vocab) and logits.shape[-1] < draft_full_vocab - - def maybe_gather_sharded_draft_logits(self, - logits, - spec_metadata, - mapping_lm_head_tp=None): - """All-gather TP-sharded draft logits to full vocab before advanced sampling. - - Advanced (non-greedy) draft sampling needs the full-vocab distribution. - Gathers shards only for a non-greedy batch when the logits are sharded - (see ``_draft_logits_are_sharded``); replicated full-vocab logits are - returned unchanged. - - Plain TP gathers vocab shards over ``self.mapping``. The LM-head-TP - stacked/sharded layout never reaches this path: an advanced-sampling - batch bypasses the LM-head-TP fast path in the worker and computes - full-vocab logits locally from the (ADP-replicated) lm_head weight, so - ``mapping_lm_head_tp`` is only ever passed alongside greedy sampling. - """ - assert mapping_lm_head_tp is None, ( - "Advanced draft sampling must not receive LM-head-TP " - "stacked/sharded logits; the worker bypasses the LM-head-TP fast " - "path for non-all-greedy batches (see _forward_linear_draft_loop)") - if (spec_metadata is None or spec_metadata.is_all_greedy_sample - or not self._draft_logits_are_sharded(logits, spec_metadata)): - return logits - - # Only plain TP (no attention DP) reaches here -- ADP variants return - # early via _draft_logits_are_sharded, since their ranks are - # data-parallel and must not gather draft logits across ranks. - from ..distributed.ops import allgather - return allgather(logits, self.mapping, dim=-1) - - def advanced_sample_draft(self, - logits: torch.Tensor, - spec_metadata: "SpecMetadata", - batch_size: int, - draft_step: Optional[int] = None): - """Per-step advanced (non-greedy) draft sampler for step workers - (MTP, DraftTarget). - - With rejection enabled and a ``draft_step``, samples via - ``sampling_batch_spec_dec_one_model_for_rejection`` and scatters this - step's proposal distribution into the slot-indexed ``draft_probs`` - buffer; otherwise uses ``sampling_batch_spec_dec_one_model`` (tokens - only). Returns tokens in draft-vocab space (the caller applies d2t). - Expects 2D ``[batch_size, vocab]`` logits (one row per request). - """ - temperatures = spec_metadata.request_temperatures[:batch_size] - top_ks = spec_metadata.request_top_ks[:batch_size] - top_ps = spec_metadata.request_top_ps[:batch_size] - - self._update_advance_draft_sampling_seed(logits.device) - if spec_metadata.use_rejection_sampling and draft_step is not None: - draft_tokens, probs = ( - sampling_batch_spec_dec_one_model_for_rejection( - logits, - temperatures, - top_ks, - top_ps, - seed=self.seed, - offset=self.offset)) - # Scatter probs into the slot-indexed buffer so each request's data - # lands at its stable py_seq_slot row regardless of batch shifts. - assert spec_metadata.batch_slot_ids is not None, ( - "batch_slot_ids must be populated by " - "populate_sampling_params_for_one_model before draft probs " - "storage") - batch_slots = spec_metadata.batch_slot_ids[:batch_size] - vocab = probs.shape[-1] - spec_metadata.draft_probs[batch_slots, draft_step, :vocab] = probs - spec_metadata.draft_probs_last_dim = vocab - else: - draft_tokens = sampling_batch_spec_dec_one_model(logits, - temperatures, - top_ks, - top_ps, - seed=self.seed, - offset=self.offset) - - return draft_tokens.type(torch.int32) - - def _reshape_draft_tokens_for_accept(self, spec_metadata, num_gens, device): - """Reshape the stored draft tokens to ``[num_gens, runtime_draft_len]`` - for acceptance. Default assumes one draft token per step (DraftTarget, - DFlash); workers with a different buffer layout (e.g. PARD's 2K-1 - entries) override this. - """ - runtime_draft_len = spec_metadata.runtime_draft_len - if spec_metadata.draft_tokens is None: - return torch.zeros((num_gens, runtime_draft_len), - dtype=torch.int, - device=device) - return spec_metadata.draft_tokens.reshape(num_gens, runtime_draft_len) - - def _reshape_logits_for_accept(self, logits, num_contexts, num_gens, - spec_metadata): - """Reshape target logits to the ``[num_contexts + num_gens*(K+1), vocab]`` - layout expected by acceptance. Default is identity (target already emits - one logit per accepted position); workers that emit extra positions - (e.g. PARD's 2K per gen request) override this. - """ - return logits - - def sample_and_accept_draft_tokens(self, logits, attn_metadata, - spec_metadata): - """Sample the golden token and verify previously proposed draft tokens. - - Default implementation for one-model workers whose acceptance differs - only in how draft tokens / target logits are reshaped (DraftTarget, - PARD, DFlash): unpack batch sizes, reshape via the overridable hooks, - then route through ``_accept_draft_tokens`` (strict or rejection). Workers - with a materially different acceptance path (MTP, Eagle3: relaxed / - THOP / extra ``input_ids``) override this method entirely. - """ - batch_size = attn_metadata.num_seqs - num_contexts = attn_metadata.num_contexts - num_gens = batch_size - num_contexts - draft_tokens = self._reshape_draft_tokens_for_accept( - spec_metadata, num_gens, logits.device) - logits = self._reshape_logits_for_accept(logits, num_contexts, num_gens, - spec_metadata) - return self._accept_draft_tokens(logits, draft_tokens, num_contexts, - batch_size, spec_metadata) - - def _rejection_buffers_valid(self, draft_tokens, draft_len, stored_vocab, - num_contexts, batch_size, logits, - spec_metadata) -> bool: - """Fail-closed guard: return True only when the slot-indexed draft-prob - buffers exist and every shape the rejection path dereferences is valid; - otherwise the caller falls back to strict acceptance. Inspects only - host-side tensor shapes -- no ``.item()`` / value read on CUDA tensors -- - so it stays CUDA-graph-capture safe. - """ - draft_probs = spec_metadata.draft_probs - batch_slot_ids = spec_metadata.batch_slot_ids - if draft_probs is None or batch_slot_ids is None: - return False - if stored_vocab <= 0: - return False - num_gens = batch_size - num_contexts - # draft_probs must cover the slice [:, :draft_len, :stored_vocab]. - if draft_probs.dim() != 3: - return False - if draft_probs.shape[1] < draft_len or draft_probs.shape[ - 2] < stored_vocab: - return False - if draft_tokens.dim() != 2 or draft_tokens.shape[0] != num_gens: - return False - # logits must cover context rows (1 each) + gen rows (draft_len + 1 each). - logits_rows = logits.shape[0] if logits.dim() > 1 else 1 - if logits_rows < num_contexts + num_gens * (draft_len + 1): - return False - # Slot ids for the gen subset must exist (range safety is guaranteed by - # construction). - if batch_slot_ids.shape[0] < batch_size: - return False - if batch_slot_ids[num_contexts:batch_size].shape[0] != num_gens: - return False - return True - def _can_use_rejection_sampling(self, spec_metadata: SpecMetadata) -> bool: # Skip rejection sampling when the whole batch is greedy: the accepted # result is identical to argmax and the base path is cheaper. Mixed # batches (context + gen) are handled via slot-indexed draft probs and # are split inside _sample_and_accept_draft_tokens_rejection. return (spec_metadata.use_rejection_sampling + and spec_metadata.draft_probs_valid and not spec_metadata.is_all_greedy_sample) def _sample_and_accept_draft_tokens_rejection( @@ -1701,17 +1343,16 @@ def _sample_and_accept_draft_tokens_rejection( assert draft_probs.shape[1] == runtime_draft_len, ( f"draft_probs draft length mismatch: {draft_probs.shape[1]} != " f"{runtime_draft_len}") - d2t = self._d2t.data if self._d2t is not None else None + d2t = getattr(spec_metadata, "d2t", None) if draft_vocab_size != vocab_size: + # Use the pre-allocated buffer from spec_metadata.prepare() + # (zero-filled once at init; untouched positions stay 0). + # Falls back to per-iter allocation if the buffer is not + # configured, e.g. when use_rejection_sampling was off at + # prepare() time. if spec_metadata.full_draft_probs is not None: - # Slice to runtime_draft_len so the max_draft_len buffer - # never passes stale extra rows to the rejection kernel. - full_draft_probs = spec_metadata.full_draft_probs[: - num_gens, : - runtime_draft_len] + full_draft_probs = spec_metadata.full_draft_probs[:num_gens] else: - # Buffer not pre-allocated (e.g. rejection off at prepare()): - # fall back to a per-iter allocation. full_draft_probs = torch.zeros( (num_gens, runtime_draft_len, vocab_size), dtype=torch.float32, @@ -1778,279 +1419,142 @@ def _sample_and_accept_draft_tokens_rejection( spec_metadata=spec_metadata) return accepted_tokens, num_accepted_tokens - def _update_advance_draft_sampling_seed(self, device): - """Increment the draft sampler's RNG seed for this draft-sampling call - (lazily initializing the seed/offset tensors on first use), so each call - samples with a fresh, deterministic seed.""" - if self.seed is None: - self.seed = torch.tensor([0], dtype=torch.int64, device=device) - self.offset = torch.tensor([0], dtype=torch.int64, device=device) - self.seed += 1 - self.seed %= (2**31) - - def _draft_sampler_greedy(self, logits: torch.Tensor): + def _draft_sampler_greedy(self, logits: torch.Tensor, d2t=None): """ Simple greedy draft token sampling using argmax. Args: logits: [num_tokens, vocab_size] - Draft model logits + d2t: Optional dictionary offset tensor for vocab mapping Returns: draft_tokens: [num_tokens] - Sampled draft token ids (int32) """ - draft_tokens = greedy_search_sampling_batch(logits, - return_probs=False)[0] + draft_tokens = greedy(logits, return_probs=False)[0] - # Apply the cached draft->target vocab offset map. - if self._d2t is not None: - draft_tokens = self._d2t[draft_tokens] + draft_tokens + # Apply d2t (offsets between draft and target model dictionaries) + if d2t is not None: + draft_tokens = d2t[draft_tokens] + draft_tokens return draft_tokens.type(torch.int32) - def _get_local_max_and_combined(self, logits, mapping_lm_tp=None): - """Pack each rank's local (global_argmax_index, max_value) for a - distributed argmax over a vocab-sharded draft LM head. - """ - local_max_values, local_argmax = torch.max(logits, dim=-1, keepdim=True) - vocab_per_rank = logits.shape[-1] - mapping_lm_tp = mapping_lm_tp if mapping_lm_tp is not None else self.mapping - max_index_per_rank = local_argmax.type( - torch.int32) + (mapping_lm_tp.tp_rank * vocab_per_rank) - max_index_per_rank_float = max_index_per_rank.float() - local_max_values_float32 = local_max_values.float() - # Interleaved layout: [idx0, val0, idx1, val1, ...] after all-gather. - combined = torch.stack( - [max_index_per_rank_float, local_max_values_float32], - dim=-1).flatten(-2) - return combined - - @torch.compile(options={"max-autotune": True}) - def _get_draft_tokens_from_gathered(self, gathered): - """Pick the global-argmax token id from the all-gathered per-rank - (index, value) pairs produced by ``_get_local_max_and_combined``. - """ - gathered_indices_float = gathered[..., 0::2] - gathered_values_float = gathered[..., 1::2] - max_indices = torch.argmax(gathered_values_float, dim=-1, keepdim=True) - draft_tokens = torch.gather(gathered_indices_float, -1, - max_indices).squeeze(-1).type(torch.int32) - return draft_tokens - - def greedy_sample_draft_with_tp_gather(self, - logits: torch.Tensor, - spec_metadata=None, - mapping_lm_head_tp=None): - """Greedy draft-token sampling with a TP all-gather of the argmax. - - When the draft LM head is vocab-sharded under plain tensor parallelism, - a per-rank argmax disagrees across ranks and desyncs speculative - decoding. Gather only each rank's local (index, value) and pick the - global argmax. Falls back to plain argmax when the logits are not - vocab-sharded (see ``_draft_logits_are_sharded``) -- e.g. a borrowed or - gathered full-vocab draft head. Returns tokens in draft-vocab space (the - caller applies d2t). Expects 2D ``[num_tokens, vocab_shard]`` logits. - - Under ADP + LM-head TP (``mapping_lm_head_tp`` given) the logits are the - LM-head-TP group's row-stacked batch (``tp_size`` segments of - ``max_num_requests`` padded rows, all-gathered along dim 0 by the MTP - shared head) with the vocab sharded across the group. The global argmax - must combine the group's vocab shards, and each rank must read its own - row segment at offset ``tp_rank * max_num_requests`` -- NOT rows - ``[:batch]``, which belong to group rank 0. + def _draft_sampler_advanced( + self, + logits: torch.Tensor, + spec_metadata: "SpecMetadata", + batch_size: int, + d2t: Optional[torch.Tensor] = None, + ): """ - if (mapping_lm_head_tp is not None - and getattr(mapping_lm_head_tp, "tp_size", 1) > 1): - from ..distributed.ops import allgather - combined = self._get_local_max_and_combined(logits, - mapping_lm_head_tp) - gathered = allgather(combined, mapping_lm_head_tp, dim=-1) - group_size = mapping_lm_head_tp.tp_size - local_rows = logits.shape[0] // group_size - own_segment = gathered.view(group_size, local_rows, - -1)[mapping_lm_head_tp.tp_rank] - return self._get_draft_tokens_from_gathered(own_segment) - mapping = self.mapping - sharded = self._draft_logits_are_sharded(logits, spec_metadata) - if (sharded and mapping is not None - and getattr(mapping, "tp_size", 1) > 1 - and not mapping.enable_attention_dp): - from ..distributed.ops import allgather - combined = self._get_local_max_and_combined(logits) - gathered = allgather(combined, mapping, dim=-1) - return self._get_draft_tokens_from_gathered(gathered) - # No cross-rank gather for plain attention-DP: each rank owns its own - # requests with replicated full-vocab logits, so a per-rank argmax is - # the correct proposal and a gather would desync the ranks (see - # _draft_logits_are_sharded). Plain argmax; caller applies d2t. - return torch.argmax(logits, dim=-1).type(torch.int32) - - def advanced_sample_draft_block(self, gen_logits: torch.Tensor, - spec_metadata: "SpecMetadata", - num_contexts: int, batch_size: int): - """Block counterpart of ``advanced_sample_draft`` for gen-only workers - (PARD, DFLASH): produces all ``K`` draft positions per gen request in one - forward from ``[num_gens, K, vocab]`` logits. - - With rejection enabled, samples via - ``sampling_batch_spec_dec_one_model_for_rejection`` and scatters the K - proposal rows into ``draft_probs[gen_slot_ids, 0:K, :]``; otherwise uses - ``sampling_batch_spec_dec_one_model`` (tokens only). Only called for a - non-greedy batch (the all-greedy path is handled by the caller). Returns - ``[num_gens, K]`` int32 tokens in draft-vocab space (the caller applies - d2t); stored probs likewise stay in draft-vocab space. + Draft token sampling using per-request sampling parameters from the + target's sampling config. Falls back to argmax when the batch is + all-greedy. + + Args: + logits: [batch_size, vocab_size] - Draft model logits (one row per + request, since each draft step emits one token per request). + spec_metadata: Source of per-request temperatures / top_k / top_p + tensors populated by populate_sampling_params_for_one_model. + batch_size: Number of active requests in the batch. + d2t: Optional dictionary offset tensor for vocab mapping. + + Returns: + draft_tokens: [batch_size] - Sampled draft token ids (int32) """ - num_gens, K, vocab = gen_logits.shape - if num_gens == 0: - return torch.empty((0, K), - dtype=torch.int32, - device=gen_logits.device) - - # Take the gen slice and repeat each request's value K times to line up - # with the flattened [num_gens*K, vocab] logits (K rows per request). - temps = spec_metadata.request_temperatures[ - num_contexts:batch_size].repeat_interleave(K) - top_ks = spec_metadata.request_top_ks[ - num_contexts:batch_size].repeat_interleave(K) - top_ps = spec_metadata.request_top_ps[ - num_contexts:batch_size].repeat_interleave(K) - - self._update_advance_draft_sampling_seed(gen_logits.device) - flat_logits = gen_logits.reshape(num_gens * K, vocab) - - if getattr(spec_metadata, "use_rejection_sampling", False): - flat_tokens, flat_probs = ( - sampling_batch_spec_dec_one_model_for_rejection( - flat_logits, - temps, - top_ks, - top_ps, - seed=self.seed, - offset=self.offset)) - # Scatter the K prob rows per gen request into its stable slot row. - if spec_metadata.draft_probs is not None: - assert spec_metadata.batch_slot_ids is not None, ( - "batch_slot_ids must be populated before block draft prob " - "storage") - gen_slot_ids = spec_metadata.batch_slot_ids[ - num_contexts:batch_size] - probs = flat_probs.reshape(num_gens, K, vocab) - spec_metadata.draft_probs[gen_slot_ids, :K, :vocab] = probs - spec_metadata.draft_probs_last_dim = vocab - else: - flat_tokens = sampling_batch_spec_dec_one_model(flat_logits, - temps, - top_ks, - top_ps, - seed=self.seed, - offset=self.offset) - - return flat_tokens.reshape(num_gens, K).type(torch.int32) - - def write_context_onehot_draft_probs(self, - spec_metadata, - num_contexts, - num_gens, - draft_len, - gen_vocab=None): - """Write a one-hot draft-prob distribution (prob 1.0 at draft-vocab token - id 0, the placeholder draft token) into each context request's stable - slot row, so the row is a legal distribution when the context request - becomes a generation request next iteration and rejection acceptance - reads ``draft_probs[slot, :draft_len, :stored_vocab]``. - - Block workers (PARD/DFlash) do not draft context requests (they get a - zero placeholder token), leaving their slot rows unwritten. Mixed - iterations reuse the vocab width the gen scatter just set (``gen_vocab``); - pure-context iterations have no gen logits, so use the buffer's full - draft-vocab width and publish it via ``draft_probs_last_dim`` (acceptance - next iter reads this scalar). No-op unless rejection is enabled and the - slot-indexed buffers exist. Static shapes -> CUDA-graph safe. + if spec_metadata.is_all_greedy_sample: + return self._draft_sampler_greedy(logits, d2t) + + temperatures = spec_metadata.request_temperatures[:batch_size] + top_ks = spec_metadata.request_top_ks[:batch_size] + top_ps = spec_metadata.request_top_ps[:batch_size] + + if self.seed is None: + self.seed = torch.tensor([0], + dtype=torch.int64, + device=logits.device) + self.offset = torch.tensor([0], + dtype=torch.int64, + device=logits.device) + self.seed += 1 + self.seed %= (2**31) + + draft_tokens = sampling_batch_spec_dec_one_model(logits, + temperatures, + top_ks, + top_ps, + seed=self.seed, + offset=self.offset) + + if d2t is not None: + draft_tokens = d2t[draft_tokens] + draft_tokens + + return draft_tokens.type(torch.int32) + + def _draft_sampler_advanced_for_rejection( + self, + logits: torch.Tensor, + spec_metadata: "SpecMetadata", + batch_size: int, + d2t: Optional[torch.Tensor] = None, + draft_step: int = 0, + ): """ - if (num_contexts <= 0 - or not getattr(spec_metadata, "use_rejection_sampling", False) - or spec_metadata.draft_probs is None): - return - ctx_slot_ids = spec_metadata.batch_slot_ids[:num_contexts] - if num_gens > 0: - onehot_vocab = gen_vocab # matches the gen scatter's width - else: - onehot_vocab = spec_metadata.draft_probs_vocab_size - spec_metadata.draft_probs_last_dim = onehot_vocab - spec_metadata.draft_probs[ctx_slot_ids, :draft_len, :onehot_vocab] = 0.0 - spec_metadata.draft_probs[ctx_slot_ids, :draft_len, 0] = 1.0 - - def sample_draft_tokens(self, - logits, - spec_metadata, - batch_size, - *, - num_contexts=0, - draft_step=None, - mapping_lm_head_tp=None): - """Unified draft-token production entry for all one-model workers. - - Branches by logits rank: 3D ``[num_gens, K, vocab]`` is the block form - (gen-only workers emitting all K positions in one forward, e.g. - PARD/DFlash); 2D ``[num_tokens, vocab]`` is the per-step form - (autoregressive workers called once per draft step, e.g. MTP/ - DraftTarget). ``draft_step`` applies only to the step form and - ``num_contexts`` only to the block form. d2t is read from ``self._d2t``. - - In a mixed (context + generation) batch the block form receives - gen-only logits (context requests draft from accepted tokens they do not - have yet), so ``num_contexts`` slices the full-batch per-request metadata - down to the gen segment. The step form receives full-batch logits - (context requests draft from target hidden states already available), so - no slicing is needed. + Rejection-sampling-aware variant of ``_draft_sampler_advanced``. + + Single-pass compute + sample + scatter: computes the per-request prob + distribution once via TRT-LLM's fused ``compute_probs_from_logits`` + (temp + top_k + top_p + softmax + greedy override in one CUDA kernel), + samples the draft token from that distribution, and scatters the same + probs into the slot-indexed ``spec_metadata.draft_probs`` buffer for + next-iter rejection verification. Replaces the previous two-stage path + (flashinfer fused sampling kernel + a redundant softmax pass to store + probs). + + All-greedy batches take the cheaper argmax path — + ``_can_use_rejection_sampling`` will bypass rejection for those anyway. """ - is_block = logits.dim() == 3 - - # Draft tokens use argmax unless rejection sampling is engaged for a - # non-greedy batch. Rejection sampling is the only path that needs the - # draft's stochastic proposal distribution (stored in draft_probs); every - # other path (all-greedy, or non-greedy with strict/exact-match - # acceptance) accepts a draft token only when it equals the target's - # choice, and there argmax maximizes the acceptance rate (E[accept] = - # max_i p_i >= sum_i p_i^2 = E[accept] for a stochastic draft). This - # matches sglang/vLLM, which draft with argmax/top-k by default and apply - # sampling params only on the target/acceptance side. - advanced = spec_metadata.wants_advanced_draft_sampling - - # All samplers below return tokens in draft-vocab space; d2t is applied - # once after the branch. - if not advanced: - # greedy_sample_draft_with_tp_gather expects 2D [tokens, vocab]; - # flatten the block form and restore its shape (step form is 2D). - batch_shape = logits.shape[:-1] - tokens = self.greedy_sample_draft_with_tp_gather( - logits.reshape(-1, logits.shape[-1]), spec_metadata, - mapping_lm_head_tp) - if mapping_lm_head_tp is None: - tokens = tokens.reshape(batch_shape) - # else: ADP+LM-head-TP (2D step form only) -- the sampler returned - # this rank's own row segment, 1/tp_size of the stacked input rows, - # so the input batch shape no longer applies. Keep as-is; the - # caller trims the max_num_requests padding to token_count. - else: - # Advanced sampling gathers the vocab-sharded draft logits to full - # vocab, then samples (scattering this step's proposal distribution - # into draft_probs). - logits = self.maybe_gather_sharded_draft_logits( - logits, spec_metadata, mapping_lm_head_tp) - if is_block: - tokens = self.advanced_sample_draft_block( - logits, spec_metadata, num_contexts, batch_size).long() - else: - tokens = self.advanced_sample_draft(logits, - spec_metadata, - batch_size, - draft_step=draft_step) + if spec_metadata.is_all_greedy_sample: + return self._draft_sampler_greedy(logits, d2t) + + temperatures = spec_metadata.request_temperatures[:batch_size] + top_ks = spec_metadata.request_top_ks[:batch_size] + top_ps = spec_metadata.request_top_ps[:batch_size] - # Map draft-vocab token ids to target vocab (no-op for shared vocab). - if self._d2t is not None: - tokens = self._d2t[tokens] + tokens + if self.seed is None: + self.seed = torch.tensor([0], + dtype=torch.int64, + device=logits.device) + self.offset = torch.tensor([0], + dtype=torch.int64, + device=logits.device) + self.seed += 1 + self.seed %= (2**31) - return tokens.type(torch.int32) + draft_tokens, probs = sampling_batch_spec_dec_one_model_for_rejection( + logits, + temperatures, + top_ks, + top_ps, + seed=self.seed, + offset=self.offset, + ) + + # Scatter probs into the slot-indexed buffer (shaped + # [max_num_requests, max_draft_len, vocab_size]). Each request's data + # always lands at its stable py_seq_slot row regardless of batch + # composition shifts across iterations. + assert spec_metadata.batch_slot_ids is not None, ( + "batch_slot_ids must be populated by " + "populate_sampling_params_for_one_model before draft probs storage") + batch_slots = spec_metadata.batch_slot_ids[:batch_size] + vocab = probs.shape[-1] + spec_metadata.draft_probs[batch_slots, draft_step, :vocab] = probs + spec_metadata.draft_probs_last_dim = vocab + + if d2t is not None: + draft_tokens = d2t[draft_tokens] + draft_tokens + + return draft_tokens.type(torch.int32) def _execute_guided_decoder_if_present(self, logits): """Execute guided decoder on target model logits if available.""" diff --git a/tensorrt_llm/_torch/speculative/mtp.py b/tensorrt_llm/_torch/speculative/mtp.py index 3dc12a669aa6..6afed0fc02e9 100644 --- a/tensorrt_llm/_torch/speculative/mtp.py +++ b/tensorrt_llm/_torch/speculative/mtp.py @@ -5,8 +5,10 @@ import torch from tensorrt_llm._utils import prefer_pinned +from tensorrt_llm.mapping import Mapping from ..attention_backend import AttentionMetadata +from ..distributed.ops import allgather from ..pyexecutor.llm_request import LlmRequest from ..pyexecutor.resource_manager import BaseResourceManager, SlotManager from ..pyexecutor.sampler import TorchSampler @@ -276,19 +278,10 @@ class MTPWorker(SpecWorkerBase): def __init__(self, spec_config: "MTPDecodingConfig", model_config=None, - use_separate_draft_kv_cache: bool = False, - *, - mapping=None): + use_separate_draft_kv_cache: bool = False): super().__init__(use_separate_draft_kv_cache) self.spec_config = spec_config self.model_config = model_config - # Use the mapping passed by get_spec_worker (the same Mapping that shards - # the draft LM head). model_config.mapping is unreliable here -- for the - # MTP worker it can be None -- so reading it caused greedy draft sampling - # to skip the TP argmax gather and desync the ranks into a hang under - # plain TP + overlap scheduler. - self.mapping = mapping if mapping is not None else getattr( - model_config, "mapping", None) self.is_thop = False self.sa_enhancer: Optional[SADraftEnhancer] = None if spec_config.sa_config is not None: @@ -298,7 +291,7 @@ def __init__(self, def max_draft_len(self) -> int: return self.spec_config.max_draft_len - def _forward_impl( + def forward( self, input_ids, position_ids, @@ -476,10 +469,7 @@ def _forward_impl( self.guided_decoder.execute_draft_batch(logits, draft_step=i) - new_draft_token = self.sample_draft_tokens(logits, - spec_metadata, - batch_size, - draft_step=i) + new_draft_token = self.draft_sampler(logits) next_draft_tokens.append(new_draft_token) # shift input_ids and hidden_states input_ids = draft_inputs["input_ids"] @@ -851,16 +841,6 @@ def sample_and_accept_draft_tokens( runtime_draft_len, spec_metadata=spec_metadata) - # Rejection sampling acceptance. _can_use_rejection_sampling() requires - # use_rejection_sampling and a non-all-greedy batch; otherwise falls - # through to strict acceptance below. Context rows take the target's - # first sampled token; gen rows run the rejection kernel. - elif self._can_use_rejection_sampling(spec_metadata): - draft_tokens = spec_metadata.draft_tokens.reshape( - num_gens, runtime_draft_len) - accepted_tokens, num_accepted_tokens = self._accept_draft_tokens( - logits, draft_tokens, num_contexts, batch_size, spec_metadata) - # Strict acceptance else: if self.is_thop: @@ -1112,3 +1092,80 @@ def prepare_drafter_inputs( "hidden_states": return_hidden_states, "attn_metadata": attn_metadata, } + + @torch.compile(options={"max-autotune": True}) + def get_local_max_and_combined(self, logits, mapping_lm_tp=None): + local_max_values, local_argmax = torch.max(logits, dim=-1, keepdim=True) + # Adjust indices based on TP rank and size + vocab_per_rank = logits.shape[-1] + mapping_lm_tp = mapping_lm_tp if mapping_lm_tp is not None else self.model_config.mapping + max_index_per_rank = local_argmax.type( + torch.int32) + (mapping_lm_tp.tp_rank * vocab_per_rank) + # Use torch.stack and flatten instead of view+cat to avoid torch.compile issues + # Convert both to float32 to ensure consistent dtype + max_index_per_rank_float = max_index_per_rank.float() + local_max_values_float32 = local_max_values.float() + + # Stack and flatten to get interleaved layout: [idx0, val0, idx1, val1, ...] + combined = torch.stack( + [max_index_per_rank_float, local_max_values_float32], + dim=-1).flatten(-2) + return combined + + @torch.compile(options={"max-autotune": True}) + def get_draft_tokens_from_gathered(self, gathered): + gathered_indices_float = gathered[..., 0::2] # Even positions: indices + gathered_values_float = gathered[..., 1::2] # Odd positions: values + + # Find the rank with maximum value + max_indices = torch.argmax(gathered_values_float, dim=-1, keepdim=True) + + # Get the corresponding token indices and convert back to int32 + draft_tokens = torch.gather(gathered_indices_float, -1, + max_indices).squeeze(-1).type(torch.int32) + return draft_tokens + + def draft_sampler( + self, + logits: torch.Tensor, + mapping_lm_head_tp: Mapping = None, + ): + ''' + Sampling draft tokens. + + Args: + logits: torch.Tensor + [num_tokens, vocab_size] + Logits produced by the draft model. + + Returns: + draft_tokens: torch.Tensor + [batch_size * max_draft_len] + Draft token ids. Flattened. + ''' + if (self.model_config is not None + and hasattr(self.model_config, 'mapping') + and self.model_config.mapping.tp_size + > 1) and not (self.model_config.mapping.enable_attention_dp): + combined = self.get_local_max_and_combined(logits) + gathered = allgather(combined, self.model_config.mapping, dim=-1) + draft_tokens = self.get_draft_tokens_from_gathered(gathered) + elif (self.model_config is not None + and hasattr(self.model_config, 'mapping') + and self.model_config.mapping.tp_size + > 1) and self.model_config.mapping.enable_lm_head_tp_in_adp: + # For ADP + LM head TP mode, we need to find the global argmax across all TP ranks + combined = self.get_local_max_and_combined(logits, + mapping_lm_head_tp) + gathered = allgather(combined, mapping_lm_head_tp, dim=-1) + batch_size = logits.shape[0] + local_batch_size = batch_size // mapping_lm_head_tp.tp_size + gathered = gathered.view(mapping_lm_head_tp.tp_size, + local_batch_size, -1) + sliced_gathered = gathered[mapping_lm_head_tp.tp_rank] + draft_tokens = self.get_draft_tokens_from_gathered(sliced_gathered) + else: + # Simple argmax if no TP or no model config + draft_tokens = self._draft_sampler_greedy(logits) + + return draft_tokens diff --git a/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py b/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py deleted file mode 100644 index 053d23362f98..000000000000 --- a/tensorrt_llm/_torch/speculative/mtp_dynamic_tree.py +++ /dev/null @@ -1,1122 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""MTP-Eagle one-model dynamic tree speculative decoding (greedy only).""" - -import math -from typing import TYPE_CHECKING, List, Optional - -import torch -import triton - -from tensorrt_llm._torch.pyexecutor.mamba_cache_manager import MambaHybridCacheManager -from tensorrt_llm._utils import get_sm_version, nvtx_range -from tensorrt_llm.mapping import Mapping - -from ..distributed.ops import allgather -from ..model_config import ModelConfig -from ..pyexecutor.llm_request import LlmRequest -from ..pyexecutor.resource_manager import BaseResourceManager -from ..pyexecutor.scheduler import ScheduledRequests -from .eagle3 import MTPEagleWorker - -# Reuse drafter-agnostic dynamic-tree helpers. -from .eagle3_dynamic_tree import ( - _build_mask_and_position, - _gather_repack_step0_kernel, - _resample_final_tokens, - _select_topk_draft_tokens, -) -from .mtp import MTPHiddenStatesManager - -if TYPE_CHECKING: - from tensorrt_llm.llmapi.llm_args import MTPDecodingConfig - - -class MTPEagleDynamicTreeWorker(MTPEagleWorker): - """MTP-Eagle worker with dynamic-tree draft and greedy verify.""" - - def __init__( - self, - spec_config: "MTPDecodingConfig", - model_config: Optional[ModelConfig] = None, - use_separate_draft_kv_cache: bool = False, - *, - mapping: Optional[Mapping] = None, - ): - super().__init__(spec_config, model_config, use_separate_draft_kv_cache, mapping=mapping) - assert getattr(spec_config, "use_dynamic_tree", False), ( - "MTPEagleDynamicTreeWorker requires use_dynamic_tree=True" - ) - - from .dynamic_tree_ops import DynamicTreeOpsConverter - - self.K = spec_config.dynamic_tree_max_topK - self.max_total_draft_tokens = spec_config.tokens_per_gen_step - 1 - self.tokens_per_gen_step = spec_config.tokens_per_gen_step - # Set by py_executor_creator from the global max_batch_size. - assert spec_config._max_batch_size is not None, ( - "MTPDecodingConfig._max_batch_size was not populated; " - "py_executor_creator should have set it from the global max_batch_size." - ) - self._max_batch_size = spec_config._max_batch_size - - K = self.K - max_draft_len = spec_config.max_draft_len - max_batch_size = self._max_batch_size - loop_max_tokens = K * max_draft_len # draft loop working size - - # spec_tree_manager is lazily bound from the resource manager. - self.spec_tree_manager = None - self._d2t = None - - # Pre-allocated draft-loop buffers (CUDA-graph safe). - self.draft_tokens_buffer = torch.zeros( - max_batch_size, loop_max_tokens, dtype=torch.int32, device="cuda" - ) - self.position_ids_buffer = torch.zeros( - max_batch_size, loop_max_tokens, dtype=torch.int32, device="cuda" - ) - self.history_draft_tokens_buffer = torch.zeros( - (max_batch_size, (K + K * K * (max_draft_len - 1))), dtype=torch.int32, device="cuda" - ) - self.history_score_buffer = torch.zeros( - (max_batch_size, K + K * K * (max_draft_len - 1)), dtype=torch.float32, device="cuda" - ) - self.history_draft_tokens_parent_buffer = torch.zeros( - (max_batch_size, max(K * (max_draft_len - 1) + 1, K + 1)), - dtype=torch.int64, - device="cuda", - ) - self.tree_mask_buffer = torch.zeros( - (max_batch_size * loop_max_tokens * loop_max_tokens), dtype=torch.int32, device="cuda" - ) - self.tree_mask_init_buffer = ( - torch.eye(K, dtype=torch.int32, device="cuda").unsqueeze(0).repeat(max_batch_size, 1, 1) - ) - self.tree_ops_converter = DynamicTreeOpsConverter( - dynamic_tree_max_topK=K, - max_draft_len=max_draft_len, - max_total_draft_tokens=self.max_total_draft_tokens, - max_batch_size=max_batch_size, - device=torch.device("cuda"), - ) - - self._max_path_len = max_draft_len + 1 - # Step-0 draft resets verify-time tree metadata to accepted-path width. - self._kv_correction = self.tokens_per_gen_step - self._max_path_len - self._step0_causal_mask = torch.tensor( - [(1 << (t + 1)) - 1 for t in range(self._max_path_len)], - dtype=torch.int32, - device="cuda", - ) - self._causal_offs = torch.arange(self._max_path_len, device="cuda", dtype=torch.int32) - self._last_selected_parents = None - self._parent_init_arange = torch.arange(-1, K, device="cuda", dtype=torch.int32) - - # Accepted-path bookkeeping for KV relocation and output. - self._accepted_draft_indices_tensor = torch.full( - (max_batch_size, max_draft_len), -1, dtype=torch.int32, device="cuda" - ) - self._kv_head_dim_bytes = None - - # === Verification buffers (greedy only) === - N = self.tokens_per_gen_step - self._accepted_tokens_buf = torch.zeros( - max_batch_size, self._max_path_len, dtype=torch.int32, device="cuda" - ) - self._num_accepted_tokens_buf = torch.ones(max_batch_size, dtype=torch.int32, device="cuda") - self._target_tokens_buf = torch.zeros(max_batch_size * N, dtype=torch.int64, device="cuda") - self._candidates_buf = torch.zeros(max_batch_size, N, dtype=torch.int32, device="cuda") - self._target_predict_buf = torch.zeros(max_batch_size, N, dtype=torch.int32, device="cuda") - - # Hidden states for the growing-context draft loop. - self._hs_write_buffer = None - self._accumulated_hs = None - self._hs_read_map = torch.zeros( - max_batch_size, loop_max_tokens, dtype=torch.long, device="cuda" - ) - self._step0_hs = None - self._hs_dim = None - - # Step-0 repack scratch for accepted-path inputs. - max_total_tokens = max_batch_size * self.tokens_per_gen_step - self._step0_input_ids_buf = torch.zeros(max_total_tokens, dtype=torch.int32, device="cuda") - self._step0_position_ids_buf = torch.zeros( - max_total_tokens, dtype=torch.int32, device="cuda" - ) - self._step0_hidden_states_buf = None - self._gather_ids_buf = torch.zeros(max_total_tokens, dtype=torch.long, device="cuda") - - # Mask repack scratch (graph-safe; avoids .contiguous() in the loop). - buf_dim = max(self.max_total_draft_tokens + 1, loop_max_tokens) - mask_width = (buf_dim + 31) // 32 - self._mask_repack_buf = torch.zeros( - max_batch_size * buf_dim * mask_width, dtype=torch.int32, device="cuda" - ) - # sm>=100 (except 120/121): prepareCustomMask keeps padded 3D; no repack. - sm = get_sm_version() - self._needs_mask_repack = sm < 100 or sm in (120, 121) - - def _prepare_attn_metadata_for_spec_dec(self, attn_metadata): - super()._prepare_attn_metadata_for_spec_dec(attn_metadata) - - batch_size = attn_metadata.num_seqs - if hasattr(attn_metadata, "kv_lens_cuda"): - # Keep kv_lens_cuda itself alive because TRTLLM attention holds a - # runtime view into it. - self._saved_kv_lens_cuda = attn_metadata.kv_lens_cuda[:batch_size].clone() - else: - self._saved_kv_lens_cuda = None - - # Restore verify metadata after the draft loop mutates it. - if attn_metadata.spec_decoding_packed_mask is not None: - self._saved_packed_mask = attn_metadata.spec_decoding_packed_mask[:batch_size].clone() - else: - self._saved_packed_mask = None - if attn_metadata.spec_decoding_position_offsets is not None: - self._saved_position_offsets = attn_metadata.spec_decoding_position_offsets.clone() - self._saved_position_offsets_cpp = attn_metadata.spec_decoding_position_offsets_cpp - else: - self._saved_position_offsets = None - self._saved_position_offsets_cpp = None - if attn_metadata.spec_decoding_generation_lengths is not None: - self._saved_generation_lengths = attn_metadata.spec_decoding_generation_lengths[ - :batch_size - ].clone() - else: - self._saved_generation_lengths = None - - def prepare_position_ids_and_last_tokens(self, position_ids, seq_lens_cuda): - position_ids = position_ids.squeeze(0) - last_tokens_idx = torch.cumsum(seq_lens_cuda, dim=0, dtype=torch.long) - 1 - return position_ids, last_tokens_idx - - def _restore_attn_metadata_from_spec_dec(self, attn_metadata): - super()._restore_attn_metadata_from_spec_dec(attn_metadata) - - if self._saved_kv_lens_cuda is not None: - batch_size = self._saved_kv_lens_cuda.shape[0] - attn_metadata.kv_lens_cuda[:batch_size].copy_(self._saved_kv_lens_cuda) - self._saved_kv_lens_cuda = None - - if self._saved_packed_mask is not None: - batch_size = self._saved_packed_mask.shape[0] - attn_metadata.spec_decoding_packed_mask[:batch_size].copy_(self._saved_packed_mask) - self._saved_packed_mask = None - if self._saved_position_offsets is not None: - attn_metadata.spec_decoding_position_offsets.copy_(self._saved_position_offsets) - attn_metadata.spec_decoding_position_offsets_cpp = self._saved_position_offsets_cpp - self._saved_position_offsets = None - self._saved_position_offsets_cpp = None - if self._saved_generation_lengths is not None: - batch_size = self._saved_generation_lengths.shape[0] - attn_metadata.spec_decoding_generation_lengths[:batch_size].copy_( - self._saved_generation_lengths - ) - self._saved_generation_lengths = None - - # ------------------------------------------------------------------ # - # Helpers # - # ------------------------------------------------------------------ # - def _apply_spec_metadata(self, attn_metadata, batch_size, query_len): - """Set spec-dec gen lengths and refresh the C++ position-offset view.""" - attn_metadata.spec_decoding_generation_lengths[:batch_size] = query_len - attn_metadata.update_position_offsets_for_cpp(query_len) - - def _refresh_blackwell_tree_mask_metadata(self, attn_metadata): - if not getattr(attn_metadata, "use_spec_decoding", False): - return - if not getattr(attn_metadata, "is_spec_dec_dynamic_tree", False): - return - - first_sparse = getattr(attn_metadata, "spec_bl_tree_first_sparse_mask_offset_kv", None) - bl_tree_mask = getattr(attn_metadata, "spec_decoding_bl_tree_mask", None) - if first_sparse is None and bl_tree_mask is None: - return - - if bl_tree_mask is not None: - bl_tree_mask.zero_() - if first_sparse is not None: - attn_metadata.update_blackwell_first_sparse_mask_offset() - - def _repack_mask_padded_to_packed(self, mask_buf, n_req, n_tok): - """Compact padded masks into the flat prefix XQA expects.""" - buf_dim = mask_buf.shape[1] - if n_tok >= buf_dim or n_req <= 1: - return - mask_width = math.ceil(n_tok / 32) - total_elems = n_req * n_tok * mask_width - scratch = self._mask_repack_buf[:total_elems].view(n_req, n_tok, mask_width) - scratch.copy_(mask_buf[:n_req, :n_tok, :mask_width]) - flat = mask_buf.view(-1) - flat[:total_elems] = scratch.view(-1) - - @nvtx_range("mtp_dyn._ensure_spec_tree_manager") - def _ensure_spec_tree_manager(self, resource_manager): - """Lazily bind spec_tree_manager and KV head metadata.""" - if self.spec_tree_manager is not None: - return - from ..pyexecutor.resource_manager import ResourceManagerType - - spec_rm = resource_manager.get_resource_manager(ResourceManagerType.SPEC_RESOURCE_MANAGER) - assert spec_rm is not None and hasattr(spec_rm, "spec_tree_manager"), ( - "Dynamic tree mode requires spec_tree_manager in resource_manager" - ) - self.spec_tree_manager = spec_rm.spec_tree_manager - - if self._kv_head_dim_bytes is None: - cache_mgr = resource_manager.get_resource_manager(ResourceManagerType.KV_CACHE_MANAGER) - if cache_mgr is not None and hasattr(cache_mgr, "head_dim"): - from tensorrt_llm.bindings import DataType - - _dtype_bytes = { - DataType.HALF: 2, - DataType.BF16: 2, - DataType.FLOAT: 4, - DataType.FP8: 1, - DataType.INT8: 1, - DataType.NVFP4: 0.5, - } - self._kv_head_dim_bytes = int( - cache_mgr.head_dim * _dtype_bytes.get(cache_mgr.dtype, 0.5) - ) - - @nvtx_range("mtp_dyn.sample") - def sample( - self, logits: torch.Tensor, max_top_k: int, draft_model=None - ) -> tuple[torch.Tensor, torch.Tensor]: - """TopK sampling for dynamic tree; all-gather sharded TP logits.""" - mapping = ( - getattr(self.model_config, "mapping", None) if self.model_config is not None else None - ) - if mapping is not None and mapping.tp_size > 1 and not mapping.enable_attention_dp: - logits = allgather(logits, mapping, dim=-1) - if draft_model is not None: - vocab_size = draft_model.lm_head.num_embeddings - logits = logits[..., :vocab_size] - probs = torch.softmax(logits, dim=-1) - topk_values, topk_indices = torch.topk(probs, k=max_top_k, dim=-1) - return topk_indices, topk_values - - def update_draft_tokens_and_scores( - self, - cur_draft_idx, - new_draft_tokens, - new_draft_scores, - previous_draft_scores, - batch_size, - attn_metadata=None, - ): - """Grow the tree and update history buffers.""" - if cur_draft_idx == 0: - new_draft_scores = new_draft_scores.reshape(batch_size, self.K) - new_draft_tokens_2d = new_draft_tokens.reshape(batch_size, self.K) - self.draft_tokens_buffer[:batch_size, : self.K] = new_draft_tokens_2d - self.history_draft_tokens_buffer[:batch_size, : self.K] = new_draft_tokens_2d - self.history_score_buffer[:batch_size, : self.K] = new_draft_scores - # Parent buffer: -1 for root, 0..K-1 for first layer. - self.history_draft_tokens_parent_buffer[:batch_size, : self.K + 1] = ( - self._parent_init_arange - ) - self.prepare_tree_mask_and_position_offset(cur_draft_idx, attn_metadata, None) - return new_draft_scores - - ( - real_draft_tokens, - topk_values, - topk_indices, - selected_parents, - new_draft_tokens, - new_draft_scores, - ) = _select_topk_draft_tokens( - new_draft_tokens, new_draft_scores, previous_draft_scores, self.K - ) - - num_tokens_previous_layer = cur_draft_idx * self.K - num_tokens_current_layer = (cur_draft_idx + 1) * self.K - self.draft_tokens_buffer[ - :batch_size, num_tokens_previous_layer:num_tokens_current_layer - ] = real_draft_tokens - - write_start = self.K + (cur_draft_idx - 1) * self.K * self.K - write_end = write_start + self.K * self.K - self.history_draft_tokens_buffer[:batch_size, write_start:write_end] = new_draft_tokens - self.history_score_buffer[:batch_size, write_start:write_end] = new_draft_scores - - self._last_selected_parents = selected_parents - self.prepare_tree_mask_and_position_offset(cur_draft_idx, attn_metadata, selected_parents) - - if cur_draft_idx < self.max_draft_len - 1: - next_layer_start = cur_draft_idx * self.K + 1 - next_layer_end = next_layer_start + self.K - parents_relative_indices = topk_indices + self.K**2 * (cur_draft_idx - 1) + self.K - self.history_draft_tokens_parent_buffer[ - :batch_size, next_layer_start:next_layer_end - ] = parents_relative_indices - return topk_values - - def resampling_final_draft_tokens(self, batch_size: int): - """Reconstruct the final tree from history buffers.""" - return _resample_final_tokens( - self.history_score_buffer[:batch_size, :], - self.history_draft_tokens_buffer[:batch_size, :], - self.max_total_draft_tokens, - ) - - def prepare_tree_mask_and_position_offset( - self, cur_draft_idx, attn_metadata, selected_parents=None - ): - """Prepare mask and position offsets for the next draft layer.""" - if attn_metadata.spec_decoding_packed_mask is None: - return - spec_tree_manager = self.spec_tree_manager - batch_size = attn_metadata.num_seqs - num_tokens_current_layer = self.K * (cur_draft_idx + 1) - num_tokens_previous_layer = self.K * cur_draft_idx - packed_mask = attn_metadata.spec_decoding_packed_mask - if cur_draft_idx == 0: - spec_tree_manager.compute_spec_dec_packed_mask( - self.tree_mask_init_buffer[:batch_size], - packed_mask[:batch_size, :num_tokens_current_layer, :], - ) - self.tree_mask_buffer[ - : batch_size * num_tokens_current_layer * num_tokens_current_layer - ].copy_(self.tree_mask_init_buffer[:batch_size].view(-1)) - attn_metadata.spec_decoding_position_offsets.fill_(0) - self._apply_spec_metadata(attn_metadata, batch_size, num_tokens_current_layer) - else: - num_parent_mask = batch_size * cur_draft_idx * self.K * cur_draft_idx * self.K - parent_mask = self.tree_mask_buffer[:num_parent_mask].reshape( - batch_size, cur_draft_idx * self.K, cur_draft_idx * self.K - ) - - prev_total = batch_size * num_tokens_previous_layer - previous_position_offsets = attn_metadata.spec_decoding_position_offsets[ - :prev_total - ].view(batch_size, num_tokens_previous_layer) - - current_mask, new_positions = _build_mask_and_position( - parent_mask, - selected_parents, - self.tree_mask_init_buffer[:batch_size], - previous_position_offsets, - self.K, - ) - - spec_tree_manager.compute_spec_dec_packed_mask( - current_mask, packed_mask[:batch_size, :num_tokens_current_layer, :] - ) - self.tree_mask_buffer[ - : batch_size * num_tokens_current_layer * num_tokens_current_layer - ].copy_(current_mask.reshape(-1)) - - cur_total = batch_size * num_tokens_current_layer - attn_metadata.spec_decoding_position_offsets[:cur_total] = new_positions.reshape(-1) - self._apply_spec_metadata(attn_metadata, batch_size, num_tokens_current_layer) - - if self._needs_mask_repack: - self._repack_mask_padded_to_packed(packed_mask, batch_size, num_tokens_current_layer) - - def update_hidden_states( - self, - cur_draft_idx, - batch_size, - step0_hs=None, - hidden_states_to_save=None, - selected_parents=None, - ): - """Manage growing-context hidden states for the MTP draft loop.""" - if cur_draft_idx == 0: - hs_dim = step0_hs.shape[-1] - self._hs_dim = hs_dim - if self._hs_write_buffer is None or self._hs_write_buffer.shape[2] != hs_dim: - self._hs_write_buffer = torch.zeros( - self._max_batch_size, - self.max_draft_len * self.K, - hs_dim, - device=step0_hs.device, - dtype=step0_hs.dtype, - ) - if self._accumulated_hs is None or self._accumulated_hs.shape[2] != hs_dim: - self._accumulated_hs = torch.zeros( - self._max_batch_size, - self.max_draft_len * self.K, - hs_dim, - device=step0_hs.device, - dtype=step0_hs.dtype, - ) - # All K depth-0 tokens share step0_hs (the parent hidden state). - self._accumulated_hs[:batch_size, : self.K] = step0_hs.unsqueeze(1).expand( - -1, self.K, -1 - ) - self._step0_hs = step0_hs - else: - num_tokens_per_req = cur_draft_idx * self.K - hs_to_save_reshaped = hidden_states_to_save.reshape(batch_size, num_tokens_per_req, -1) - self._hs_write_buffer[:batch_size, :num_tokens_per_req] = hs_to_save_reshaped - parent_offset = (cur_draft_idx - 1) * self.K - self._hs_read_map[ - :batch_size, cur_draft_idx * self.K : (cur_draft_idx + 1) * self.K - ] = parent_offset + selected_parents - num_tokens_next = (cur_draft_idx + 1) * self.K - read_idx = self._hs_read_map[:batch_size, self.K : num_tokens_next] - hs_dim = self._hs_write_buffer.shape[2] - self._accumulated_hs[:batch_size, self.K : num_tokens_next] = torch.gather( - self._hs_write_buffer[:batch_size], 1, read_idx.unsqueeze(-1).expand(-1, -1, hs_dim) - ) - - # ------------------------------------------------------------------ # - # Verification (greedy only) # - # ------------------------------------------------------------------ # - @nvtx_range("mtp_dyn.sample_and_accept_draft_tokens") - def sample_and_accept_draft_tokens(self, input_ids, logits, spec_metadata, attn_metadata): - """Greedy verification of the previous dynamic tree.""" - batch_size = attn_metadata.num_seqs - num_contexts = attn_metadata.num_contexts - num_gens = batch_size - num_contexts - N = self.tokens_per_gen_step - max_path_len = self._max_path_len - - if logits.dim() == 1: - logits = logits.unsqueeze(0) - - # Reset output buffers. - self._accepted_tokens_buf[:batch_size].zero_() - accepted_tokens = self._accepted_tokens_buf[:batch_size, :max_path_len] - self._num_accepted_tokens_buf[:batch_size].fill_(1) - num_accepted_tokens = self._num_accepted_tokens_buf[:batch_size] - self._accepted_draft_indices_tensor[:batch_size].fill_(-1) - - num_flat_tokens = logits.shape[0] - torch.argmax(logits, dim=-1, out=self._target_tokens_buf[:num_flat_tokens]) - target_tokens = self._target_tokens_buf[:num_flat_tokens] - - # Context requests: accept the sampled golden token only. - accepted_tokens[:num_contexts, 0].copy_(target_tokens[:num_contexts]) - - if num_gens > 0: - spec_tree_manager = self.spec_tree_manager - target_predict = self._target_predict_buf[:num_gens] - target_predict.copy_(target_tokens[num_contexts:].reshape(num_gens, N)) - - # No prior tree exists on bootstrap/warmup; accept the golden token. - if spec_tree_manager is None: - num_accepted_tokens[num_contexts:batch_size] = 1 - accepted_tokens[num_contexts:batch_size, 0] = target_predict[:, 0] - self._accepted_draft_indices_tensor[num_contexts:batch_size] = -1 - return accepted_tokens, num_accepted_tokens - - # candidates[:, 0] = golden token, candidates[:, 1:] = draft tokens. - candidates = self._candidates_buf[:num_gens] - candidates[:, 1:] = spec_metadata.draft_tokens.reshape(num_gens, N - 1) - candidates[:, 0] = target_predict[:, 0] - - slot_storage = spec_tree_manager.slot_storage - gen_slot_ids = slot_storage.all_ids_buf[num_contexts : num_contexts + num_gens] - tree_valid = slot_storage.has_tree[gen_slot_ids] - retrieve_packed = slot_storage.pack_retrieve_from_slots(gen_slot_ids, num_gens) - - accept_index, accept_token_num, accept_token = ( - self.tree_ops_converter.verify_dynamic_tree_greedy_out_packed( - candidates, - retrieve_packed, - target_predict, - num_gens, - self._max_path_len, - tree_valid=tree_valid, - ) - ) - tree_valid_i = tree_valid[:num_gens] - accepted_draft_count = torch.where( - tree_valid_i, - accept_token_num[:num_gens], - torch.zeros_like(accept_token_num[:num_gens]), - ) - num_accepted_tokens[num_contexts:batch_size] = (accepted_draft_count + 1).to( - torch.int32 - ) - - gen_accepted_tokens = accept_token[:num_gens].to(torch.int32) - bootstrap_accepted_tokens = torch.zeros_like(gen_accepted_tokens) - bootstrap_accepted_tokens[:, 0] = target_predict[:, 0] - accepted_tokens[num_contexts:batch_size] = torch.where( - tree_valid_i.unsqueeze(1), gen_accepted_tokens, bootstrap_accepted_tokens - ) - # Convert root/padding index 0 to draft-node sentinel -1. - gen_accepted_indices = (accept_index[:num_gens, 1:max_path_len] - 1).to(torch.int32) - self._accepted_draft_indices_tensor[num_contexts:batch_size] = torch.where( - tree_valid_i.unsqueeze(1), - gen_accepted_indices, - torch.full_like(gen_accepted_indices, -1), - ).to(torch.int32) - - num_accepted_tokens = self._apply_force_accepted_tokens( - num_accepted_tokens, num_contexts, self.max_draft_len - ) - return accepted_tokens, num_accepted_tokens - - def _accepted_leaf_intermediate_positions(self, num_accepted_tokens, num_contexts, num_gens): - """Return each accepted leaf's position in the Mamba state buffer.""" - accepted = num_accepted_tokens[num_contexts : num_contexts + num_gens].to(torch.int64) - # Column of the deepest accepted draft node, clamped to >=0 for the - # golden-only case (its value is ignored via the mask below). - draft_idx = self._accepted_draft_indices_tensor[num_contexts : num_contexts + num_gens].to( - torch.int64 - ) - last_col = (accepted - 2).clamp_(min=0, max=draft_idx.shape[1] - 1) - leaf = torch.gather(draft_idx, 1, last_col.unsqueeze(1)).squeeze(1) + 1 - # Golden-only requests (num_accepted == 1) take the root at position 0. - return torch.where(accepted > 1, leaf, torch.zeros_like(leaf)) - - @nvtx_range("mtp_dyn._relocate_kv_eagerly") - def _relocate_kv_eagerly(self, attn_metadata, batch_size): - """Move accepted draft KV from tree positions to the linear prefix.""" - cache_mgr = getattr(attn_metadata, "kv_cache_manager", None) - if cache_mgr is None or self._kv_head_dim_bytes is None: - return - if not hasattr(cache_mgr, "num_kv_heads_per_layer"): - return - - # Mamba layers have zero KV heads; relocate attention-layer KV only. - kv_heads = cache_mgr.num_kv_heads_per_layer - attn_heads = set(h for h in kv_heads if h > 0) - assert len(attn_heads) == 1, ( - "update_kv_cache_draft_token_location_2d requires uniform " - f"num_kv_heads across attention layers, got {list(kv_heads)}" - ) - attn_num_heads = attn_heads.pop() - attn_layer_offsets = [i for i, h in enumerate(kv_heads) if h > 0] - attn_num_layers = len(attn_layer_offsets) - - # Resolve the attention KV pool used by the relocation op. - pool_mapping = getattr(cache_mgr, "kv_cache_pool_mapping", None) - if pool_mapping is not None: - attn_pool_indices = set(int(pool_mapping[off][0]) for off in attn_layer_offsets) - assert len(attn_pool_indices) == 1, ( - "update_kv_cache_draft_token_location_2d requires all attention " - f"layers in one KV pool, got pools {sorted(attn_pool_indices)}" - ) - attn_pool_idx = attn_pool_indices.pop() - else: - attn_pool_idx = 0 - - pool_pointers = cache_mgr.kv_cache_pool_pointers[attn_pool_idx] - block_offsets = attn_metadata.kv_cache_block_offsets[attn_pool_idx] - - torch.ops.tensorrt_llm.update_kv_cache_draft_token_location_2d( - self._accepted_draft_indices_tensor[:batch_size], - self._num_accepted_tokens_buf[:batch_size], - attn_metadata.kv_lens_cuda[:batch_size], - True, - attn_num_layers, - attn_num_heads, - self._kv_head_dim_bytes, - cache_mgr.max_total_draft_tokens, - cache_mgr.max_attention_window_vec[0], - pool_pointers, - block_offsets, - cache_mgr.max_blocks_per_seq, - cache_mgr.tokens_per_block, - None, - ) - - # ------------------------------------------------------------------ # - # Top-level forward # - # ------------------------------------------------------------------ # - @nvtx_range("mtp_dyn.forward") - def _forward_impl( - self, - input_ids, - position_ids, - hidden_states, - logits, - attn_metadata, - spec_metadata, - draft_model, - resource_manager=None, - ): - """Run verify, cache promotion, and next-tree drafting.""" - if resource_manager is not None: - self._ensure_spec_tree_manager(resource_manager) - - batch_size = attn_metadata.num_seqs - num_contexts = attn_metadata.num_contexts - num_gens = batch_size - num_contexts - raw_logits = logits - - self._execute_guided_decoder_if_present(logits) - - # (a) Verify previous tree (greedy). Also relocates accepted KV. - accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( - input_ids, logits, spec_metadata, attn_metadata - ) - if num_gens > 0: - self._relocate_kv_eagerly(attn_metadata, batch_size) - - # Dynamic-tree Mamba states are stored by tree-node position. - if self._is_mamba_hybrid_cache is None: - self._is_mamba_hybrid_cache = isinstance( - attn_metadata.kv_cache_manager, MambaHybridCacheManager - ) - if num_gens > 0 and self._is_mamba_hybrid_cache: - accepted_leaf_positions = self._accepted_leaf_intermediate_positions( - num_accepted_tokens, num_contexts, num_gens - ) - attn_metadata.kv_cache_manager.update_mamba_states( - attn_metadata=attn_metadata, - num_accepted_tokens=num_accepted_tokens, - state_indices=attn_metadata.mamba_metadata.state_indices, - accepted_leaf_positions=accepted_leaf_positions, - ) - - # Save attn/spec metadata before the draft loop mutates it. - original_all_rank_num_tokens = attn_metadata.all_rank_num_tokens - original_force_prepare_spec_dec_tree_mask = attn_metadata.force_prepare_spec_dec_tree_mask - self._prepare_attn_metadata_for_spec_dec(attn_metadata) - attn_metadata.force_prepare_spec_dec_tree_mask = True - - # (c) Run the MTP draft tree loop -> build + store the next tree. - draft_kv_cache_manager = self.get_draft_kv_cache_manager(resource_manager) - next_draft_tokens = self._forward_draft_loop( - input_ids=input_ids, - position_ids=position_ids, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - num_accepted_tokens=num_accepted_tokens, - attn_metadata=attn_metadata, - spec_metadata=spec_metadata, - draft_model=draft_model, - draft_kv_cache_manager=draft_kv_cache_manager, - num_contexts=num_contexts, - num_gens=num_gens, - batch_size=batch_size, - ) - - # Restore attn metadata to support cuda graph. - self._restore_attn_metadata_from_spec_dec(attn_metadata) - attn_metadata.all_rank_num_tokens = original_all_rank_num_tokens - attn_metadata.force_prepare_spec_dec_tree_mask = original_force_prepare_spec_dec_tree_mask - attn_metadata.use_spec_decoding = True - - # (d) Prepare next_new_tokens for overlap scheduler. - next_new_tokens = self._prepare_next_new_tokens( - accepted_tokens, - next_draft_tokens, - spec_metadata.batch_indices_cuda, - batch_size, - num_accepted_tokens, - ) - - return { - "logits": raw_logits, - "new_tokens": accepted_tokens, - "new_tokens_lens": num_accepted_tokens, - "next_draft_tokens": next_draft_tokens, - "next_new_tokens": next_new_tokens, - "accepted_draft_tokens_indices": self._accepted_draft_indices_tensor[:batch_size], - } - - # ------------------------------------------------------------------ # - # Step-0 drafter-input repack (dynamic tree) # - # ------------------------------------------------------------------ # - @nvtx_range("mtp_dyn._prepare_step0_drafter_inputs") - def _prepare_step0_drafter_inputs( - self, - input_ids, - position_ids, - last_tokens_idx, - hidden_states, - accepted_tokens, - attn_metadata, - ): - """Repack step-0 drafter inputs to accepted-path layout.""" - num_contexts = attn_metadata.num_contexts - batch_size = attn_metadata.num_seqs - num_gens = batch_size - num_contexts - num_ctx_tokens = attn_metadata.num_ctx_tokens - - # Match MTPEagleWorker context input repack. - input_ids_ctx = self._prepare_context_input_ids( - input_ids, num_ctx_tokens, last_tokens_idx, accepted_tokens, num_contexts - ) - - if num_gens > 0: - max_path_len = self._max_path_len - num_gen_tokens = num_gens * max_path_len - - hidden_dim = hidden_states.shape[-1] - if ( - self._step0_hidden_states_buf is None - or self._step0_hidden_states_buf.shape[-1] != hidden_dim - ): - self._step0_hidden_states_buf = torch.zeros( - self._step0_input_ids_buf.shape[0], - hidden_dim, - dtype=hidden_states.dtype, - device="cuda", - ) - - # Accepted path includes the golden token at column 0. - accept_token = accepted_tokens[num_contexts:batch_size] - - BLOCK_H = triton.next_power_of_2(hidden_dim) - _gather_repack_step0_kernel[(num_gens * max_path_len,)]( - hidden_states, - accept_token, - position_ids, - self._accepted_draft_indices_tensor[num_contexts:batch_size], - self._num_accepted_tokens_buf, - self._step0_hidden_states_buf, - self._step0_input_ids_buf, - self._step0_position_ids_buf, - self._gather_ids_buf, - num_ctx_tokens, - num_contexts, - self.tokens_per_gen_step, - max_path_len, - self.max_draft_len, - hidden_dim, - num_ctx_tokens, # gather_id references combined [ctx|gen] tensor - BLOCK_H=BLOCK_H, - ) - - input_ids = torch.cat( - [input_ids_ctx, self._step0_input_ids_buf[:num_gen_tokens]], dim=0 - ) - position_ids = torch.cat( - [position_ids[:num_ctx_tokens], self._step0_position_ids_buf[:num_gen_tokens]], - dim=0, - ) - hidden_states = torch.cat( - [hidden_states[:num_ctx_tokens], self._step0_hidden_states_buf[:num_gen_tokens]], - dim=0, - ) - - attn_metadata._seq_lens[num_contexts:batch_size].fill_(max_path_len) - attn_metadata._seq_lens_cuda[num_contexts:batch_size].fill_(max_path_len) - attn_metadata.on_update() - else: - # Context-only (warmup): no gen tokens to repack. - input_ids = input_ids_ctx - - return { - "input_ids": input_ids, - "position_ids": position_ids, - "hidden_states": hidden_states, - "attn_metadata": attn_metadata, - } - - # ------------------------------------------------------------------ # - # MTP draft tree loop # - # ------------------------------------------------------------------ # - def _forward_draft_loop( - self, - input_ids, - position_ids, - hidden_states, - accepted_tokens, - num_accepted_tokens, - attn_metadata, - spec_metadata, - draft_model, - draft_kv_cache_manager, - num_contexts, - num_gens, - batch_size, - ): - """Draft the next dynamic tree with growing context.""" - spec_tree_manager = self.spec_tree_manager - - assert batch_size <= self._max_batch_size, ( - f"batch_size {batch_size} exceeds pre-allocated max_batch_size {self._max_batch_size}" - ) - - # Step 0: run MTP over accepted-path rows. - position_ids, last_tokens_idx = self.prepare_position_ids_and_last_tokens( - position_ids, attn_metadata.seq_lens_cuda - ) - inputs = self._prepare_step0_drafter_inputs( - input_ids=input_ids, - position_ids=position_ids, - last_tokens_idx=last_tokens_idx, - hidden_states=hidden_states, - accepted_tokens=accepted_tokens, - attn_metadata=attn_metadata, - ) - - # Reset verify-time tree metadata to accepted-path width. - num_step0_tokens = self._max_path_len - if attn_metadata.spec_decoding_generation_lengths is not None: - total = num_gens * num_step0_tokens - dst = attn_metadata.spec_decoding_position_offsets[:total].view( - num_gens, num_step0_tokens - ) - dst.copy_(self._causal_offs[:num_step0_tokens].unsqueeze(0).expand(num_gens, -1)) - self._apply_spec_metadata(attn_metadata, num_gens, num_step0_tokens) - packed_mask = attn_metadata.spec_decoding_packed_mask - packed_mask[:num_gens].zero_() - packed_mask[:num_gens, :num_step0_tokens, 0] = self._step0_causal_mask[ - :num_step0_tokens - ] - if self._needs_mask_repack: - self._repack_mask_padded_to_packed(packed_mask, num_gens, num_step0_tokens) - attn_metadata.use_spec_decoding = num_gens > 0 - if num_gens > 0 and hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= self._kv_correction - self._refresh_blackwell_tree_mask_metadata(attn_metadata) - if spec_metadata.all_rank_num_tokens is not None: - # Keep attention/MoE token counts aligned with step-0 repack. - attn_metadata.all_rank_num_tokens = spec_metadata.all_rank_num_tokens - - with self.draft_kv_cache_context(attn_metadata, draft_kv_cache_manager): - hidden_states = draft_model.mtp_layers[0]( - embed_tokens=draft_model.embed_tokens, - all_rank_num_tokens=spec_metadata.all_rank_num_tokens, - **inputs, - ) - - # Gather each request's root hidden state for depth-0 expansion. - self._gather_ids_buf[:num_contexts].copy_(last_tokens_idx[:num_contexts]) - gather_ids = self._gather_ids_buf[:batch_size] - - step0_hs = hidden_states[gather_ids] - logits = draft_model.mtp_layers[0].shared_head( - step0_hs, draft_model.lm_head, attn_metadata, True - ) - - new_draft_tokens, new_draft_scores = self.sample( - logits, self.K, draft_model=draft_model - ) - previous_draft_scores = self.update_draft_tokens_and_scores( - cur_draft_idx=0, - new_draft_tokens=new_draft_tokens, - new_draft_scores=new_draft_scores, - previous_draft_scores=None, - batch_size=batch_size, - attn_metadata=attn_metadata, - ) - self.update_hidden_states(cur_draft_idx=0, batch_size=batch_size, step0_hs=step0_hs) - self._prepare_draft_layer_metadata( - 0, - attn_metadata, - batch_size, - gather_ids, - num_contexts, - num_gens, - num_accepted_tokens, - inputs, - ) - - # Subsequent layers grow the tree. - for layer_idx in range(1, self.max_draft_len): - num_tokens_per_req = layer_idx * self.K - num_infer_tokens = batch_size * num_tokens_per_req - subseq_all_rank_num_tokens = None - if spec_metadata.all_rank_num_seqs is not None: - # Token counts scale with the current tree width. - subseq_all_rank_num_tokens = [ - n * num_tokens_per_req for n in spec_metadata.all_rank_num_seqs - ] - attn_metadata.all_rank_num_tokens = subseq_all_rank_num_tokens - - inp_hs = self._accumulated_hs[:batch_size, :num_tokens_per_req, :].reshape( - num_infer_tokens, -1 - ) - inp_ids = self.draft_tokens_buffer[:batch_size, :num_tokens_per_req].reshape(-1) - inp_pos = self.position_ids_buffer[:batch_size, :num_tokens_per_req].reshape(-1) - layer_inputs = { - "input_ids": inp_ids, - "position_ids": inp_pos, - "hidden_states": inp_hs, - "attn_metadata": attn_metadata, - } - hidden_states = draft_model.mtp_layers[0]( - embed_tokens=draft_model.embed_tokens, - all_rank_num_tokens=subseq_all_rank_num_tokens - or spec_metadata.subseq_all_rank_num_tokens, - **layer_inputs, - ) - - # Take the last K hidden states per request (the new leaves). - hs_reshaped = hidden_states.reshape(batch_size, num_tokens_per_req, -1) - selected_hs = hs_reshaped[:, -self.K :, :].reshape(batch_size * self.K, -1) - logits = draft_model.mtp_layers[0].shared_head( - selected_hs, draft_model.lm_head, attn_metadata, True - ) - - new_draft_tokens, new_draft_scores = self.sample( - logits, self.K, draft_model=draft_model - ) - new_draft_tokens = new_draft_tokens.reshape(batch_size, self.K, self.K) - new_draft_scores = new_draft_scores.reshape(batch_size, self.K, self.K) - - previous_draft_scores = self.update_draft_tokens_and_scores( - cur_draft_idx=layer_idx, - new_draft_tokens=new_draft_tokens, - new_draft_scores=new_draft_scores, - previous_draft_scores=previous_draft_scores, - batch_size=batch_size, - attn_metadata=attn_metadata, - ) - self.update_hidden_states( - cur_draft_idx=layer_idx, - batch_size=batch_size, - hidden_states_to_save=hidden_states, - selected_parents=self._last_selected_parents, - ) - self._prepare_draft_layer_metadata(layer_idx, attn_metadata, batch_size) - - # Resample the final tree and build it into slot_storage. - real_draft_tokens, topk_score_indices = self.resampling_final_draft_tokens(batch_size) - - if spec_tree_manager is not None and num_gens > 0: - self.tree_ops_converter.build_dynamic_tree( - history_draft_tokens_parent_buffer=self.history_draft_tokens_parent_buffer[ - num_contexts:batch_size - ], - topk_score_indices=topk_score_indices[num_contexts:], - tree_mask=spec_tree_manager.spec_dec_packed_mask[:num_gens], - positions=spec_tree_manager.spec_dec_position_offsets[:num_gens], - retrieve_index=spec_tree_manager.retrieve_index[:num_gens], - retrieve_next_token=spec_tree_manager.retrieve_next_token[:num_gens], - retrieve_next_sibling=spec_tree_manager.retrieve_next_sibling[:num_gens], - use_packed_mask=True, - ) - slot_storage = spec_tree_manager.slot_storage - gen_slots = slot_storage.all_ids_buf[num_contexts:batch_size] - spec_tree_manager.scatter_to_slot_storage(slot_storage, gen_slots, num_gens) - - return real_draft_tokens - - def _prepare_draft_layer_metadata( - self, - cur_draft_idx, - attn_metadata, - batch_size, - gather_ids=None, - num_contexts=0, - num_gens=0, - num_accepted_tokens=None, - inputs=None, - ): - """Set attn_metadata seq_lens/kv_lens for the next draft layer.""" - if cur_draft_idx == 0: - base_pos = inputs["position_ids"][gather_ids] + 1 - self.position_ids_buffer[:batch_size, : self.K] = base_pos.unsqueeze(1).expand( - -1, self.K - ) - - attn_metadata._seq_lens[:batch_size].fill_(self.K) - attn_metadata._seq_lens_cuda[:batch_size].fill_(self.K) - attn_metadata.on_update() - - if inputs["attn_metadata"].kv_cache_manager is not None: - attn_metadata.host_request_types[: attn_metadata.num_contexts].fill_(1) - attn_metadata.num_contexts = 0 - - if hasattr(attn_metadata, "kv_lens_cuda"): - # Rewind only unaccepted verify tokens; draft KV is added later. - if num_gens > 0: - attn_metadata.kv_lens_cuda[num_contexts:batch_size] -= ( - self._max_path_len - ) - num_accepted_tokens[num_contexts:batch_size] - attn_metadata.kv_lens_cuda[:batch_size] += self.K - attn_metadata.use_spec_decoding = True - self._refresh_blackwell_tree_mask_metadata(attn_metadata) - else: - num_tokens_previous_layer = cur_draft_idx * self.K - num_tokens_current_layer = self.K * (cur_draft_idx + 1) - prev_pos = self.position_ids_buffer[:batch_size, :num_tokens_previous_layer] - self.position_ids_buffer[ - :batch_size, num_tokens_previous_layer:num_tokens_current_layer - ] = prev_pos[:, -self.K :] + 1 - attn_metadata._seq_lens[:batch_size].fill_(num_tokens_current_layer) - attn_metadata._seq_lens_cuda[:batch_size].fill_(num_tokens_current_layer) - attn_metadata.on_update() - if hasattr(attn_metadata, "kv_lens_cuda"): - attn_metadata.kv_lens_cuda[:batch_size] += self.K - self._refresh_blackwell_tree_mask_metadata(attn_metadata) - - -class MTPEagleDynamicTreeResourceManager(BaseResourceManager): - """Resource manager for MTP dynamic-tree mode.""" - - hidden_states: Optional[torch.Tensor] = None - - def __init__( - self, - config: "MTPDecodingConfig", - dtype: torch.dtype, - hidden_size: int, - max_num_requests: int, - sa_manager=None, - ): - from .spec_tree_manager import SpecTreeManager - - self.max_num_requests = max_num_requests - self.spec_tree_manager = SpecTreeManager( - max_num_requests=max_num_requests, - use_dynamic_tree=True, - max_draft_len=config.max_draft_len, - max_total_draft_tokens=config.tokens_per_gen_step - 1, - eagle_choices=None, - dynamic_tree_max_topK=config.dynamic_tree_max_topK, - ) - # MTP hidden-state slot pools (needed by MTPEagleWorker drafter inputs). - self._mtp_hidden_states_manager = MTPHiddenStatesManager( - config, dtype, hidden_size, max_num_requests, sa_manager=sa_manager - ) - - # Expose the MTPHiddenStatesManager surface MTPSpecMetadata expects. - @property - def slot_manager(self): - return self._mtp_hidden_states_manager.slot_manager - - @property - def mtp_past_hidden_states_pool(self): - return self._mtp_hidden_states_manager.mtp_past_hidden_states_pool - - @property - def mtp_past_tokens_pool(self): - return self._mtp_hidden_states_manager.mtp_past_tokens_pool - - @property - def sa_manager(self): - return self._mtp_hidden_states_manager.sa_manager - - def prepare_resources(self, scheduled_batch: ScheduledRequests): - self._mtp_hidden_states_manager.prepare_resources(scheduled_batch) - - def update_resources(self, scheduled_batch: ScheduledRequests): - self._mtp_hidden_states_manager.update_resources(scheduled_batch) - - def free_resources(self, request: LlmRequest): - # Clear tree validity for the freed slot, then free the MTP slot. - if request.py_seq_slot is not None: - self.spec_tree_manager.slot_storage.mark_invalid(request.py_seq_slot) - self._mtp_hidden_states_manager.free_resources(request) - - def add_dummy_requests(self, request_ids: List[int]): - # Dummies still need MTP hidden-state slots. - self._mtp_hidden_states_manager.add_dummy_requests(request_ids) - - def shutdown(self): - self._mtp_hidden_states_manager.shutdown() - - def get_max_resource_count(self) -> int: - return self.max_num_requests - - def get_needed_resource_to_completion(self, request: LlmRequest): - return 0 diff --git a/tensorrt_llm/_torch/speculative/pard.py b/tensorrt_llm/_torch/speculative/pard.py index 603fa26639c6..34a4509314cd 100644 --- a/tensorrt_llm/_torch/speculative/pard.py +++ b/tensorrt_llm/_torch/speculative/pard.py @@ -91,12 +91,6 @@ def __init__( self.sa_enhancer: Optional[SADraftEnhancer] = None if getattr(spec_config, "sa_config", None) is not None: self.sa_enhancer = SADraftEnhancer(spec_config.sa_config.threshold) - # Deferred kv_lens_cuda rewind state (see _prepare_kv_for_draft_forward, - # _apply_kv_rewind_after_draft, _ensure_spec_dec_state_restored). - self._kv_rewind_pending = False - self._kv_rewind_amount = None - self._kv_rewind_nc = None - self._kv_rewind_bs = None logger.info( f"PARDWorker initialized with use_separate_draft_kv_cache={use_separate_draft_kv_cache}" ) @@ -150,7 +144,6 @@ def _prepare_kv_for_draft_forward( if batch_size > num_contexts: attn_metadata.kv_lens_cuda[num_contexts:batch_size] += 1 - self._kv_rewind_pending = True attn_metadata.update_for_spec_dec() @@ -162,32 +155,17 @@ def _apply_kv_rewind_after_draft(self, attn_metadata, spec_metadata): by prepare_for_spec_dec) to avoid cumulative shrinkage. Applied during capture and normal inference. """ - self._kv_rewind_pending = False is_warmup = spec_metadata.is_cuda_graph and not torch.cuda.is_current_stream_capturing() if is_warmup: - # kv_lens_cuda was saved by prepare_for_spec_dec in this mode and - # is restored wholesale, so no rewind is needed. return - if self._kv_rewind_amount is not None and hasattr(attn_metadata, "kv_lens_cuda"): + if hasattr(self, "_kv_rewind_amount") and hasattr(attn_metadata, "kv_lens_cuda"): nc = self._kv_rewind_nc bs = self._kv_rewind_bs attn_metadata.kv_lens_cuda[nc:bs] -= self._kv_rewind_amount attn_metadata.kv_lens_cuda[nc:bs].clamp_(min=0) - def _ensure_spec_dec_state_restored(self, attn_metadata, spec_metadata): - # Restore first (in warmup mode kv_lens_cuda was saved and comes back - # wholesale), then apply any pending rewind for the other modes so a - # failed draft forward does not leave kv_lens_cuda incremented. - super()._ensure_spec_dec_state_restored(attn_metadata, spec_metadata) - if ( - getattr(self, "_kv_rewind_pending", False) - and attn_metadata is not None - and spec_metadata is not None - ): - self._apply_kv_rewind_after_draft(attn_metadata, spec_metadata) - - def _forward_impl( + def forward( self, input_ids, position_ids, @@ -218,8 +196,25 @@ def _forward_impl( self._execute_guided_decoder_if_present(logits) - accepted_tokens, num_accepted_tokens = self.sample_and_accept_draft_tokens( - logits, attn_metadata, spec_metadata + # draft_tokens buffer has (2K-1) entries per gen request; extract the K real drafts + if num_gens > 0: + draft_tokens = spec_metadata.draft_tokens[: num_gens * (2 * K - 1)] + draft_tokens = draft_tokens.reshape(num_gens, 2 * K - 1)[:, :K] + else: + draft_tokens = spec_metadata.draft_tokens.reshape(0, K) + + # logits have 2K entries per gen request; extract K+1 for acceptance + if num_gens > 0: + ctx_logits = logits[:num_contexts] + vocab_size = logits.shape[-1] + gen_logits_2k = logits[num_contexts:].reshape(num_gens, 2 * K, vocab_size) + gen_logits_kp1 = gen_logits_2k[:, : K + 1, :].reshape(-1, vocab_size) + logits_for_accept = torch.cat([ctx_logits, gen_logits_kp1], dim=0) + else: + logits_for_accept = logits + + accepted_tokens, num_accepted_tokens = self._sample_and_accept_draft_tokens_base( + logits_for_accept, draft_tokens, num_contexts, batch_size, spec_metadata ) # Pad accepted_tokens from (batch, K+1) to (batch, 2K) to match sampler buffer @@ -290,12 +285,14 @@ def _forward_impl( vocab_size = gen_logits.shape[-1] gen_logits = gen_logits.reshape(num_gens, K, vocab_size) - gen_draft_tokens = self.sample_draft_tokens( - gen_logits, - spec_metadata, - batch_size, - num_contexts=num_contexts, - ) + # Use torch.argmax directly to avoid cute_argmax stride issues + d2t = getattr(draft_model.model, "d2t", None) + gen_draft_tokens = torch.argmax(gen_logits, dim=-1, keepdim=False).long() + + if d2t is not None: + gen_draft_tokens = d2t[gen_draft_tokens] + gen_draft_tokens + + gen_draft_tokens = gen_draft_tokens.type(torch.int32) if self.sa_enhancer is not None and sa_manager is not None: gen_draft_tokens = self.sa_enhancer.maybe_override_all_draft_tokens( @@ -317,12 +314,6 @@ def _forward_impl( else: gen_draft_tokens = torch.empty((0, 2 * K - 1), dtype=torch.int32, device="cuda") - # Context requests are not drafted by the block worker (zero placeholder - # token); fill their draft-prob slot rows with a one-hot placeholder so - # they are a legal distribution when they become gen requests next iter. - gen_vocab = vocab_size if num_gens > 0 else None - self.write_context_onehot_draft_probs(spec_metadata, num_contexts, num_gens, K, gen_vocab) - if num_contexts > 0 and num_gens > 0: ctx_draft_tokens = torch.zeros( (num_contexts, 2 * K - 1), dtype=torch.int32, device="cuda" @@ -356,26 +347,23 @@ def _forward_impl( "next_new_tokens": next_new_tokens, } - def _reshape_draft_tokens_for_accept(self, spec_metadata, num_gens, device): - # draft_tokens buffer has (2K-1) entries per gen request; extract the K real drafts. - K = spec_metadata.runtime_draft_len - if num_gens > 0: - draft_tokens = spec_metadata.draft_tokens[: num_gens * (2 * K - 1)] - return draft_tokens.reshape(num_gens, 2 * K - 1)[:, :K] - # Context-only batch: return an empty view without reshaping the - # (possibly preallocated, non-empty) draft_tokens buffer. - return torch.empty((0, K), dtype=torch.int32, device=device) - - def _reshape_logits_for_accept(self, logits, num_contexts, num_gens, spec_metadata): - # logits have 2K entries per gen request; extract K+1 for acceptance. - if num_gens == 0: - return logits - K = spec_metadata.runtime_draft_len - ctx_logits = logits[:num_contexts] - vocab_size = logits.shape[-1] - gen_logits_2k = logits[num_contexts:].reshape(num_gens, 2 * K, vocab_size) - gen_logits_kp1 = gen_logits_2k[:, : K + 1, :].reshape(-1, vocab_size) - return torch.cat([ctx_logits, gen_logits_kp1], dim=0) + def draft_decoder( + self, + logits: torch.Tensor, + draft_model: nn.Module, + ): + """ + Sample draft tokens using greedy decoding. + + Args: + logits: [num_tokens, vocab_size] from the draft model. + draft_model: The draft model (used to read the d2t mapping). + + Returns: + draft_tokens: [batch_size * max_draft_len] flattened token ids. + """ + d2t = getattr(draft_model.model, "d2t", None) + return self._draft_sampler_greedy(logits, d2t) def prepare_1st_drafter_inputs( self, diff --git a/tensorrt_llm/_torch/speculative/sa_worker.py b/tensorrt_llm/_torch/speculative/sa_worker.py index 07023fe6d661..90cd617cef2a 100644 --- a/tensorrt_llm/_torch/speculative/sa_worker.py +++ b/tensorrt_llm/_torch/speculative/sa_worker.py @@ -119,7 +119,7 @@ def __init__(self, spec_config: "SADecodingConfig", model_config=None): def max_draft_len(self) -> int: return self._max_draft_len - def _forward_impl( + def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, diff --git a/tensorrt_llm/_torch/speculative/spec_tree_manager.py b/tensorrt_llm/_torch/speculative/spec_tree_manager.py index 545b5bc7bb06..74dd622826f1 100644 --- a/tensorrt_llm/_torch/speculative/spec_tree_manager.py +++ b/tensorrt_llm/_torch/speculative/spec_tree_manager.py @@ -16,40 +16,24 @@ class DynamicTreeSlotStorage: Buffers are [S, ...] where S = num_slots + 1 (+1 for CUDA graph dummy). """ - def __init__(self, - num_slots: int, - n_dt: int, - mask_width: int, - top_k: int = 1): + def __init__(self, num_slots: int, n_dt: int, mask_width: int): S = num_slots + 1 self.dummy_slot_id = num_slots - # Bootstrap/reused slots may not have a tree yet; keep their metadata - # as a valid linear chain so verification kernels can read it directly. - no_tree_position_offsets, no_tree_packed_mask = self._make_kary_tree_metadata( - n_dt, mask_width, top_k=1) - self.position_offsets = no_tree_position_offsets.unsqueeze(0).repeat( - S, 1).contiguous() - self.packed_mask = no_tree_packed_mask.unsqueeze(0).repeat( - S, 1, 1).contiguous() - self._no_tree_position_offsets = no_tree_position_offsets - self._no_tree_packed_mask = no_tree_packed_mask - - # CUDA-graph dummies use a deterministic K-ary tree, matching real - # dynamic-tree mask/position shapes without depending on request state. - dummy_position_offsets, dummy_packed_mask = self._make_kary_tree_metadata( - n_dt, mask_width, top_k) - self.position_offsets[self.dummy_slot_id] = dummy_position_offsets - self.packed_mask[self.dummy_slot_id] = dummy_packed_mask + # Slot buffers — C++ kernel writes directly via slotIds + self.packed_mask = torch.zeros((S, n_dt, mask_width), + dtype=torch.int32, + device='cuda') + self.position_offsets = torch.zeros((S, n_dt), + dtype=torch.int32, + device='cuda') self.retrieve_index = torch.zeros((S, n_dt), dtype=torch.int32, device='cuda') - - # Mamba verify reads next links unconditionally, so no-tree rows must be - # valid linear chains instead of sentinels. - self._no_tree_next_token = self._make_no_tree_next_token(n_dt) - self.retrieve_next_token = self._no_tree_next_token.unsqueeze(0).repeat( - S, 1) + self.retrieve_next_token = torch.full((S, n_dt), + -1, + dtype=torch.int32, + device='cuda') self.retrieve_next_sibling = torch.full((S, n_dt), -1, dtype=torch.int32, @@ -73,43 +57,6 @@ def __init__(self, dtype=torch.int32, device='cuda') - @staticmethod - def _make_kary_tree_metadata( - n_dt: int, mask_width: int, - top_k: int) -> tuple[torch.Tensor, torch.Tensor]: - top_k = max(int(top_k), 1) - token_ids = torch.arange(n_dt, device='cuda') - parents = torch.where(token_ids > 0, (token_ids - 1) // top_k, - token_ids) - ancestor_chain = torch.empty((n_dt, n_dt), - dtype=torch.long, - device='cuda') - current = token_ids - for depth in range(n_dt): - ancestor_chain[:, depth] = current - current = parents[current] - - # Pack bits directly from the parent chain instead of materializing a - # dense bool mask and repacking it. - valid_ancestors = torch.ones((n_dt, n_dt), - dtype=torch.bool, - device='cuda') - valid_ancestors[:, 1:] = ancestor_chain[:, 1:] != ancestor_chain[:, :-1] - bit_values = (1 << (ancestor_chain % 32)).to(torch.int32) - bit_values.masked_fill_(~valid_ancestors, 0) - packed_mask = torch.zeros((n_dt, mask_width), - dtype=torch.int32, - device='cuda') - packed_mask.scatter_add_(1, ancestor_chain // 32, bit_values) - position_offsets = valid_ancestors.sum(-1).to(torch.int32) - 1 - return position_offsets, packed_mask - - @staticmethod - def _make_no_tree_next_token(n_dt: int) -> torch.Tensor: - next_token = torch.arange(1, n_dt + 1, dtype=torch.int32, device='cuda') - next_token[n_dt - 1] = -1 - return next_token - def fill_all_slot_ids(self, context_requests, generation_requests): """Fill all_ids_buf for full batch [ctx | gen] via one HtoD copy.""" dummy_slot = self.dummy_slot_id @@ -134,12 +81,12 @@ def mark_valid(self, slot_ids, count): self.has_tree.narrow(0, self.dummy_slot_id, 1).fill_(False) def mark_invalid(self, slot_id): - """Clear validity and restore valid no-tree metadata.""" + """Clear validity and reset slot data.""" self.has_tree[slot_id] = False - self.packed_mask[slot_id] = self._no_tree_packed_mask - self.position_offsets[slot_id] = self._no_tree_position_offsets + self.packed_mask[slot_id] = 0 + self.position_offsets[slot_id] = 0 self.retrieve_index[slot_id] = 0 - self.retrieve_next_token[slot_id] = self._no_tree_next_token + self.retrieve_next_token[slot_id] = -1 self.retrieve_next_sibling[slot_id] = -1 def pack_retrieve_from_slots(self, slot_ids, count): @@ -337,7 +284,6 @@ def init_tree_info_for_dynamic_tree(self): num_slots=self.num_trees, n_dt=num_draft_with_root, mask_width=mask_width, - top_k=self.dynamic_tree_max_topK, ) def scatter_to_slot_storage(self, ss, gen_slots, num_gens): diff --git a/tensorrt_llm/_torch/speculative/suffix_automaton.py b/tensorrt_llm/_torch/speculative/suffix_automaton.py index 12bb77208fe4..1eb7be0bcea5 100644 --- a/tensorrt_llm/_torch/speculative/suffix_automaton.py +++ b/tensorrt_llm/_torch/speculative/suffix_automaton.py @@ -87,17 +87,6 @@ class SuffixAutomatonManager(BaseResourceManager): and are CUDA graph compatible. Used as the resource manager for both NGram and MTP+SA speculative decoding. - - Rejection sampling is not supported for NGram or SA drafting. Both are - retrieval-based drafters: they propose draft tokens by matching against - previously seen context (n-gram lookup / suffix-automaton traversal) instead - of sampling from a model head, so they emit only token ids with no per-token - proposal distribution ``q(x)`` over the vocabulary. Rejection sampling's - acceptance test needs ``q`` to form the ratio ``min(1, p(x)/q(x))`` and the - residual ``(p - q)+`` correction on rejection; without ``q`` the target - distribution cannot be preserved. The gate lives in - ``TorchLlmArgs.validate_speculative_config`` (NGram falls outside the - supported-mode whitelist; SA is rejected via ``rs_sa_active``). """ def __init__( diff --git a/tensorrt_llm/_torch/speculative/utils.py b/tensorrt_llm/_torch/speculative/utils.py index 62189035cafb..d7a0e8185da5 100644 --- a/tensorrt_llm/_torch/speculative/utils.py +++ b/tensorrt_llm/_torch/speculative/utils.py @@ -20,7 +20,6 @@ from .draft_target import (DraftTargetOneModelSampler, DraftTargetOneModelSpecMetadata, DraftTargetOneModelWorker) -from .dspark import DSparkSpecMetadata, DSparkWorker from .eagle3 import (Eagle3OneModelDynamicTreeResourceManager, Eagle3OneModelSampler, Eagle3OneModelSpecMetadata, Eagle3OneModelWorker, Eagle3ResourceManager, @@ -28,8 +27,6 @@ from .eagle3_dynamic_tree import Eagle3OneModelDynamicTreeWorker from .model_drafter import ModelDrafter from .mtp import MTPHiddenStatesManager, MTPSampler, MTPSpecMetadata, MTPWorker -from .mtp_dynamic_tree import (MTPEagleDynamicTreeResourceManager, - MTPEagleDynamicTreeWorker) from .ngram import NGramDrafter, NGramPoolManager from .pard import PARDSpecMetadata, PARDWorker from .sa_worker import SASampler, SASpecMetadata, SAWorker @@ -45,37 +42,6 @@ def _is_effective_dynamic_tree(spec_config) -> bool: and getattr(spec_config, 'dynamic_tree_max_topK', 0) > 1) -def _get_draft_vocab_size(spec_config, target_vocab_size: int) -> int: - """Draft-model vocab size, used to decide whether rejection sampling needs - the d2t-expanded ``full_draft_probs`` buffer (only when it differs from the - target vocab). - - Reads the draft model's ``config.json`` from ``spec_config.speculative_model``. - Eagle3 configs store the target vocab in ``vocab_size`` and the reduced head - width in ``draft_vocab_size``, so ``draft_vocab_size`` is read first, falling - back to ``vocab_size`` (or a nested ``text_config.vocab_size``). Returns - ``target_vocab_size`` (shared vocab, no buffer needed) when there is no - separate draft model or the config cannot be read. - """ - draft_dir = getattr(spec_config, "speculative_model", None) - if not draft_dir: - return target_vocab_size - try: - import json - import os - cfg_path = os.path.join(str(draft_dir), "config.json") - if not os.path.isfile(cfg_path): - return target_vocab_size - with open(cfg_path) as f: - cfg = json.load(f) - vs = cfg.get("draft_vocab_size") or cfg.get("vocab_size") - if vs is None: - vs = (cfg.get("text_config") or {}).get("vocab_size") - return int(vs) if vs else target_vocab_size - except (OSError, ValueError, TypeError, AttributeError): - return target_vocab_size - - def get_spec_metadata(spec_config, model_config, max_num_requests, @@ -91,9 +57,6 @@ def get_spec_metadata(spec_config, num_seq_slots = (num_seq_slots if num_seq_slots is not None else max_num_requests) vocab_size = getattr(model_config, "vocab_size", 0) - # Draft-model vocab size, used to gate the d2t-expanded full_draft_probs - # buffer allocation (see SpecMetadata.prepare_rejection_sampling_buffers). - draft_vocab_size = _get_draft_vocab_size(spec_config, vocab_size) if spec_config.spec_dec_mode.is_mtp_eagle_one_model(): # MTP Eagle one-model reuses Eagle3 one-model metadata for the # unified worker/sampler/slot_ids plumbing, but skips per-layer @@ -113,9 +76,7 @@ def get_spec_metadata(spec_config, use_rejection_sampling=use_rejection_sampling, vocab_size=vocab_size, num_seq_slots=num_seq_slots, - draft_vocab_size=draft_vocab_size, spec_resource_manager=spec_resource_manager, - use_dynamic_tree=getattr(spec_config, 'use_dynamic_tree', False), ) if spec_config.spec_dec_mode.is_mtp_vanilla(): return MTPSpecMetadata( @@ -125,9 +86,6 @@ def get_spec_metadata(spec_config, mtp_num_modules=spec_config.max_draft_len, max_num_requests=max_num_requests, mtp_hidden_states_manager=spec_resource_manager, - use_rejection_sampling=use_rejection_sampling, - vocab_size=vocab_size, - draft_vocab_size=draft_vocab_size, ) if spec_config.spec_dec_mode.is_mtp_eagle(): return Eagle3SpecMetadata( @@ -176,7 +134,6 @@ def get_spec_metadata(spec_config, layers_to_capture=spec_config.eagle3_layers_to_capture, use_rejection_sampling=use_rejection_sampling, vocab_size=vocab_size, - draft_vocab_size=draft_vocab_size, spec_resource_manager=spec_resource_manager, use_dynamic_tree=_is_effective_dynamic_tree(spec_config), eagle_choices=spec_config.eagle_choices, @@ -188,9 +145,6 @@ def get_spec_metadata(spec_config, spec_dec_mode=spec_config.spec_dec_mode, max_num_requests=max_num_requests, spec_resource_manager=spec_resource_manager, - use_rejection_sampling=use_rejection_sampling, - vocab_size=vocab_size, - draft_vocab_size=draft_vocab_size, ) if spec_config.spec_dec_mode.is_dflash(): target_layer_ids = getattr(spec_config, 'target_layer_ids', None) @@ -203,24 +157,6 @@ def get_spec_metadata(spec_config, hidden_size=model_config.hidden_size, max_num_tokens=max_num_tokens, dtype=model_config.torch_dtype, - use_rejection_sampling=use_rejection_sampling, - vocab_size=vocab_size, - draft_vocab_size=draft_vocab_size, - ) - if spec_config.spec_dec_mode.is_dspark(): - target_layer_ids = getattr(spec_config, 'target_layer_ids', None) - return DSparkSpecMetadata( - max_draft_len=spec_config.max_draft_len, - max_total_draft_tokens=spec_config.tokens_per_gen_step - 1, - spec_dec_mode=spec_config.spec_dec_mode, - max_num_requests=max_num_requests, - layers_to_capture=target_layer_ids, - hidden_size=model_config.hidden_size, - max_num_tokens=max_num_tokens, - dtype=model_config.torch_dtype, - use_rejection_sampling=use_rejection_sampling, - vocab_size=vocab_size, - draft_vocab_size=draft_vocab_size, ) if spec_config.spec_dec_mode.is_draft_target_one_model(): return DraftTargetOneModelSpecMetadata( @@ -229,9 +165,6 @@ def get_spec_metadata(spec_config, spec_dec_mode=spec_config.spec_dec_mode, max_num_requests=max_num_requests, max_num_tokens=max_num_tokens, - use_rejection_sampling=use_rejection_sampling, - vocab_size=vocab_size, - draft_vocab_size=draft_vocab_size, ) if spec_config.spec_dec_mode.is_save_hidden_states(): return SaveHiddenStatesSpecMetadata( @@ -292,15 +225,6 @@ def get_spec_resource_manager(model_engine, draft_model_engine=None): if sa_cfg is not None: sa_manager = SuffixAutomatonManager(sa_cfg, max_num_requests, max_seq_len) - # Dynamic tree combines SpecTreeManager with MTP hidden-state slots. - if getattr(spec_config, 'use_dynamic_tree', False): - return MTPEagleDynamicTreeResourceManager( - spec_config, - model_config.torch_dtype, - model_config.hidden_size, - max_num_requests, - sa_manager=sa_manager, - ) if spec_config.use_relaxed_acceptance_for_thinking or sa_manager is not None: # Unified resource manager: the unified worker reads # ``relaxed_delta_pool`` from ``Eagle3ResourceManager`` (mirrors the @@ -389,10 +313,7 @@ def get_spec_decoder( # MTP Eagle one-model now uses the same sampler as Eagle3 one-model. return Eagle3OneModelSampler(sampler_args, spec_config=spec_config) if spec_config.spec_dec_mode.is_mtp_vanilla(): - nextn = spec_config.max_draft_len - if getattr(spec_config, "use_dynamic_tree", False): - nextn = spec_config.max_total_draft_tokens - return MTPSampler(sampler_args, nextn=nextn) + return MTPSampler(sampler_args, nextn=spec_config.max_draft_len) if spec_config.spec_dec_mode.is_eagle3( ) or spec_config.spec_dec_mode.is_mtp_eagle(): # TorchSampler handles Eagle3 gracefully, by integrating d2t into the sampling process @@ -475,20 +396,10 @@ def get_spec_worker(spec_config, use_separate_draft_kv_cache: bool = False): spec_dec_mode = spec_config.spec_dec_mode if spec_dec_mode.is_mtp_vanilla(): - return MTPWorker(spec_config, - model_config, - use_separate_draft_kv_cache, - mapping=mapping) + return MTPWorker(spec_config, model_config, use_separate_draft_kv_cache) if spec_dec_mode.is_mtp_eagle_one_model(): - if getattr(spec_config, 'use_dynamic_tree', False): - return MTPEagleDynamicTreeWorker(spec_config, - model_config, - use_separate_draft_kv_cache, - mapping=mapping) - return MTPEagleWorker(spec_config, - model_config, - use_separate_draft_kv_cache, - mapping=mapping) + return MTPEagleWorker(spec_config, model_config, + use_separate_draft_kv_cache) if spec_dec_mode.is_eagle3_one_model(): if _is_effective_dynamic_tree(spec_config): return Eagle3OneModelDynamicTreeWorker(spec_config, mapping, @@ -501,8 +412,6 @@ def get_spec_worker(spec_config, return PARDWorker(spec_config, mapping, use_separate_draft_kv_cache) if spec_dec_mode.is_dflash(): return DFlashWorker(spec_config, mapping, use_separate_draft_kv_cache) - if spec_dec_mode.is_dspark(): - return DSparkWorker(spec_config, mapping, use_separate_draft_kv_cache) if spec_dec_mode.is_sa(): return SAWorker(spec_config, model_config) if spec_dec_mode.is_draft_target_one_model(): @@ -574,8 +483,7 @@ def update_spec_config_from_model_config(spec_config, model_config): f"using max_draft_len={effective_draft_len} draft tokens.") spec_config.max_draft_len = effective_draft_len - if not spec_config.use_dynamic_tree: - spec_config.max_total_draft_tokens = spec_config.max_draft_len + spec_config.max_total_draft_tokens = spec_config.max_draft_len def update_spec_config_from_loaded_model(spec_config, model) -> None: diff --git a/tensorrt_llm/_torch/tensor_lru_cache.py b/tensorrt_llm/_torch/tensor_lru_cache.py deleted file mode 100644 index 2d94e75a1868..000000000000 --- a/tensorrt_llm/_torch/tensor_lru_cache.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2026 NVIDIA CORPORATION & AFFILIATES -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from collections import OrderedDict -from collections.abc import Hashable -from dataclasses import dataclass -from threading import RLock -from typing import Generic, NamedTuple, TypeVar - -import torch - -from tensorrt_llm.logger import logger - -K = TypeVar("K", bound=Hashable) - - -class _Entry(NamedTuple): - value: torch.Tensor - size_bytes: int - - -class TensorLRUCacheStats(NamedTuple): - max_bytes: int - current_bytes: int - item_count: int - hits: int - misses: int - insertions: int - replacements: int - evictions: int - rejected_insertions: int - hit_rate: float - - -@dataclass -class _CacheCounters: - hits: int = 0 - misses: int = 0 - insertions: int = 0 - replacements: int = 0 - evictions: int = 0 - rejected_insertions: int = 0 - - @property - def hit_rate(self) -> float: - total_gets = self.hits + self.misses - return self.hits / total_gets if total_gets else 0.0 - - -class TensorLRUCache(Generic[K]): - """Thread-safe LRU cache from hashable keys to tensor values. - - Size accounting uses logical tensor bytes: `tensor.numel() * tensor.element_size()`. - Returned tensors alias the cache-owned tensor objects. Callers must treat them as immutable - while they remain cache-owned. - - The cache owns a detached copy rather than the caller's tensor or view. This prevents later - caller mutations from changing a cached value and prevents a small cached view from retaining - the caller's larger backing allocation. The copy is made before acquiring the lock and before - replacement or eviction, preserving existing cache entries if copying fails. Consequently, - `max_bytes` bounds steady-state logical cache contents, not peak allocation: insertion - temporarily needs both the source tensor and its copy and may exceed the cache limit until - eviction completes. - - Args: - max_bytes: Maximum logical tensor bytes held by this cache. - name: Short label used in debug log messages. - """ - - def __init__( - self, - max_bytes: int, - *, - name: str = "tensor_lru_cache", - ) -> None: - if max_bytes <= 0: - raise ValueError("max_bytes must be positive") - - self._max_bytes = max_bytes - self._name = name - self._current_bytes = 0 - self._items: OrderedDict[K, _Entry] = OrderedDict() - self._lock = RLock() - self._counters = _CacheCounters() - - @property - def max_bytes(self) -> int: - return self._max_bytes - - @property - def current_bytes(self) -> int: - with self._lock: - return self._current_bytes - - def __len__(self) -> int: - with self._lock: - return len(self._items) - - def get(self, key: K) -> torch.Tensor | None: - """Return a cache-owned, immutable tensor and promote it to most-recently-used. - - The returned tensor aliases the cached value. Callers must not mutate it. - """ - with self._lock: - entry = self._items.get(key) - if entry is None: - self._counters.misses += 1 - return None - - self._counters.hits += 1 - self._items.move_to_end(key) - return entry.value - - def put(self, key: K, value: torch.Tensor) -> bool: - """Insert or replace a tensor. - - Returns `False` and leaves the cache unchanged when `value` is larger than the full - cache capacity. - """ - size_bytes = self._tensor_size_bytes(value) - - if size_bytes > self._max_bytes: - with self._lock: - self._counters.rejected_insertions += 1 - logger.debug( - f"{self._name}: rejected oversized tensor insertion, size_bytes={size_bytes}, " - f"max_bytes={self._max_bytes}" - ) - return False - - stored_value = value.detach().clone() - - with self._lock: - old_entry = self._items.pop(key, None) - if old_entry is not None: - self._current_bytes -= old_entry.size_bytes - self._counters.replacements += 1 - else: - self._counters.insertions += 1 - - self._items[key] = _Entry(value=stored_value, size_bytes=size_bytes) - self._current_bytes += size_bytes - - evicted_count, evicted_bytes = self._evict_until_within_limit() - if evicted_count: - self._counters.evictions += evicted_count - logger.debug( - f"{self._name}: evicted {evicted_count} LRU entries, " - f"freed_bytes={evicted_bytes}, current_bytes={self._current_bytes}, " - f"max_bytes={self._max_bytes}" - ) - return True - - def pop(self, key: K) -> torch.Tensor | None: - """Remove one key and return its tensor, or `None` on miss.""" - with self._lock: - entry = self._items.pop(key, None) - if entry is None: - return None - - self._current_bytes -= entry.size_bytes - return entry.value - - def clear(self) -> None: - with self._lock: - self._items.clear() - self._current_bytes = 0 - - def stats(self) -> TensorLRUCacheStats: - with self._lock: - return TensorLRUCacheStats( - max_bytes=self._max_bytes, - current_bytes=self._current_bytes, - item_count=len(self._items), - hits=self._counters.hits, - misses=self._counters.misses, - insertions=self._counters.insertions, - replacements=self._counters.replacements, - evictions=self._counters.evictions, - rejected_insertions=self._counters.rejected_insertions, - hit_rate=self._counters.hit_rate, - ) - - def log_stats(self, reason: str) -> None: - stats = self.stats() - logger.debug( - f"{self._name}: stats after {reason}: items={stats.item_count}, " - f"bytes={stats.current_bytes}/{stats.max_bytes}, hits={stats.hits}, " - f"misses={stats.misses}, hit_rate={stats.hit_rate:.3f}, " - f"insertions={stats.insertions}, replacements={stats.replacements}, " - f"evictions={stats.evictions}, rejected_insertions={stats.rejected_insertions}" - ) - - @staticmethod - def _tensor_size_bytes(tensor: torch.Tensor) -> int: - return tensor.numel() * tensor.element_size() - - def _evict_until_within_limit(self) -> tuple[int, int]: - evicted_count = 0 - evicted_bytes = 0 - while self._current_bytes > self._max_bytes: - _, entry = self._items.popitem(last=False) - self._current_bytes -= entry.size_bytes - evicted_count += 1 - evicted_bytes += entry.size_bytes - return evicted_count, evicted_bytes diff --git a/tensorrt_llm/_torch/utils.py b/tensorrt_llm/_torch/utils.py index ba0453fed790..beb6c1c9f075 100644 --- a/tensorrt_llm/_torch/utils.py +++ b/tensorrt_llm/_torch/utils.py @@ -4,7 +4,7 @@ import threading from dataclasses import dataclass from enum import Enum, IntEnum -from typing import Dict, List, Optional +from typing import Dict, List import torch from torch.nn import functional as F @@ -38,11 +38,6 @@ ) -def is_gdn_replay_enabled() -> bool: - """Return whether GDN replay was explicitly enabled.""" - return os.environ.get("TRTLLM_USE_GDN_REPLAY", "0") == "1" - - # IMPORTANT: Keep the same order of activation functions in this enum and the enum in # cpp/tensorrt_llm/kernels/cutlass_kernels/include/common.h class ActivationType(IntEnum): @@ -165,13 +160,6 @@ class Fp4QuantizedTensor: fp4_tensor: torch.Tensor scaling_factor: torch.Tensor is_sf_swizzled: bool = True - # Optional un-quantized (BF16/FP16) hidden-state view of the same logical - # activation. When the FP4 tensor is produced by a fused - # (add+)RMSNorm+NVFP4-quant that also returns the un-quantized - # (post-RMSNorm) value, this carries that tensor so downstream consumers - # needing the un-quantized form (e.g. DSv3.2's DSA indexer at - # sparse/dsa.py:pre_indexer_proj) can use it without dequantizing FP4. - unquantized_hidden_states: Optional[torch.Tensor] = None @property def shape(self): diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/__init__.py b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/__init__.py index 9b70421c3b81..292be3036d5b 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/__init__.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/__init__.py @@ -15,7 +15,7 @@ """ CuTe DSL attention backend family for visual generation models. - fmha.py — CuTeDSLAttention (dense and blockscaled JIT FMHA) + fmha.py — CuTeDSLAttention (dense cubin path, head_dim=128) vsa.py — VSAAttention (Video Sparse Attention, CuTe JIT + SDPA fallback) """ diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/fmha.py b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/fmha.py index b0e5b882afdc..c15f8b2ac47d 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/fmha.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/cute_dsl/fmha.py @@ -15,568 +15,40 @@ """ CuTe DSL (NVIDIA kernels) Dense FMHA Backend for Visual Generation Models -JIT-compiles the dense FMHA kernel and caches the compiled artifact for each kernel configuration. -Expects NHD layout ([B, S, H, D]) and supports float16/bfloat16 inputs. The VSA sparse path uses -VSAAttention from vsa.py instead. +Uses pre-compiled cubins derived from CUTLASS CuTe DSL FMHA. +Expects NHD layout ([B, S, H, D]) and supports float16/bfloat16. +For the VSA sparse path use VSAAttention in vsa.py. """ import math -from typing import Any, NamedTuple, Tuple +from typing import Optional, Tuple import torch -from tensorrt_llm.logger import logger from tensorrt_llm.visual_gen.args import QuantAttentionConfig from ....attention_backend.interface import PredefinedAttentionMask from ..interface import AttentionBackend, AttentionTensorLayout -_cute_dsl_import_error: BaseException | None = None +_cute_dsl_import_error = None try: - import cutlass - from cuda.bindings import driver as cuda_driver - from cutlass import cute - from cutlass.cute import typing as cute_typing - from cutlass.cute.runtime import from_dlpack - - from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention import ( - BlackwellFusedMultiHeadAttentionForward, - BlackwellFusedMultiHeadBlockScaledAttentionForward, + import tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention as cute_dsl + from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.blackwell.attention.fmha import ( + _cute_runtime_import_error, ) - from tensorrt_llm._torch.visual_gen.cute_dsl_kernels.helpers import fmha_helpers as fmha_utils + + if _cute_runtime_import_error is not None: + raise ImportError(_cute_runtime_import_error) except (ImportError, OSError) as e: - cutlass = None - cuda_driver = None - cute = None - cute_typing = None - from_dlpack = None - BlackwellFusedMultiHeadAttentionForward = None - BlackwellFusedMultiHeadBlockScaledAttentionForward = None - fmha_utils = None + cute_dsl = None _cute_dsl_import_error = e -SUPPORTED_GPU_ARCHS: Tuple[str, ...] = ("sm_100a", "sm_103a") - - -# ============================================================================ -# Runtime helpers -# ============================================================================ - - -def _check_cute_runtime_available() -> None: - if _cute_dsl_import_error is None: - return - raise ImportError( - f"CuTe DSL runtime is not available. Import error: {_cute_dsl_import_error}" - ) from _cute_dsl_import_error - - -def _get_gpu_arch(device: torch.device | None = None) -> str: - capability = torch.cuda.get_device_capability(device) - gpu_arch = f"sm_{capability[0]}{capability[1]}a" - if gpu_arch not in SUPPORTED_GPU_ARCHS: - supported = ", ".join(SUPPORTED_GPU_ARCHS) - raise ValueError( - f"Unsupported GPU architecture {gpu_arch}. Supported architectures: {supported}." - ) - return gpu_arch - - -def _validate_inputs( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - o: torch.Tensor, -) -> None: - """Validate the (B, S, H, D) layout the backend feeds the kernel.""" - if not (q.dim() == k.dim() == v.dim() == o.dim() == 4): - raise ValueError("FMHA expects 4D (B, S, H, D) Q/K/V/O tensors.") - if q.dtype != k.dtype: - raise ValueError(f"Q/K dtype mismatch: {q.dtype} vs {k.dtype}") - if q.shape[0] != k.shape[0]: - raise ValueError(f"Batch size mismatch: q={q.shape[0]} vs k={k.shape[0]}") - if q.shape[-1] != k.shape[-1]: - raise ValueError(f"Q/K head dim mismatch: {q.shape[-1]} vs {k.shape[-1]}") - if k.shape[:-1] != v.shape[:-1]: - raise ValueError(f"K/V shape mismatch: {k.shape[:-1]} vs {v.shape[:-1]}") - expected_o_shape = (*q.shape[:-1], v.shape[-1]) - if tuple(o.shape) != expected_o_shape: - raise ValueError(f"Output shape mismatch: {tuple(o.shape)} vs {expected_o_shape}") - - -def _to_cute_tensor(tensor: torch.Tensor, leading_dim: int, cutlass_element_type=None): - """Wrap a torch tensor as a CuTe tensor. - - For sub-byte / non-torch-dtype elements (FP4 packed as uint8, MXFP8 SF exponents stored as - uint8), pass `cutlass_element_type` to override the interpretation; the tensor storage must be - byte-addressable (uint8 / int8). Otherwise the FP8-e4m3 path and the default - direct-from-dlpack path apply. - """ - # Match cutlass.torch.cute_tensor_like: set element_type BEFORE mark_layout_dynamic so the - # layout transformation sees the override-typed tensor and carries it forward. - if cutlass_element_type is not None: - cute_tensor = from_dlpack(tensor.view(torch.int8), assumed_align=16) - cute_tensor = cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) - cute_tensor.element_type = cutlass_element_type - return cute_tensor - if tensor.dtype == torch.float8_e4m3fn: - cute_tensor = from_dlpack(tensor.view(torch.int8), assumed_align=16) - cute_tensor = cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) - cute_tensor.element_type = cutlass.Float8E4M3FN - return cute_tensor - return from_dlpack(tensor, assumed_align=16).mark_layout_dynamic(leading_dim=leading_dim) - - -# ============================================================================ -# JIT compile + cache -# ============================================================================ - - -def _torch_to_cutlass_dtype(t: torch.dtype): - table = { - torch.float16: cutlass.Float16, - torch.bfloat16: cutlass.BFloat16, - torch.float32: cutlass.Float32, - torch.float8_e4m3fn: cutlass.Float8E4M3FN, - } - try: - return table[t] - except KeyError as exc: - raise ValueError(f"Unsupported torch dtype for CuTe DSL FMHA: {t}") from exc - - -class _CacheKey(NamedTuple): - qk_cutlass_dtype: Any - pv_cutlass_dtype: Any - out_cutlass_dtype: Any - qk_acc_dtype: Any - pv_acc_dtype: Any - head_dim: int - head_dim_v: int - mma_tiler_mn: Tuple[int, int] - qk_sf_vec: int # 0 = dense Q/K; 32 = MXFP8; 16 = NVFP4 - is_persistent: bool - mask_type: Any # fmha_utils.MaskEnum - with_lse: bool - with_sink: bool - with_scale_v_channels: bool - has_window: bool - has_skip_softmax: bool - use_tma_store: bool - enable_ex2_emulation: bool - enable_skip_correction: bool - gpu_arch_str: str - - -_COMPILE_CACHE: dict = {} - - -def clear_cute_dsl_fmha_cache() -> None: - """Drop all compiled CuTe DSL FMHA kernels (for tests / teardown).""" - _COMPILE_CACHE.clear() - - -def _get_or_compile(key: _CacheKey, compile_args: tuple): - cached = _COMPILE_CACHE.get(key) - if cached is not None: - return cached - hd, hd_v = key.head_dim, key.head_dim_v - head_dim_arg = hd if hd == hd_v else (hd, hd_v) - if key.qk_sf_vec != 0: - fmha = BlackwellFusedMultiHeadBlockScaledAttentionForward( - key.qk_acc_dtype, - key.pv_acc_dtype, - key.mma_tiler_mn, - head_dim_arg, - key.is_persistent, - key.mask_type, - key.enable_ex2_emulation, - key.enable_skip_correction, - key.qk_sf_vec, - use_tma_store=key.use_tma_store, - ) - else: - fmha = BlackwellFusedMultiHeadAttentionForward( - key.qk_acc_dtype, - key.pv_acc_dtype, - key.mma_tiler_mn, - head_dim_arg, - key.is_persistent, - key.mask_type, - key.enable_ex2_emulation, - key.enable_skip_correction, - use_tma_store=key.use_tma_store, - ) - logger.info( - f"Compiling CuTe DSL FMHA kernel for {key.qk_cutlass_dtype.__name__}/" - f"{key.pv_cutlass_dtype.__name__} head_dim={key.head_dim} " - f"mask={key.mask_type.name} persistent={key.is_persistent} " - f"lse={key.with_lse} on {key.gpu_arch_str} ..." - f"qk_sf_vec={key.qk_sf_vec} " - ) - compiled = cute.compile(fmha, *compile_args) - _COMPILE_CACHE[key] = compiled - return compiled - - -@torch.compiler.disable -def cute_dsl_fmha_fwd( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - o: torch.Tensor, - *, - is_causal: bool = False, - sm_scale: float | None = None, - window_left: int = -1, - window_right: int = -1, - lse: torch.Tensor | None = None, - scale_q: float | torch.Tensor = 1.0, - scale_k: float | torch.Tensor = 1.0, - scale_v: float | torch.Tensor = 1.0, - scale_v_channels: torch.Tensor | None = None, - scale_o: float | torch.Tensor = 1.0, - is_persistent: bool = True, - skip_softmax_threshold_scale_factor: float | None = None, - qk_sf_vec: int = 0, - q_sf: torch.Tensor | None = None, - k_sf: torch.Tensor | None = None, - qk_cutlass_dtype: Any = None, -) -> None: - """JIT-compile (or fetch from cache) and launch the CuTe DSL FMHA kernel. - - Expects contiguous 4D (B, S, H, D) Q/K/V/O tensors with uniform per-batch sequence lengths. - Varlen via indptr is intentionally not exposed — callers pack uniform-length sequences. - - When `qk_sf_vec` is non-zero, dispatches to the block-scaled kernel class: - 32 selects MXFP8 (Q/K stored as FP8 e4m3, SFs as Float8E8M0FNU uint8 storage); - 16 selects NVFP4 (Q/K stored as packed FP4 in torch.uint8, SFs as Float8E4M3FN in uint8 storage). - """ - _check_cute_runtime_available() - _validate_inputs(q, k, v, o) - if qk_sf_vec != 0: - if q_sf is None or k_sf is None: - raise ValueError("Block-scaled path (qk_sf_vec != 0) requires q_sf and k_sf tensors.") - if not q_sf.is_contiguous() or not k_sf.is_contiguous(): - raise ValueError("q_sf and k_sf must be contiguous.") - elif scale_v_channels is not None: - raise ValueError("scale_v_channels is only supported by MXFP8 and NVFP4 kernels.") - - # The kernel hard-codes dense strides in its CuTe layout (fmha.py:447-461) and ignores the - # input tensor's actual strides, so non-dense inputs (e.g. `qkv.split(...)` views) would - # be read at wrong offsets. .contiguous() is a no-op when the tensor is already dense. - q = q.contiguous() - k = k.contiguous() - v = v.contiguous() - if not o.is_contiguous(): - raise ValueError("Output tensor `o` must be contiguous (writes happen in place).") - if lse is not None and not lse.is_contiguous(): - raise ValueError("LSE tensor must be contiguous (writes happen in place).") - - # Delay scalar extraction to inside @torch.compiler.disable decoration - def _scalar_float(t): - if isinstance(t, torch.Tensor): - return t.item() - else: - return t - - scale_q = _scalar_float(scale_q) - scale_k = _scalar_float(scale_k) - scale_v = _scalar_float(scale_v) - scale_o = _scalar_float(scale_o) - - # Reshape (B, S, H, D) → (B, S, h_kv, h_r, D) for Q/O and (B, S, h_kv, 1, D) for K/V; LSE - # goes (B, S, H) → (B, S, h_kv, h_r). Layout matches fmha.py:run() (no head_dim split). - # For NVFP4 (qk_cutlass_dtype == Float4E2M1FN), Q/K's storage last-dim is head_dim//2. - # We keep the packed shape through the view; the kernel's FP4 element_type override resolves - # element-unit strides to the right byte addresses. - batch_size, seq_len_q, num_heads_q, qk_storage_dim = q.shape - _, seq_len_kv, num_heads_kv, _ = k.shape - value_head_dim = v.shape[-1] - num_head_groups = num_heads_q // num_heads_kv - is_fp4 = qk_cutlass_dtype is cutlass.Float4E2M1FN - head_dim = qk_storage_dim * 2 if is_fp4 else qk_storage_dim - if qk_sf_vec != 0 and head_dim != 128: - raise ValueError( - f"MXFP8 / NVFP4 (qk_sf_vec={qk_sf_vec}) currently requires head_dim=128, " - f"got head_dim={head_dim}." - ) - if scale_v_channels is not None: - expected_scale_shape = (num_heads_kv, value_head_dim) - if tuple(scale_v_channels.shape) != expected_scale_shape: - raise ValueError( - f"scale_v_channels must have shape {expected_scale_shape}; " - f"got {tuple(scale_v_channels.shape)}." - ) - if scale_v_channels.dtype != torch.float32: - raise ValueError("scale_v_channels must use torch.float32.") - if scale_v_channels.device != v.device: - raise ValueError("scale_v_channels must be on the same device as V.") - if not scale_v_channels.is_contiguous(): - raise ValueError("scale_v_channels must be contiguous.") - - q_5d = q.view(batch_size, seq_len_q, num_heads_kv, num_head_groups, qk_storage_dim) - o_5d = o.view(batch_size, seq_len_q, num_heads_kv, num_head_groups, value_head_dim) - k_5d = k.view(batch_size, seq_len_kv, num_heads_kv, 1, qk_storage_dim) - v_5d = v.view(batch_size, seq_len_kv, num_heads_kv, 1, value_head_dim) - lse_4d = ( - lse.view(batch_size, seq_len_q, num_heads_kv, num_head_groups) if lse is not None else None - ) - - # Map is_causal / window args onto the published MaskEnum surface. - has_window = is_causal or window_left != -1 or window_right != -1 - if is_causal: - mask_type = fmha_utils.MaskEnum.WINDOW_MASK - ws_l_int = None if window_left == -1 else window_left - ws_r_int = 0 if window_right == -1 else window_right - elif has_window: - mask_type = fmha_utils.MaskEnum.WINDOW_MASK - ws_l_int = None if window_left == -1 else window_left - ws_r_int = None if window_right == -1 else window_right - else: - mask_type = fmha_utils.MaskEnum.RESIDUAL_MASK - ws_l_int = None - ws_r_int = None - - if sm_scale is None: - sm_scale = 1.0 / math.sqrt(head_dim) - scale_softmax = scale_q * scale_k * sm_scale - scale_softmax_log2 = scale_softmax * math.log2(math.exp(1.0)) - scale_output = scale_v / scale_o - - use_skip_softmax = ( - skip_softmax_threshold_scale_factor is not None and skip_softmax_threshold_scale_factor > 0 - ) - skip_threshold_log2 = ( - cute_typing.Float32(math.log2(skip_softmax_threshold_scale_factor / seq_len_kv)) - if use_skip_softmax - else None - ) - - # For the block-scaled paths Q/K may be FP4 (stored as torch.uint8) or FP8 e4m3; pass - # qk_cutlass_dtype to override the inferred element type for FP4 (FP8 e4m3 is auto-detected). - q_cute = _to_cute_tensor(q_5d, leading_dim=4, cutlass_element_type=qk_cutlass_dtype) - k_cute = _to_cute_tensor(k_5d, leading_dim=4, cutlass_element_type=qk_cutlass_dtype) - v_cute = _to_cute_tensor(v_5d, leading_dim=4) - o_cute = _to_cute_tensor(o_5d, leading_dim=4) - if qk_sf_vec != 0: - # MXFP8 SFs are Float8E8M0FNU (uint8 storage); NVFP4 SFs are Float8E4M3FN. - sf_dtype = cutlass.Float8E8M0FNU if qk_sf_vec == 32 else cutlass.Float8E4M3FN - q_sf_cute = _to_cute_tensor(q_sf, leading_dim=0, cutlass_element_type=sf_dtype) - k_sf_cute = _to_cute_tensor(k_sf, leading_dim=0, cutlass_element_type=sf_dtype) - scale_v_channels_cute = ( - _to_cute_tensor(scale_v_channels.view(-1), leading_dim=0) - if scale_v_channels is not None - else None - ) - else: - q_sf_cute = None - k_sf_cute = None - scale_v_channels_cute = None - # lse_4d is (B, S_q, h_kv, h_r) contiguous → h_r is the stride-1 inner dim (index 3). - lse_cute = ( - from_dlpack(lse_4d, assumed_align=16).mark_layout_dynamic(leading_dim=3) - if lse_4d is not None - else None - ) - - ws_left = None if ws_l_int is None else cute_typing.Int32(ws_l_int) - ws_right = None if ws_r_int is None else cute_typing.Int32(ws_r_int) - - # problem_size = (b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d, dv); with cum_seqlen_* = None, - # s_lse_max collapses to s_q (per fmha.py:run()). - problem_size = ( - batch_size, - seq_len_q, - seq_len_q, - seq_len_kv, - num_heads_q, - num_heads_kv, - head_dim, - value_head_dim, - ) - stream = cuda_driver.CUstream(torch.cuda.current_stream(q.device).cuda_stream) - - gpu_arch_str = _get_gpu_arch(q.device) - qk_dtype_cache = ( - qk_cutlass_dtype if qk_cutlass_dtype is not None else _torch_to_cutlass_dtype(q.dtype) - ) - # SM100 needs ex2 emulation; SM103 (and any other SM10X SKU) does not. - enable_ex2_emulation = gpu_arch_str == "sm_100a" - - key = _CacheKey( - qk_cutlass_dtype=qk_dtype_cache, - pv_cutlass_dtype=_torch_to_cutlass_dtype(v.dtype), - out_cutlass_dtype=_torch_to_cutlass_dtype(o.dtype), - qk_acc_dtype=cutlass.Float32, - pv_acc_dtype=cutlass.Float32, - head_dim=head_dim, - head_dim_v=value_head_dim, - mma_tiler_mn=(128, 128), - qk_sf_vec=qk_sf_vec, - is_persistent=is_persistent, - mask_type=mask_type, - with_lse=lse is not None, - with_sink=False, - with_scale_v_channels=scale_v_channels is not None, - has_window=has_window, - has_skip_softmax=use_skip_softmax, - use_tma_store=True, - enable_ex2_emulation=enable_ex2_emulation, - enable_skip_correction=True, - gpu_arch_str=gpu_arch_str, - ) - - if qk_sf_vec != 0: - launch_args = ( - q_cute, - k_cute, - q_sf_cute, - k_sf_cute, - v_cute, - o_cute, - problem_size, - None, # cum_seqlen_q - None, # cum_seqlen_k - lse_cute, - None, # sink - cute_typing.Float32(scale_softmax_log2), - cute_typing.Float32(scale_softmax), - cute_typing.Float32(scale_output), - scale_v_channels_cute, - skip_threshold_log2, - ws_left, - ws_right, - None, # skip_softmax_count - None, # total_softmax_count - stream, - False, # use_pdl - ) - else: - launch_args = ( - q_cute, - k_cute, - v_cute, - o_cute, - problem_size, - None, # cum_seqlen_q - None, # cum_seqlen_k - lse_cute, - None, # sink - cute_typing.Float32(scale_softmax_log2), - cute_typing.Float32(scale_softmax), - cute_typing.Float32(scale_output), - skip_threshold_log2, - ws_left, - ws_right, - None, # skip_softmax_count - None, # total_softmax_count - stream, - False, # use_pdl - ) - - compiled = _get_or_compile(key, launch_args) - compiled(*launch_args) - - -# ============================================================================ -# Block-scaled (MXFP8 / NVFP4) Q/K quantization -# ============================================================================ - - -_FP8_E4M3_MAX = 448.0 # FP8 e4m3 max magnitude -_FP4_E2M1_MAX = 6.0 # FP4 e2m1 max magnitude - - -def _quantize_fp8_v( - v_bshd: torch.Tensor, per_head_channel: bool -) -> Tuple[torch.Tensor, float | torch.Tensor, torch.Tensor | None]: - """Quantize V to FP8 with either one tensor scale or an (H, D) scale tensor.""" - if per_head_channel: - v_qscale = _FP8_E4M3_MAX / v_bshd.float().abs().amax(dim=(0, 1)).clamp(min=1e-3) - v_quantized = (v_bshd * v_qscale).to(torch.float8_e4m3fn) - return v_quantized, 1.0, v_qscale.reciprocal().contiguous() - - v_qscale = _FP8_E4M3_MAX / v_bshd.abs().amax().clamp(min=1e-3) - v_quantized = (v_bshd * v_qscale).to(torch.float8_e4m3fn) - return v_quantized, v_qscale.reciprocal(), None - - -def _quantize_blockscaled_one( - x_bshd: torch.Tensor, qk_sf_vec: int -) -> Tuple[torch.Tensor, torch.Tensor, float]: - """Quantize a single (B, S, H, D) bf16/fp16 tensor for the block-scaled kernel. - - Steps: - 1. Transpose to (B, H, S, D) — GEMM-compatible (num_heads is batch-like). - 2. Pad S -> S_pad (multiple of 128) along the row axis. - 3. Invoke the TRT-LLM op with proper options. - 4. Reshape the quantized data, unpad to S, and transpose back to BSHD. - The SF tensor stays at S_pad per kernel constraints. - - The returned data tensor's last dim is the *storage* dim: D for MXFP8 ; D/2 for NVFP4. - - Returns: - x_q: quantized data in (B, S, H, D_storage); - x_sf: swizzled SF tensor produced by the op (kept padded to S_pad). - scale: scalar dequant factor to fold into scale_softmax. - 1.0 for MXFP8 (per-block SFs carry the full range); - amax/(448*6) for NVFP4 (the per-tensor global scale). - """ - if x_bshd.dim() != 4: - raise ValueError(f"_quantize_blockscaled_one expects (B, S, H, D); got {x_bshd.shape}") - batch_size, seq_len, num_heads, head_dim = x_bshd.shape - - x_bhsd = x_bshd.transpose(1, 2).contiguous() # (B, H, S, D) - s_pad = ((seq_len + 127) // 128) * 128 - if s_pad != seq_len: - pad = torch.zeros( - batch_size, - num_heads, - s_pad - seq_len, - head_dim, - dtype=x_bhsd.dtype, - device=x_bhsd.device, - ) - x_bhsd = torch.cat([x_bhsd, pad], dim=2) - x_2d = x_bhsd.reshape(batch_size * num_heads * s_pad, head_dim) - - if qk_sf_vec == 32: - # MXFP8: per-32-element block, UE8M0 SFs in swizzled layout. - x_q_2d, x_sf = torch.ops.trtllm.mxfp8_quantize(x_2d, True, alignment=32) - # x_q_2d shape: (M, D) fp8_e4m3fn; storage last-dim == logical head_dim. - x_q = x_q_2d.view(batch_size, num_heads, s_pad, head_dim) - x_q = x_q[:, :, :seq_len, :].transpose(1, 2).contiguous() # (B, S, H, D) - return x_q, x_sf, 1.0 - - if qk_sf_vec == 16: - # NVFP4: per-16-element block, E4M3 SFs in swizzled layout; per-tensor global scale folded - # into the returned `scale` (caller multiplies it into scale_softmax via scale_q / scale_k). - amax = x_2d.float().abs().amax().clamp(min=1e-6) - global_sf = (_FP8_E4M3_MAX * _FP4_E2M1_MAX) / amax - global_sf_t = global_sf.to(torch.float32).reshape(1) - x_q_2d, x_sf = torch.ops.trtllm.fp4_quantize(x_2d, global_sf_t, 16, False) - # x_q_2d shape: (M, D/2) uint8 — natural packed layout (2 FP4 / byte). - head_dim_packed = head_dim // 2 - x_q = x_q_2d.view(batch_size, num_heads, s_pad, head_dim_packed) - x_q = x_q[:, :, :seq_len, :].transpose(1, 2).contiguous() # (B, S, H, D/2) - return x_q, x_sf, amax / (_FP8_E4M3_MAX * _FP4_E2M1_MAX) - - raise ValueError(f"Unsupported qk_sf_vec={qk_sf_vec}; expected 0, 16, or 32.") - - -# ============================================================================ -# VisualGen AttentionBackend class -# ============================================================================ - - class CuTeDSLAttention(AttentionBackend): """ CuTe DSL (NVIDIA kernels) backend for diffusion models. - JIT-compiles BlackwellFusedMultiHeadAttentionForward on first use and caches the - compiled artifact per (dtype, mask, head_dim, ...) configuration. + Uses pre-compiled cubin kernels (head_dim=128 only). """ def __init__( @@ -584,12 +56,15 @@ def __init__( layer_idx: int = 0, num_heads: int = 8, head_dim: int = 64, - num_kv_heads: int | None = None, - dtype: torch.dtype | None = None, - quant_attention_config: QuantAttentionConfig | None = None, - skip_softmax_threshold_scale: float | None = None, + num_kv_heads: Optional[int] = None, + dtype: Optional[torch.dtype] = None, + quant_attention_config: Optional[QuantAttentionConfig] = None, + skip_softmax_threshold_scale: Optional[float] = None, **kwargs, ): + # Only head_dim=128 cubins are packaged. + if head_dim != 128: + raise ValueError(f"CUTEDSL cubins require head_dim=128, got head_dim={head_dim}.") self.layer_idx = layer_idx self.num_heads = num_heads self.head_dim = head_dim @@ -602,31 +77,6 @@ def __init__( # CuTe DSL expects [B, S, H, D] format self._preferred_layout = AttentionTensorLayout.NHD - def _smooth_qk( - self, - q: torch.Tensor, - k: torch.Tensor, - alpha: float = 0.7, - ): - """Smooth-Smooth technique: - Performs SmoothQuant-like handling for Qk, leveraging the additional fact that - K - K_mean doesn't affect attention result. - """ - k = k.unflatten(2, (-1, 1)) # (B, S, H_K, 1, D) - q = q.unflatten(2, (k.shape[2], -1)) # (B, S, H_K, H_R, D) - q_max = ( - q.abs().amax(dim=(0, 1, 3), keepdim=True).float().clamp_min(1e-4) - ) # (1, 1, H_K, 1, D) - k_max = ( - k.abs().amax(dim=(0, 1, 3), keepdim=True).float().clamp_min(1e-4) - ) # (1, 1, H_K, 1, D) - s = (q_max.pow(alpha) / k_max.pow(1 - alpha)).clamp(1e-4, 1e4) - q = q * s.reciprocal().bfloat16() - # Per-channel shift commutes with the per-channel smooth scale `s`. - k = k - k.mean(dim=(0, 1, 3), keepdim=True) - k = k * s.bfloat16() - return q.flatten(2, 3), k.flatten(2, 3) - def _prepare_inputs( self, q: torch.Tensor, @@ -642,22 +92,16 @@ def _prepare_inputs( is_causal = attention_mask == PredefinedAttentionMask.CAUSAL - # Perform QK-smoothing if Bmm1 is to be quantized. - qac = self.quant_attention_config - smooth_qk = qac is not None and qac.qk_dtype not in ["bf16", "fp16"] - - # Published kernel supports float16 and bfloat16 only. + # Packaged cubins support float16 and bfloat16 only. origin_dtype = q.dtype if q.dtype not in (torch.float16, torch.bfloat16): q = q.to(torch.bfloat16) k = k.to(torch.bfloat16) v = v.to(torch.bfloat16) - if smooth_qk: - q, k = self._smooth_qk(q, k) return q, k, v, is_causal, origin_dtype - # cute_dsl_fmha_fwd is already @torch.compiler.disable'd, so torch.compile may still fuse - # preceding linear/norm with the V quantization below. + # cute_dsl.cute_dsl_fmha_fwd is already decorated with @torch.compiler.disable + # Allow torch.compile to fuse preceding linear/norm with quantization of V / seq-preprocess def _fwd( self, q: torch.Tensor, @@ -667,7 +111,7 @@ def _fwd( **kwargs, ) -> Tuple[torch.Tensor, torch.Tensor]: batch_size, seq_len_q, num_heads, _ = q.shape - value_head_dim = v.shape[-1] + _, seq_len_kv, _, value_head_dim = v.shape out = torch.empty( batch_size, seq_len_q, @@ -684,50 +128,42 @@ def _fwd( device=q.device, ) - # V is tensor-scaled by default. MXFP8/NVFP4 with v_block_size=1 use an (H, D) scale. + # Options that instructs quantization of V scale_v = kwargs.get("scale_v", 1.0) - scale_q = kwargs.get("scale_q", 1.0) - scale_k = kwargs.get("scale_k", 1.0) - qac = self.quant_attention_config - q_sf = k_sf = qk_cutlass_dtype = None - qk_sf_vec = 0 - scale_v_channels = None - if qac is not None: - if qac.qk_dtype in ("mxfp8", "nvfp4"): - qk_sf_vec = 32 if qac.qk_dtype == "mxfp8" else 16 - q, q_sf, gs_q = _quantize_blockscaled_one(q, qk_sf_vec) - k, k_sf, gs_k = _quantize_blockscaled_one(k, qk_sf_vec) - scale_q = scale_q * gs_q - scale_k = scale_k * gs_k - qk_cutlass_dtype = cutlass.Float4E2M1FN if qk_sf_vec == 16 else cutlass.Float8E4M3FN - v, v_dequant_scale, scale_v_channels = _quantize_fp8_v( - v, per_head_channel=qk_sf_vec != 0 and qac.v_block_size == 1 - ) - scale_v = scale_v * v_dequant_scale + if self.quant_attention_config is not None: + v_qscale = 448.0 / v.abs().amax().clamp(min=1e-3) + v = (v * v_qscale).to(torch.float8_e4m3fn) + scale_v = scale_v / v_qscale + + # Sequence preproc. + qo_indptr_host = [i * seq_len_q for i in range(batch_size + 1)] + qo_indptr = torch.tensor(qo_indptr_host).to(device=q.device, dtype=torch.int32) + kv_indptr_host = [i * seq_len_kv for i in range(batch_size + 1)] + kv_indptr = torch.tensor(kv_indptr_host).to(device=q.device, dtype=torch.int32) # Skip softmax. skip_softmax_threshold_scale = self.skip_softmax_threshold_scale if skip_softmax_threshold_scale is not None and skip_softmax_threshold_scale <= 0.0: skip_softmax_threshold_scale = None - cute_dsl_fmha_fwd( - q, - k, - v, - out, + cute_dsl.cute_dsl_fmha_fwd( + q.flatten(0, 1).contiguous(), + k.flatten(0, 1).contiguous(), + v.flatten(0, 1).contiguous(), + out.flatten(0, 1), + qo_indptr=qo_indptr, + kv_indptr=kv_indptr, is_causal=is_causal, sm_scale=self.scale, - lse=lse, - scale_q=scale_q, - scale_k=scale_k, + lse=lse.flatten(0, 1).contiguous(), + scale_q=kwargs.get("scale_q", 1.0), + scale_k=kwargs.get("scale_k", 1.0), scale_v=scale_v, - scale_v_channels=scale_v_channels, scale_o=kwargs.get("scale_o", 1.0), + max_qo_len=seq_len_q, + max_kv_len=seq_len_kv, + is_persistent=False, skip_softmax_threshold_scale_factor=skip_softmax_threshold_scale, - qk_sf_vec=qk_sf_vec, - q_sf=q_sf, - k_sf=k_sf, - qk_cutlass_dtype=qk_cutlass_dtype, ) return out, lse diff --git a/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py b/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py index ce012e16aea3..e52f52c87247 100644 --- a/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py +++ b/tensorrt_llm/_torch/visual_gen/attention_backend/utils.py @@ -49,8 +49,8 @@ def get_visual_gen_attention_backend( Better performance but requires fused QKV - "FA4": Flash Attention 4; provides higher speedup on Blackwell GPUs (sm100) Requires flash-attn package with cute interface - - "CUTEDSL": CuTe DSL kernels. create_attention selects dense FMHA or VSA from - AttentionConfig.sparse_attention_config. + - "CUTEDSL": CuTe DSL FMHA kernels; uses packaged cubins when present, + otherwise compiles from CuTe DSL source """ # Lazy imports to avoid circular dependency from .cute_dsl import CuTeDSLAttention @@ -105,8 +105,8 @@ def create_attention( will automatically reallocate if larger batches are encountered. max_seq_len: Initial sequence length for metadata pre-allocation. The backend will automatically reallocate if longer sequences are encountered. - attention_config: Optional AttentionConfig used to select the attention algorithm and - forward its quantization or sparsity configuration. + attention_config: Optional AttentionConfig; quant_attention_config is + extracted and forwarded to the TRTLLM backend when present. attention_metadata_state: Optional model-scoped metadata state from visual-gen config. Required for TRTLLM backend. **kwargs: Additional backend-specific arguments @@ -116,7 +116,9 @@ def create_attention( """ attn_cls = get_visual_gen_attention_backend(backend) - # Forward the validated quantization recipe to TRTLLM or the dense CuTe DSL FMHA backend. + # Extract quant_attention_config from AttentionConfig and pass to backends + # that support it (TRTLLM SAGE recipes, CUTEDSL QK16PV8). AttentionConfig + # validation rejects unsupported (backend, recipe) combinations upstream. if attention_config is not None and attention_config.quant_attention_config is not None: kwargs["quant_attention_config"] = attention_config.quant_attention_config if backend.upper() == "TRTLLM": @@ -131,6 +133,7 @@ def create_attention( attention_config.sparse_attention_config is not None and getattr(attention_config.sparse_attention_config, "algorithm", None) == "vsa" ): + # VSA sparse path: use VSAAttention from .cute_dsl.vsa import VSAAttention attn_cls = VSAAttention diff --git a/tensorrt_llm/_torch/visual_gen/cache/cache_dit_enablers.py b/tensorrt_llm/_torch/visual_gen/cache/cache_dit_enablers.py index 4f4c4c267711..a859e50d70a8 100644 --- a/tensorrt_llm/_torch/visual_gen/cache/cache_dit_enablers.py +++ b/tensorrt_llm/_torch/visual_gen/cache/cache_dit_enablers.py @@ -9,7 +9,6 @@ from typing import Any, Callable, List, Optional import cache_dit -import torch import torch.nn as nn from cache_dit import ( BlockAdapter, @@ -42,20 +41,6 @@ class CacheDiTEnableResult: summary_modules: List[nn.Module] -def _has_compiled_blocks(*block_lists: Any) -> bool: - """True when any transformer block is a torch.compile OptimizedModule wrapper. - - torch.compile'd blocks expose a ``(*args, **kwargs)`` forward signature, so - cache_dit's inspect-based forward-pattern checks cannot match them and must - be skipped (the underlying blocks still follow the declared pattern). - """ - return any( - isinstance(block, torch._dynamo.OptimizedModule) - for blocks in block_lists - for block in blocks - ) - - def _resolved_enable_separate_cfg(cfg: CacheDiTConfig, default: bool) -> bool: if cfg.enable_separate_cfg is not None: return cfg.enable_separate_cfg @@ -273,18 +258,12 @@ def enable_cache_dit_for_flux( forward_pattern = [ForwardPattern.Pattern_1, ForwardPattern.Pattern_1] tag = "FLUX.1" - compiled_blocks = _has_compiled_blocks(*block_lists) - if compiled_blocks: - logger.info( - f"Cache-DiT: {tag} blocks are torch.compile'd; skipping forward-pattern checks." - ) adapter = BlockAdapter( transformer=pipeline.transformer, blocks=block_lists, forward_pattern=forward_pattern, params_modifiers=[modifier], - check_forward_pattern=not compiled_blocks, - check_num_outputs=not compiled_blocks, + check_forward_pattern=True, ) logger.info( diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py index ec83a7d6a30b..fdf84901e4c5 100644 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/__init__.py @@ -13,17 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .fmha import BlackwellFusedMultiHeadAttentionForward, make_thread_cooperative_group -from .fmha_blockscaled import ( - BlackwellFusedMultiHeadBlockScaledAttentionForward, - compact_fp4_data, - create_scale_factor_tensor, -) +from .fmha import cute_dsl_fmha_fwd -__all__ = [ - "BlackwellFusedMultiHeadAttentionForward", - "BlackwellFusedMultiHeadBlockScaledAttentionForward", - "compact_fp4_data", - "create_scale_factor_tensor", - "make_thread_cooperative_group", -] +__all__ = ["cute_dsl_fmha_fwd"] diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..93e56313b689 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bde8e845611c0ff97738d3822da3ab406169596065bc703a8c8f1af58aeb4fbd +size 711184 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..8f7b825447a4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d962a68d6d0c89ca5e71d0514b2ea5f1b1da3099c7c89ea4c05cf7c5468f6b0c +size 686552 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..6f2a475d0922 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bc6c9ff687d1a49bdff39dffcf9e08f74c0114dc63921b458695d1b1add12df +size 702960 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..8c94270cef08 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d71eff9707d7e4464e6b2847ece251c352c783b60056086841cfc1ad5767489 +size 682416 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..b12c8c09af05 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:117b99d4733a4cf500515a22f5fb2c87b5c89ffdf7736bc302fff48b27b68289 +size 645672 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..be106aa226d8 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed8d36370827808578ee1b2a7927a8bb964772ba1446a84d7eb66ae11857c28 +size 625128 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..df218788b1ad --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:909f85670f5cfeae51fb7e7ca2ec6ee648f272c72c0e65545c2f94a084bbd8d7 +size 637440 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..67cd56345e01 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb1bd11b767de8d4b039a4456f799a0461cc0cfa423e4ce97d6bddf1f289a81d +size 616896 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..05f14144f81e --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4451124313741a07ee5c616c5fbaf3db110a1ab531bfc5955eeb2a126eb33d3e +size 711096 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..86aef559d4ab --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bad02325167d25248a381c9e8cfa4479d93e4f7f2d2bbfff0816db8b03e52ea +size 690552 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..2bd77cbe433b --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e71c573fad3a61051b4037cd38ca42daaa2cc6f2c1a007af74c7d5e0107b71 +size 702872 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..a15c97e7c4ff --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40cb6caa255f53e90c8c84fb5b5cf13b4776863d76f1d0ead0553fb3b574b169 +size 686424 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..9b939a3c3917 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d48995a1b50324fb7f103a543bd8aa2d48ab8617a972e7bc594a834f78e7888 +size 645576 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..6273d4c74ad7 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fff35882b5e0bfbebf2b313a67c7a80398a68df411a1fb30542f11a2e9212f1 +size 629136 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..ba3f0e067efe --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1436b06493487e07c260e5ce283dcd85bfd9551db272d96d2e395788ed34ba5b +size 637352 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..daf2c2df15b8 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:460134c26759d391ac0d4df94627e63794898425f887a210871259fc030d2157 +size 620904 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..10e981459b3c --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6ea5c21b35d5562eec533b601348f527da2ed159c8d6f2a413f02d8b3dfce4 +size 657936 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..e153954ec5ee --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed589e4c6f369afb05903be7bee828ab5f8293736cca734390d6eab9ac609f56 +size 637400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..a8b2717c21dc --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99801a8fb3c547c9f5778764b14049229e07a7395a2f26e098112ca3968884eb +size 653808 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..0691e9326f5a --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:834109925ad81a8b50d44a436c34b9b9ff2f37bf87dc1c3420da10082f951f71 +size 629168 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..a33382575553 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:962849852e70fd20d8f086b9257173ddda4f6ca654ed4fa165768a5398e50e75 +size 596520 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..6a76fed84298 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a040d0f9333c99a78cdf2baa42f2dc71cf2d8252ac3d1b7005f3afd7b4b536 +size 575976 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..46d5cb25f686 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c241a35bb7ab27b9b4fe26c03f1a8ac4ba1eeb28cfc68fcc0f0d5133a74f393 +size 588288 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..073aa5c4860d --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:285541f2d3d081f8726d79c7e6f484af55a06e52e7580fb58a360f9aad5e846b +size 567744 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..02af20762774 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7284ff86f4d04fedf14e87947411e1765b55851ee9d9f12450f0314644abd8e +size 657848 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..4eb755d3c573 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f26d9fb521fea7260587558e4a5ec016c2c21b53f0fca580e65c46f7599c898 +size 641400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..a82de70cf9be --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d951a82391c4df1933bf1ff13bf7c455d90c95b19ebf3360c34a3151763ad9cc +size 653720 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..07ab962faa79 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce18821322bdb617a5c26e52465ff1c02d6bd03f6f24ce7604a538a2862e107a +size 633176 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..5396cdeac4a3 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5af2f95b9ff7328f1dce2863345ddbf72a39ec4cc0c3ce7601ad904d1c279bb3 +size 592328 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..ec033517f17f --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afe75060cd7f3dd1a37704e2a4fe70aec59bcf60e83f6cc77c510fae8adb1209 +size 575888 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..490ac3c2eec1 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:807fc151395aabe1f4fb2bd0451d6d4f8def14e591cff121dc5426c2ee24b841 +size 588200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..d708b622be2a --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/aarch64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0faf35561762e27fbf861fe3385298f08da3aa9cc9564b9d3cd99c7fb2f74e5 +size 567656 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..334d4de2da9a --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20f59a88225f55116a6269dde1eb677bd527cb91eb1200058765c8c64356b9fb +size 716248 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..ffb16280b537 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7666214fc5357f11f4db5cdae11c11f2c3a42679b30c1534535e12916aed578 +size 694232 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..7a3a7cf23675 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd18be0827da75961d58cb09d0c63fe0826ad9d1a83e16a44b74db43fb7b183b +size 705936 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..86656cfff50e --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3a3da59ea39eb5d41e345a04ba0ab573fb6ef7d950091d4757ac27af2c736e5 +size 683984 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..4ce0c3cc1752 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecd8a2048373f40ddf71a9706f7a16aef9fb0e5b6487937be695f6e7d0744161 +size 651128 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..4b6997fefec4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3407c3c3f177c8ab0b02c69c7cae9968710041d06acef9a10fc5ffa534ec274f +size 630200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..2c01e5f00009 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f64ccf41ae8680369363660c1c350386a8b9784141853832fb49d2767e283a9e +size 639728 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..986c6a5db0d5 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eca236d41155a0e2c5b0f3289c5399b60589ffb5b679abff456202566e2bdf0 +size 618192 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..14609fab4bbc --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d5ad69ef4a76110e48c4f1b6642e3e8b321dc1d794e83957b6228a8c68edfd4 +size 714872 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..6c4ef0b8a4c7 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1896d588ebbbb6ffde8331432dafccbf996507504b6b86a0ba17bf539f2e8f9 +size 697240 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..40433af18781 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ce41d196fa54fb89e5848891e1e402d6511d0e5635098e98f00b0312ce3ad77 +size 704296 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..14d97d8463c3 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f45b423c9980f32c116718dca45f1b05e65ad9abd1e0a70e37a07c9106c6827 +size 686888 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..faabf383e21f --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:150cd0b8fa1834d00aa2de8ead15916bc08c4e4f5ba002683ceccca5dc15a793 +size 650496 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..2500b613f8e4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c0eab2d74850c29abd02aa927cc53da3ac730ebed1f8d62e5836c68c3c86a42 +size 635008 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..8c867ef9b986 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d044e658d6d3fda52a038a0d0d8b9cce502f7e383bb96deec344d10e217a6e98 +size 640056 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..5449e7917a22 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_100a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20bc7f069ea456ff8274d71dccaceeb979ba877e19a29d956753ebfa8e6000ad +size 624504 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..e22d76d563cc --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77fee6189e75f9b0233f7ccce9eec7d2f70b277f2f62d599c01f057e1c179235 +size 665400 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..cc6eab46060c --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:182a606d2ca126fc83c2f914a6e5547d84a9aa668e3526710ca3e7396aac2f8e +size 642776 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..9a06ad3879df --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96052f92a4a4e9b8edc152a2e4e60d0b3c9ffc3e8cad0f72716f4be1072cea5e +size 656112 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..04c07f32cdd9 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c4ca62124eb16d29903dde0f9cf9a46b8a72d3480fee3447736b6c4f2d71895 +size 632560 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..c27e7a6a19dc --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9b8919f036acd199b4aed790563b63de6d1cfa3ef37bfeec00751a25ee5c29d +size 600408 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..5f9f6cd54c9d --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eff71a1a8838b98ec684ada9e693db4f39a69527d8f072bb52ad3b136e2ad966 +size 580200 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..62349117d032 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e7d9c9e4ca723169a115029633811b97d6a57880274f668678218466bef9ee0 +size 591488 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..4b09aab76f59 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_e4m3_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30f0681109e2520f4135d1bb4910d31c11b6d0aae64e09efc6705ba461e2eede +size 568848 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..17b50b8f2a9b --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a761c98700c565c98a0e7c7a67a569cf165cdf1fe1262c541ec066f60ffa7c1 +size 663704 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..a4d1b3ecae30 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f21a9a2fc02f9ada3d126ee7c054f130fad0b16b62daadd63c881dc63991cf6b +size 644472 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..883e738091b4 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9edebdc6f4c23e407f057fba17144122f625db87a29bdd61d14951b8d05aadf +size 655176 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..a22d88458722 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_causal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9becebe7efb3c0342b41668b6d82888e5a100bccdf906a25c10402ee76b56115 +size 635160 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so new file mode 100644 index 000000000000..836913616f49 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8239bc1f03bef805f46a356a9dcffa449dfa1f112ce8d39297f7cc272da60e48 +size 599536 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so new file mode 100644 index 000000000000..226972ea4776 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_lse_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deb5f65d94e50b7466c027004c9fbfe0c7a5b9b67eec6c139496b70a7a0a85eb +size 581984 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so new file mode 100644 index 000000000000..bca39e1a5e2e --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_skipsm_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dca9eeb79fe6a289db4cee5e9c2e8700dd9e138c149da1ec1299ea471efc3c8e +size 589096 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so new file mode 100644 index 000000000000..da1001774b1e --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/cubins/x86_64/sm_103a/cute_dsl_fmha_bf16_h128_nocausal_nonpersistent_varlen_tvmffi.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c24bd23172aa6d42fc3df1f7221e9ca3ce9c2b4e608386879f1734d58956fa45 +size 571544 diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py index bca82e086559..3de28390c569 100644 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py +++ b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha.py @@ -1,3210 +1,494 @@ -# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: BSD-3-Clause - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# ruff: noqa: I001, E501, F841 - +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import functools import math -from typing import Callable, Type, Tuple, Union, Optional -from functools import partial - -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -from cutlass.cute.nvgpu import tcgen05 -from cutlass.cute.nvgpu.common import OperandMajorMode -import cutlass.utils as utils -import cutlass.pipeline as pipeline -from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait -import cutlass.utils.blackwell_helpers as sm100_utils -from cutlass.cute.typing import Int8, Int32, Int64, Float32 -from cutlass.base_dsl.arch import Arch -from cutlass.cutlass_dsl import BaseDSL - -from ...helpers import fmha_helpers as fmha_utils - -""" -A fused multi-head attention (FMHA) example for the NVIDIA Blackwell SM100 architecture using CUTE DSL - -This example demonstrates an implementation of fused multi-head attention using a TMA + Blackwell SM100 -TensorCore warp-specialized persistent kernel. The implementation integrates the Q*K^T matrix multiplication, -softmax normalization, and softmax(Q*K^T)*V into a single kernel, avoiding intermediate data movement between -global memory and shared memory, thus improving computational efficiency. - -The kernel implements key optimizations including: -- Warp specialization for different computation phases (load, MMA, softmax, correction, epilogue) -- Pipeline stages between different warps for overlapping computation and memory access -- Support for different precision data types -- Optional causal masking for autoregressive models - -To run this example: - -.. code-block:: bash - - python examples/cute/blackwell/kernel/attention/fmha/fmha.py \ - --qk_acc_dtype Float32 --pv_acc_dtype Float32 \ - --mma_tiler_mn 128,128 \ - --q_shape 4,1024,8,64 --k_shape 4,1024,8,64 \ - --is_persistent - -The above example runs FMHA with batch size 4, sequence length 1024, 8 attention heads, and head -dimension 64. The Blackwell tcgen05 MMA tile shape is (128, 128), and the kernel uses fp16 for input/output -with fp32 for accumulation. - -To collect performance with NCU profiler: - -.. code-block:: bash - - ncu python examples/cute/blackwell/kernel/attention/fmha/fmha.py \ - --qk_acc_dtype Float32 --pv_acc_dtype Float32 \ - --mma_tiler_mn 128,128 \ - --q_shape 4,1024,8,64 --k_shape 4,1024,8,64 \ - --is_persistent --warmup_iterations 10 \ - --iterations 10 --skip_ref_check - -Constraints for this example: -* Supported head dimensions: 32, 64, and 128 -* Number of heads in Q must be divisible by number of heads in K -* mma_tiler_mn must be 128,128 -* Batch size must be the same for Q, K, and V tensors -* For causal masking, use --is_causal (note: specify without =True/False) -* For persistent scheduling, use --is_persistent (note: specify without =True/False) - -For details on the skip softmax algorithm, please refer to the paper: https://arxiv.org/abs/2512.12087. -""" - - -def make_thread_cooperative_group(size: int): - return pipeline.CooperativeGroup(pipeline.Agent.Thread, size) - - -class BlackwellFusedMultiHeadAttentionForward: - arch_str: str = "sm_100" - arch_name: str = "Blackwell SM100" - - def __init__( - self, - qk_acc_dtype: Type[cutlass.Numeric], - pv_acc_dtype: Type[cutlass.Numeric], - mma_tiler: Tuple[int, int], - head_dim: Union[int, Tuple[int, int]], - is_persistent: bool, - mask_type: fmha_utils.MaskEnum, - enable_ex2_emulation: bool, - enable_skip_correction: bool, - use_tma_store: bool = True, - ): - """Initializes the configuration for a Blackwell Fused Multi-Head Attention (FMHA) kernel. - - This configuration includes several key aspects: - - 1. Data Type Settings: - - qk_acc_dtype: Data type for Q*K^T matrix multiplication accumulator - - pv_acc_dtype: Data type for P*V matrix multiplication accumulator - - 2. MMA Instruction Settings: - - mma_tiler: The shape of the MMA instruction unit: (M, N) for BMM1 and (M, K) for BMM2 - - head_dim: The head dimension, it can be a single integer or a tuple of two integers (D, Dv). - If it is a tuple, Dv is the head dimension of the value & output tensors. - It also determines the K dimension of the BMM1's MMA instruction unit - & N dimension of the BMM2's MMA instruction unit. - - qk_mma_tiler: MMA shape for Q*K^T computation - - pv_mma_tiler: MMA shape for P*V computation - - 3. Kernel Execution Mode: - - is_persistent: Boolean indicating whether to use persistent kernel mode - - mask_type: Specifies the type of mask to use (no mask, residual mask, or causal mask) - - window_size_left/right: Sliding window size for attention masking - - enable_ex2_emulation: Whether to enable exp2 emulation - - enable_skip_correction: Whether to skip the correction when rowmax is not updated larger than a threshold - - :param qk_acc_dtype: Data type for Q*K^T matrix multiplication accumulator - :type qk_acc_dtype: Type[cutlass.Numeric] - :param pv_acc_dtype: Data type for P*V matrix multiplication accumulator - :type pv_acc_dtype: Type[cutlass.Numeric] - :param mma_tiler: The (M, N) shape of the MMA instruction - :type mma_tiler: Tuple[int, int] - :param head_dim: The head dimension, it can be a single integer or a tuple of two integers (D, Dv). - :type head_dim: Union[int, Tuple[int, int]] - :param is_persistent: Whether to use persistent kernel mode - :type is_persistent: bool - :param mask_type: Type of mask to use - :type mask_type: fmha_utils.MaskEnum - :param window_size_left: Left-side sliding window size for attention masking - :type window_size_left: int - :param window_size_right: Right-side sliding window size for attention masking - :type window_size_right: int - """ - - self.qk_acc_dtype = qk_acc_dtype - self.pv_acc_dtype = pv_acc_dtype - if isinstance(head_dim, tuple): - self.head_dim = head_dim[0] - self.head_dim_v = head_dim[1] - assert self.head_dim == 192 and self.head_dim_v == 128, ( - f"When Headdim is a tuple, it's for MLA. Must be (192, 128), but got {head_dim}" - ) - else: - self.head_dim = head_dim - self.head_dim_v = head_dim - self.cta_tiler = ( - 2 * mma_tiler[0], # 2 O tile per CTA - mma_tiler[1], - self.head_dim_v, - ) - self.qk_mma_tiler = ( - *mma_tiler, - self.head_dim, - ) - self.pv_mma_tiler = ( - mma_tiler[0], - self.head_dim_v, - mma_tiler[1], - ) - self.cluster_shape_mn = (1, 1) - self.is_persistent = is_persistent - self.mask_type = mask_type - self.enable_skip_correction = enable_skip_correction - self.enable_ex2_emulation = enable_ex2_emulation - self.use_tma_store = use_tma_store - - self.softmax0_warp_ids = (0, 1, 2, 3) - self.softmax1_warp_ids = (4, 5, 6, 7) - self.correction_warp_ids = (8, 9, 10, 11) - self.mma_warp_id = 12 - self.load_warp_id = 13 - self.epilogue_warp_id = 14 - self.empty_warp_id = 15 - self.num_tmem_alloc_cols = cute.arch.get_max_tmem_alloc_cols(self.arch_str) - - self.threads_per_warp = 32 - self.threads_per_cta = self.threads_per_warp * len( - ( - *self.softmax0_warp_ids, - *self.softmax1_warp_ids, - *self.correction_warp_ids, - self.mma_warp_id, - self.load_warp_id, - self.epilogue_warp_id, - self.empty_warp_id, - ) - ) - self.tmem_alloc_barrier = pipeline.NamedBarrier( - barrier_id=2, - num_threads=self.threads_per_warp - * sum( - ( - len((self.mma_warp_id,)), - len(self.softmax0_warp_ids), - len(self.softmax1_warp_ids), - len(self.correction_warp_ids), - ) - ), - ) - self.sequence_s0_s1_barrier = pipeline.NamedBarrier( - barrier_id=3, - num_threads=self.threads_per_warp - * len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)), - ) - self.sequence_s1_s0_barrier = pipeline.NamedBarrier( - barrier_id=4, - num_threads=self.threads_per_warp - * len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)), - ) - self.s0_warpgroup_barrier = pipeline.NamedBarrier( - barrier_id=5, - num_threads=self.threads_per_warp * len(self.softmax0_warp_ids), - ) - self.s1_warpgroup_barrier = pipeline.NamedBarrier( - barrier_id=6, - num_threads=self.threads_per_warp * len(self.softmax1_warp_ids), - ) - self.tmem_dealloc_barrier = pipeline.NamedBarrier( - barrier_id=7, - num_threads=self.threads_per_warp * len(self.correction_warp_ids), - ) - - self.tmem_s0_offset = 0 - self.tmem_s1_offset = 128 - self.tmem_o0_offset = 256 - self.tmem_o1_offset = 384 - # inplaced with s1 - self.tmem_p0_offset = 160 - # inplaced with s0 - self.tmem_p1_offset = 32 - # vec buffer for row_max & row_sum - # inplaced with s0 - self.tmem_vec0_offset = 0 - # inplaced with s1 - self.tmem_vec1_offset = 128 - # skip mma pv flag offset regarding to the vec buffer - # inplaced with s1 - self.tmem_skip_softmax0_offset = 136 - # inplaced with s0 - self.tmem_skip_softmax1_offset = 8 - - self.num_regs_softmax = 192 - self.num_regs_correction = 96 - self.num_regs_other = 32 - self.buffer_align_bytes = 1024 - self.arch = BaseDSL._get_dsl().get_arch_enum() - - if self.arch >= Arch.sm_103: - assert self.enable_ex2_emulation == False, ( # noqa - f"Don't enable exp2 emulation for {self.arch}, it doesn't help performance" - ) - - num_warps_per_warpgroup = 4 - self.softmax_warpgroup_count = ( - len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)) // num_warps_per_warpgroup - ) - - def _make_qk_tiled_mma(self, cta_group): - """Build the QK tiled MMA. Override in subclasses to target a different arch.""" - return sm100_utils.make_trivial_tiled_mma( - self.q_dtype, - self.q_dtype, - self.q_major_mode, - self.k_major_mode, - self.qk_acc_dtype, - cta_group, - self.qk_mma_tiler[:2], - ) - - def _make_pv_tiled_mma(self, cta_group, p_major_mode, p_source): - """Build the PV tiled MMA. Override in subclasses to target a different arch.""" - return sm100_utils.make_trivial_tiled_mma( - self.v_dtype, - self.v_dtype, - p_major_mode, - self.v_major_mode, - self.pv_acc_dtype, - cta_group, - self.pv_mma_tiler[:2], - p_source, - ) - - def _setup_attributes(self, enable_skip_softmax: bool) -> None: - """Set up configurations and parameters for the FMHA kernel operation. - - This method initializes and configures various attributes required for the - execution of the fused multi-head attention kernel, mainly about the pipeline stages: - - - Sets up staging parameters for Q, K, V inputs and accumulator data - - Configures pipeline stages for softmax, correction, and epilogue operations - """ - - self.q_stage = 2 - k_stage = 4 if self.q_dtype.width == 8 else 3 - v_stage = 4 if self.v_dtype.width == 8 else 3 - self.kv_stage = min(k_stage, v_stage) - # For D192, the smem usage of Q & K is larger. So, we need to reduce the stage count. - if self.head_dim == 192 and self.q_dtype.width == 16: - self.kv_stage = 2 - self.p_mma_stage = 1 - self.acc_stage = 1 - self.softmax_corr_stage = 1 - self.mma_corr_stage = 2 - self.mma_softmax_stage = 1 - self.epi_stage = 2 - - # Tunable parameters - if not self.enable_skip_correction: - self.rescale_threshold = 0.0 - elif enable_skip_softmax: - self.rescale_threshold = 1.0 - else: - self.rescale_threshold = 8.0 - # FP8 P pre-scale: offset added to exp2 exponent so that P*2^offset fills - # more of E4M3's [0, 448] range, improving quantization precision. - # Derived from rescale_threshold to guarantee P*2^offset <= 448. - self.p_fp8_prescale_log2 = max(0.0, math.floor(math.log2(448) - self.rescale_threshold)) - # ln(2) * offset correction for LSE when pre-scale is active - self.p_fp8_prescale_lse_correction = self.p_fp8_prescale_log2 * math.log(2) - # For most cases, seq barrier is needed to help keep the pipeline stable - # But sometimes, compiler will schedule the barrier at an unexpected place - # if it hurts perf a lot, try to quickly fix it by disabling seq barrier - self.enable_sequence_barrier = False - # Optional double buffering for correction rescale. - self.enable_correction_double_buffer = False - - @cute.jit - def __call__( - self, - q_tensor: cute.Tensor, - k_tensor: cute.Tensor, - v_tensor: cute.Tensor, - o_tensor: cute.Tensor, - problem_size: Tuple[Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32], - cum_seqlen_q: Optional[cute.Tensor], - cum_seqlen_k: Optional[cute.Tensor], - lse_tensor: Optional[cute.Tensor], - sink_tensor: Optional[cute.Tensor], - scale_softmax_log2: Float32, - scale_softmax: Float32, - scale_output: Float32, - skip_softmax_threshold_log2: Optional[Float32], - window_size_left: Optional[Int32], - window_size_right: Optional[Int32], - skip_softmax_count: Optional[cute.Tensor], - total_softmax_count: Optional[cute.Tensor], - stream: cuda.CUstream, - use_pdl: bool, - ): - """Execute the Fused Multi-Head Attention operation on the provided tensors. - - This method prepares the input tensors for processing, validates their shapes and types, - configures the computation parameters, and launches the CUDA kernel. - - The method handles: - 1. Tensor layout transformations for specific memory access patterns - 2. Validation of tensor shapes and data types - 3. Initialization of hardware-specific parameters and memory layouts - 4. Configuration of TMA (Tensor Memory Access) operations - 5. Grid and work scheduling computation - 6. Kernel launch with appropriate parameters - - :param q_tensor: The query tensor with shape (b, s_q, h_k, h_r, d) - :type q_tensor: cute.Tensor - :param k_tensor: The key tensor with shape (b, s_k, h_k, 1, d) - :type k_tensor: cute.Tensor - :param v_tensor: The value tensor with shape (b, s_v, h_k, 1, dv) - :type v_tensor: cute.Tensor - :param o_tensor: The output tensor with shape (b, s_q, h_k, h_r, dv) - :type o_tensor: cute.Tensor - :param problem_size: The problem size with shape [b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d, dv]. If cum_seqlen_q or cum_seqlen_k is not None, s_q_max and s_k_max are the max of the per-batch sequence lengths respectively. - :type problem_size: Tuple[Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32] - :param cum_seqlen_q: The cumulative sequence length tensor for query - :type cum_seqlen_q: Optional[cute.Tensor] - :param cum_seqlen_k: The cumulative sequence length tensor for key - :type cum_seqlen_k: Optional[cute.Tensor] - :param scale_softmax_log2: The log2 scale factor for softmax - :type scale_softmax_log2: Float32 - :param scale_softmax: The scale factor for softmax - :type scale_softmax: Float32 - :param scale_output: The scale factor for the output - :type scale_output: Float32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - :param stream: The CUDA stream to execute the kernel on - :type stream: cuda.CUstream - :raises TypeError: If tensor data types don't match or aren't supported - :raises RuntimeError: If tensor layouts aren't in supported formats - """ - b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d, dv = problem_size - h_r = h_q // h_k - # setup static attributes before smem/grid/tma computation - self.q_dtype = q_tensor.element_type - self.k_dtype = k_tensor.element_type - self.v_dtype = v_tensor.element_type - self.o_dtype = o_tensor.element_type - - # s_q, s_k, s_v are the actual tensor dimensions (total seqlen for varlen) - s_q = q_tensor.shape[1] - s_k = k_tensor.shape[1] - s_v = v_tensor.shape[1] - s_lse = s_lse_max - # Important for performance - align = 256 // self.o_dtype.width - d = cute.assume(Int32(d), align) - dv = cute.assume(Int32(dv), align) - - stride_b_q = h_r * h_k * s_q * d if cum_seqlen_q is None else 0 - stride_b_o = h_r * h_k * s_q * dv if cum_seqlen_q is None else 0 - stride_b_k = h_k * s_k * d if cum_seqlen_k is None else 0 - stride_b_v = h_k * s_v * dv if cum_seqlen_k is None else 0 - stride_b_lse = h_r * h_k * s_lse if cum_seqlen_q is None else 0 - - # (b, s_q, h_k, h_r, d) -> (s_q, d, ((h_r, h_k), b)) - q_layout = cute.make_layout( - (s_q, d, ((h_r, h_k), b)), - stride=(d * h_r * h_k, 1, ((d, d * h_r), stride_b_q)), - ) - q = cute.make_tensor(q_tensor.iterator, q_layout) - # (b, s_k, h_k, 1, d) -> (s_k, d, ((h_r, h_k), b)), 0-stride for h_r to broadcast - k_layout = cute.make_layout( - (s_k, d, ((h_r, h_k), b)), - stride=(d * h_k, 1, ((0, d), stride_b_k)), - ) - k = cute.make_tensor(k_tensor.iterator, k_layout) - # (b, s_v, h_k, 1, dv) -> (dv, s_v, ((h_r, h_k), b)), 0-stride for h_r to broadcast - v_layout = cute.make_layout( - (dv, s_v, ((h_r, h_k), b)), - stride=(1, dv * h_k, ((0, dv), stride_b_v)), - ) - v = cute.make_tensor(v_tensor.iterator, v_layout) - # (b, s_q, h_k, h_r, dv) -> (s_q, dv, ((h_r, h_k), b)) - o_layout = cute.make_layout( - (s_q, dv, ((h_r, h_k), b)), - stride=(dv * h_r * h_k, 1, ((dv, dv * h_r), stride_b_o)), - ) - o = cute.make_tensor(o_tensor.iterator, o_layout) - if cutlass.const_expr(lse_tensor is not None): - # (s, ((h_r, h_k), b)) - head stride=1 to match FlashInfer (total_q, h_q) convention - lse_layout = cute.make_layout( - (s_lse, ((h_r, h_k), b)), - stride=(h_r * h_k, ((1, h_r), stride_b_lse)), - ) - lse = cute.make_tensor(lse_tensor.iterator, lse_layout) - else: - lse = None - - if cutlass.const_expr(sink_tensor is not None): - # sink_tensor is 1D with shape (h_q,) = (h_k * h_r,) - # Create layout ((h_r, h_k), b) with stride 0 for batch so blk_coord[2] works - sink_layout = cute.make_layout( - ((h_r, h_k), b), - stride=((1, h_r), 0), - ) - sink = cute.make_tensor(sink_tensor.iterator, sink_layout) - else: - sink = None - - self.tile_sched_params, grid = fmha_utils.compute_grid( - cute.shape((s_q_max, d, ((h_r, h_k), b))), - self.cta_tiler, - self.is_persistent, - ) - self.q_major_mode = utils.LayoutEnum.from_tensor(q).mma_major_mode() - self.k_major_mode = utils.LayoutEnum.from_tensor(k).mma_major_mode() - self.v_major_mode = utils.LayoutEnum.from_tensor(v).mma_major_mode() - self.o_layout = utils.LayoutEnum.from_tensor(o) - - if cutlass.const_expr(self.q_major_mode != OperandMajorMode.K): - raise RuntimeError("The layout of q is not supported") - if cutlass.const_expr(self.k_major_mode != OperandMajorMode.K): - raise RuntimeError("The layout of k is not supported") - if cutlass.const_expr(self.v_major_mode != OperandMajorMode.MN): - raise RuntimeError("The layout of v is not supported") - - # check type consistency: Q and K must share the same dtype (qk_dtype); - # V may use a different dtype (pv_dtype) to allow qk_dtype != pv_dtype. - if cutlass.const_expr(self.q_dtype != self.k_dtype): - raise TypeError(f"Type mismatch: {self.q_dtype} != {self.k_dtype}") - self._setup_attributes(skip_softmax_threshold_log2 is not None) - - cta_group = tcgen05.CtaGroup.ONE - # the intermediate tensor p is from tmem & k-major - p_source = tcgen05.OperandSource.TMEM - p_major_mode = cute.nvgpu.OperandMajorMode.K - qk_tiled_mma = self._make_qk_tiled_mma(cta_group) - pv_tiled_mma = self._make_pv_tiled_mma(cta_group, p_major_mode, p_source) - - self.cluster_shape_mnk = (*self.cluster_shape_mn, 1) - self.cluster_layout_vmnk = cute.tiled_divide( - cute.make_layout(self.cluster_shape_mnk), - (qk_tiled_mma.thr_id.shape,), - ) - self.epi_tile = self.pv_mma_tiler[:2] - - q_smem_layout_staged = sm100_utils.make_smem_layout_a( - qk_tiled_mma, - self.qk_mma_tiler, - self.q_dtype, - self.q_stage, - ) - k_smem_layout_staged = sm100_utils.make_smem_layout_b( - qk_tiled_mma, - self.qk_mma_tiler, - self.k_dtype, - self.kv_stage, - ) - p_tmem_layout_staged = sm100_utils.make_smem_layout_a( - pv_tiled_mma, - self.pv_mma_tiler, - self.v_dtype, - self.acc_stage, - ) - v_smem_layout_staged_origin = sm100_utils.make_smem_layout_b( - pv_tiled_mma, - self.pv_mma_tiler, - self.v_dtype, - self.kv_stage, - ) - # k & v share the same smem buffer. Pad the smaller-tile operand's stage stride to - # match the larger tile. Stride is scaled to the target operand's element width. - # sK_cosize covers all kv_stage slots at the larger tile's byte footprint. - if cutlass.const_expr(self.k_dtype.width >= self.v_dtype.width): - # k tile >= v tile: give v k's stage stride in v_dtype units - k_tile_cosize = cute.cosize(cute.select(k_smem_layout_staged, mode=[0, 1, 2])) - v_stage_stride = k_tile_cosize * self.k_dtype.width // self.v_dtype.width - v_smem_layout_staged = cute.append( - cute.select(v_smem_layout_staged_origin, mode=[0, 1, 2]), - cute.make_layout(self.kv_stage, stride=v_stage_stride), - ) - sK_cosize = cute.cosize(k_smem_layout_staged) - else: - # v tile > k tile: give k v's stage stride in k_dtype units - v_tile_cosize = cute.cosize(cute.select(v_smem_layout_staged_origin, mode=[0, 1, 2])) - k_stage_stride = v_tile_cosize * self.v_dtype.width // self.k_dtype.width - k_smem_layout_staged = cute.append( - cute.select(k_smem_layout_staged, mode=[0, 1, 2]), - cute.make_layout(self.kv_stage, stride=k_stage_stride), - ) - v_smem_layout_staged = v_smem_layout_staged_origin - # cute.cosize gives (kv_stage-1)*k_stage_stride + k_tile_cosize, but the - # last stage must also accommodate a full V tile, so size for kv_stage slots. - sK_cosize = self.kv_stage * k_stage_stride - - o_smem_layout_staged = sm100_utils.make_smem_layout_epi( - self.o_dtype, - self.o_layout, - self.epi_tile, - self.epi_stage, - ) - - # TMA load for Q - tma_load_op = cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp(cta_group) - tma_store_op = cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp() - - q_smem_layout = cute.select(q_smem_layout_staged, mode=[0, 1, 2]) - tma_atom_q, tma_tensor_q = cute.nvgpu.make_tiled_tma_atom_A( - tma_load_op, - q, - q_smem_layout, - self.qk_mma_tiler, - qk_tiled_mma, - self.cluster_layout_vmnk.shape, - ) - - # TMA load for K - k_smem_layout = cute.select(k_smem_layout_staged, mode=[0, 1, 2]) - tma_atom_k, tma_tensor_k = cute.nvgpu.make_tiled_tma_atom_B( - tma_load_op, - k, - k_smem_layout, - self.qk_mma_tiler, - qk_tiled_mma, - self.cluster_layout_vmnk.shape, - ) - # TMA load for V - v_smem_layout = cute.select(v_smem_layout_staged, mode=[0, 1, 2]) - tma_atom_v, tma_tensor_v = cute.nvgpu.make_tiled_tma_atom_B( - tma_load_op, - v, - v_smem_layout, - self.pv_mma_tiler, - pv_tiled_mma, - self.cluster_layout_vmnk.shape, - ) - - o_smem_layout = cute.select(o_smem_layout_staged, mode=[0, 1]) - tma_atom_o, tma_tensor_o = cute.nvgpu.cpasync.make_tiled_tma_atom( - tma_store_op, - o, - o_smem_layout, - self.epi_tile, - ) - - q_copy_size = cute.size_in_bytes(self.q_dtype, q_smem_layout) - k_copy_size = cute.size_in_bytes(self.k_dtype, k_smem_layout) - v_copy_size = cute.size_in_bytes(self.v_dtype, v_smem_layout) - self.tma_copy_q_bytes = q_copy_size - self.tma_copy_k_bytes = k_copy_size - self.tma_copy_v_bytes = v_copy_size - - @cute.struct - class SharedStorage: - # Pipeline barriers - load_q_mbar_ptr: cute.struct.MemRange[Int64, self.q_stage * 2] - load_kv_mbar_ptr: cute.struct.MemRange[Int64, self.kv_stage * 2] - mma_s0_mbar_ptr: cute.struct.MemRange[Int64, self.mma_softmax_stage * 2] - mma_s1_mbar_ptr: cute.struct.MemRange[Int64, self.mma_softmax_stage * 2] - p0_mma_mbar_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] - p1_mma_mbar_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] - s0_corr_mbar_ptr: cute.struct.MemRange[Int64, self.softmax_corr_stage * 2] - s1_corr_mbar_ptr: cute.struct.MemRange[Int64, self.softmax_corr_stage * 2] - corr_epi_mbar_ptr: cute.struct.MemRange[Int64, self.epi_stage * 2] - mma_corr_mbar_ptr: cute.struct.MemRange[Int64, self.mma_corr_stage * 2] - s0_p1_inplace_barrier_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] - s1_p0_inplace_barrier_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] - # Tmem holding buffer - tmem_holding_buf: Int32 - # Smem tensors - sO: cute.struct.Align[ - cute.struct.MemRange[self.o_dtype, cute.cosize(o_smem_layout_staged)], - self.buffer_align_bytes, - ] - sQ: cute.struct.Align[ - cute.struct.MemRange[self.q_dtype, cute.cosize(q_smem_layout_staged)], - self.buffer_align_bytes, - ] - sK: cute.struct.Align[ - cute.struct.MemRange[self.k_dtype, sK_cosize], - self.buffer_align_bytes, - ] - # Skip softmax and PV warpgroup votes - s0_warp_wants_skip_softmax_exchange: cute.struct.MemRange[Int8, 4] - s1_warp_wants_skip_softmax_exchange: cute.struct.MemRange[Int8, 4] - - self.shared_storage = SharedStorage - - # Launch the kernel synchronously - self.kernel( - qk_tiled_mma, - pv_tiled_mma, - tma_atom_q, - tma_tensor_q, - tma_atom_k, - tma_tensor_k, - tma_atom_v, - tma_tensor_v, - tma_atom_o, - tma_tensor_o, - o, - cum_seqlen_q, - cum_seqlen_k, - lse, - sink, - scale_softmax_log2, - scale_softmax, - scale_output, - skip_softmax_threshold_log2, - window_size_left, - window_size_right, - q_smem_layout_staged, - k_smem_layout_staged, - p_tmem_layout_staged, - v_smem_layout_staged, - o_smem_layout_staged, - skip_softmax_count, - total_softmax_count, - self.tile_sched_params, - ).launch( - grid=grid, - block=[self.threads_per_cta, 1, 1], - cluster=self.cluster_shape_mnk, - stream=stream, - min_blocks_per_mp=1, - use_pdl=use_pdl, - ) - - # GPU device kernel - @cute.kernel - def kernel( - self, - qk_tiled_mma: cute.TiledMma, - pv_tiled_mma: cute.TiledMma, - tma_atom_q: cute.CopyAtom, - mQ_qdl: cute.Tensor, - tma_atom_k: cute.CopyAtom, - mK_kdl: cute.Tensor, - tma_atom_v: cute.CopyAtom, - mV_dkl: cute.Tensor, - tma_atom_o: cute.CopyAtom, - mO_qdl: cute.Tensor, - mO: cute.Tensor, - cum_seqlen_q: Optional[cute.Tensor], - cum_seqlen_k: Optional[cute.Tensor], - mLSE: Optional[cute.Tensor], - mSink: Optional[cute.Tensor], - scale_softmax_log2: Float32, - scale_softmax: Float32, - scale_output: Float32, - skip_softmax_threshold_log2: Optional[Float32], - window_size_left: Optional[Int32], - window_size_right: Optional[Int32], - q_smem_layout_staged: cute.ComposedLayout, - k_smem_layout_staged: cute.ComposedLayout, - p_tmem_layout_staged: cute.ComposedLayout, - v_smem_layout_staged: cute.ComposedLayout, - o_smem_layout_staged: cute.ComposedLayout, - skip_softmax_count: Optional[cute.Tensor], - total_softmax_count: Optional[cute.Tensor], - tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams, - ): - """The device kernel implementation of the Fused Multi-Head Attention. - - This kernel coordinates multiple specialized warps to perform different phases of the FMHA computation: - 1. Load warp: Loads Q, K, V data from global memory to shared memory using TMA - 2. MMA warp: Performs matrix multiplications (Q*K^T and P*V) - 3. Softmax warps: Compute softmax normalization on attention scores - 4. Correction warps: Apply adjustments to intermediate results - 5. Epilogue warp: Handles final output transformation and storage - - The kernel implements a complex pipeline with overlapping computation and memory operations, - using tensor memory access (TMA) for efficient data loading, warp specialization for different - computation phases, and optional attention masking. - - :param qk_tiled_mma: Tiled MMA for Q*K^T - :type qk_tiled_mma: cute.TiledMma - :param pv_tiled_mma: Tiled MMA for P*V - :type pv_tiled_mma: cute.TiledMma - :param tma_atom_q: TMA copy atom for query tensor - :type tma_atom_q: cute.CopyAtom - :param mQ_qdl: Partitioned query tensor - :type mQ_qdl: cute.Tensor - :param tma_atom_k: TMA copy atom for key tensor - :type tma_atom_k: cute.CopyAtom - :param mK_kdl: Partitioned key tensor - :type mK_kdl: cute.Tensor - :param tma_atom_v: TMA copy atom for value tensor - :type tma_atom_v: cute.CopyAtom - :param mV_dkl: Partitioned value tensor - :type mV_dkl: cute.Tensor - :param tma_atom_o: TMA copy atom for output tensor - :type tma_atom_o: cute.CopyAtom - :param mO_qdl: Partitioned output tensor - :type mO_qdl: cute.Tensor - :param mO: Non-partitioned output tensor - :type mO: cute.Tensor - :param scale_softmax_log2: The log2 scale factor for softmax - :type scale_softmax_log2: Float32 - :param scale_output: The scale factor for the output - :type scale_output: Float32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - :param q_smem_layout_staged: Shared memory layout for query tensor - :type q_smem_layout_staged: cute.ComposedLayout - :param k_smem_layout_staged: Shared memory layout for key tensor - :type k_smem_layout_staged: cute.ComposedLayout - :param p_tmem_layout_staged: Tensor memory layout for probability matrix - :type p_tmem_layout_staged: cute.ComposedLayout - :param v_smem_layout_staged: Shared memory layout for value tensor - :type v_smem_layout_staged: cute.ComposedLayout - :param o_smem_layout_staged: Shared memory layout for output tensor - :type o_smem_layout_staged: cute.ComposedLayout - :param tile_sched_params: Scheduling parameters for work distribution - :type tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams - """ - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) - # coord inside cta - tidx, _, _ = cute.arch.thread_idx() - - # - # Prefetch tma desc - # - if warp_idx == self.load_warp_id: - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_q) - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_k) - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_v) - if cutlass.const_expr(self.use_tma_store): - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_o) - - # Alloc - smem = utils.SmemAllocator() - storage = smem.allocate(self.shared_storage) - - load_q_producer, load_q_consumer = pipeline.PipelineTmaUmma.create( - num_stages=self.q_stage, - producer_group=make_thread_cooperative_group(len([self.load_warp_id])), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - tx_count=self.tma_copy_q_bytes, - barrier_storage=storage.load_q_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - load_kv_producer, load_kv_consumer = pipeline.PipelineTmaUmma.create( - num_stages=self.kv_stage, - producer_group=make_thread_cooperative_group(len([self.load_warp_id])), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - tx_count=self.tma_copy_k_bytes, - barrier_storage=storage.load_kv_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - load_kv_full_mbar_ptr = storage.load_kv_mbar_ptr.data_ptr() - load_kv_empty_mbar_ptr = load_kv_full_mbar_ptr + self.kv_stage - mma_s0_producer, mma_s0_consumer = pipeline.PipelineUmmaAsync.create( - num_stages=self.mma_softmax_stage, - producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - barrier_storage=storage.mma_s0_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - mma_s1_producer, mma_s1_consumer = pipeline.PipelineUmmaAsync.create( - num_stages=self.mma_softmax_stage, - producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - barrier_storage=storage.mma_s1_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - p0_mma_producer, p0_mma_consumer = pipeline.PipelineAsyncUmma.create( - num_stages=self.p_mma_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - barrier_storage=storage.p0_mma_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - p1_mma_producer, p1_mma_consumer = pipeline.PipelineAsyncUmma.create( - num_stages=self.p_mma_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - barrier_storage=storage.p1_mma_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - s0_corr_producer, s0_corr_consumer = pipeline.PipelineAsync.create( - num_stages=self.softmax_corr_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len((*self.softmax0_warp_ids, self.mma_warp_id)) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.correction_warp_ids) - ), - barrier_storage=storage.s0_corr_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - s1_corr_producer, s1_corr_consumer = pipeline.PipelineAsync.create( - num_stages=self.softmax_corr_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len((*self.softmax1_warp_ids, self.mma_warp_id)) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.correction_warp_ids) - ), - barrier_storage=storage.s1_corr_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - corr_epi_producer, corr_epi_consumer = pipeline.PipelineAsync.create( - num_stages=self.epi_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.correction_warp_ids) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len([self.epilogue_warp_id]) - ), - barrier_storage=storage.corr_epi_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - mma_corr_producer, mma_corr_consumer = pipeline.PipelineUmmaAsync.create( - num_stages=self.mma_corr_stage, - producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.correction_warp_ids) - ), - barrier_storage=storage.mma_corr_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - s0_p1_inplace_producer, s0_p1_inplace_consumer = pipeline.PipelineAsync.create( - num_stages=self.p_mma_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - barrier_storage=storage.s0_p1_inplace_barrier_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - s1_p0_inplace_producer, s1_p0_inplace_consumer = pipeline.PipelineAsync.create( - num_stages=self.p_mma_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - barrier_storage=storage.s1_p0_inplace_barrier_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - tmem = utils.TmemAllocator( - storage.tmem_holding_buf.ptr, - barrier_for_retrieve=self.tmem_alloc_barrier, - # Correction warp is the last one that accesses tmem - allocator_warp_id=self.correction_warp_ids[0], - arch=self.arch_str, - ) - pipeline_init_arrive(is_relaxed=True) - - # Generate smem tensor Q/K/V/O - # (MMA, MMA_Q, MMA_D, PIPE) - sQ = storage.sQ.get_tensor(q_smem_layout_staged.outer, swizzle=q_smem_layout_staged.inner) - # (MMA, MMA_K, MMA_D, PIPE) - sK = storage.sK.get_tensor(k_smem_layout_staged.outer, swizzle=k_smem_layout_staged.inner) - # (MMA, MMA_K, MMA_D, PIPE) - # Reuse k's smem buffer for v. Recast element type so MMA descriptor matches v_dtype. - sV_ptr = cute.recast_ptr( - cute.recast_ptr(sK.iterator, dtype=self.v_dtype), - v_smem_layout_staged.inner, - ) - sV = cute.make_tensor(sV_ptr, v_smem_layout_staged.outer) - sO = storage.sO.get_tensor(o_smem_layout_staged.outer, swizzle=o_smem_layout_staged.inner) - s0_warp_wants_skip_softmax_exchange = ( - storage.s0_warp_wants_skip_softmax_exchange.get_tensor(cute.make_layout((4,))) - ) - s1_warp_wants_skip_softmax_exchange = ( - storage.s1_warp_wants_skip_softmax_exchange.get_tensor(cute.make_layout((4,))) - ) - - qk_thr_mma = qk_tiled_mma.get_slice(0) # default 1sm - pv_thr_mma = pv_tiled_mma.get_slice(0) # default 1sm - tSrQ = qk_thr_mma.make_fragment_A(sQ) - tSrK = qk_thr_mma.make_fragment_B(sK) - tOrV = pv_thr_mma.make_fragment_B(sV) - - def make_tmem_tensors( - self, - qk_thr_mma: cute.TiledMma, - pv_thr_mma: cute.TiledMma, - p_tmem_layout_staged: cute.Layout, - tmem_ptr: cute.Pointer, - ): - qk_acc_shape = qk_thr_mma.partition_shape_C( - (self.qk_mma_tiler[0], self.qk_mma_tiler[1]) - ) - tStS_fake = qk_thr_mma.make_fragment_C(qk_acc_shape) - tStS = cute.make_tensor(tmem_ptr + self.tmem_s0_offset, tStS_fake.layout) - pv_acc_shape = pv_thr_mma.partition_shape_C( - (self.pv_mma_tiler[0], self.pv_mma_tiler[1]) - ) - tOtO = pv_thr_mma.make_fragment_C(pv_acc_shape) - tStS0 = cute.make_tensor(tmem_ptr + self.tmem_s0_offset, tStS.layout) - tStS1 = cute.make_tensor(tmem_ptr + self.tmem_s1_offset, tStS.layout) - tOtO0 = cute.make_tensor(tmem_ptr + self.tmem_o0_offset, tOtO.layout) - tOtO1 = cute.make_tensor(tmem_ptr + self.tmem_o1_offset, tOtO.layout) - tP = cute.make_tensor(tStS.iterator, p_tmem_layout_staged.outer) - tOrP = pv_thr_mma.make_fragment_A(tP)[None, None, None, 0] - tOrP0 = cute.make_tensor( - cute.recast_ptr( - tmem_ptr + self.tmem_p0_offset, - dtype=tOrP.dtype, - ), - tOrP.layout, - ) - tOrP1 = cute.make_tensor( - cute.recast_ptr( - tmem_ptr + self.tmem_p1_offset, - dtype=tOrP.dtype, - ), - tOrP.layout, - ) - return tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 - - tile_sched = fmha_utils.create_fmha_static_tile_scheduler( - tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() - ) - work_tile = tile_sched.initial_work_tile_info() - pipeline_init_wait() - softmax_fn = partial( - self.softmax, - qk_thr_mma=qk_thr_mma, - value_args=( - mK_kdl.shape[0], - mQ_qdl.shape[0], - scale_softmax_log2, - skip_softmax_threshold_log2, - ), - mask_args=(window_size_left, window_size_right), - sched_args=(tile_sched, work_tile), - ) - # /////////////////////////////////////////////////////////////////////////////// - # EMPTY - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.empty_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_other) - - # /////////////////////////////////////////////////////////////////////////////// - # LOAD - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.load_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_other) - cute.arch.griddepcontrol_wait() - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - continue_cond = False - cuseqlen_q = Int32(0) - seqlen_q = mQ_qdl.shape[0] - seqlen_k = mK_kdl.shape[0] - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q, - ) - ) - if not continue_cond: - if cutlass.const_expr(cum_seqlen_k is not None): - seqlen_k = cum_seqlen_k[batch_coord + 1] - cum_seqlen_k[batch_coord] - continue_cond = seqlen_k <= 0 - if not continue_cond: - mQ_qdl_ = mQ_qdl - mK_kdl_ = mK_kdl - mV_dkl_ = mV_dkl - if cutlass.const_expr(cum_seqlen_q is not None): - mQ_qdl_ = cute.domain_offset( - (cum_seqlen_q[batch_coord], 0, ((0, 0), 0)), mQ_qdl - ) - if cutlass.const_expr(cum_seqlen_k is not None): - mK_kdl_ = cute.domain_offset( - (cum_seqlen_k[batch_coord], 0, ((0, 0), 0)), mK_kdl - ) - mV_dkl_ = cute.domain_offset( - (0, cum_seqlen_k[batch_coord], ((0, 0), 0)), mV_dkl - ) - # Local tile partition global tensors - gQ_qdl = cute.flat_divide(mQ_qdl_, cute.select(self.qk_mma_tiler, mode=[0, 2])) - tSgQ_qdl = qk_thr_mma.partition_A(gQ_qdl) - tQsQ, tQgQ_qdl = cute.nvgpu.cpasync.tma_partition( - tma_atom_q, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sQ, 0, 3), - cute.group_modes(tSgQ_qdl, 0, 3), - ) - tQgQ = tQgQ_qdl[None, None, 0, curr_block_coord[2]] - gK_kdl = cute.flat_divide(mK_kdl_, cute.select(self.qk_mma_tiler, mode=[1, 2])) - tSgK_kdl = qk_thr_mma.partition_B(gK_kdl) - tKsK, tKgK_kdl = cute.nvgpu.cpasync.tma_partition( - tma_atom_k, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sK, 0, 3), - cute.group_modes(tSgK_kdl, 0, 3), - ) - tKgK = tKgK_kdl[None, None, 0, curr_block_coord[2]] - gV_dkl = cute.flat_divide(mV_dkl_, cute.select(self.pv_mma_tiler, mode=[1, 2])) - tSgV_dkl = pv_thr_mma.partition_B(gV_dkl) - tVsV, tVgV_dkl = cute.nvgpu.cpasync.tma_partition( - tma_atom_v, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sV, 0, 3), - cute.group_modes(tSgV_dkl, 0, 3), - ) - tVgV = tVgV_dkl[None, 0, None, curr_block_coord[2]] - seqlen_kv_loop_start = fmha_utils.FusedMask.get_trip_start( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q, - seqlen_k, - window_size_left, - ) - # Q0 - q0_coord = 2 * curr_block_coord[0] - q0_handle = load_q_producer.acquire_and_advance() - cute.copy( - tma_atom_q, - tQgQ[None, q0_coord], - tQsQ[None, q0_handle.index], - tma_bar_ptr=q0_handle.barrier, - ) - seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - # K0 - kv_coord = seqlen_kv_loop_start - k_handle = load_kv_producer.acquire_and_advance() - cute.copy( - tma_atom_k, - tKgK[None, kv_coord], - tKsK[None, k_handle.index], - tma_bar_ptr=k_handle.barrier, - ) - # Q1 - q1_coord = q0_coord + 1 - q1_handle = load_q_producer.acquire_and_advance() - cute.copy( - tma_atom_q, - tQgQ[None, q1_coord], - tQsQ[None, q1_handle.index], - tma_bar_ptr=q1_handle.barrier, - ) - kv_coord += 1 - - for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): - # Ki - k_handle = load_kv_producer.acquire_and_advance() - cute.copy( - tma_atom_k, - tKgK[None, kv_coord], - tKsK[None, k_handle.index], - tma_bar_ptr=k_handle.barrier, - ) - # Vi-1 - v_handle, load_kv_producer = self.kv_producer_update_tx_acquire_and_advance( - load_kv_producer, - load_kv_empty_mbar_ptr, - load_kv_full_mbar_ptr, - self.tma_copy_v_bytes, - ) - cute.copy( - tma_atom_v, - tVgV[None, kv_coord - 1], - tVsV[None, v_handle.index], - tma_bar_ptr=load_kv_full_mbar_ptr + v_handle.index, - ) - kv_coord += 1 - # End of seqlen_kv loop - # Vi_end - v_handle, load_kv_producer = self.kv_producer_update_tx_acquire_and_advance( - load_kv_producer, - load_kv_empty_mbar_ptr, - load_kv_full_mbar_ptr, - self.tma_copy_v_bytes, - ) - cute.copy( - tma_atom_v, - tVgV[None, kv_coord - 1], - tVsV[None, v_handle.index], - tma_bar_ptr=load_kv_full_mbar_ptr + v_handle.index, - ) - # End of if not continue_cond - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - # End of persistent scheduler loop - # /////////////////////////////////////////////////////////////////////////////// - # MMA - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.mma_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_other) - tmem.wait_for_alloc() - tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) - tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( - self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr - ) - enable_skip_softmax = skip_softmax_threshold_log2 is not None - tiled_tmem_load_v = None - tTMEM_LOADtS_v0, tTMEM_LOADtS_v1 = None, None - tTMEM_LOADrS_v0, tTMEM_LOADrS_v1 = None, None - if cutlass.const_expr(enable_skip_softmax): - cS = cute.make_identity_tensor(cute.select(self.qk_mma_tiler, mode=[0, 1])) - tScS = qk_thr_mma.partition_C(cS) - tStS_v = cute.composition(tStS, cute.make_layout((self.threads_per_warp, 1))) - tScS_v = cute.composition(tScS, cute.make_layout((self.threads_per_warp, 1))) - tmem_load_v_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(1)), - self.qk_acc_dtype, - ) - thread_idx = tidx % self.threads_per_warp - - tiled_tmem_load_v = tcgen05.make_tmem_copy(tmem_load_v_atom, tStS_v) - thr_tmem_load_v = tiled_tmem_load_v.get_slice(thread_idx) - tTMEM_LOADtS_v = thr_tmem_load_v.partition_S(tStS_v) - tTMEM_LOADcS_v = thr_tmem_load_v.partition_D(tScS_v) - tTMEM_LOADrS_v0 = cute.make_rmem_tensor(tTMEM_LOADcS_v.shape, self.qk_acc_dtype) - tTMEM_LOADrS_v1 = cute.make_rmem_tensor(tTMEM_LOADcS_v.shape, self.qk_acc_dtype) - tTMEM_LOADtS_v0 = cute.make_tensor( - tTMEM_LOADtS_v.iterator + self.tmem_skip_softmax0_offset, - tTMEM_LOADtS_v.layout, - ) - tTMEM_LOADtS_v1 = cute.make_tensor( - tTMEM_LOADtS_v.iterator + self.tmem_skip_softmax1_offset, - tTMEM_LOADtS_v.layout, - ) - - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - continue_cond = False - seqlen_q = mQ_qdl.shape[0] - seqlen_k = mK_kdl.shape[0] - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q, - ) - ) - if not continue_cond: - if cutlass.const_expr(cum_seqlen_k is not None): - cuseqlen_k = cum_seqlen_k[batch_coord] - seqlen_k = cum_seqlen_k[batch_coord + 1] - cuseqlen_k - continue_cond = seqlen_k <= 0 - if not continue_cond: - # Wait for Q0 - q0_handle = load_q_consumer.wait_and_advance() - tSrQ0 = tSrQ[None, None, None, q0_handle.index] - # Wait for K0 - k_handle = load_kv_consumer.wait_and_advance() - tSrK0 = tSrK[None, None, None, k_handle.index] - # GEMM_QK00 (Q0 * K0 -> S0) - mma_s0_producer, s0_corr_producer = self.mma_qk( - qk_tiled_mma, - (tSrQ0, tSrK0, tStS0), - (mma_s0_producer, s0_corr_producer), - ) - # Wait for Q1 - q1_handle = load_q_consumer.wait_and_advance() - tSrQ1 = tSrQ[None, None, None, q1_handle.index] - # GEMM_QK10 (Q1 * K0 -> S1), K0 is ready in GEMM_QK00 - mma_s1_producer, s1_corr_producer = self.mma_qk( - qk_tiled_mma, - (tSrQ1, tSrK0, tStS1), - (mma_s1_producer, s1_corr_producer), - ) - # Release K0 - k_handle.release() - # Note: Q0 & Q1 are still needed in the seqlen_kv loop - # so we need to release them after the seqlen_kv loop - seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - # O1 hasn't been accumulated yet, its first MMA calculation doesn't need to accumulate - pv_whether_acc = False - for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): - # Wait for Ki - k_handle = load_kv_consumer.wait_and_advance() - tSrKi = tSrK[None, None, None, k_handle.index] - # GEMM_QK0i (Q0 * Ki -> S0) - mma_s0_producer, s0_corr_producer = self.mma_qk( - qk_tiled_mma, - (tSrQ0, tSrKi, tStS0), - (mma_s0_producer, s0_corr_producer), - ) - # Wait for Vi-1 - v_handle = load_kv_consumer.wait_and_advance() - tOrVi = tOrV[None, None, None, v_handle.index] - # GEMM_PV0(i-1) (P0 * Vi-1 -> O0_partial) - mma_corr_producer, p0_mma_consumer = self.mma_pv( - pv_tiled_mma, - pv_whether_acc, - (tOrP0, tOrVi, tOtO0), - (mma_corr_producer, p0_mma_consumer), - ( - enable_skip_softmax, - tiled_tmem_load_v, - tTMEM_LOADtS_v0, - tTMEM_LOADrS_v0, - ), - ) - # GEMM_QK1i (Q1 * Ki -> S1) - mma_s1_producer, s1_corr_producer = self.mma_qk( - qk_tiled_mma, - (tSrQ1, tSrKi, tStS1), - (mma_s1_producer, s1_corr_producer), - ) - # Release Ki - k_handle.release() - # GEMM_PV1(i-1) (P1 * Vi-1 -> O1_partial) - mma_corr_producer, p1_mma_consumer = self.mma_pv( - pv_tiled_mma, - pv_whether_acc, - (tOrP1, tOrVi, tOtO1), - (mma_corr_producer, p1_mma_consumer), - ( - enable_skip_softmax, - tiled_tmem_load_v, - tTMEM_LOADtS_v1, - tTMEM_LOADrS_v1, - ), - ) - pv_whether_acc = True - # Release Vi-1 - v_handle.release() - # End of seqlen_kv loop - # release Q0 & Q1 - q0_handle.release() - q1_handle.release() - # Wait for Vi_end - v_handle = load_kv_consumer.wait_and_advance() - tOrVi = tOrV[None, None, None, v_handle.index] - # GEMM_PV0(i_end) (P0 * Vi_end -> O0) - mma_corr_producer, p0_mma_consumer = self.mma_pv( - pv_tiled_mma, - pv_whether_acc, - (tOrP0, tOrVi, tOtO0), - (mma_corr_producer, p0_mma_consumer), - ( - enable_skip_softmax, - tiled_tmem_load_v, - tTMEM_LOADtS_v0, - tTMEM_LOADrS_v0, - ), - ) - # GEMM_PV1(i_end) (P1 * Vi_end -> O1) - mma_corr_producer, p1_mma_consumer = self.mma_pv( - pv_tiled_mma, - pv_whether_acc, - (tOrP1, tOrVi, tOtO1), - (mma_corr_producer, p1_mma_consumer), - ( - enable_skip_softmax, - tiled_tmem_load_v, - tTMEM_LOADtS_v1, - tTMEM_LOADrS_v1, - ), - ) - # Release Vi_end - v_handle.release() - # Empty step for correction epilog - vec0_handle = s0_corr_producer.acquire_and_advance() - vec0_handle.commit() - vec1_handle = s1_corr_producer.acquire_and_advance() - vec1_handle.commit() - # End of if not continue_cond - # Advance to next tile - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - # End of persistent scheduler loop - # /////////////////////////////////////////////////////////////////////////////// - # Epilogue (TMA store path only) - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.epilogue_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_other) - if cutlass.const_expr(self.use_tma_store): - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - continue_cond = False - cuseqlen_q = Int32(0) - seqlen_q = mQ_qdl.shape[0] - - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q, - ) - ) - if not continue_cond: - mO_qdl_ = mO_qdl - if cutlass.const_expr(cum_seqlen_q is not None): - mO_qdl_ = cute.domain_offset( - (cum_seqlen_q[batch_coord], 0, ((0, 0), 0)), mO_qdl - ) - - o0_coord = 2 * curr_block_coord[0] - o1_coord = o0_coord + 1 - gO_qdl = cute.flat_divide( - mO_qdl_, cute.select(self.pv_mma_tiler, mode=[0, 1]) - ) - gO = gO_qdl[None, None, None, 0, curr_block_coord[2]] - tOsO, tOgO = cute.nvgpu.cpasync.tma_partition( - tma_atom_o, - 0, - cute.make_layout(1), - cute.group_modes(sO, 0, 2), - cute.group_modes(gO, 0, 2), - ) - - # O0 O1 using the same pipeline - # wait from corr, issue tma store on smem - # O0 - # 1. Wait for O0 final - o0_handle = corr_epi_consumer.wait_and_advance() - # 2. Copy O0 to gmem - cute.copy(tma_atom_o, tOsO[None, 0], tOgO[None, o0_coord]) - cute.arch.cp_async_bulk_commit_group() - # O1 - # 1. Wait for O1 final - o1_handle = corr_epi_consumer.wait_and_advance() - # 2. Copy O1 to gmem - cute.copy(tma_atom_o, tOsO[None, 1], tOgO[None, o1_coord]) - cute.arch.cp_async_bulk_commit_group() - - # Ensure O0 buffer is ready to be released - cute.arch.cp_async_bulk_wait_group(1, read=True) - o0_handle.release() - # Ensure O1 buffer is ready to be released - cute.arch.cp_async_bulk_wait_group(0, read=True) - o1_handle.release() - - # Advance to next tile - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - cute.arch.griddepcontrol_launch_dependents() - # End of persistent scheduler loop - # /////////////////////////////////////////////////////////////////////////////// - # Softmax0 - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx < self.softmax1_warp_ids[0]: - cute.arch.setmaxregister_increase(self.num_regs_softmax) - tmem.wait_for_alloc() - tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) - tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( - self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr - ) - softmax_fn( - stage=0, - tensor_args=( - tStS, - tStS0, - cum_seqlen_k, - cum_seqlen_q, - s0_warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ), - pipeline_args=(mma_s0_consumer, s0_corr_producer, p0_mma_producer), - inplace_args=(s0_p1_inplace_producer, s1_p0_inplace_consumer), - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Softmax1 - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx < self.correction_warp_ids[0] and warp_idx >= self.softmax1_warp_ids[0]: - cute.arch.setmaxregister_increase(self.num_regs_softmax) - tmem.wait_for_alloc() - tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) - tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( - self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr - ) - softmax_fn( - stage=1, - tensor_args=( - tStS, - tStS1, - cum_seqlen_k, - cum_seqlen_q, - s1_warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ), - pipeline_args=(mma_s1_consumer, s1_corr_producer, p1_mma_producer), - inplace_args=(s1_p0_inplace_producer, s0_p1_inplace_consumer), - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Correction - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx >= self.correction_warp_ids[0] and warp_idx < self.mma_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_correction) - tmem.allocate(self.num_tmem_alloc_cols) - tmem.wait_for_alloc() - tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) - tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( - self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr - ) - cS = cute.make_identity_tensor((self.qk_mma_tiler[0], self.qk_mma_tiler[1])) - tScS = qk_thr_mma.partition_C(cS) - - tStS_vec_layout = cute.composition(tStS.layout, cute.make_layout((128, 2))) - - tStS_vec0 = cute.make_tensor(tStS.iterator + self.tmem_vec0_offset, tStS_vec_layout) - tStS_vec1 = cute.make_tensor(tStS.iterator + self.tmem_vec1_offset, tStS_vec_layout) - - tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) - tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) - tmem_load_v_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(2)), - self.qk_acc_dtype, - ) - tiled_tmem_load_vec = tcgen05.make_tmem_copy(tmem_load_v_atom, tStS_vec0) - thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) - thr_tmem_load_vec = tiled_tmem_load_vec.get_slice(thread_idx) - tTMEM_LOAD_VECtS0 = thr_tmem_load_vec.partition_S(tStS_vec0) - tTMEM_LOAD_VECtS1 = thr_tmem_load_vec.partition_S(tStS_vec1) - tTMEM_LOAD_VECcS = thr_tmem_load_vec.partition_D(tScS_vec) - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - seqlen_k = mK_kdl.shape[0] - row_idx = Int32(0) - continue_cond = False - cuseqlen_q = Int32(0) - seqlen_q = mQ_qdl.shape[0] - - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q, - ) - ) - if not continue_cond: - row_idx = curr_block_coord[0] * self.cta_tiler[0] + tTMEM_LOAD_VECcS[0][0] - if cutlass.const_expr(cum_seqlen_k is not None): - cuseqlen_k = cum_seqlen_k[batch_coord] - seqlen_k = cum_seqlen_k[batch_coord + 1] - cuseqlen_k - continue_cond = seqlen_k <= 0 - if not continue_cond: - # Ignore first signal from softmax as no correction is required - vec0_handle = s0_corr_consumer.wait_and_advance() - vec0_handle.release() - vec1_handle = s1_corr_consumer.wait_and_advance() - vec1_handle.release() - # O0/O1 share the same mma_corr consumer state, so the Oi - # peek token rolls from O0 -> O1 -> next O0. Seed with a - # blocking token; the rescale helper refreshes it near the - # end of each iteration. - oi_peek_status = cutlass.Boolean(False) - seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): - # Rescale O0 - ( - (s0_corr_consumer, mma_corr_consumer), - oi_peek_status, - ) = self.correction_rescale( - pv_thr_mma, - tiled_tmem_load_vec, - scale_softmax_log2, - (tOtO0, tTMEM_LOAD_VECtS0, tTMEM_LOAD_VECcS), - (s0_corr_consumer, mma_corr_consumer), - oi_peek_status, - ) - # Rescale O1 - ( - (s1_corr_consumer, mma_corr_consumer), - oi_peek_status, - ) = self.correction_rescale( - pv_thr_mma, - tiled_tmem_load_vec, - scale_softmax_log2, - (tOtO1, tTMEM_LOAD_VECtS1, tTMEM_LOAD_VECcS), - (s1_corr_consumer, mma_corr_consumer), - oi_peek_status, - ) - # End of seqlen_corr_loop_steps - value_args = ( - cuseqlen_q, - seqlen_q, - curr_block_coord, - scale_softmax, - scale_output, - ) - if cutlass.const_expr(self.use_tma_store): - # TMA store path: write to sO, signal epilogue warp - # Normalize O0 - s0_corr_consumer, mma_corr_consumer, corr_epi_producer = ( - self.correction_epilog( - pv_thr_mma, - tiled_tmem_load_vec, - ( - tOtO0, - tTMEM_LOAD_VECtS0, - tTMEM_LOAD_VECcS, - sO[None, None, 0], - mLSE, - mSink, - ), - ( - s0_corr_consumer, - mma_corr_consumer, - corr_epi_producer, - ), - (row_idx, *value_args), - ) - ) - row_idx += self.qk_mma_tiler[0] - # Normalize O1 - s1_corr_consumer, mma_corr_consumer, corr_epi_producer = ( - self.correction_epilog( - pv_thr_mma, - tiled_tmem_load_vec, - ( - tOtO1, - tTMEM_LOAD_VECtS1, - tTMEM_LOAD_VECcS, - sO[None, None, 1], - mLSE, - mSink, - ), - ( - s1_corr_consumer, - mma_corr_consumer, - corr_epi_producer, - ), - (row_idx, *value_args), - ) - ) - else: - # st.global path: store directly to global memory - block_offset_o = Int32(0) - if cutlass.const_expr(cum_seqlen_q is not None): - block_offset_o = cum_seqlen_q[batch_coord] - mO_ = cute.make_tensor( - mO.iterator + block_offset_o * mO.stride[0], - cute.make_layout( - (seqlen_q, mO.shape[1], mO.shape[2]), - stride=mO.stride, - ), - ) - o0_coord = 2 * curr_block_coord[0] - o1_coord = o0_coord + 1 - gO_stg = cute.local_tile( - mO_, - (self.pv_mma_tiler[0], self.pv_mma_tiler[1]), - (None, None, None), - ) - gO0 = gO_stg[None, None, o0_coord, 0, curr_block_coord[2]] - gO1 = gO_stg[None, None, o1_coord, 0, curr_block_coord[2]] - # Normalize O0 and store to global memory - s0_corr_consumer, mma_corr_consumer = self.correction_epilog( - pv_thr_mma, - tiled_tmem_load_vec, - ( - tOtO0, - tTMEM_LOAD_VECtS0, - tTMEM_LOAD_VECcS, - gO0, - mLSE, - mSink, - ), - (s0_corr_consumer, mma_corr_consumer), - (row_idx, *value_args), - ) - row_idx += self.qk_mma_tiler[0] - # Normalize O1 and st.global to global memory - s1_corr_consumer, mma_corr_consumer = self.correction_epilog( - pv_thr_mma, - tiled_tmem_load_vec, - ( - tOtO1, - tTMEM_LOAD_VECtS1, - tTMEM_LOAD_VECcS, - gO1, - mLSE, - mSink, - ), - (s1_corr_consumer, mma_corr_consumer), - (row_idx, *value_args), - ) - # End of if not continue_cond - # Advance to next tile - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - if cutlass.const_expr(not self.use_tma_store): - cute.arch.griddepcontrol_launch_dependents() - # End of persistent scheduler loop - tmem.relinquish_alloc_permit() - # Synchronize before TMEM dealloc (done by the caller) - self.tmem_dealloc_barrier.arrive_and_wait() - tmem.free(tmem_ptr) +import platform +from pathlib import Path +from typing import Optional, Union + +import torch + +_cute_runtime_import_error = None +try: + import cutlass + from cuda.bindings import driver as cuda_driver + from cutlass import cute + from cutlass.cute import typing as cute_typing + from cutlass.cute.runtime import from_dlpack +except (ImportError, OSError) as e: + cutlass = None + cute = None + from_dlpack = None + cuda_driver = None + _cute_runtime_import_error = e + + +CUBINS_ROOT = Path(__file__).resolve().parent / "cubins" +SUPPORTED_GPU_ARCHS = ("sm_100a", "sm_103a") + + +def _dtype_to_str(dtype: torch.dtype) -> str: + float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) + dtype_map = { + torch.float16: "fp16", + torch.bfloat16: "bf16", + torch.float32: "fp32", + } + if float8_e4m3fn is not None: + dtype_map[float8_e4m3fn] = "e4m3" + try: + return dtype_map[dtype] + except KeyError as exc: + raise ValueError(f"Unsupported CuTe DSL FMHA dtype: {dtype}") from exc + + +def _gpu_cpu_arch() -> str: + arch = platform.machine().lower() + if arch in ("amd64", "x64"): + return "x86_64" + if arch in ("arm64",): + return "aarch64" + return arch + + +def _get_gpu_arch(device: torch.device | str | None = None) -> str: + capability = torch.cuda.get_device_capability(device) + gpu_arch = f"sm_{capability[0]}{capability[1]}a" + if gpu_arch not in SUPPORTED_GPU_ARCHS: + supported = ", ".join(SUPPORTED_GPU_ARCHS) + raise ValueError( + f"Unsupported GPU architecture {gpu_arch}. Supported architectures: {supported}." + ) + return gpu_arch + + +def _get_cubins_dir(gpu_arch: str) -> Path: + return CUBINS_ROOT / _gpu_cpu_arch() / gpu_arch + + +def _get_variant_name( + qk_dtype: torch.dtype, + pv_dtype: torch.dtype, + out_dtype: torch.dtype, + head_dim: int, + is_causal: bool, + is_persistent: bool = True, + varlen: bool = False, + with_lse: bool = False, + enable_skip_softmax: bool = False, + enable_tvm_ffi: bool = False, +) -> str: + qk_str = _dtype_to_str(qk_dtype) + pv_str = _dtype_to_str(pv_dtype) + out_str = _dtype_to_str(out_dtype) + if qk_dtype != pv_dtype: + dtype_str = f"{qk_str}_{pv_str}_{out_str}" + elif qk_dtype != out_dtype: + dtype_str = f"{qk_str}_{out_str}" + else: + dtype_str = qk_str + + causal_str = "causal" if is_causal else "nocausal" + persist_str = "persistent" if is_persistent else "nonpersistent" + varlen_str = "_varlen" if varlen else "" + lse_str = "_lse" if with_lse else "" + skip_str = "_skipsm" if enable_skip_softmax else "" + ffi_str = "_tvmffi" if enable_tvm_ffi else "" + return ( + f"cute_dsl_fmha_{dtype_str}_h{head_dim}_{causal_str}_{persist_str}" + f"{varlen_str}{lse_str}{skip_str}{ffi_str}" + ) + + +def _get_candidate_paths(variant_name: str, gpu_arch: str | None) -> list[Path]: + names = [f"{variant_name}.so", f"{variant_name}.o"] + if gpu_arch is not None: + return [_get_cubins_dir(gpu_arch) / name for name in names] + + host_dir = CUBINS_ROOT / _gpu_cpu_arch() + return [ + host_dir / supported_gpu_arch / name + for supported_gpu_arch in SUPPORTED_GPU_ARCHS + for name in names + ] + + +def _resolve_cubin_path( + variant_name: str, + gpu_arch: str | None = None, +) -> Path: + tried = [] + for candidate in _get_candidate_paths(variant_name, gpu_arch): + tried.append(candidate) + if candidate.exists(): + return candidate.resolve() + + searched = "\n".join(f" - {path}" for path in tried) + default_dir = ( + _get_cubins_dir(gpu_arch) + if gpu_arch is not None + else CUBINS_ROOT / _gpu_cpu_arch() / "" + ) + raise FileNotFoundError( + f"Could not find packaged CuTe DSL FMHA cubins for '{variant_name}'.\n" + f"Expected a .so or .o under {default_dir}.\nSearched:\n{searched}" + ) + + +def _check_cute_runtime_available() -> None: + if cute is not None: return - - @cute.jit - def kv_producer_update_tx_acquire_and_advance( - self, tma_producer, empty_mbar_ptr, full_mbar_ptr, tx_bytes - ): - # This utility function is a special version of tma_producer.acquire_and_advance(). - # This is used to customize the tx bytes which is different from - # the initialized tx bytes of tma_producer. - state = tma_producer._PipelineProducer__state.clone() - cute.arch.mbarrier_wait(empty_mbar_ptr + state.index, state.phase) - with cute.arch.elect_one(): - cute.arch.mbarrier_arrive_and_expect_tx( - full_mbar_ptr + state.index, - tx_bytes, - ) - tma_producer.advance() - return state, tma_producer - - @cute.jit - def get_skip_softmax_flag(self, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v): - cute.copy(tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v) - tTMEM_LOADrS_v_i32 = cute.recast_tensor(tTMEM_LOADrS_v, dtype=cutlass.Int32) - skip_softmax_flag = cute.arch.make_warp_uniform(tTMEM_LOADrS_v_i32[0]) - return skip_softmax_flag - - @cute.jit - def mma_qk( - self, - tiled_mma: cute.TiledMma, - tensor_args: Tuple, - pipeline_args: Tuple, - pipeline_tokens: Tuple = (None, None), - ) -> Tuple[pipeline.PipelineProducer, pipeline.PipelineProducer]: - """Perform a single step of the QK GEMM computation on a block of attention scores. - - :param tiled_mma: Tiled MMA for QK GEMM - :type tiled_mma: cute.TiledMma - :param tensor_args: Tuple containing Qi, K, and Si - :type tensor_args: Tuple - :param pipeline_args: Tuple containing mma_si_producer and si_corr_producer - :type pipeline_args: Tuple - :param pipeline_tokens: Optional non-blocking peek tokens for the Si and - vec_i producers, in the form ``(si_peek_status, veci_peek_status)``. - ``None`` for either token falls back to a blocking acquire. - :type pipeline_tokens: Tuple - :return: Tuple containing mma_si_producer and si_corr_producer - :rtype: Tuple[pipeline.PipelineProducer, pipeline.PipelineProducer] - """ - tSrQi, tSrK, tStSi = tensor_args - mma_si_producer, si_corr_producer = pipeline_args - si_peek_status, veci_peek_status = pipeline_tokens - # 0. Make sure Qi & K are ready when calling mma_qk - # 1. acquire S0 - si_handle = mma_si_producer.acquire_and_advance(si_peek_status) - # 2. make sure vec is already released in corr - veci_handle = si_corr_producer.acquire_and_advance(veci_peek_status) - veci_handle.commit() - # 3. gemm - num_kphases = cute.size(tSrQi, mode=[2]) - for kphase_idx in cutlass.range(num_kphases, unroll_full=True): - kphase_coord = (None, None, kphase_idx) - tiled_mma.set(tcgen05.Field.ACCUMULATE, kphase_idx != 0) - cute.gemm( - tiled_mma, - tStSi, - tSrQi[kphase_coord], - tSrK[kphase_coord], - tStSi, - ) - # 4. release S0 - si_handle.commit() - return mma_si_producer, si_corr_producer - - @cute.jit - def mma_pv( - self, - tiled_mma: cute.TiledMma, - whether_acc: bool, - tensor_args: Tuple, - pipeline_args: Tuple, - skip_pv_args: Tuple, - ) -> Tuple[ - pipeline.PipelineProducer, - pipeline.PipelineConsumer, - ]: - """Perform a single step of the PV GEMM computation on accumulating O. - - :param tiled_mma: Tiled MMA for PV GEMM - :type tiled_mma: cute.TiledMma - :param whether_acc: Whether to accumulate O - :type whether_acc: bool - :param tensor_args: Tuple containing Pi, Vi, and Oi - :type tensor_args: Tuple - :param pipeline_args: Tuple containing mma_corr_producer and pi_mma_consumer - :type pipeline_args: Tuple - :param skip_pv_args: Tuple containing enable_skip_softmax, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v - :type skip_pv_args: Tuple - :return: Tuple containing mma_corr_producer and pi_mma_consumer - :rtype: Tuple[pipeline.PipelineProducer, pipeline.PipelineConsumer] - """ - tOrPi, tOrVi, tOtOi = tensor_args - mma_corr_producer, pi_mma_consumer = pipeline_args - enable_skip_softmax, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v = skip_pv_args - # 0. Make sure Vi is ready when calling mma_pv - # 1. acquire Oi - oi_handle = mma_corr_producer.acquire_and_advance() - # 2. wait for Pi - pi_handle = pi_mma_consumer.wait_and_advance() - # 3. gemm - num_kphases = cute.size(tOrPi, mode=[2]) - if cutlass.const_expr(enable_skip_softmax): - skip_pv = self.get_skip_softmax_flag(tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v) - if not skip_pv: - for kphase_idx in cutlass.range(num_kphases, unroll_full=True): - kphase_coord = (None, None, kphase_idx) - tiled_mma.set(tcgen05.Field.ACCUMULATE, whether_acc or kphase_idx != 0) - cute.gemm( - tiled_mma, - tOtOi, - tOrPi[kphase_coord], - tOrVi[kphase_coord], - tOtOi, - ) - else: - for kphase_idx in cutlass.range(num_kphases, unroll_full=True): - kphase_coord = (None, None, kphase_idx) - tiled_mma.set(tcgen05.Field.ACCUMULATE, whether_acc or kphase_idx != 0) - cute.gemm( - tiled_mma, - tOtOi, - tOrPi[kphase_coord], - tOrVi[kphase_coord], - tOtOi, - ) - # 4. commit Pi - pi_handle.release() - # 5. commit Oi - oi_handle.commit() - return mma_corr_producer, pi_mma_consumer - - @cute.jit - def calculate_skip_softmax_flag( - self, - row_max, - tile_row_max, - scale_softmax_log2, - skip_softmax_threshold_log2, - seqlen_q, - thread_idx, - logical_offset, - warp_wants_skip_softmax_exchange, - stage, - skip_softmax_count, - total_softmax_count, - ) -> Tuple[bool, float]: - """Calculate the skip softmax flag and the row maximum. - - :param row_max: The row maximum. - :type row_max: float - :param tile_row_max: The tile row maximum. - :type tile_row_max: float - :param scale_softmax_log2: The scale softmax log2. - :type scale_softmax_log2: float - :param skip_softmax_threshold_log2: The skip softmax threshold log2. - :type skip_softmax_threshold_log2: float - :param seqlen_q: The sequence length q. - :type seqlen_q: int - :param thread_idx: The thread index. - :type thread_idx: int - :param logical_offset: The logical offset. - :type logical_offset: Tuple[int, int] - :param warp_wants_skip_softmax_exchange: The warp wants skip softmax exchange. - :type warp_wants_skip_softmax_exchange: cute.Tensor - :param stage: The stage. - :type stage: int - :param skip_softmax_count: The skip softmax count. - :type skip_softmax_count: cute.Tensor - :param total_softmax_count: The total softmax count. - :type total_softmax_count: cute.Tensor - :return: Tuple containing the skip softmax flag and the row maximum. - :rtype: Tuple[bool, float] - """ - thread_wants_skip = ( - tile_row_max * scale_softmax_log2 - row_max * scale_softmax_log2 - ) < skip_softmax_threshold_log2 - thread_wants_skip = thread_wants_skip or ((logical_offset[0] + thread_idx) >= seqlen_q) - warp_wants_skip = cute.arch.vote_all_sync(thread_wants_skip) - - with cute.arch.elect_one(): - warp_wants_skip_softmax_exchange[cute.arch.warp_idx() % 4] = warp_wants_skip - softmax_barrier = self.s0_warpgroup_barrier if stage == 0 else self.s1_warpgroup_barrier - softmax_barrier.arrive_and_wait() - warp_wants_skip_softmax_exchange_i32 = cute.make_tensor( - cute.recast_ptr(warp_wants_skip_softmax_exchange.iterator, dtype=cutlass.Int32), - cute.make_layout((1,)), + raise ImportError( + f"CuTe DSL runtime is not available. Import error: {_cute_runtime_import_error}" + ) from _cute_runtime_import_error + + +def _load_cubin_from_path( + path: str | Path, + variant_name: str | None = None, + enable_tvm_ffi: bool = True, +): + _check_cute_runtime_available() + + cubin_path = Path(path).expanduser().resolve() + if variant_name is None: + variant_name = cubin_path.stem + + module = cute.runtime.load_module(str(cubin_path), enable_tvm_ffi=enable_tvm_ffi) + try: + return getattr(module, variant_name) + except AttributeError as exc: + raise AttributeError( + f"Loaded {cubin_path}, but symbol '{variant_name}' was not found. " + "The symbol name must match the .so filename stem / function prefix." + ) from exc + + +@functools.lru_cache(maxsize=None) +def _load_cute_dsl_fmha_cubin_cached( + variant_name: str, + gpu_arch: str | None, + enable_tvm_ffi: bool, +): + path = _resolve_cubin_path(variant_name, gpu_arch) + return _load_cubin_from_path(path, variant_name, enable_tvm_ffi) + + +def get_cute_dsl_fmha_cubin( + qk_dtype: torch.dtype, + pv_dtype: torch.dtype, + out_dtype: torch.dtype, + head_dim: int, + is_causal: bool, + is_persistent: bool = True, + enable_tvm_ffi: bool = True, + varlen: bool = False, + with_lse: bool = False, + enable_skip_softmax: bool = False, + gpu_arch: str | None = None, +): + variant_name = _get_variant_name( + qk_dtype, + pv_dtype, + out_dtype, + head_dim, + is_causal, + is_persistent, + varlen, + with_lse, + enable_skip_softmax, + enable_tvm_ffi, + ) + return _load_cute_dsl_fmha_cubin_cached( + variant_name, + gpu_arch, + enable_tvm_ffi, + ) + + +def _check_inputs( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + o: torch.Tensor, +) -> bool: + if q.dim() not in (3, 4) or k.dim() != q.dim() or v.dim() != q.dim(): + raise ValueError("Expected 3D or 4D Q/K/V tensors with matching ranks.") + if q.dtype != k.dtype: + raise ValueError(f"Q/K dtype mismatch: {q.dtype} vs {k.dtype}") + if q.shape[-1] != k.shape[-1]: + raise ValueError(f"Q/K head dim mismatch: {q.shape[-1]} vs {k.shape[-1]}") + if k.shape[:-1] != v.shape[:-1]: + raise ValueError(f"K/V shape mismatch: {k.shape[:-1]} vs {v.shape[:-1]}") + expected_o_shape = (*q.shape[:-1], v.shape[-1]) + if tuple(o.shape) != expected_o_shape: + raise ValueError(f"Output shape mismatch: {tuple(o.shape)} vs {expected_o_shape}") + return q.dim() == 3 + + +def _to_cint_contiguous(tensor: torch.Tensor) -> torch.Tensor: + return tensor.to(torch.int32).contiguous() + + +def _to_cute_tensor(tensor: torch.Tensor, leading_dim: int): + float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) + if tensor.dtype == float8_e4m3fn: + cute_tensor = from_dlpack(tensor.view(torch.int8), assumed_align=16) + cute_tensor = cute_tensor.mark_layout_dynamic(leading_dim=leading_dim) + cute_tensor.element_type = cutlass.Float8E4M3FN + return cute_tensor + return from_dlpack(tensor, assumed_align=16).mark_layout_dynamic(leading_dim=leading_dim) + + +def _get_runtime_problem( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + o: torch.Tensor, + lse: Optional[torch.Tensor], + qo_indptr: Optional[torch.Tensor], + kv_indptr: Optional[torch.Tensor], + max_qo_len: Optional[int], + max_kv_len: Optional[int], + varlen: bool, +) -> tuple[ + int, + int, + int, + int, + int, + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, +]: + if varlen: + if qo_indptr is None or kv_indptr is None or max_qo_len is None or max_kv_len is None: + raise ValueError("Varlen FMHA requires indptr tensors and max sequence lengths.") + if qo_indptr.dim() != 1 or kv_indptr.dim() != 1: + raise ValueError("Varlen FMHA indptr tensors must be 1D.") + if qo_indptr.numel() != kv_indptr.numel() or qo_indptr.numel() < 2: + raise ValueError("Varlen FMHA indptr tensors must have matching non-empty sizes.") + total_q, num_heads_q, head_dim = q.shape + _, num_heads_kv, _ = k.shape + batch_size = qo_indptr.numel() - 1 + max_s_q = max_qo_len + max_s_k = max_kv_len + q_4d = q.unsqueeze(0) + k_4d = k.unsqueeze(0) + v_4d = v.unsqueeze(0) + o_4d = o.unsqueeze(0) + lse_3d = lse.unsqueeze(0) if lse is not None else None + else: + batch_size, max_s_q, num_heads_q, head_dim = q.shape + _, max_s_k, num_heads_kv, _ = k.shape + total_q = batch_size * max_s_q + q_4d = q + k_4d = k + v_4d = v + o_4d = o + lse_3d = lse + value_head_dim = v.shape[-1] + return ( + batch_size, + max_s_q, + total_q, + max_s_k, + num_heads_q, + num_heads_kv, + head_dim, + value_head_dim, + q_4d, + k_4d, + v_4d, + o_4d, + lse_3d, + ) + + +@torch.compiler.disable +def cute_dsl_fmha_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + o: torch.Tensor, + qo_indptr: Optional[torch.Tensor] = None, + kv_indptr: Optional[torch.Tensor] = None, + is_causal: bool = False, + sm_scale: Optional[float] = None, + window_left: int = -1, + window_right: int = -1, + lse: Optional[torch.Tensor] = None, + scale_q: Union[float, torch.Tensor] = 1.0, + scale_k: Union[float, torch.Tensor] = 1.0, + scale_v: Union[float, torch.Tensor] = 1.0, + scale_o: Union[float, torch.Tensor] = 1.0, + enable_tvm_ffi: bool = True, + is_persistent: bool = False, + max_qo_len: Optional[int] = None, + max_kv_len: Optional[int] = None, + kernel_fn=None, + skip_softmax_threshold_scale_factor: Optional[float] = None, +) -> None: + varlen = _check_inputs(q, k, v, o) + + scale_q = float(scale_q.item()) if isinstance(scale_q, torch.Tensor) else scale_q + scale_k = float(scale_k.item()) if isinstance(scale_k, torch.Tensor) else scale_k + scale_v = float(scale_v.item()) if isinstance(scale_v, torch.Tensor) else scale_v + scale_o = float(scale_o.item()) if isinstance(scale_o, torch.Tensor) else scale_o + + ( + batch_size, + max_s_q, + total_q, + max_s_k, + num_heads_q, + num_heads_kv, + head_dim, + value_head_dim, + q_4d, + k_4d, + v_4d, + o_4d, + lse_3d, + ) = _get_runtime_problem(q, k, v, o, lse, qo_indptr, kv_indptr, max_qo_len, max_kv_len, varlen) + use_skip_softmax = ( + skip_softmax_threshold_scale_factor is not None and skip_softmax_threshold_scale_factor > 0 + ) + problem_size = ( + batch_size, + max_s_q, + total_q, + max_s_k, + num_heads_q, + num_heads_kv, + head_dim, + value_head_dim, + ) + + if kernel_fn is None: + kernel_fn = get_cute_dsl_fmha_cubin( + q.dtype, + v.dtype, + o.dtype, + head_dim, + is_causal, + is_persistent=is_persistent, + varlen=varlen, + enable_tvm_ffi=enable_tvm_ffi, + with_lse=lse is not None, + enable_skip_softmax=use_skip_softmax, + gpu_arch=_get_gpu_arch(q.device), + ) + + if sm_scale is None: + sm_scale = 1.0 / math.sqrt(head_dim) + scale_softmax = scale_q * scale_k * sm_scale + scale_softmax_log2 = scale_softmax * math.log2(math.exp(1.0)) + scale_output = scale_v / scale_o + + skip_threshold_log2 = None + if use_skip_softmax: + threshold = skip_softmax_threshold_scale_factor / max_s_k + skip_threshold_log2 = cute_typing.Float32(math.log2(threshold)) + + ws_left = None if window_left == -1 else cute_typing.Int32(window_left) + ws_right = None if window_right == -1 else cute_typing.Int32(window_right) + if is_causal and ws_right is None: + ws_right = cute_typing.Int32(0) + + # CUBIN path + num_head_groups = num_heads_q // num_heads_kv + q_5d = q_4d.unflatten(2, (num_heads_kv, num_head_groups)) + k_5d = k_4d.unsqueeze(3) + v_5d = v_4d.unsqueeze(3) + o_5d = o_4d.unflatten(2, (num_heads_kv, num_head_groups)) + lse_4d = lse_3d.unflatten(2, (num_heads_kv, num_head_groups)) if lse is not None else None + + if enable_tvm_ffi: + qo_indptr_i32 = _to_cint_contiguous(qo_indptr) if varlen else None + kv_indptr_i32 = _to_cint_contiguous(kv_indptr) if varlen else None + kernel_fn( + q_5d, + k_5d, + v_5d, + o_5d, + problem_size, + qo_indptr_i32, + kv_indptr_i32, + lse_4d, + None, # sink + cute_typing.Float32(scale_softmax_log2), + cute_typing.Float32(scale_softmax), + cute_typing.Float32(scale_output), + skip_threshold_log2, + ws_left, + ws_right, + None, + None, + False, # reserved ) - skip_softmax = cute.arch.popc(warp_wants_skip_softmax_exchange_i32[0]) == 4 - - if not skip_softmax: - row_max = max(row_max, tile_row_max) - - if cutlass.const_expr(skip_softmax_count is not None): - if thread_idx == 0: - if skip_softmax: - cute.arch.atomic_add(skip_softmax_count.iterator.llvm_ptr, Int32(1)) - cute.arch.atomic_add(total_softmax_count.iterator.llvm_ptr, Int32(1)) - return skip_softmax, row_max - - @cute.jit - def apply_exp_and_cvt( - self, - tTMEM_LOADrS, - tTMEM_LOADrS_cvt, - tTMEM_STORErS_x4_e_cvt, - stage, - scale, - minus_row_max_scale, - local_row_sum, - inplace_consumer, - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_COUNT, - CVT_COUNT, - CVT_PER_STEP, - FMA_COUNT, - ARV_COUNT, - ): - """Apply the exp and conversion to the P data type on fragment. - - :param tTMEM_LOADrS: The tTMEM_LOADrS tensor. - :type tTMEM_LOADrS: cute.Tensor - :param tTMEM_LOADrS_cvt: The tTMEM_LOADrS_cvt tensor. - :type tTMEM_LOADrS_cvt: cute.Tensor - :param tTMEM_STORErS_x4_e_cvt: The tTMEM_STORErS_x4_e_cvt tensor. - :type tTMEM_STORErS_x4_e_cvt: cute.Tensor - :param stage: The stage. - :type stage: int - :param scale: The scale. - :type scale: float - :param minus_row_max_scale: The minus row maximum scale. - :type minus_row_max_scale: float - :param local_row_sum: The local row sum. - :type local_row_sum: float - :param inplace_consumer: The inplace consumer. - :type inplace_consumer: cute.Tensor - :param EXP2_EMULATION_OFFSET: The exp2 emulation offset. - :type EXP2_EMULATION_OFFSET: int - :param EXP2_EMULATION_COUNT: The exp2 emulation count. - :type EXP2_EMULATION_COUNT: int - :param CVT_COUNT: The cvt count. - :type CVT_COUNT: int - :param CVT_PER_STEP: The cvt per step. - :type CVT_PER_STEP: int - :param FMA_COUNT: The fma count. - :type FMA_COUNT: int - :param ARV_COUNT: The arv count. - :type ARV_COUNT: int - :return: The local row sum and the inplace consumer. - :rtype: Tuple[float, cute.Tensor] - """ - for i in cutlass.range_constexpr(0, EXP2_EMULATION_OFFSET, 2): - if cutlass.const_expr(i >= CVT_COUNT): - if cutlass.const_expr(i % CVT_PER_STEP == 0): - if cutlass.const_expr(self.v_dtype.width == 8): - fmha_utils.cvt_f32x4_to_f8x4( - tTMEM_LOADrS_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP], - tTMEM_STORErS_x4_e_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP], - ) - else: - s_vec = tTMEM_LOADrS_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP].load() - tTMEM_STORErS_x4_e_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP].store( - s_vec.to(self.v_dtype) - ) - local_row_sum = cute.arch.add_packed_f32x2( - local_row_sum, - ( - tTMEM_LOADrS[i - CVT_COUNT], - tTMEM_LOADrS[i - CVT_COUNT + 1], - ), - ) - tTMEM_LOADrS[i] = cute.math.exp2(tTMEM_LOADrS[i], fastmath=True) - if cutlass.const_expr(i + FMA_COUNT < EXP2_EMULATION_OFFSET): - ( - tTMEM_LOADrS[i + FMA_COUNT], - tTMEM_LOADrS[i + FMA_COUNT + 1], - ) = cute.arch.fma_packed_f32x2( - ( - tTMEM_LOADrS[i + FMA_COUNT], - tTMEM_LOADrS[i + FMA_COUNT + 1], - ), - (scale, scale), - (minus_row_max_scale, minus_row_max_scale), - ) - tTMEM_LOADrS[i + 1] = cute.math.exp2(tTMEM_LOADrS[i + 1], fastmath=True) - if cutlass.const_expr(i == EXP2_EMULATION_OFFSET - ARV_COUNT): - if cutlass.const_expr(self.enable_sequence_barrier): - if cutlass.const_expr(stage == 0): - self.sequence_s1_s0_barrier.arrive() - else: - self.sequence_s0_s1_barrier.arrive() - - # The remaining conversion steps - for i in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET - CVT_COUNT, - EXP2_EMULATION_OFFSET, - 2, - ): - if cutlass.const_expr(i % CVT_PER_STEP == 0): - if cutlass.const_expr(self.v_dtype.width == 8): - fmha_utils.cvt_f32x4_to_f8x4( - tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], - ) - else: - s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) - local_row_sum = cute.arch.add_packed_f32x2( - local_row_sum, (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]) - ) - for i in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET, EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT // 2, 2 - ): - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( - (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), - (scale, scale), - (minus_row_max_scale, minus_row_max_scale), - ) - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = fmha_utils.ex2_emulation_packed_f32x2( - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] - ) - if cutlass.const_expr((i + 2) % CVT_PER_STEP == 0): - if cutlass.const_expr(self.v_dtype.width == 8): - fmha_utils.cvt_f32x4_to_f8x4( - tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], - ) - else: - s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) - - inplace_peek_status = inplace_consumer.try_wait() - for i in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT // 2, - EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, - 2, - ): - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( - (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), - (scale, scale), - (minus_row_max_scale, minus_row_max_scale), - ) - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = fmha_utils.ex2_emulation_packed_f32x2( - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] - ) - if cutlass.const_expr((i + 2) % CVT_PER_STEP == 0): - if cutlass.const_expr(self.v_dtype.width == 8): - fmha_utils.cvt_f32x4_to_f8x4( - tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], - ) - else: - s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) - inplace_consumer.wait_and_advance(inplace_peek_status) - return local_row_sum, inplace_consumer - - @cute.jit - def softmax_step( - self, - stage: int, - whether_apply_mask: bool, - iter_args: Tuple, - stats_args: Tuple, - pipeline_args: Tuple, - value_args: Tuple, - atom_args: Tuple, - tensor_args: Tuple, - ) -> Tuple[Tuple, Tuple]: - """Perform a single step of the softmax computation on a block of attention scores. - - This method processes one block of the attention matrix, computing numerically stable - softmax by first finding the row maximum, subtracting it from all elements, applying - exponential function, and then normalizing by the sum of exponentials. It also handles - optional masking of attention scores. - - The method involves several key operations: - 1. Loading attention scores from tensor memory - 2. Applying optional masking based on position - 3. Computing row-wise maximum values for numerical stability - 4. Transforming scores using exp2(x*scale - max*scale) - 5. Computing row sums for normalization - 6. Coordinating pipeline synchronization between different processing stages - - :param stage: Processing stage (0 for first half, 1 for second half) - :type stage: int - :param whether_apply_mask: Whether to apply attention masking - :type whether_apply_mask: bool - :param iter_args: Tuple containing the counting tensor, row_max, row_sum, and vector buffer's handle for current iteration - :type iter_args: Tuple - :param stats_args: Tuple containing row_sum and row_max - :type stats_args: Tuple - :param pipeline_args: Tuple containing pipeline related arguments for MMA, correction, and sequence synchronization - :type pipeline_args: Tuple - :param value_args: Tuple containing seqlen_k, seqlen_q, and scale_softmax_log2 - :type value_args: Tuple - :param atom_args: Tuple containing mma & copy atoms - :type atom_args: Tuple - :param tensor_args: Tuple containing softmax related tensors - :type tensor_args: Tuple - :param fused_mask: Compute trip counts and apply masking for attention blocks - :type fused_mask: fmha_utils.FusedMask - :return: Updated stats_args and pipeline_args - :rtype: Tuple[Tuple, Tuple] - """ - row_sum, row_max = stats_args - cS, is_last_iter = iter_args - ( - seqlen_k, - seqlen_q, - scale_softmax_log2, - window_size_left, - window_size_right, - skip_softmax_threshold_log2, - thread_idx, - logical_offset, - ) = value_args - ( - si_peek_status, - mma_si_consumer, - si_corr_producer, - pi_mma_producer, - inplace_producer, - inplace_consumer, - ) = pipeline_args - ( - qk_thr_mma, - tiled_tmem_load, - tiled_tmem_store, - tiled_tmem_store_vec, - thr_tmem_load, - thr_tmem_store, - thr_tmem_store_vec, - ) = atom_args - ( - tTMEM_LOADtS, - tTMEM_STORE_VECtS, - tTMEM_STORE_SKIP_SOFTMAX, - tTMEM_STOREtS_x4, - warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ) = tensor_args - tilePlikeFP32 = self.qk_mma_tiler[1] // Float32.width * self.o_dtype.width - tScS = qk_thr_mma.partition_C(cS) - enable_skip_softmax = skip_softmax_threshold_log2 is not None - tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) - tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) - tScS_P_layout = cute.composition(tScS.layout, cute.make_layout((128, tilePlikeFP32))) - tScS_P = cute.make_tensor(tScS.iterator, tScS_P_layout) - tTMEM_LOADcS = thr_tmem_load.partition_D(tScS) - tTMEM_STORE_VECcS = thr_tmem_store_vec.partition_S(tScS_vec) - tTMEM_STOREcS = thr_tmem_store.partition_S(tScS_P) - # Wait for Si - si_handle = mma_si_consumer.wait_and_advance(si_peek_status) - tTMEM_LOADrS = cute.make_rmem_tensor(tTMEM_LOADcS.shape, self.qk_acc_dtype) - old_row_max = row_max - skip_softmax = cutlass.Boolean(False) - if whether_apply_mask: - if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): - cute.copy(tiled_tmem_load, tTMEM_LOADtS, tTMEM_LOADrS) - else: - tTMEM_LOADrMax = cute.make_rmem_tensor( - cute.make_layout((1, cute.size(tTMEM_LOADrS, mode=[1]))), - self.qk_acc_dtype, - ) - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[1])): - cute.copy_atom_call( - tiled_tmem_load, - tTMEM_LOADtS[None, i, 0, 0], - (tTMEM_LOADrS[None, i, 0, 0], tTMEM_LOADrMax[None, i]), - ) - fmha_utils.FusedMask.apply_mask( - self.mask_type, - tTMEM_LOADrS, - tTMEM_LOADcS, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - tile_row_max = tTMEM_LOADrS.load().reduce(cute.ReductionOp.MAX, -cutlass.Float32.inf, 0) - if cutlass.const_expr(not enable_skip_softmax): - row_max = cute.arch.fmax(row_max, tile_row_max) - else: - skip_softmax, row_max = self.calculate_skip_softmax_flag( - row_max, - tile_row_max, - scale_softmax_log2, - skip_softmax_threshold_log2, - seqlen_q, - thread_idx, - logical_offset, - warp_wants_skip_softmax_exchange, - stage, - skip_softmax_count, - total_softmax_count, - ) - si_handle.release() - # S0 -> P1 / S1 -> P0 - inplace_producer.commit() - inplace_producer.advance() - else: - if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): - cute.copy( - tiled_tmem_load, - tTMEM_LOADtS[None, 0, None, None], - tTMEM_LOADrS[None, 0, None, None], - ) - cute.copy( - tiled_tmem_load, - tTMEM_LOADtS[None, 1, None, None], - tTMEM_LOADrS[None, 1, None, None], - ) - tile_row_max = -cutlass.Float32.inf - tile_row_max_ = tile_row_max - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 0, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 0, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 0, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 0, 0, 0]) - cute.copy( - tiled_tmem_load, - tTMEM_LOADtS[None, 2, None, None], - tTMEM_LOADrS[None, 2, None, None], - ) - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 1, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 1, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 1, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 1, 0, 0]) - cute.copy( - tiled_tmem_load, - tTMEM_LOADtS[None, 3, None, None], - tTMEM_LOADrS[None, 3, None, None], - ) - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 2, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 2, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 2, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 2, 0, 0]) - cute.arch.fence_view_async_tmem_store() - si_handle.release() - # S0 -> P1 / S1 -> P0 - inplace_producer.commit() - inplace_producer.advance() - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 3, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 3, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 3, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 3, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tile_row_max_) - if cutlass.const_expr(not enable_skip_softmax): - row_max = cute.arch.fmax(tile_row_max, row_max) - else: - skip_softmax, row_max = self.calculate_skip_softmax_flag( - row_max, - tile_row_max, - scale_softmax_log2, - skip_softmax_threshold_log2, - seqlen_q, - thread_idx, - logical_offset, - warp_wants_skip_softmax_exchange, - stage, - skip_softmax_count, - total_softmax_count, - ) - else: - tTMEM_LOADrMax = cute.make_rmem_tensor( - cute.make_layout((1, cute.size(tTMEM_LOADrS, mode=[1]))), - self.qk_acc_dtype, - ) - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[1])): - cute.copy_atom_call( - tiled_tmem_load, - tTMEM_LOADtS[None, i, 0, 0], - (tTMEM_LOADrS[None, i, 0, 0], tTMEM_LOADrMax[None, i]), - ) - cute.arch.fence_view_async_tmem_store() - tile_row_max = tTMEM_LOADrMax.load().reduce( - cute.ReductionOp.MAX, -cutlass.Float32.inf, 0 - ) - if cutlass.const_expr(not enable_skip_softmax): - row_max = cute.arch.fmax(tile_row_max, row_max) - else: - skip_softmax, row_max = self.calculate_skip_softmax_flag( - row_max, - tile_row_max, - scale_softmax_log2, - skip_softmax_threshold_log2, - seqlen_q, - thread_idx, - logical_offset, - warp_wants_skip_softmax_exchange, - stage, - skip_softmax_count, - total_softmax_count, - ) - si_handle.release() - # S0 -> P1 / S1 -> P0 - inplace_producer.commit() - inplace_producer.advance() - - row_max_safe = row_max - if row_max == -cutlass.Float32.inf: - row_max_safe = 0.0 - if cutlass.const_expr(self.rescale_threshold > 0.0): - if (row_max_safe - old_row_max) * scale_softmax_log2 <= self.rescale_threshold: - row_max_safe = old_row_max - tTMEM_STORE_VECrS = cute.make_rmem_tensor(tTMEM_STORE_VECcS.shape, self.qk_acc_dtype) - tTMEM_STORE_VECrS[0] = old_row_max - tTMEM_STORE_VECrS[1] = row_max_safe - vec_i_peek_status = si_corr_producer.try_acquire() - tTMEM_STORErS_x4 = cute.make_rmem_tensor(tTMEM_STOREcS.shape, self.qk_acc_dtype) - tTMEM_STORErS_x4_e = cute.make_tensor( - cute.recast_ptr(tTMEM_STORErS_x4.iterator, dtype=self.v_dtype), - tTMEM_LOADrS.layout, - ) - scale = scale_softmax_log2 - minus_row_max_scale = (0.0 - row_max_safe) * scale - if cutlass.const_expr(self.v_dtype.width == 8 and self.p_fp8_prescale_log2 > 0): - minus_row_max_scale = minus_row_max_scale + self.p_fp8_prescale_log2 - - ARV_COUNT = 4 - FMA_COUNT = 8 - CVT_COUNT = 8 if self.v_dtype.width == 8 else 4 - CVT_PER_STEP = 4 if self.v_dtype.width == 8 else 2 - assert CVT_COUNT % CVT_PER_STEP == 0, ( - f"CVT_COUNT {CVT_COUNT} must be divisible by CVT_PER_STEP {CVT_PER_STEP}" - ) - tTMEM_LOADrS_cvt = cute.logical_divide(tTMEM_LOADrS, cute.make_layout(CVT_PER_STEP)) - tTMEM_STORErS_x4_e_cvt = cute.logical_divide( - tTMEM_STORErS_x4_e, cute.make_layout(CVT_PER_STEP) - ) - for i in cutlass.range_constexpr(0, FMA_COUNT, 2): - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( - (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), - (scale, scale), - (minus_row_max_scale, minus_row_max_scale), - ) - vec_i_handle = si_corr_producer.acquire_and_advance(vec_i_peek_status) - cute.copy(tiled_tmem_store_vec, tTMEM_STORE_VECrS, tTMEM_STORE_VECtS) - cute.arch.fence_view_async_tmem_store() - # Notify correction wg that row_max is ready - vec_i_handle.commit() - - EXP2_EMULATION_COUNT = 20 if self.enable_ex2_emulation and not whether_apply_mask else 0 - EXP2_EMULATION_OFFSET = cute.size(tTMEM_LOADrS) - EXP2_EMULATION_COUNT - acc_scale_ = scale * (old_row_max - row_max_safe) - acc_scale = cute.math.exp2(acc_scale_, fastmath=True) * 0.5 - if cutlass.const_expr(self.enable_sequence_barrier): - if cutlass.const_expr(stage == 0): - self.sequence_s0_s1_barrier.arrive_and_wait() - else: - self.sequence_s1_s0_barrier.arrive_and_wait() - - if cutlass.const_expr(enable_skip_softmax): - if not skip_softmax: - row_sum *= acc_scale - local_row_sum = (row_sum, row_sum) - local_row_sum, inplace_consumer = self.apply_exp_and_cvt( - tTMEM_LOADrS, - tTMEM_LOADrS_cvt, - tTMEM_STORErS_x4_e_cvt, - stage, - scale, - minus_row_max_scale, - local_row_sum, - inplace_consumer, - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_COUNT, - CVT_COUNT, - CVT_PER_STEP, - FMA_COUNT, - ARV_COUNT, - ) - tTMEM_STORE_VECrS_i32 = cute.recast_tensor(tTMEM_STORE_VECrS, dtype=cutlass.Int32) - tTMEM_STORE_VECrS_i32[0] = 0 - pi_handle = pi_mma_producer.acquire_and_advance() - # store skip softmax flag - cute.copy( - tiled_tmem_store_vec, - tTMEM_STORE_VECrS_i32, - tTMEM_STORE_SKIP_SOFTMAX, - ) - # store P - cute.copy(tiled_tmem_store, tTMEM_STORErS_x4, tTMEM_STOREtS_x4) - cute.arch.fence_view_async_tmem_store() - pi_handle.commit() - for j in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, - 2, - ): - local_row_sum = cute.arch.add_packed_f32x2( - (tTMEM_LOADrS[j], tTMEM_LOADrS[j + 1]), - local_row_sum, - ) - row_sum = local_row_sum[0] + local_row_sum[1] - cute.arch.fence_view_async_tmem_store() - else: - if cutlass.const_expr(self.enable_sequence_barrier): - if cutlass.const_expr(stage == 0): - self.sequence_s1_s0_barrier.arrive() - else: - self.sequence_s0_s1_barrier.arrive() - inplace_peek_status = inplace_consumer.try_wait() - inplace_consumer.wait_and_advance(inplace_peek_status) - tTMEM_STORE_VECrS_i32 = cute.recast_tensor(tTMEM_STORE_VECrS, dtype=cutlass.Int32) - tTMEM_STORE_VECrS_i32[0] = 1 - pi_handle = pi_mma_producer.acquire_and_advance() - # store skip softmax flag - cute.copy( - tiled_tmem_store_vec, - tTMEM_STORE_VECrS_i32, - tTMEM_STORE_SKIP_SOFTMAX, - ) - cute.arch.fence_view_async_tmem_store() - pi_handle.commit() - else: - row_sum *= acc_scale - local_row_sum = (row_sum, row_sum) - local_row_sum, inplace_consumer = self.apply_exp_and_cvt( - tTMEM_LOADrS, - tTMEM_LOADrS_cvt, - tTMEM_STORErS_x4_e_cvt, - stage, - scale, - minus_row_max_scale, - local_row_sum, - inplace_consumer, - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_COUNT, - CVT_COUNT, - CVT_PER_STEP, - FMA_COUNT, - ARV_COUNT, - ) - pi_handle = pi_mma_producer.acquire_and_advance() - # store P - cute.copy(tiled_tmem_store, tTMEM_STORErS_x4, tTMEM_STOREtS_x4) - cute.arch.fence_view_async_tmem_store() - for j in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, - 2, - ): - local_row_sum = cute.arch.add_packed_f32x2( - (tTMEM_LOADrS[j], tTMEM_LOADrS[j + 1]), - local_row_sum, - ) - row_sum = local_row_sum[0] + local_row_sum[1] - cute.arch.fence_view_async_tmem_store() - # Notify tensor core warp that softmax(S->P) is ready - pi_handle.commit() - if not is_last_iter: - si_peek_status = mma_si_consumer.try_wait() - - stats_args = (row_sum, row_max_safe) - pipeline_args = ( - si_peek_status, - mma_si_consumer, - si_corr_producer, - pi_mma_producer, - inplace_producer, - inplace_consumer, - ) - return stats_args, pipeline_args - - # For both softmax0 and softmax1 warp group - @cute.jit - def softmax( - self, - stage: int, - tensor_args: Tuple, - pipeline_args: Tuple, - inplace_args: Tuple, - qk_thr_mma: cute.ThrMma, - value_args: Tuple, - mask_args: Tuple, - sched_args: Tuple, - ): - """Compute softmax on attention scores from QK matrix multiplication. - - This method handles the softmax computation for either the first or second half of the - attention matrix, depending on the 'stage' parameter. It calculates row-wise maximum - and sum values needed for stable softmax computation, applies optional masking, and - transforms raw attention scores into probability distributions. - - The implementation uses specialized memory access patterns and efficient math operations - for computing exp(x) using exp2 functions. It also coordinates pipeline - synchronization between MMA, correction, and sequence processing stages. - - :param stage: Processing stage (0 for first half, 1 for second half of attention matrix) - :type stage: int - :param seqlen_k: Length of the key sequence - :type seqlen_k: Int32 - :param seqlen_q: Length of the query sequence - :type seqlen_q: Int32 - :param cum_seqlen_q: Cumulative sequence lengths for queries - :type cum_seqlen_q: cute.Tensor | None - :param cum_seqlen_k: Cumulative sequence lengths for keys - :type cum_seqlen_k: cute.Tensor | None - :param scale_softmax_log2: Log2 scale factor for softmax operation - :type scale_softmax_log2: Float32 - :param qk_thr_mma: Thread MMA operation for QK matrix multiplication - :type qk_thr_mma: cute.ThrMma - :param tStS: Shared tensor for softmax input/output - :type tStS: cute.Tensor - :param tStSi: Input tensor containing attention scores - :type tStSi: cute.Tensor - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - :param mma_si_consumer: Pipeline for synchronizing with Si tensors - :type mma_si_consumer: pipeline.PipelineConsumer - :param si_corr_producer: Pipeline for synchronizing with correction operations - :type si_corr_producer: pipeline.PipelineProducer - :param pi_mma_producer: Pipeline for synchronizing with Pi tensors - :type pi_mma_producer: pipeline.PipelineProducer - :param tile_sched_params: Parameters for tile scheduling - :type tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams - :param fused_mask: Compute trip counts and apply masking for attention blocks - :type fused_mask: fmha_utils.FusedMask - """ - ( - tStS, - tStSi, - cum_seqlen_k, - cum_seqlen_q, - warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ) = tensor_args - mma_si_consumer, si_corr_producer, pi_mma_producer = pipeline_args - inplace_producer, inplace_consumer = inplace_args - ( - seqlen_k, - seqlen_q, - scale_softmax_log2, - skip_softmax_threshold_log2, - ) = value_args - window_size_left, window_size_right = mask_args - tile_sched, work_tile = sched_args - - tidx, _, _ = cute.arch.thread_idx() - thread_idx = tidx % (self.threads_per_warp * len(self.softmax0_warp_ids)) - - cS_base = cute.make_identity_tensor((self.qk_mma_tiler[0], self.qk_mma_tiler[1])) - tilePlikeFP32 = self.qk_mma_tiler[1] // 32 * self.o_dtype.width - tScS = qk_thr_mma.partition_C(cS_base) - tStS_vec_layout = cute.composition(tStS.layout, cute.make_layout((128, 2))) - tmem_vec_offset = self.tmem_vec0_offset if stage == 0 else self.tmem_vec1_offset - tStS_vec = cute.make_tensor(tStS.iterator + tmem_vec_offset, tStS_vec_layout) - tmem_skip_softmax_offset = ( - self.tmem_skip_softmax0_offset if stage == 0 else self.tmem_skip_softmax1_offset - ) - tStS_skip_softmax = cute.make_tensor( - tStS.iterator + tmem_skip_softmax_offset, tStS_vec_layout - ) - tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) - tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) - tStS_P_layout = cute.composition(tStS.layout, cute.make_layout((128, tilePlikeFP32))) - tmem_p_offset = self.tmem_p0_offset if stage == 0 else self.tmem_p1_offset - tStS_P = cute.make_tensor(tStS.iterator + tmem_p_offset, tStS_P_layout) - if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), - self.qk_acc_dtype, - ) - else: - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.LdRed32x32bOp( - tcgen05.copy.Repetition(32), redOp=tcgen05.TmemLoadRedOp.MAX - ), - self.qk_acc_dtype, - ) - - tiled_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStSi) - thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) - tTMEM_LOADtS = thr_tmem_load.partition_S(tStSi) - tmem_store_vec_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(2)), - self.qk_acc_dtype, - ) - tiled_tmem_store_vec = tcgen05.make_tmem_copy(tmem_store_vec_atom, tStS_vec) - thr_tmem_store_vec = tiled_tmem_store_vec.get_slice(thread_idx) - tTMEM_STORE_VECtS = thr_tmem_store_vec.partition_D(tStS_vec) - tTMEM_STORE_VECcS = thr_tmem_store_vec.partition_S(tScS_vec) - tTMEM_STORE_SKIP_SOFTMAX = thr_tmem_store_vec.partition_D(tStS_skip_softmax) - tmem_store_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(32)), - self.qk_acc_dtype, - ) - tiled_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tStS_P) - thr_tmem_store = tiled_tmem_store.get_slice(thread_idx) - tTMEM_STOREtS_x4 = thr_tmem_store.partition_D(tStS_P) - if cutlass.const_expr(self.enable_sequence_barrier): - if cutlass.const_expr(stage == 1): - self.sequence_s0_s1_barrier.arrive() - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - seqlen_k_ = seqlen_k - seqlen_q_ = seqlen_q - continue_cond = False - cuseqlen_q = Int32(0) - seqlen_q_ = seqlen_q - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q_ = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q_, - ) - ) - if not continue_cond: - if cutlass.const_expr(cum_seqlen_k is not None): - cuseqlen_k = cum_seqlen_k[batch_coord] - seqlen_k_ = cum_seqlen_k[batch_coord + 1] - cuseqlen_k - continue_cond = seqlen_k_ <= 0 - if not continue_cond: - logical_offset = ( - curr_block_coord[0] * self.cta_tiler[0] + stage * self.qk_mma_tiler[0], - 0, - ) - cS = cute.domain_offset(logical_offset, cS_base) - value_args_ = ( - seqlen_k_, - seqlen_q_, - scale_softmax_log2, - window_size_left, - window_size_right, - skip_softmax_threshold_log2, - thread_idx, - logical_offset, - ) - atom_args = ( - qk_thr_mma, - tiled_tmem_load, - tiled_tmem_store, - tiled_tmem_store_vec, - thr_tmem_load, - thr_tmem_store, - thr_tmem_store_vec, - ) - tensor_args_ = ( - tTMEM_LOADtS, - tTMEM_STORE_VECtS, - tTMEM_STORE_SKIP_SOFTMAX, - tTMEM_STOREtS_x4, - warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ) - st_cnt, end_cnt, ld_mask_cnt, unmask_cnt, tl_mask_cnt = ( - fmha_utils.FusedMask.get_masked_info( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q_, - seqlen_k_, - window_size_left, - window_size_right, - ) - ) - row_max = -Float32.inf - row_sum = 0.0 - stats_args = (row_sum, row_max) - - def softmax_loop( - whether_apply_mask: bool, - loop_args: Tuple, - stats_args: Tuple, - pipeline_args: Tuple, - inner_fn: Callable, - value_args: Tuple, - atom_args: Tuple, - tensor_args: Tuple, - cS: cute.Tensor, - ) -> Tuple[Tuple, Tuple]: - start_index, iter_num, upper_bound = loop_args - for i in cutlass.range(start_index, start_index + iter_num, 1, unroll=1): - cS_iter = cute.domain_offset((0, i * self.qk_mma_tiler[1]), cS) - iter_args = (cS_iter, i == upper_bound - 1) - stats_args, pipeline_args = inner_fn( - stage, - whether_apply_mask, - iter_args, - stats_args, - pipeline_args, - value_args, - atom_args, - tensor_args, - ) - return stats_args, pipeline_args - - softmax_step_fn = self.softmax_step - softmax_loop_fn = partial( - softmax_loop, - inner_fn=softmax_step_fn, - value_args=value_args_, - atom_args=atom_args, - tensor_args=tensor_args_, - cS=cS, - ) - si_peek_status = mma_si_consumer.try_wait() - if cutlass.const_expr(stage == 1): - inplace_consumer.wait_and_advance() - pipeline_args_ = ( - si_peek_status, - mma_si_consumer, - si_corr_producer, - pi_mma_producer, - inplace_producer, - inplace_consumer, - ) - # 1. Leading mask loop - loop_args = (st_cnt, ld_mask_cnt, end_cnt) - stats_args, pipeline_args_ = softmax_loop_fn( - True, loop_args, stats_args, pipeline_args_ - ) - # 2. Unmasked loop - loop_args = (st_cnt + ld_mask_cnt, unmask_cnt, end_cnt) - stats_args, pipeline_args_ = softmax_loop_fn( - False, loop_args, stats_args, pipeline_args_ - ) - # 3. Trailing mask loop - loop_args = (st_cnt + ld_mask_cnt + unmask_cnt, tl_mask_cnt, end_cnt) - stats_args, pipeline_args_ = softmax_loop_fn( - True, loop_args, stats_args, pipeline_args_ - ) - - # Unpack pipeline_args - ( - _, - mma_si_consumer, - si_corr_producer, - pi_mma_producer, - inplace_producer, - inplace_consumer, - ) = pipeline_args_ - if cutlass.const_expr(stage == 0): - inplace_producer.commit() - inplace_producer.advance() - # 4. Copy the final stats for correction epilog - tTMEM_STORE_VECrS = cute.make_rmem_tensor( - tTMEM_STORE_VECcS.shape, self.qk_acc_dtype - ) - tTMEM_STORE_VECrS[0] = stats_args[0] - tTMEM_STORE_VECrS[1] = stats_args[1] - vec_i_handle = si_corr_producer.acquire_and_advance() - cute.copy(tiled_tmem_store_vec, tTMEM_STORE_VECrS, tTMEM_STORE_VECtS) - cute.arch.fence_view_async_tmem_store() - vec_i_handle.commit() - # End of if not continue_cond - # Advance to next tile - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - # End of persistent scheduler loop - - @cute.jit - def correction_rescale( - self, - thr_mma: cute.ThrMma, - tiled_tmem_load_vec: cute.TiledCopy, - scale_softmax_log2: Float32, - tensor_args: Tuple, - pipeline_args: Tuple, - oi_peek_status=None, - ): - """Rescale intermediate attention results based on softmax normalization factor. - - This method performs a crucial correction step in the attention computation pipeline. - When processing attention in blocks, the softmax normalization factors may change - as new blocks are processed. This method rescales previously computed partial - output values to account for updated normalization factors. - - The implementation uses efficient tensor memory operations to: - 1. Load existing partial attention output from tensor memory - 2. Apply the scaling factor to all elements - 3. Store the rescaled results back to tensor memory - - When ``self.enable_correction_double_buffer`` is True, the rescale loop - runs as a 2-buffer tensor-memory load / multiply / store pipeline. - - :param thr_mma: Thread MMA operation for the computation - :type thr_mma: cute.ThrMma - :param tiled_tmem_load_vec: Tiled memory load operation for the vectorized row-wise max - :type tiled_tmem_load_vec: cute.TiledCopy - :param scale_softmax_log2: Log2 of the softmax factor - :type scale_softmax_log2: Float32 - :param tensor_args: Tuple containing the tensors for the correction - :type tensor_args: Tuple[cute.Tensor, cute.Tensor, cute.Tensor] - :param pipeline_args: Tuple containing the pipeline arguments for the correction - :type pipeline_args: Tuple[pipeline.PipelineConsumer, pipeline.PipelineConsumer] - :param oi_peek_status: Optional non-blocking token for the Oi consumer - wait. ``None`` or ``False`` falls back to a blocking wait. - :return: ``((si_corr_consumer, mma_corr_consumer), next_oi_peek_status)`` - where ``next_oi_peek_status`` is the peek for the next Oi (only - refreshed when a rescale actually ran; otherwise stays - ``cutlass.Boolean(False)``). - """ - tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS = tensor_args - si_corr_consumer, mma_corr_consumer = pipeline_args - - pv_tiled_mma_shape = ( - self.pv_mma_tiler[0], - self.pv_mma_tiler[1], - ) - cO = cute.make_identity_tensor(pv_tiled_mma_shape) - tOcO = thr_mma.partition_C(cO) - corr_tile_size = 16 # tuneable parameter - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), - self.pv_acc_dtype, - ) - tmem_store_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), - self.pv_acc_dtype, - ) - tOtO_i_layout = cute.composition(tOtO.layout, cute.make_layout((128, corr_tile_size))) - tOcO_i_layout = cute.composition(tOcO.layout, cute.make_layout((128, corr_tile_size))) - tOtO_i = cute.make_tensor(tOtO.iterator, tOtO_i_layout) - tOcO_i = cute.make_tensor(tOcO.iterator, tOcO_i_layout) - tiled_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tOtO_i) - tiled_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tOtO_i) - tidx, _, _ = cute.arch.thread_idx() - thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) - thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) - thr_tmem_store = tiled_tmem_store.get_slice(thread_idx) - tTMEM_LOADtO = thr_tmem_load.partition_S(tOtO_i) - tTMEM_LOADcO = thr_tmem_load.partition_D(tOcO_i) - tTMEM_STOREtO = thr_tmem_store.partition_D(tOtO_i) - num_tiles = self.cta_tiler[2] // corr_tile_size - if cutlass.const_expr(self.enable_correction_double_buffer): - # Double buffer: 2 register buffers to pipeline tensor-memory load / multiply / store. - tTMrO = cute.make_rmem_tensor((tTMEM_LOADcO.shape, 2), self.pv_acc_dtype) - # Rank-matching views for cute.copy (TMEM partitions are rank-3, - # raw tTMrO[None, idx] is rank-1; composition restores the rank). - copy_layout = cute.make_layout(tTMrO.shape[0]) - view_0 = tTMrO[None, 0] - view_1 = tTMrO[None, 1] - tTMrO_copy = ( - cute.make_tensor( - view_0.iterator, - cute.composition(view_0.layout, copy_layout), - ), - cute.make_tensor( - view_1.iterator, - cute.composition(view_1.layout, copy_layout), - ), - ) - else: - tTMrO = cute.make_rmem_tensor((tTMEM_LOADcO.shape, num_tiles), self.pv_acc_dtype) - tTMEM_LOAD_VECrS = cute.make_rmem_tensor(tTMEM_LOAD_VECcS.shape, self.qk_acc_dtype) - # Wait for vec_i (row_wise current max & previous max) - vec_i_handle = si_corr_consumer.wait_and_advance() - cute.copy(tiled_tmem_load_vec, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECrS) - cute.arch.fence_view_async_tmem_load() - vec_i_handle.release() - # Wait for Oi (peek-token aware: None/False falls back to blocking). - oi_handle = mma_corr_consumer.wait_and_advance(oi_peek_status) - next_oi_peek_status = cutlass.Boolean(False) - vote_ballot_cnt = cute.arch.vote_ballot_sync(tTMEM_LOAD_VECrS[0] != tTMEM_LOAD_VECrS[1]) - should_rescale = vote_ballot_cnt != 0 - if should_rescale: - scale_ = scale_softmax_log2 * (tTMEM_LOAD_VECrS[0] - tTMEM_LOAD_VECrS[1]) - scale = cute.math.exp2(scale_, fastmath=True) - if cutlass.const_expr(self.enable_correction_double_buffer): - num_elems = cute.size(tTMrO, mode=[0]) - # Prologue: load first tile into buffer 0 - cute.copy(tiled_tmem_load, tTMEM_LOADtO, tTMrO_copy[0]) - # Steady state. Refresh the next Oi probe near the end of the - # current rescale pipeline. - for i in cutlass.range_constexpr(1, num_tiles): - cute.copy( - tiled_tmem_load, - cute.make_tensor( - tTMEM_LOADtO.iterator + i * corr_tile_size, - tTMEM_LOADtO.layout, - ), - tTMrO_copy[i % 2], - ) - for j in range(0, num_elems, 2): - tTMrO[j, (i - 1) % 2], tTMrO[j + 1, (i - 1) % 2] = ( - cute.arch.mul_packed_f32x2( - ( - tTMrO[j, (i - 1) % 2], - tTMrO[j + 1, (i - 1) % 2], - ), - (scale, scale), - ) - ) - cute.copy( - tiled_tmem_store, - tTMrO_copy[(i - 1) % 2], - cute.make_tensor( - tTMEM_STOREtO.iterator + (i - 1) * corr_tile_size, - tTMEM_STOREtO.layout, - ), - ) - next_oi_peek_status = mma_corr_consumer.try_wait() - # Epilogue: compute and store last tile - last = (num_tiles - 1) % 2 - for j in range(0, num_elems, 2): - tTMrO[j, last], tTMrO[j + 1, last] = cute.arch.mul_packed_f32x2( - (tTMrO[j, last], tTMrO[j + 1, last]), - (scale, scale), - ) - cute.copy( - tiled_tmem_store, - tTMrO_copy[last], - cute.make_tensor( - tTMEM_STOREtO.iterator + (num_tiles - 1) * corr_tile_size, - tTMEM_STOREtO.layout, - ), - ) - else: - for i in cutlass.range_constexpr(0, num_tiles): - tTMrO_i_ = tTMrO[None, i] - tTMrO_i = cute.make_tensor( - tTMrO_i_.iterator, - cute.composition(tTMrO_i_.layout, cute.make_layout(tTMrO.shape[0])), - ) - cute.copy( - tiled_tmem_load, - cute.make_tensor( - tTMEM_LOADtO.iterator + i * corr_tile_size, - tTMEM_LOADtO.layout, - ), - tTMrO_i, - ) - for j in range(0, cute.size(tTMrO_i), 2): - tTMrO_i[j], tTMrO_i[j + 1] = cute.arch.mul_packed_f32x2( - (tTMrO_i[j], tTMrO_i[j + 1]), - (scale, scale), - ) - cute.copy( - tiled_tmem_store, - tTMrO_i, - cute.make_tensor( - tTMEM_STOREtO.iterator + i * corr_tile_size, - tTMEM_STOREtO.layout, - ), - ) - next_oi_peek_status = mma_corr_consumer.try_wait() - # Release Oi - cute.arch.fence_view_async_tmem_store() - oi_handle.release() - return (si_corr_consumer, mma_corr_consumer), next_oi_peek_status - - @cute.jit - def correction_epilog( - self, - thr_mma: cute.ThrMma, - tiled_tmem_load_vec: cute.TiledCopy, - tensor_args: Tuple, - pipeline_args: Tuple, - value_args: Tuple, - ): - """Apply final scaling and transformation to attention output. - - When use_tma_store=True: writes to shared memory and signals epilogue warp for TMA store. - When use_tma_store=False: writes directly to global memory via st.global. - - :param thr_mma: Thread MMA operation for the computation - :type thr_mma: cute.ThrMma - :param tiled_tmem_load_vec: Tiled memory load operation for the vectorized row-wise max - :type tiled_tmem_load_vec: cute.TiledCopy - :param tensor_args: Tuple containing (tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS, sO_or_gO, mLSE, mSink) - :type tensor_args: Tuple - :param pipeline_args: When use_tma_store: (si_corr_consumer, mma_corr_consumer, corr_epi_producer). - When not use_tma_store: (si_corr_consumer, mma_corr_consumer). - :type pipeline_args: Tuple - :param value_args: Tuple containing (row_idx, cuseqlen_q, seqlen_q, blk_coord, scale_softmax, scale_output) - :type value_args: Tuple - """ - tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS, dest_O, mLSE, mSink = tensor_args - row_idx, cuseqlen_q, seqlen_q, blk_coord, scale_softmax, scale_output = value_args - - pv_tiled_mma_shape = ( - self.pv_mma_tiler[0], - self.pv_mma_tiler[1], - ) - cO = cute.make_identity_tensor(pv_tiled_mma_shape) - - corr_tile_size = 32 * 8 // self.o_dtype.width - tOdO = thr_mma.partition_C(dest_O) - tOcO = thr_mma.partition_C(cO) - tOtO_i = cute.logical_divide(tOtO, cute.make_layout((128, corr_tile_size))) - tOcO_i = cute.logical_divide(tOcO, cute.make_layout((128, corr_tile_size))) - tOdO_i = cute.logical_divide(tOdO, cute.make_layout((128, corr_tile_size))) - - tidx, _, _ = cute.arch.thread_idx() - thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) - epi_subtile = (self.epi_tile[0], corr_tile_size) - tmem_copy_atom = sm100_utils.get_tmem_load_op( - self.pv_mma_tiler, - self.o_layout, - self.o_dtype, - self.pv_acc_dtype, - epi_subtile, - use_2cta_instrs=False, - ) - tiled_tmem_load = tcgen05.make_tmem_copy(tmem_copy_atom, tOtO_i[(None, None), 0]) - thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) - tTMEM_LOADtO = thr_tmem_load.partition_S(tOtO_i[(None, None), None]) - tTMEM_LOADdO = thr_tmem_load.partition_D(tOdO_i[(None, None), None]) - tTMEM_LOADoO = thr_tmem_load.partition_D(tOcO_i[(None, None), None]) - - if cutlass.const_expr(self.use_tma_store): - si_corr_consumer, mma_corr_consumer, corr_epi_producer = pipeline_args - smem_copy_atom = sm100_utils.get_smem_store_op( - self.o_layout, self.o_dtype, self.pv_acc_dtype, tiled_tmem_load - ) - tiled_smem_store = cute.make_tiled_copy_D(smem_copy_atom, tiled_tmem_load) - else: - si_corr_consumer, mma_corr_consumer = pipeline_args - - # Wait for vec_i (row_wise global sum) - vec_i_handle = si_corr_consumer.wait_and_advance() - tTMEM_LOAD_VECrS = cute.make_rmem_tensor(tTMEM_LOAD_VECcS.shape, self.qk_acc_dtype) - cute.copy(tiled_tmem_load_vec, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECrS) - cute.arch.fence_view_async_tmem_load() - vec_i_handle.release() - - # Wait for Oi - oi_handle = mma_corr_consumer.wait_and_advance() - if cutlass.const_expr(self.use_tma_store): - oi_final_handle = corr_epi_producer.acquire_and_advance() - row_sum = tTMEM_LOAD_VECrS[0] - if cutlass.const_expr(mSink is not None): - sink_val = mSink[blk_coord[2]] - row_max_raw = tTMEM_LOAD_VECrS[1] - # sink is already in scaled logit space, row_max_raw is unscaled - # exp2((sink - max_scaled) * log2(e)) = exp(sink - max_scaled) - log2_e = Float32(1.4426950408889634) - sink_exp = cute.math.exp2( - (sink_val - row_max_raw * scale_softmax) * log2_e, fastmath=True - ) - row_sum = row_sum + sink_exp - scale = scale_output / row_sum - - for i in range(self.cta_tiler[2] // corr_tile_size): - tTMEM_LOADtO_i = tTMEM_LOADtO[None, 0, 0, i] - tTMEM_LOADdO_i = tTMEM_LOADdO[None, 0, 0, i] - tTMrO = cute.make_rmem_tensor(tTMEM_LOADoO[None, 0, 0, i].shape, self.pv_acc_dtype) - cute.copy(tiled_tmem_load, tTMEM_LOADtO_i, tTMrO) - for j in range(0, cute.size(tTMrO), 2): - tTMrO[j], tTMrO[j + 1] = cute.arch.mul_packed_f32x2( - (tTMrO[j], tTMrO[j + 1]), - (scale, scale), - ) - tDMrO = cute.make_rmem_tensor(tTMrO.shape, self.o_dtype) - o_vec = tTMrO.load() - tDMrO.store(o_vec.to(self.o_dtype)) - if cutlass.const_expr(self.use_tma_store): - # TMA store path: write to shared memory - cute.copy(tiled_smem_store, tDMrO, tTMEM_LOADdO_i) - else: - # st.global path: write directly to global memory with bounds check - if row_idx < seqlen_q: - cute.autovec_copy(tDMrO, tTMEM_LOADdO_i) - - if cutlass.const_expr(mLSE is not None): - scaled_tmp = scale_softmax * tTMEM_LOAD_VECrS[1] - # Convert LSE from natural log to log2 space, consistent with flashinfer trtllm-gen backend - lse = (cute.math.log(row_sum, fastmath=True) + scaled_tmp) * Float32(1.4426950408889634) - # Pre-scale correction: row_sum was inflated by 2^offset, so the - # log2-space LSE is too large by exactly `p_fp8_prescale_log2`. - if cutlass.const_expr(self.v_dtype.width == 8 and self.p_fp8_prescale_log2 > 0): - lse = lse - self.p_fp8_prescale_log2 - if row_idx < seqlen_q: - mLSE[row_idx + cuseqlen_q, blk_coord[2]] = lse - if cutlass.const_expr(self.use_tma_store): - # fence view async shared - cute.arch.fence_view_async_shared() - oi_handle.release() - oi_final_handle.commit() - return (si_corr_consumer, mma_corr_consumer, corr_epi_producer) - else: - oi_handle.release() - return (si_corr_consumer, mma_corr_consumer) - - def check_supported_dtypes( - self, - qk_dtype: Type[cutlass.Numeric], - pv_dtype: Type[cutlass.Numeric], - out_dtype: Type[cutlass.Numeric], - qk_acc_dtype: Type[cutlass.Numeric], - pv_acc_dtype: Type[cutlass.Numeric], - ): - supported = {cutlass.Float8E4M3FN, cutlass.Float16, cutlass.BFloat16} - if qk_dtype not in supported: - raise NotImplementedError("Unsupported qk_dtype") - if pv_dtype not in supported: - raise NotImplementedError("Unsupported pv_dtype") - if out_dtype not in {cutlass.Float8E4M3FN, cutlass.Float16, cutlass.BFloat16}: - raise NotImplementedError("Unsupported out_dtype") - if qk_acc_dtype not in {cutlass.Float32}: - raise NotImplementedError("Unsupported qk_acc_dtype") - if pv_acc_dtype not in {cutlass.Float32}: - raise NotImplementedError("Unsupported pv_acc_dtype") - - def check_invalid_shape( - self, - qk_dtype: Type[cutlass.Numeric], - q_shape: Tuple[int, int, int, int], - k_shape: Tuple[int, int, int, int], - ): - # Shapes are passed around this example as (batch, seq_len, num_heads, head_dim). - b, s_q, h_q, d = q_shape - b_, s_k, h_k, d_ = k_shape - - if b != b_: - raise NotImplementedError("q & k must have the same batch size") - if d != d_: - raise NotImplementedError("q & k must have the same head dimension") - if d not in {32, 64, 128, 192}: - raise NotImplementedError("Unsupported head dimension") - if h_q % h_k != 0: - raise NotImplementedError("h_q must be divisible by h_k") - if isinstance(s_q, tuple) and len(s_q) != b: - raise NotImplementedError("variable_seqlen s_q must have the length of batch size") - if isinstance(s_k, tuple) and len(s_k) != b: - raise NotImplementedError("variable_seqlen s_k must have the length of batch size") - if d == 192 and qk_dtype not in {cutlass.Float8E4M3FN}: - raise NotImplementedError("unimplemented dtypes for headdim 192") + return - def can_implement( - self, - q_shape: Tuple[int, int, int, int], - k_shape: Tuple[int, int, int, int], - qk_dtype: Type[cutlass.Numeric], - pv_dtype: Type[cutlass.Numeric], - out_dtype: Type[cutlass.Numeric], - qk_acc_dtype: Type[cutlass.Numeric], - pv_acc_dtype: Type[cutlass.Numeric], - ) -> bool: - """ - :param q_shape: Shape of the query tensor. - :type q_shape: Tuple[int, int, int, int] - :param k_shape: Shape of the key tensor. - :type k_shape: Tuple[int, int, int, int] - :param qk_dtype: Data type for Q and K (Bmm1 inputs). - :type qk_dtype: Type[cutlass.Numeric] - :param pv_dtype: Data type for P and V (Bmm2 inputs). - :type pv_dtype: Type[cutlass.Numeric] - :param out_dtype: Data type of the output tensor. - :type out_dtype: Type[cutlass.Numeric] - :param qk_acc_dtype: Data type of the qk accumulator tensor. - :type qk_acc_dtype: Type[cutlass.Numeric] - :param pv_acc_dtype: Data type of the pv accumulator tensor. - :type pv_acc_dtype: Type[cutlass.Numeric] - :return: True if the kernel can be implemented, False otherwise. - :rtype: bool - """ - try: - # Skip unsupported types - self.check_supported_dtypes( - qk_dtype, - pv_dtype, - out_dtype, - qk_acc_dtype, - pv_acc_dtype, - ) - # Skip invalid shape - self.check_invalid_shape( - qk_dtype, - q_shape, - k_shape, - ) - except NotImplementedError: - return False - return True + q_cute = _to_cute_tensor(q_5d, leading_dim=4) + k_cute = _to_cute_tensor(k_5d, leading_dim=4) + v_cute = _to_cute_tensor(v_5d, leading_dim=4) + o_cute = _to_cute_tensor(o_5d, leading_dim=4) + + cum_seqlen_q_cute = None + cum_seqlen_k_cute = None + if varlen: + qo_indptr_i32 = _to_cint_contiguous(qo_indptr) + kv_indptr_i32 = _to_cint_contiguous(kv_indptr) + cum_seqlen_q_cute = from_dlpack(qo_indptr_i32, assumed_align=16).mark_layout_dynamic( + leading_dim=0 + ) + cum_seqlen_k_cute = from_dlpack(kv_indptr_i32, assumed_align=16).mark_layout_dynamic( + leading_dim=0 + ) + + lse_iter = None + if lse is not None: + lse_cute = from_dlpack(lse_4d, assumed_align=16).mark_layout_dynamic(leading_dim=2) + lse_iter = lse_cute.iterator + + stream = cuda_driver.CUstream(torch.cuda.current_stream(q).cuda_stream) + kernel_fn( + q_cute.iterator, + k_cute.iterator, + v_cute.iterator, + o_cute.iterator, + problem_size, + cum_seqlen_q_cute, + cum_seqlen_k_cute, + lse_iter, + None, # sink_iter + cute_typing.Float32(scale_softmax_log2), + cute_typing.Float32(scale_softmax), + cute_typing.Float32(scale_output), + skip_threshold_log2, + ws_left, + ws_right, + None, + None, + False, # reserved + stream, + ) diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha_blockscaled.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha_blockscaled.py deleted file mode 100644 index 01be74595ac9..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/blackwell/attention/fmha_blockscaled.py +++ /dev/null @@ -1,3768 +0,0 @@ -# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: BSD-3-Clause - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# ruff: noqa: I001, E501, F841, E712, E741 - -import math -from typing import Callable, Type, Tuple, Union, Optional -from functools import partial - -import torch -import cuda.bindings.driver as cuda - -import cutlass -import cutlass.cute as cute -from cutlass.cute.nvgpu import tcgen05 -from cutlass.cute.nvgpu.common import OperandMajorMode -import cutlass.utils as utils -import cutlass.pipeline as pipeline -from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait -import cutlass.torch as cutlass_torch -import cutlass.utils.blackwell_helpers as sm100_utils -import cutlass.utils.blockscaled_layout as blockscaled_utils -from cutlass.cute.typing import Int8, Int32, Int64, Float32 -from cutlass.base_dsl.arch import Arch -from cutlass.cutlass_dsl import BaseDSL - -from ...helpers import fmha_helpers as fmha_utils - -""" -A fused multi-head attention (FMHA) example where Bmm1(Q@K) is block-scaled with MXFP8 or NVFP4 for -NVIDIA Blackwell SM100 architecture using CUTE DSL - -This example demonstrates an implementation of fused multi-head attention using a TMA + Blackwell SM100 -TensorCore warp-specialized persistent kernel. The implementation integrates the Q*K^T matrix multiplication, -softmax normalization, and softmax(Q*K^T)*V into a single kernel, avoiding intermediate data movement between -global memory and shared memory, thus improving computational efficiency. - -The kernel implements key optimizations including: -- Warp specialization for different computation phases (load, MMA, softmax, correction, epilogue) -- Pipeline stages between different warps for overlapping computation and memory access -- Support for different precision data types -- Optional causal masking for autoregressive models - -To run this example: - -.. code-block:: bash - - python nvidia-internal/fmha_blockscaled.py \ - --qk_mode MXFP8 \ - --qk_acc_dtype Float32 --pv_acc_dtype Float32 \ - --mma_tiler_mn 128,128 \ - --q_shape 4,1024,8,64 --k_shape 4,1024,8,64 \ - --is_persistent - -The above example runs FMHA with batch size 4, sequence length 1024, 8 attention heads, and head -dimension 64. The Blackwell tcgen05 MMA tile shape is (128, 128), and the default runner uses BF16 -for V/output with FP32 for accumulation. - -To collect performance with NCU profiler: - -.. code-block:: bash - - ncu python nvidia-internal/fmha_blockscaled.py \ - --qk_mode NVFP4 \ - --qk_acc_dtype Float32 --pv_acc_dtype Float32 \ - --mma_tiler_mn 128,128 \ - --q_shape 4,1024,8,64 --k_shape 4,1024,8,64 \ - --is_persistent --warmup_iterations 10 \ - --iterations 10 --skip_ref_check - -Constraints for this example: -* Q and K use supports two microscaling schema: - MXFP8 (qk_dtype in {Float8E4M3FN, Float8E5M2}, qk_sf_dtype=Float8E8M0FNU, qk_sf_vec_size=32) - and NVFP4 (qk_dtype=Float4E2M1FN, qk_sf_dtype=Float8E4M3FN, qk_sf_vec_size=16) -* Bmm2 (P@V) remains dense FP8 or BF16 through pv_dtype -* Supported QK head dimensions: 128 (due to current limitations in blockscaled_utils) -* Number of heads in Q must be divisible by number of heads in K -* mma_tiler_mn must be 128,128 -* Batch size must be the same for Q, K, and V tensors -* For causal masking, use --is_causal (note: specify without =True/False) -* For persistent scheduling, use --is_persistent (note: specify without =True/False) - -For details on the skip softmax algorithm, please refer to the paper: https://arxiv.org/abs/2512.12087. -""" - - -def make_thread_cooperative_group(size: int): - return pipeline.CooperativeGroup(pipeline.Agent.Thread, size) - - -def create_scale_factor_tensor( - mn: int, - k: int, - l: int, - sf_vec_size: int, - sf_dtype: Type[cutlass.Numeric], -): - """ - Create dense reference SFs and blocked device SF storage for QK block-scaled MMA. - - Generates per-position SFs sampled from {0.5, 1.0, 2.0} (which are exactly representable in both - E8M0 and E4M3) and arranges them into the blocked storage that tile_atom_to_shape_SF, - BlockScaledBasicChunk consume on the kernel side. The SF atom layout is: - ((32,4),(sf_vec,4)) : ((16,4),(0,1)) - so for a logical mn in [0,128) inside an atom, the SF byte lives at offset: - (mn % 32) * 16 + (mn // 32) * 4 + k_inner - """ - - def ceil_div(a: int, b: int) -> int: - return (a + b - 1) // b - - atom_m = 32 * 4 - atom_k = 4 - m_atoms = ceil_div(mn, atom_m) - sf_k = ceil_div(k, sf_vec_size) - sf_k_atoms = ceil_div(sf_k, atom_k) - - # Generate SFs in the LOGICAL (mn, sf_k, l) layout first, then re-arrange into the blocked - # storage layout explained in docstring above. - # CuTe decomposes mn in [0,128) column-major as (mn % 32, mn // 32), so the size-32 axis - # gets the LOW 5 bits of mn (stride 16) and the size-4 axis gets the HIGH 2 bits (stride 4). - mn_padded = m_atoms * atom_m - sf_k_padded = sf_k_atoms * atom_k - sf_choices = torch.tensor([0.5, 1.0, 2.0], dtype=torch.float32) - sf_logical_padded = sf_choices[torch.randint(0, len(sf_choices), (mn_padded, sf_k_padded, l))] - sf_storage_f32 = ( - # K-mode: k_padded -> (sf_k_atoms, atom_k=4) - # MN-mode: mn_padded -> (sf_m_atoms, proton_m=4, neutron_m=32) - sf_logical_padded.reshape(m_atoms, 4, 32, sf_k_atoms, atom_k, l) - # Arrangement (last-idx-major): (l, sf_m_atoms, sf_k_atoms, neutron_m, proton_m, atom_k) - # (l, sf_m_atoms, sf_k_atoms) follows a traditional K-major interblock layout - # (atom_k) is the last index -> reproduces (sf_vec,atom_k):(0,1) - # (neutron_m, proton_m) -> reproduces - # prod((neutron_m,proton_m):(proton_m:1), 1:atom_k) = (32,4):(16,4) - .permute(5, 0, 3, 2, 1, 4) - .contiguous() - ) - - sf_tensor, _ = cutlass_torch.cute_tensor_like( - sf_storage_f32, - sf_dtype, - is_dynamic_layout=True, - assumed_align=16, - ) - - sf_logical = sf_logical_padded[:mn, :sf_k, :] - # Broadcast each SF across its sf_vec_size K elements to build a elementwise (mn, k, l) - # reference whose (m, j, b) entry is the SF the kernel multiplies into the same logical position - ref_elementwise = ( - sf_logical.unsqueeze(2) - .expand(-1, -1, sf_vec_size, -1) - .reshape(mn, sf_k * sf_vec_size, l)[:, :k, :] - .contiguous() - ) - - return ref_elementwise, sf_tensor - - -def compact_fp4_data(torch_underlying: torch.Tensor, dtype: Type[cutlass.Numeric]) -> None: - """Compact packed FP4 rows to match CuTe's sub-byte stride interpretation.""" - if dtype is not cutlass.Float4E2M1FN: - return - - # torch lacks fill_/copy_ kernels for Float4_e2m1fn_x2 on CUDA, but the - # storage is byte-packed (two FP4 per byte), so a uint8 view is a free - # reinterpret and supports the in-place primitives we need. - d = torch_underlying.shape[-1] - packed_size = d // (8 // dtype.width) - underlying_u8 = torch_underlying.view(torch.uint8) - rows = underlying_u8.reshape(-1, d) - packed = rows[:, :packed_size].contiguous() - underlying_u8.fill_(0) - underlying_u8.flatten()[: packed.numel()].copy_(packed.flatten()) - - -class BlackwellFusedMultiHeadBlockScaledAttentionForward: - arch_str: str = "sm_100" - arch_name: str = "Blackwell SM100" - - def __init__( - self, - qk_acc_dtype: Type[cutlass.Numeric], - pv_acc_dtype: Type[cutlass.Numeric], - mma_tiler: Tuple[int, int], - head_dim: Union[int, Tuple[int, int]], - is_persistent: bool, - mask_type: fmha_utils.MaskEnum, - enable_ex2_emulation: bool, - enable_skip_correction: bool, - qk_sf_vec_size: int, - use_tma_store: bool = True, - ): - """Initializes the configuration for a Blackwell Fused Multi-Head Attention (FMHA) kernel. - - This configuration includes several key aspects: - - 1. Data Type Settings: - - qk_acc_dtype: Data type for Q*K^T matrix multiplication accumulator - - pv_acc_dtype: Data type for P*V matrix multiplication accumulator - - 2. MMA Instruction Settings: - - mma_tiler: The shape of the MMA instruction unit: (M, N) for BMM1 and (M, K) for BMM2 - - head_dim: The head dimension, it can be a single integer or a tuple of two integers (D, Dv). - If it is a tuple, Dv is the head dimension of the value & output tensors. - It also determines the K dimension of the BMM1's MMA instruction unit - & N dimension of the BMM2's MMA instruction unit. - - qk_mma_tiler: MMA shape for Q*K^T computation - - pv_mma_tiler: MMA shape for P*V computation - - 3. Kernel Execution Mode: - - is_persistent: Boolean indicating whether to use persistent kernel mode - - mask_type: Specifies the type of mask to use (no mask, residual mask, or causal mask) - - window_size_left/right: Sliding window size for attention masking - - enable_ex2_emulation: Whether to enable exp2 emulation - - enable_skip_correction: Whether to skip the correction when rowmax is not updated larger than a threshold - - :param qk_acc_dtype: Data type for Q*K^T matrix multiplication accumulator - :type qk_acc_dtype: Type[cutlass.Numeric] - :param pv_acc_dtype: Data type for P*V matrix multiplication accumulator - :type pv_acc_dtype: Type[cutlass.Numeric] - :param mma_tiler: The (M, N) shape of the MMA instruction - :type mma_tiler: Tuple[int, int] - :param head_dim: The head dimension, it can be a single integer or a tuple of two integers (D, Dv). - :type head_dim: Union[int, Tuple[int, int]] - :param is_persistent: Whether to use persistent kernel mode - :type is_persistent: bool - :param mask_type: Type of mask to use - :type mask_type: fmha_utils.MaskEnum - :param window_size_left: Left-side sliding window size for attention masking - :type window_size_left: int - :param window_size_right: Right-side sliding window size for attention masking - :type window_size_right: int - """ - - self.qk_acc_dtype = qk_acc_dtype - self.pv_acc_dtype = pv_acc_dtype - if mma_tiler != (128, 128): - raise ValueError( - "This standalone kernel uses a static TMEM map and currently supports only mma_tiler=(128, 128)" - ) - if isinstance(head_dim, tuple): - self.head_dim = head_dim[0] - self.head_dim_v = head_dim[1] - assert self.head_dim == 192 and self.head_dim_v == 128, ( - f"When Headdim is a tuple, it's for MLA. Must be (192, 128), but got {head_dim}" - ) - else: - self.head_dim = head_dim - self.head_dim_v = head_dim - self.cta_tiler = ( - 2 * mma_tiler[0], # 2 O tile per CTA - mma_tiler[1], - self.head_dim_v, - ) - self.qk_mma_tiler = ( - *mma_tiler, - self.head_dim, - ) - self.pv_mma_tiler = ( - mma_tiler[0], - self.head_dim_v, - mma_tiler[1], - ) - self.cluster_shape_mn = (1, 1) - self.is_persistent = is_persistent - self.mask_type = mask_type - self.enable_skip_correction = enable_skip_correction - self.enable_ex2_emulation = enable_ex2_emulation - self.qk_sf_vec_size = qk_sf_vec_size - self.qk_mma_inst_bits_k = 256 - if qk_sf_vec_size == 16: - # NVFP4: a 256-bit MMA operand tile covers 64 logical K values. - self.qk_mma_inst_tile_k = self.head_dim // (self.qk_mma_inst_bits_k // 4) - elif qk_sf_vec_size == 32: - # MXFP8: a 256-bit MMA operand tile covers 32 logical K values. - self.qk_mma_inst_tile_k = self.head_dim // (self.qk_mma_inst_bits_k // 8) - else: - self.qk_mma_inst_tile_k = 0 - self.use_tma_store = use_tma_store - - self.softmax0_warp_ids = (0, 1, 2, 3) - self.softmax1_warp_ids = (4, 5, 6, 7) - self.correction_warp_ids = (8, 9, 10, 11) - self.mma_warp_id = 12 - self.load_warp_id = 13 - self.epilogue_warp_id = 14 - self.empty_warp_id = 15 - self.num_tmem_alloc_cols = cute.arch.get_max_tmem_alloc_cols(self.arch_str) - - self.threads_per_warp = 32 - self.threads_per_cta = self.threads_per_warp * len( - ( - *self.softmax0_warp_ids, - *self.softmax1_warp_ids, - *self.correction_warp_ids, - self.mma_warp_id, - self.load_warp_id, - self.epilogue_warp_id, - self.empty_warp_id, - ) - ) - self.tmem_alloc_barrier = pipeline.NamedBarrier( - barrier_id=2, - num_threads=self.threads_per_warp - * sum( - ( - len((self.mma_warp_id,)), - len(self.softmax0_warp_ids), - len(self.softmax1_warp_ids), - len(self.correction_warp_ids), - ) - ), - ) - self.sequence_s0_s1_barrier = pipeline.NamedBarrier( - barrier_id=3, - num_threads=self.threads_per_warp - * len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)), - ) - self.sequence_s1_s0_barrier = pipeline.NamedBarrier( - barrier_id=4, - num_threads=self.threads_per_warp - * len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)), - ) - self.s0_warpgroup_barrier = pipeline.NamedBarrier( - barrier_id=5, - num_threads=self.threads_per_warp * len(self.softmax0_warp_ids), - ) - self.s1_warpgroup_barrier = pipeline.NamedBarrier( - barrier_id=6, - num_threads=self.threads_per_warp * len(self.softmax1_warp_ids), - ) - self.tmem_dealloc_barrier = pipeline.NamedBarrier( - barrier_id=7, - num_threads=self.threads_per_warp * len(self.correction_warp_ids), - ) - - self.tmem_s0_offset = 0 - self.tmem_s1_offset = 128 - self.tmem_o0_offset = 256 - self.tmem_o1_offset = 384 - # inplaced with s1 - self.tmem_p0_offset = 160 - # inplaced with s0 - self.tmem_p1_offset = 32 - # Block-scaled QK scale factors are live only while issuing QK MMA. - # Keep each stage's scale buffers in the opposite S/P tile's tail columns. - self.tmem_qk0_sf_offset = 224 - self.tmem_qk1_sf_offset = 96 - # vec buffer for row_max & row_sum - # inplaced with s0 - self.tmem_vec0_offset = 0 - # inplaced with s1 - self.tmem_vec1_offset = 128 - # skip mma pv flag offset regarding to the vec buffer - # inplaced with s1 - self.tmem_skip_softmax0_offset = 136 - # inplaced with s0 - self.tmem_skip_softmax1_offset = 8 - - self.num_regs_softmax = 192 - self.num_regs_correction = 96 - self.num_regs_other = 32 - self.buffer_align_bytes = 1024 - self.arch = BaseDSL._get_dsl().get_arch_enum() - - if self.arch >= Arch.sm_103: - assert self.enable_ex2_emulation == False, ( - f"Don't enable exp2 emulation for {self.arch}, it doesn't help performance" - ) - - num_warps_per_warpgroup = 4 - self.softmax_warpgroup_count = ( - len((*self.softmax0_warp_ids, *self.softmax1_warp_ids)) // num_warps_per_warpgroup - ) - - def _make_qk_tiled_mma(self, cta_group): - """Build the QK tiled MMA. Override in subclasses to target a different arch.""" - return sm100_utils.make_blockscaled_trivial_tiled_mma( - self.q_dtype, - self.k_dtype, - self.q_major_mode, - self.k_major_mode, - self.qk_sf_dtype, - self.qk_sf_vec_size, - cta_group, - self.qk_mma_tiler[:2], - ) - - def _make_pv_tiled_mma(self, cta_group, p_major_mode, p_source): - """Build the PV tiled MMA. Override in subclasses to target a different arch.""" - return sm100_utils.make_trivial_tiled_mma( - self.v_dtype, - self.v_dtype, - p_major_mode, - self.v_major_mode, - self.pv_acc_dtype, - cta_group, - self.pv_mma_tiler[:2], - p_source, - ) - - def _setup_attributes(self, enable_skip_softmax: bool) -> None: - """Set up configurations and parameters for the FMHA kernel operation. - - This method initializes and configures various attributes required for the - execution of the fused multi-head attention kernel, mainly about the pipeline stages: - - - Sets up staging parameters for Q, K, V inputs and accumulator data - - Configures pipeline stages for softmax, correction, and epilogue operations - """ - - self.q_stage = 2 - k_stage = 4 if self.q_dtype.width == 8 else 3 - v_stage = 4 if self.v_dtype.width == 8 else 3 - self.kv_stage = min(k_stage, v_stage) - # For D192, the smem usage of Q & K is larger. So, we need to reduce the stage count. - if self.head_dim == 192 and self.q_dtype.width == 16: - self.kv_stage = 2 - self.p_mma_stage = 1 - self.acc_stage = 1 - self.softmax_corr_stage = 1 - self.mma_corr_stage = 2 - self.mma_softmax_stage = 1 - self.epi_stage = 2 - - # Tunable parameters - if not self.enable_skip_correction: - self.rescale_threshold = 0.0 - elif enable_skip_softmax: - self.rescale_threshold = 1.0 - else: - self.rescale_threshold = 8.0 - # FP8 P pre-scale: offset added to exp2 exponent so that P*2^offset fills - # more of E4M3's [0, 448] range, improving quantization precision. - # Derived from rescale_threshold to guarantee P*2^offset <= 448. - self.p_fp8_prescale_log2 = max(0.0, math.floor(math.log2(448) - self.rescale_threshold)) - # ln(2) * offset correction for LSE when pre-scale is active - self.p_fp8_prescale_lse_correction = self.p_fp8_prescale_log2 * math.log(2) - # For most cases, seq barrier is needed to help keep the pipeline stable - # But sometimes, compiler will schedule the barrier at an unexpected place - # if it hurts perf a lot, try to quickly fix it by disabling seq barrier - self.enable_sequence_barrier = False - # Optional double buffering for correction rescale. - self.enable_correction_double_buffer = False - - @cute.jit - def __call__( - self, - q_tensor: cute.Tensor, - k_tensor: cute.Tensor, - q_sf_tensor: cute.Tensor, - k_sf_tensor: cute.Tensor, - v_tensor: cute.Tensor, - o_tensor: cute.Tensor, - problem_size: Tuple[Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32], - cum_seqlen_q: Optional[cute.Tensor], - cum_seqlen_k: Optional[cute.Tensor], - lse_tensor: Optional[cute.Tensor], - sink_tensor: Optional[cute.Tensor], - scale_softmax_log2: Float32, - scale_softmax: Float32, - scale_output: Float32, - scale_v_channels: Optional[cute.Tensor], - skip_softmax_threshold_log2: Optional[Float32], - window_size_left: Optional[Int32], - window_size_right: Optional[Int32], - skip_softmax_count: Optional[cute.Tensor], - total_softmax_count: Optional[cute.Tensor], - stream: cuda.CUstream, - use_pdl: bool, - ): - """Execute the Fused Multi-Head Attention operation on the provided tensors. - - This method prepares the input tensors for processing, validates their shapes and types, - configures the computation parameters, and launches the CUDA kernel. - - The method handles: - 1. Tensor layout transformations for specific memory access patterns - 2. Validation of tensor shapes and data types - 3. Initialization of hardware-specific parameters and memory layouts - 4. Configuration of TMA (Tensor Memory Access) operations - 5. Grid and work scheduling computation - 6. Kernel launch with appropriate parameters - - :param q_tensor: The query tensor with shape (b, s_q, h_k, h_r, d) - :type q_tensor: cute.Tensor - :param k_tensor: The key tensor with shape (b, s_k, h_k, 1, d) - :type k_tensor: cute.Tensor - :param v_tensor: The value tensor with shape (b, s_v, h_k, 1, dv) - :type v_tensor: cute.Tensor - :param o_tensor: The output tensor with shape (b, s_q, h_k, h_r, dv) - :type o_tensor: cute.Tensor - :param problem_size: The problem size with shape [b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d, dv]. If cum_seqlen_q or cum_seqlen_k is not None, s_q_max and s_k_max are the max of the per-batch sequence lengths respectively. - :type problem_size: Tuple[Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32] - :param cum_seqlen_q: The cumulative sequence length tensor for query - :type cum_seqlen_q: Optional[cute.Tensor] - :param cum_seqlen_k: The cumulative sequence length tensor for key - :type cum_seqlen_k: Optional[cute.Tensor] - :param scale_softmax_log2: The log2 scale factor for softmax - :type scale_softmax_log2: Float32 - :param scale_softmax: The scale factor for softmax - :type scale_softmax: Float32 - :param scale_output: The scale factor for the output - :type scale_output: Float32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - :param stream: The CUDA stream to execute the kernel on - :type stream: cuda.CUstream - :raises TypeError: If tensor data types don't match or aren't supported - :raises RuntimeError: If tensor layouts aren't in supported formats - """ - b, s_q_max, s_lse_max, s_k_max, h_q, h_k, d_, dv_ = problem_size - h_r = h_q // h_k - # setup static attributes before smem/grid/tma computation - self.q_dtype = q_tensor.element_type - self.k_dtype = k_tensor.element_type - self.qk_sf_dtype = q_sf_tensor.element_type - self.v_dtype = v_tensor.element_type - self.o_dtype = o_tensor.element_type - - # s_q, s_k, s_v are the actual tensor dimensions (total seqlen for varlen) - s_q = q_tensor.shape[1] - s_k = k_tensor.shape[1] - s_v = v_tensor.shape[1] - s_lse = s_lse_max - d = self.head_dim - dv = self.head_dim_v - # Important for performance - align = 256 // self.o_dtype.width - assert d % align == 0, f"head_dim must be multiple of {align} for the given datatypes." - assert dv % align == 0, f"head_dim_v must be multiple of {align} for the given datatypes." - - stride_b_q = h_r * h_k * s_q * d if cum_seqlen_q is None else 0 - stride_b_o = h_r * h_k * s_q * dv if cum_seqlen_q is None else 0 - stride_b_k = h_k * s_k * d if cum_seqlen_k is None else 0 - stride_b_v = h_k * s_v * dv if cum_seqlen_k is None else 0 - stride_b_lse = h_r * h_k * s_lse if cum_seqlen_q is None else 0 - - # (b, s_q, h_k, h_r, d) -> (s_q, d, ((h_r, h_k), b)) - q_layout = cute.make_layout( - (s_q, d, ((h_r, h_k), b)), - stride=(d * h_r * h_k, 1, ((d, d * h_r), stride_b_q)), - ) - q = cute.make_tensor(q_tensor.iterator, q_layout) - # (b, s_k, h_k, 1, d) -> (s_k, d, ((h_r, h_k), b)), 0-stride for h_r to broadcast - k_layout = cute.make_layout( - (s_k, d, ((h_r, h_k), b)), - stride=(d * h_k, 1, ((0, d), stride_b_k)), - ) - k = cute.make_tensor(k_tensor.iterator, k_layout) - q_sf_layout = blockscaled_utils.tile_atom_to_shape_SF(q.shape, self.qk_sf_vec_size) - q_sf = cute.make_tensor(q_sf_tensor.iterator, q_sf_layout) - k_sf_layout = blockscaled_utils.tile_atom_to_shape_SF( - (s_k, d, (h_k, b)), self.qk_sf_vec_size - ) - k_sf = cute.make_tensor(k_sf_tensor.iterator, k_sf_layout) - # (b, s_v, h_k, 1, dv) -> (dv, s_v, ((h_r, h_k), b)), 0-stride for h_r to broadcast - v_layout = cute.make_layout( - (dv, s_v, ((h_r, h_k), b)), - stride=(1, dv * h_k, ((0, dv), stride_b_v)), - ) - v = cute.make_tensor(v_tensor.iterator, v_layout) - # (b, s_q, h_k, h_r, dv) -> (s_q, dv, ((h_r, h_k), b)) - o_layout = cute.make_layout( - (s_q, dv, ((h_r, h_k), b)), - stride=(dv * h_r * h_k, 1, ((dv, dv * h_r), stride_b_o)), - ) - o = cute.make_tensor(o_tensor.iterator, o_layout) - if cutlass.const_expr(lse_tensor is not None): - # (s, ((h_r, h_k), b)) - head stride=1 to match FlashInfer (total_q, h_q) convention - lse_layout = cute.make_layout( - (s_lse, ((h_r, h_k), b)), - stride=(h_r * h_k, ((1, h_r), stride_b_lse)), - ) - lse = cute.make_tensor(lse_tensor.iterator, lse_layout) - else: - lse = None - - if cutlass.const_expr(sink_tensor is not None): - # sink_tensor is 1D with shape (h_q,) = (h_k * h_r,) - # Create layout ((h_r, h_k), b) with stride 0 for batch so blk_coord[2] works - sink_layout = cute.make_layout( - ((h_r, h_k), b), - stride=((1, h_r), 0), - ) - sink = cute.make_tensor(sink_tensor.iterator, sink_layout) - else: - sink = None - - if cutlass.const_expr(scale_v_channels is not None): - # scale_v_channels is per (h_k, dv): shape (h_k * dv,) row-major. - # Expose as (dv, ((h_r, h_k), b)) where h_r and b are 0-stride broadcasts. - scale_v_channels_layout = cute.make_layout( - (dv, ((h_r, h_k), b)), - stride=(1, ((0, dv), 0)), - ) - m_scale_v_channels = cute.make_tensor( - scale_v_channels.iterator, scale_v_channels_layout - ) - else: - m_scale_v_channels = None - - self.tile_sched_params, grid = fmha_utils.compute_grid( - cute.shape((s_q_max, d, ((h_r, h_k), b))), - self.cta_tiler, - self.is_persistent, - ) - self.q_major_mode = utils.LayoutEnum.from_tensor(q).mma_major_mode() - self.k_major_mode = utils.LayoutEnum.from_tensor(k).mma_major_mode() - self.v_major_mode = utils.LayoutEnum.from_tensor(v).mma_major_mode() - self.o_layout = utils.LayoutEnum.from_tensor(o) - - if cutlass.const_expr(self.q_major_mode != OperandMajorMode.K): - raise RuntimeError("The layout of q is not supported") - if cutlass.const_expr(self.k_major_mode != OperandMajorMode.K): - raise RuntimeError("The layout of k is not supported") - if cutlass.const_expr(self.v_major_mode != OperandMajorMode.MN): - raise RuntimeError("The layout of v is not supported") - - # check type consistency: Q and K must share the same dtype (qk_dtype); - # V may use a different dtype (pv_dtype) to allow qk_dtype != pv_dtype. - if cutlass.const_expr(self.q_dtype != self.k_dtype): - raise TypeError(f"Type mismatch: {self.q_dtype} != {self.k_dtype}") - if cutlass.const_expr(self.qk_sf_dtype != k_sf_tensor.element_type): - raise TypeError( - f"Q/K scale type mismatch: {self.qk_sf_dtype} != {k_sf_tensor.element_type}" - ) - if cutlass.const_expr(self.qk_mma_inst_tile_k <= 0): - raise RuntimeError( - f"Invalid QK scale-factor tiling for head_dim={self.head_dim}, qk_sf_vec_size={self.qk_sf_vec_size}" - ) - # SFQ + SFK share one 32-column TMEM slot per stage (see kernel TMEM layout - # below). Validate the static footprint on the host so we don't smuggle a - # `raise` into @cute.kernel. - _sf_atom_mn = 32 - _qk_sf_tmem_cols = (self.qk_mma_tiler[0] // _sf_atom_mn) * self.qk_mma_inst_tile_k - _kqk_sf_tmem_cols = ( - ((self.qk_mma_tiler[1] + 127) // 128 * 128) // _sf_atom_mn - ) * self.qk_mma_inst_tile_k - if cutlass.const_expr(_qk_sf_tmem_cols + _kqk_sf_tmem_cols > 32): - raise RuntimeError( - "QK scale-factor TMEM footprint exceeds the static 32-column slot " - f"(qk_sf_tmem_cols={_qk_sf_tmem_cols}, k_sf_tmem_cols={_kqk_sf_tmem_cols})" - ) - self._setup_attributes(skip_softmax_threshold_log2 is not None) - - cta_group = tcgen05.CtaGroup.ONE - # the intermediate tensor p is from tmem & k-major - p_source = tcgen05.OperandSource.TMEM - p_major_mode = cute.nvgpu.OperandMajorMode.K - qk_tiled_mma = self._make_qk_tiled_mma(cta_group) - pv_tiled_mma = self._make_pv_tiled_mma(cta_group, p_major_mode, p_source) - - self.cluster_shape_mnk = (*self.cluster_shape_mn, 1) - self.cluster_layout_vmnk = cute.tiled_divide( - cute.make_layout(self.cluster_shape_mnk), - (qk_tiled_mma.thr_id.shape,), - ) - self.epi_tile = self.pv_mma_tiler[:2] - - q_smem_layout_staged = sm100_utils.make_smem_layout_a( - qk_tiled_mma, - self.qk_mma_tiler, - self.q_dtype, - self.q_stage, - ) - k_smem_layout_staged = sm100_utils.make_smem_layout_b( - qk_tiled_mma, - self.qk_mma_tiler, - self.k_dtype, - self.kv_stage, - ) - q_sf_smem_layout_staged = blockscaled_utils.make_smem_layout_sfa( - qk_tiled_mma, - self.qk_mma_tiler, - self.qk_sf_vec_size, - self.q_stage, - ) - k_sf_smem_layout_staged = blockscaled_utils.make_smem_layout_sfb( - qk_tiled_mma, - self.qk_mma_tiler, - self.qk_sf_vec_size, - self.kv_stage, - ) - p_tmem_layout_staged = sm100_utils.make_smem_layout_a( - pv_tiled_mma, - self.pv_mma_tiler, - self.v_dtype, - self.acc_stage, - ) - v_smem_layout_staged_origin = sm100_utils.make_smem_layout_b( - pv_tiled_mma, - self.pv_mma_tiler, - self.v_dtype, - self.kv_stage, - ) - # k & v share the same smem buffer. Pad the smaller-tile operand's stage stride to - # match the larger tile. Stride is scaled to the target operand's element width. - # sK_cosize covers all kv_stage slots at the larger tile's byte footprint. - if cutlass.const_expr(self.k_dtype.width >= self.v_dtype.width): - # k tile >= v tile: give v k's stage stride in v_dtype units - k_tile_cosize = cute.cosize(cute.select(k_smem_layout_staged, mode=[0, 1, 2])) - v_stage_stride = k_tile_cosize * self.k_dtype.width // self.v_dtype.width - v_smem_layout_staged = cute.append( - cute.select(v_smem_layout_staged_origin, mode=[0, 1, 2]), - cute.make_layout(self.kv_stage, stride=v_stage_stride), - ) - sK_cosize = cute.cosize(k_smem_layout_staged) - else: - # v tile > k tile: give k v's stage stride in k_dtype units - v_tile_cosize = cute.cosize(cute.select(v_smem_layout_staged_origin, mode=[0, 1, 2])) - k_stage_stride = v_tile_cosize * self.v_dtype.width // self.k_dtype.width - k_smem_layout_staged = cute.append( - cute.select(k_smem_layout_staged, mode=[0, 1, 2]), - cute.make_layout(self.kv_stage, stride=k_stage_stride), - ) - v_smem_layout_staged = v_smem_layout_staged_origin - # cute.cosize gives (kv_stage-1)*k_stage_stride + k_tile_cosize, but the - # last stage must also accommodate a full V tile, so size for kv_stage slots. - sK_cosize = self.kv_stage * k_stage_stride - - o_smem_layout_staged = sm100_utils.make_smem_layout_epi( - self.o_dtype, - self.o_layout, - self.epi_tile, - self.epi_stage, - ) - - # TMA load for Q - tma_load_op = cute.nvgpu.cpasync.CopyBulkTensorTileG2SOp(cta_group) - tma_store_op = cute.nvgpu.cpasync.CopyBulkTensorTileS2GOp() - - q_smem_layout = cute.select(q_smem_layout_staged, mode=[0, 1, 2]) - tma_atom_q, tma_tensor_q = cute.nvgpu.make_tiled_tma_atom_A( - tma_load_op, - q, - q_smem_layout, - self.qk_mma_tiler, - qk_tiled_mma, - self.cluster_layout_vmnk.shape, - ) - q_sf_smem_layout = cute.slice_(q_sf_smem_layout_staged, (None, None, None, 0)) - tma_atom_q_sf, tma_tensor_q_sf = cute.nvgpu.make_tiled_tma_atom_A( - tma_load_op, - q_sf, - q_sf_smem_layout, - self.qk_mma_tiler, - qk_tiled_mma, - self.cluster_layout_vmnk.shape, - internal_type=cutlass.Int16, - ) - - # TMA load for K - k_smem_layout = cute.select(k_smem_layout_staged, mode=[0, 1, 2]) - tma_atom_k, tma_tensor_k = cute.nvgpu.make_tiled_tma_atom_B( - tma_load_op, - k, - k_smem_layout, - self.qk_mma_tiler, - qk_tiled_mma, - self.cluster_layout_vmnk.shape, - ) - k_sf_smem_layout = cute.slice_(k_sf_smem_layout_staged, (None, None, None, 0)) - tma_atom_k_sf, tma_tensor_k_sf = cute.nvgpu.make_tiled_tma_atom_B( - tma_load_op, - k_sf, - k_sf_smem_layout, - self.qk_mma_tiler, - qk_tiled_mma, - self.cluster_layout_vmnk.shape, - internal_type=cutlass.Int16, - ) - # TMA load for V - v_smem_layout = cute.select(v_smem_layout_staged, mode=[0, 1, 2]) - tma_atom_v, tma_tensor_v = cute.nvgpu.make_tiled_tma_atom_B( - tma_load_op, - v, - v_smem_layout, - self.pv_mma_tiler, - pv_tiled_mma, - self.cluster_layout_vmnk.shape, - ) - - o_smem_layout = cute.select(o_smem_layout_staged, mode=[0, 1]) - tma_atom_o, tma_tensor_o = cute.nvgpu.cpasync.make_tiled_tma_atom( - tma_store_op, - o, - o_smem_layout, - self.epi_tile, - ) - - q_copy_size = cute.size_in_bytes(self.q_dtype, q_smem_layout) - k_copy_size = cute.size_in_bytes(self.k_dtype, k_smem_layout) - q_sf_copy_size = cute.size_in_bytes(self.qk_sf_dtype, q_sf_smem_layout) - k_sf_copy_size = cute.size_in_bytes(self.qk_sf_dtype, k_sf_smem_layout) - v_copy_size = cute.size_in_bytes(self.v_dtype, v_smem_layout) - self.tma_copy_q_bytes = q_copy_size + q_sf_copy_size - self.tma_copy_k_bytes = k_copy_size + k_sf_copy_size - self.tma_copy_v_bytes = v_copy_size - - @cute.struct - class SharedStorage: - # Pipeline barriers - load_q_mbar_ptr: cute.struct.MemRange[Int64, self.q_stage * 2] - load_kv_mbar_ptr: cute.struct.MemRange[Int64, self.kv_stage * 2] - mma_s0_mbar_ptr: cute.struct.MemRange[Int64, self.mma_softmax_stage * 2] - mma_s1_mbar_ptr: cute.struct.MemRange[Int64, self.mma_softmax_stage * 2] - p0_mma_mbar_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] - p1_mma_mbar_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] - s0_corr_mbar_ptr: cute.struct.MemRange[Int64, self.softmax_corr_stage * 2] - s1_corr_mbar_ptr: cute.struct.MemRange[Int64, self.softmax_corr_stage * 2] - corr_epi_mbar_ptr: cute.struct.MemRange[Int64, self.epi_stage * 2] - mma_corr_mbar_ptr: cute.struct.MemRange[Int64, self.mma_corr_stage * 2] - s0_p1_inplace_barrier_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] - s1_p0_inplace_barrier_ptr: cute.struct.MemRange[Int64, self.p_mma_stage * 2] - # Softmax_{1-j} signals MMA that S_{1-j}'s TMEM region (whose tail - # columns hold SFQ_j / SFK_j) is no longer being read, so mma_qk's - # SFQ/SFK S2T copy is safe to issue. One pipeline per QK stage. - qk_sf_inplace_0_barrier_ptr: cute.struct.MemRange[Int64, 1 * 2] - qk_sf_inplace_1_barrier_ptr: cute.struct.MemRange[Int64, 1 * 2] - # Tmem holding buffer - tmem_holding_buf: Int32 - # Smem tensors - sO: cute.struct.Align[ - cute.struct.MemRange[self.o_dtype, cute.cosize(o_smem_layout_staged)], - self.buffer_align_bytes, - ] - sQ: cute.struct.Align[ - cute.struct.MemRange[self.q_dtype, cute.cosize(q_smem_layout_staged)], - self.buffer_align_bytes, - ] - sQSF: cute.struct.Align[ - cute.struct.MemRange[self.qk_sf_dtype, cute.cosize(q_sf_smem_layout_staged)], - self.buffer_align_bytes, - ] - sK: cute.struct.Align[ - cute.struct.MemRange[self.k_dtype, sK_cosize], - self.buffer_align_bytes, - ] - sKSF: cute.struct.Align[ - cute.struct.MemRange[self.qk_sf_dtype, cute.cosize(k_sf_smem_layout_staged)], - self.buffer_align_bytes, - ] - # Skip softmax and PV warpgroup votes - s0_warp_wants_skip_softmax_exchange: cute.struct.MemRange[Int8, 4] - s1_warp_wants_skip_softmax_exchange: cute.struct.MemRange[Int8, 4] - - self.shared_storage = SharedStorage - - # Launch the kernel synchronously - self.kernel( - qk_tiled_mma, - pv_tiled_mma, - tma_atom_q, - tma_tensor_q, - tma_atom_q_sf, - tma_tensor_q_sf, - tma_atom_k, - tma_tensor_k, - tma_atom_k_sf, - tma_tensor_k_sf, - tma_atom_v, - tma_tensor_v, - tma_atom_o, - tma_tensor_o, - o, - cum_seqlen_q, - cum_seqlen_k, - lse, - sink, - scale_softmax_log2, - scale_softmax, - scale_output, - m_scale_v_channels, - skip_softmax_threshold_log2, - window_size_left, - window_size_right, - q_smem_layout_staged, - k_smem_layout_staged, - q_sf_smem_layout_staged, - k_sf_smem_layout_staged, - p_tmem_layout_staged, - v_smem_layout_staged, - o_smem_layout_staged, - skip_softmax_count, - total_softmax_count, - self.tile_sched_params, - ).launch( - grid=grid, - block=[self.threads_per_cta, 1, 1], - cluster=self.cluster_shape_mnk, - stream=stream, - min_blocks_per_mp=1, - use_pdl=use_pdl, - ) - - # GPU device kernel - @cute.kernel - def kernel( - self, - qk_tiled_mma: cute.TiledMma, - pv_tiled_mma: cute.TiledMma, - tma_atom_q: cute.CopyAtom, - mQ_qdl: cute.Tensor, - tma_atom_q_sf: cute.CopyAtom, - mQSF_qdl: cute.Tensor, - tma_atom_k: cute.CopyAtom, - mK_kdl: cute.Tensor, - tma_atom_k_sf: cute.CopyAtom, - mKSF_kdl: cute.Tensor, - tma_atom_v: cute.CopyAtom, - mV_dkl: cute.Tensor, - tma_atom_o: cute.CopyAtom, - mO_qdl: cute.Tensor, - mO: cute.Tensor, - cum_seqlen_q: Optional[cute.Tensor], - cum_seqlen_k: Optional[cute.Tensor], - mLSE: Optional[cute.Tensor], - mSink: Optional[cute.Tensor], - scale_softmax_log2: Float32, - scale_softmax: Float32, - scale_output: Float32, - m_scale_v_channels: Optional[cute.Tensor], - skip_softmax_threshold_log2: Optional[Float32], - window_size_left: Optional[Int32], - window_size_right: Optional[Int32], - q_smem_layout_staged: cute.ComposedLayout, - k_smem_layout_staged: cute.ComposedLayout, - q_sf_smem_layout_staged: cute.Layout, - k_sf_smem_layout_staged: cute.Layout, - p_tmem_layout_staged: cute.ComposedLayout, - v_smem_layout_staged: cute.ComposedLayout, - o_smem_layout_staged: cute.ComposedLayout, - skip_softmax_count: Optional[cute.Tensor], - total_softmax_count: Optional[cute.Tensor], - tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams, - ): - """The device kernel implementation of the Fused Multi-Head Attention. - - This kernel coordinates multiple specialized warps to perform different phases of the FMHA computation: - 1. Load warp: Loads Q, K, V data from global memory to shared memory using TMA - 2. MMA warp: Performs matrix multiplications (Q*K^T and P*V) - 3. Softmax warps: Compute softmax normalization on attention scores - 4. Correction warps: Apply adjustments to intermediate results - 5. Epilogue warp: Handles final output transformation and storage - - The kernel implements a complex pipeline with overlapping computation and memory operations, - using tensor memory access (TMA) for efficient data loading, warp specialization for different - computation phases, and optional attention masking. - - :param qk_tiled_mma: Tiled MMA for Q*K^T - :type qk_tiled_mma: cute.TiledMma - :param pv_tiled_mma: Tiled MMA for P*V - :type pv_tiled_mma: cute.TiledMma - :param tma_atom_q: TMA copy atom for query tensor - :type tma_atom_q: cute.CopyAtom - :param mQ_qdl: Partitioned query tensor - :type mQ_qdl: cute.Tensor - :param tma_atom_k: TMA copy atom for key tensor - :type tma_atom_k: cute.CopyAtom - :param mK_kdl: Partitioned key tensor - :type mK_kdl: cute.Tensor - :param tma_atom_v: TMA copy atom for value tensor - :type tma_atom_v: cute.CopyAtom - :param mV_dkl: Partitioned value tensor - :type mV_dkl: cute.Tensor - :param tma_atom_o: TMA copy atom for output tensor - :type tma_atom_o: cute.CopyAtom - :param mO_qdl: Partitioned output tensor - :type mO_qdl: cute.Tensor - :param mO: Non-partitioned output tensor - :type mO: cute.Tensor - :param scale_softmax_log2: The log2 scale factor for softmax - :type scale_softmax_log2: Float32 - :param scale_output: The scale factor for the output - :type scale_output: Float32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - :param q_smem_layout_staged: Shared memory layout for query tensor - :type q_smem_layout_staged: cute.ComposedLayout - :param k_smem_layout_staged: Shared memory layout for key tensor - :type k_smem_layout_staged: cute.ComposedLayout - :param p_tmem_layout_staged: Tensor memory layout for probability matrix - :type p_tmem_layout_staged: cute.ComposedLayout - :param v_smem_layout_staged: Shared memory layout for value tensor - :type v_smem_layout_staged: cute.ComposedLayout - :param o_smem_layout_staged: Shared memory layout for output tensor - :type o_smem_layout_staged: cute.ComposedLayout - :param tile_sched_params: Scheduling parameters for work distribution - :type tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams - """ - warp_idx = cute.arch.make_warp_uniform(cute.arch.warp_idx()) - # coord inside cta - tidx, _, _ = cute.arch.thread_idx() - - # - # Prefetch tma desc - # - if warp_idx == self.load_warp_id: - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_q) - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_q_sf) - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_k) - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_k_sf) - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_v) - if cutlass.const_expr(self.use_tma_store): - cute.nvgpu.cpasync.prefetch_descriptor(tma_atom_o) - - # Alloc - smem = utils.SmemAllocator() - storage = smem.allocate(self.shared_storage) - - load_q_producer, load_q_consumer = pipeline.PipelineTmaUmma.create( - num_stages=self.q_stage, - producer_group=make_thread_cooperative_group(len([self.load_warp_id])), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - tx_count=self.tma_copy_q_bytes, - barrier_storage=storage.load_q_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - load_kv_producer, load_kv_consumer = pipeline.PipelineTmaUmma.create( - num_stages=self.kv_stage, - producer_group=make_thread_cooperative_group(len([self.load_warp_id])), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - tx_count=self.tma_copy_k_bytes, - barrier_storage=storage.load_kv_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - load_kv_full_mbar_ptr = storage.load_kv_mbar_ptr.data_ptr() - load_kv_empty_mbar_ptr = load_kv_full_mbar_ptr + self.kv_stage - mma_s0_producer, mma_s0_consumer = pipeline.PipelineUmmaAsync.create( - num_stages=self.mma_softmax_stage, - producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - barrier_storage=storage.mma_s0_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - mma_s1_producer, mma_s1_consumer = pipeline.PipelineUmmaAsync.create( - num_stages=self.mma_softmax_stage, - producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - barrier_storage=storage.mma_s1_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - p0_mma_producer, p0_mma_consumer = pipeline.PipelineAsyncUmma.create( - num_stages=self.p_mma_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - barrier_storage=storage.p0_mma_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - p1_mma_producer, p1_mma_consumer = pipeline.PipelineAsyncUmma.create( - num_stages=self.p_mma_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - barrier_storage=storage.p1_mma_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - s0_corr_producer, s0_corr_consumer = pipeline.PipelineAsync.create( - num_stages=self.softmax_corr_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len((*self.softmax0_warp_ids, self.mma_warp_id)) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.correction_warp_ids) - ), - barrier_storage=storage.s0_corr_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - s1_corr_producer, s1_corr_consumer = pipeline.PipelineAsync.create( - num_stages=self.softmax_corr_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len((*self.softmax1_warp_ids, self.mma_warp_id)) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.correction_warp_ids) - ), - barrier_storage=storage.s1_corr_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - corr_epi_producer, corr_epi_consumer = pipeline.PipelineAsync.create( - num_stages=self.epi_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.correction_warp_ids) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len([self.epilogue_warp_id]) - ), - barrier_storage=storage.corr_epi_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - mma_corr_producer, mma_corr_consumer = pipeline.PipelineUmmaAsync.create( - num_stages=self.mma_corr_stage, - producer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.correction_warp_ids) - ), - barrier_storage=storage.mma_corr_mbar_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - s0_p1_inplace_producer, s0_p1_inplace_consumer = pipeline.PipelineAsync.create( - num_stages=self.p_mma_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - barrier_storage=storage.s0_p1_inplace_barrier_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - s1_p0_inplace_producer, s1_p0_inplace_consumer = pipeline.PipelineAsync.create( - num_stages=self.p_mma_stage, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - consumer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - barrier_storage=storage.s1_p0_inplace_barrier_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - # QK SF TMEM-slot release pipelines: softmax_{1-j} (producer) signals - # MMA (consumer) once its T2R-load of S_{1-j} is done, so mma_qk(j)'s - # SFQ/SFK S2T copy is safe to overwrite the tail of TMEM[S_{1-j}]. - qk_sf_inplace_0_producer, qk_sf_inplace_0_consumer = pipeline.PipelineAsync.create( - num_stages=1, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax1_warp_ids) - ), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - barrier_storage=storage.qk_sf_inplace_0_barrier_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - qk_sf_inplace_1_producer, qk_sf_inplace_1_consumer = pipeline.PipelineAsync.create( - num_stages=1, - producer_group=make_thread_cooperative_group( - self.threads_per_warp * len(self.softmax0_warp_ids) - ), - consumer_group=make_thread_cooperative_group(len([self.mma_warp_id])), - barrier_storage=storage.qk_sf_inplace_1_barrier_ptr.data_ptr(), - defer_sync=True, - ).make_participants() - tmem = utils.TmemAllocator( - storage.tmem_holding_buf.ptr, - barrier_for_retrieve=self.tmem_alloc_barrier, - # Correction warp is the last one that accesses tmem - allocator_warp_id=self.correction_warp_ids[0], - arch=self.arch_str, - ) - pipeline_init_arrive(is_relaxed=True) - - # Generate smem tensor Q/K/V/O - # (MMA, MMA_Q, MMA_D, PIPE) - sQ = storage.sQ.get_tensor(q_smem_layout_staged.outer, swizzle=q_smem_layout_staged.inner) - # (MMA, MMA_Q, MMA_D, PIPE) - sQSF = storage.sQSF.get_tensor(q_sf_smem_layout_staged) - # (MMA, MMA_K, MMA_D, PIPE) - sK = storage.sK.get_tensor(k_smem_layout_staged.outer, swizzle=k_smem_layout_staged.inner) - # (MMA, MMA_K, MMA_D, PIPE) - sKSF = storage.sKSF.get_tensor(k_sf_smem_layout_staged) - # (MMA, MMA_K, MMA_D, PIPE) - # Reuse k's smem buffer for v. Recast element type so MMA descriptor matches v_dtype. - sV_ptr = cute.recast_ptr( - cute.recast_ptr(sK.iterator, dtype=self.v_dtype), - v_smem_layout_staged.inner, - ) - sV = cute.make_tensor(sV_ptr, v_smem_layout_staged.outer) - sO = storage.sO.get_tensor(o_smem_layout_staged.outer, swizzle=o_smem_layout_staged.inner) - s0_warp_wants_skip_softmax_exchange = ( - storage.s0_warp_wants_skip_softmax_exchange.get_tensor(cute.make_layout((4,))) - ) - s1_warp_wants_skip_softmax_exchange = ( - storage.s1_warp_wants_skip_softmax_exchange.get_tensor(cute.make_layout((4,))) - ) - - qk_thr_mma = qk_tiled_mma.get_slice(0) # default 1sm - pv_thr_mma = pv_tiled_mma.get_slice(0) # default 1sm - tSrQ = qk_thr_mma.make_fragment_A(sQ) - tSrK = qk_thr_mma.make_fragment_B(sK) - tOrV = pv_thr_mma.make_fragment_B(sV) - - def make_tmem_tensors( - self, - qk_thr_mma: cute.TiledMma, - pv_thr_mma: cute.TiledMma, - p_tmem_layout_staged: cute.Layout, - tmem_ptr: cute.Pointer, - ): - qk_acc_shape = qk_thr_mma.partition_shape_C( - (self.qk_mma_tiler[0], self.qk_mma_tiler[1]) - ) - tStS_fake = qk_thr_mma.make_fragment_C(qk_acc_shape) - tStS = cute.make_tensor(tmem_ptr + self.tmem_s0_offset, tStS_fake.layout) - pv_acc_shape = pv_thr_mma.partition_shape_C( - (self.pv_mma_tiler[0], self.pv_mma_tiler[1]) - ) - tOtO = pv_thr_mma.make_fragment_C(pv_acc_shape) - tStS0 = cute.make_tensor(tmem_ptr + self.tmem_s0_offset, tStS.layout) - tStS1 = cute.make_tensor(tmem_ptr + self.tmem_s1_offset, tStS.layout) - tOtO0 = cute.make_tensor(tmem_ptr + self.tmem_o0_offset, tOtO.layout) - tOtO1 = cute.make_tensor(tmem_ptr + self.tmem_o1_offset, tOtO.layout) - tP = cute.make_tensor(tStS.iterator, p_tmem_layout_staged.outer) - tOrP = pv_thr_mma.make_fragment_A(tP)[None, None, None, 0] - tOrP0 = cute.make_tensor( - cute.recast_ptr( - tmem_ptr + self.tmem_p0_offset, - dtype=tOrP.dtype, - ), - tOrP.layout, - ) - tOrP1 = cute.make_tensor( - cute.recast_ptr( - tmem_ptr + self.tmem_p1_offset, - dtype=tOrP.dtype, - ), - tOrP.layout, - ) - return tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 - - tile_sched = fmha_utils.create_fmha_static_tile_scheduler( - tile_sched_params, cute.arch.block_idx(), cute.arch.grid_dim() - ) - work_tile = tile_sched.initial_work_tile_info() - pipeline_init_wait() - softmax_fn = partial( - self.softmax, - qk_thr_mma=qk_thr_mma, - value_args=( - mK_kdl.shape[0], - mQ_qdl.shape[0], - scale_softmax_log2, - skip_softmax_threshold_log2, - ), - mask_args=(window_size_left, window_size_right), - sched_args=(tile_sched, work_tile), - # Each softmax warpgroup picks its producer by stage: softmax0 - # (stage=0) produces on qk_sf_inplace_1, softmax1 (stage=1) on - # qk_sf_inplace_0. - qk_sf_inplace_producers=( - qk_sf_inplace_1_producer, - qk_sf_inplace_0_producer, - ), - ) - # /////////////////////////////////////////////////////////////////////////////// - # EMPTY - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.empty_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_other) - - # /////////////////////////////////////////////////////////////////////////////// - # LOAD - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.load_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_other) - cute.arch.griddepcontrol_wait() - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - continue_cond = False - cuseqlen_q = Int32(0) - seqlen_q = mQ_qdl.shape[0] - seqlen_k = mK_kdl.shape[0] - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q, - ) - ) - if not continue_cond: - if cutlass.const_expr(cum_seqlen_k is not None): - seqlen_k = cum_seqlen_k[batch_coord + 1] - cum_seqlen_k[batch_coord] - continue_cond = seqlen_k <= 0 - if not continue_cond: - mQ_qdl_ = mQ_qdl - mK_kdl_ = mK_kdl - mQSF_qdl_ = mQSF_qdl - mKSF_kdl_ = mKSF_kdl - mV_dkl_ = mV_dkl - if cutlass.const_expr(cum_seqlen_q is not None): - mQ_qdl_ = cute.domain_offset( - (cum_seqlen_q[batch_coord], 0, ((0, 0), 0)), mQ_qdl - ) - mQSF_qdl_ = cute.domain_offset( - (cum_seqlen_q[batch_coord], 0, (0, 0)), mQSF_qdl - ) - if cutlass.const_expr(cum_seqlen_k is not None): - mK_kdl_ = cute.domain_offset( - (cum_seqlen_k[batch_coord], 0, ((0, 0), 0)), mK_kdl - ) - mKSF_kdl_ = cute.domain_offset( - (cum_seqlen_k[batch_coord], 0, (0, 0)), mKSF_kdl - ) - mV_dkl_ = cute.domain_offset( - (0, cum_seqlen_k[batch_coord], ((0, 0), 0)), mV_dkl - ) - # Local tile partition global tensors - gQ_qdl = cute.flat_divide(mQ_qdl_, cute.select(self.qk_mma_tiler, mode=[0, 2])) - tSgQ_qdl = qk_thr_mma.partition_A(gQ_qdl) - tQsQ, tQgQ_qdl = cute.nvgpu.cpasync.tma_partition( - tma_atom_q, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sQ, 0, 3), - cute.group_modes(tSgQ_qdl, 0, 3), - ) - tQgQ = tQgQ_qdl[None, None, 0, curr_block_coord[2]] - gQSF_qdl = cute.local_tile( - mQSF_qdl_, - cute.select(self.qk_mma_tiler, mode=[0, 2]), - (None, None, None), - ) - tSgQSF_qdl = qk_thr_mma.partition_A(gQSF_qdl) - tQsQSF, tQgQSF_qdl = cute.nvgpu.cpasync.tma_partition( - tma_atom_q_sf, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sQSF, 0, 3), - cute.group_modes(tSgQSF_qdl, 0, 3), - ) - tQsQSF = cute.filter_zeros(tQsQSF) - tQgQSF_qdl = cute.filter_zeros(tQgQSF_qdl) - # tile_atom_to_shape_SF coalesces Q's ((h_r, h_k), b) mode into (1, h_r*h_k*b), - # One needs to reconstruct the correct layout here to correctly map block_coord - # onto this linearlized L-like mode. - sf_l_q = cute.make_layout( - mQ_qdl.shape[2], - stride=( - (1, mQ_qdl.shape[2][0][0]), - cute.size(mQ_qdl.shape[2][0]), - ), - )(curr_block_coord[2]) - tQgQSF = tQgQSF_qdl[None, None, 0, sf_l_q] - gK_kdl = cute.flat_divide(mK_kdl_, cute.select(self.qk_mma_tiler, mode=[1, 2])) - tSgK_kdl = qk_thr_mma.partition_B(gK_kdl) - tKsK, tKgK_kdl = cute.nvgpu.cpasync.tma_partition( - tma_atom_k, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sK, 0, 3), - cute.group_modes(tSgK_kdl, 0, 3), - ) - tKgK = tKgK_kdl[None, None, 0, curr_block_coord[2]] - gKSF_kdl = cute.local_tile( - mKSF_kdl_, - cute.select(self.qk_mma_tiler, mode=[1, 2]), - (None, None, None), - ) - tSgKSF_kdl = qk_thr_mma.partition_B(gKSF_kdl) - tKsKSF, tKgKSF_kdl = cute.nvgpu.cpasync.tma_partition( - tma_atom_k_sf, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sKSF, 0, 3), - cute.group_modes(tSgKSF_kdl, 0, 3), - ) - tKsKSF = cute.filter_zeros(tKsKSF) - tKgKSF_kdl = cute.filter_zeros(tKgKSF_kdl) - # Same L-mode trick as Q's SF (see comment above). K's SF is shared across h_r heads - # (it indexes h_k, not h_q), which we express by a 0-stride for the h_r sub-mode. - sf_l_k = cute.make_layout( - mK_kdl.shape[2], - stride=((0, 1), mK_kdl.shape[2][0][1]), - )(curr_block_coord[2]) - tKgKSF = tKgKSF_kdl[None, None, 0, sf_l_k] - gV_dkl = cute.flat_divide(mV_dkl_, cute.select(self.pv_mma_tiler, mode=[1, 2])) - tSgV_dkl = pv_thr_mma.partition_B(gV_dkl) - tVsV, tVgV_dkl = cute.nvgpu.cpasync.tma_partition( - tma_atom_v, - 0, # no multicast - cute.make_layout(1), - cute.group_modes(sV, 0, 3), - cute.group_modes(tSgV_dkl, 0, 3), - ) - tVgV = tVgV_dkl[None, 0, None, curr_block_coord[2]] - seqlen_kv_loop_start = fmha_utils.FusedMask.get_trip_start( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q, - seqlen_k, - window_size_left, - ) - # Q0 - q0_coord = 2 * curr_block_coord[0] - q0_handle = load_q_producer.acquire_and_advance() - cute.copy( - tma_atom_q, - tQgQ[None, q0_coord], - tQsQ[None, q0_handle.index], - tma_bar_ptr=q0_handle.barrier, - ) - cute.copy( - tma_atom_q_sf, - tQgQSF[None, q0_coord], - tQsQSF[None, q0_handle.index], - tma_bar_ptr=q0_handle.barrier, - ) - seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - # K0 - kv_coord = seqlen_kv_loop_start - k_handle = load_kv_producer.acquire_and_advance() - cute.copy( - tma_atom_k, - tKgK[None, kv_coord], - tKsK[None, k_handle.index], - tma_bar_ptr=k_handle.barrier, - ) - cute.copy( - tma_atom_k_sf, - tKgKSF[None, kv_coord], - tKsKSF[None, k_handle.index], - tma_bar_ptr=k_handle.barrier, - ) - # Q1 - q1_coord = q0_coord + 1 - q1_handle = load_q_producer.acquire_and_advance() - cute.copy( - tma_atom_q, - tQgQ[None, q1_coord], - tQsQ[None, q1_handle.index], - tma_bar_ptr=q1_handle.barrier, - ) - cute.copy( - tma_atom_q_sf, - tQgQSF[None, q1_coord], - tQsQSF[None, q1_handle.index], - tma_bar_ptr=q1_handle.barrier, - ) - kv_coord += 1 - - for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): - # Ki - k_handle = load_kv_producer.acquire_and_advance() - cute.copy( - tma_atom_k, - tKgK[None, kv_coord], - tKsK[None, k_handle.index], - tma_bar_ptr=k_handle.barrier, - ) - cute.copy( - tma_atom_k_sf, - tKgKSF[None, kv_coord], - tKsKSF[None, k_handle.index], - tma_bar_ptr=k_handle.barrier, - ) - # Vi-1 - v_handle, load_kv_producer = self.kv_producer_update_tx_acquire_and_advance( - load_kv_producer, - load_kv_empty_mbar_ptr, - load_kv_full_mbar_ptr, - self.tma_copy_v_bytes, - ) - cute.copy( - tma_atom_v, - tVgV[None, kv_coord - 1], - tVsV[None, v_handle.index], - tma_bar_ptr=load_kv_full_mbar_ptr + v_handle.index, - ) - kv_coord += 1 - # End of seqlen_kv loop - # Vi_end - v_handle, load_kv_producer = self.kv_producer_update_tx_acquire_and_advance( - load_kv_producer, - load_kv_empty_mbar_ptr, - load_kv_full_mbar_ptr, - self.tma_copy_v_bytes, - ) - cute.copy( - tma_atom_v, - tVgV[None, kv_coord - 1], - tVsV[None, v_handle.index], - tma_bar_ptr=load_kv_full_mbar_ptr + v_handle.index, - ) - # End of if not continue_cond - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - # End of persistent scheduler loop - # /////////////////////////////////////////////////////////////////////////////// - # MMA - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.mma_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_other) - tmem.wait_for_alloc() - tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) - tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( - self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr - ) - q_sf_tmem_layout = blockscaled_utils.make_tmem_layout_sfa( - qk_tiled_mma, - self.qk_mma_tiler, - self.qk_sf_vec_size, - cute.slice_(q_sf_smem_layout_staged, (None, None, None, 0)), - ) - k_sf_tmem_layout = blockscaled_utils.make_tmem_layout_sfb( - qk_tiled_mma, - self.qk_mma_tiler, - self.qk_sf_vec_size, - cute.slice_(k_sf_smem_layout_staged, (None, None, None, 0)), - ) - # TMEM offsets are in u32 columns. Follow FA4's explicit SFQ footprint - # calculation instead of deriving the offset from the recast FP8 tensor - # view, which can under-count for block-scaled layouts. - sf_atom_mn = 32 - q_sf_tmem_cols = (self.qk_mma_tiler[0] // sf_atom_mn) * self.qk_mma_inst_tile_k - k_sf_tmem_cols = ( - ((self.qk_mma_tiler[1] + 127) // 128 * 128) // sf_atom_mn - ) * self.qk_mma_inst_tile_k - tCtQSF0 = cute.make_tensor( - cute.recast_ptr(tmem_ptr + self.tmem_qk0_sf_offset, dtype=self.qk_sf_dtype), - q_sf_tmem_layout, - ) - tCtKSF0 = cute.make_tensor( - cute.recast_ptr( - tmem_ptr + self.tmem_qk0_sf_offset + q_sf_tmem_cols, - dtype=self.qk_sf_dtype, - ), - k_sf_tmem_layout, - ) - tCtQSF1 = cute.make_tensor( - cute.recast_ptr(tmem_ptr + self.tmem_qk1_sf_offset, dtype=self.qk_sf_dtype), - q_sf_tmem_layout, - ) - tCtKSF1 = cute.make_tensor( - cute.recast_ptr( - tmem_ptr + self.tmem_qk1_sf_offset + q_sf_tmem_cols, - dtype=self.qk_sf_dtype, - ), - k_sf_tmem_layout, - ) - tiled_copy_s2t_qsf0, tCsQSF_s2t, tCtQSF0_s2t = self.mainloop_s2t_copy_and_partition( - sQSF, tCtQSF0 - ) - tiled_copy_s2t_ksf0, tCsKSF_s2t, tCtKSF0_s2t = self.mainloop_s2t_copy_and_partition( - sKSF, tCtKSF0 - ) - tiled_copy_s2t_qsf1, tCsQSF1_s2t, tCtQSF1_s2t = self.mainloop_s2t_copy_and_partition( - sQSF, tCtQSF1 - ) - tiled_copy_s2t_ksf1, tCsKSF1_s2t, tCtKSF1_s2t = self.mainloop_s2t_copy_and_partition( - sKSF, tCtKSF1 - ) - enable_skip_softmax = skip_softmax_threshold_log2 is not None - tiled_tmem_load_v = None - tTMEM_LOADtS_v0, tTMEM_LOADtS_v1 = None, None - tTMEM_LOADrS_v0, tTMEM_LOADrS_v1 = None, None - if cutlass.const_expr(enable_skip_softmax): - cS = cute.make_identity_tensor(cute.select(self.qk_mma_tiler, mode=[0, 1])) - tScS = qk_thr_mma.partition_C(cS) - tStS_v = cute.composition(tStS, cute.make_layout((self.threads_per_warp, 1))) - tScS_v = cute.composition(tScS, cute.make_layout((self.threads_per_warp, 1))) - tmem_load_v_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(1)), - self.qk_acc_dtype, - ) - thread_idx = tidx % self.threads_per_warp - - tiled_tmem_load_v = tcgen05.make_tmem_copy(tmem_load_v_atom, tStS_v) - thr_tmem_load_v = tiled_tmem_load_v.get_slice(thread_idx) - tTMEM_LOADtS_v = thr_tmem_load_v.partition_S(tStS_v) - tTMEM_LOADcS_v = thr_tmem_load_v.partition_D(tScS_v) - tTMEM_LOADrS_v0 = cute.make_rmem_tensor(tTMEM_LOADcS_v.shape, self.qk_acc_dtype) - tTMEM_LOADrS_v1 = cute.make_rmem_tensor(tTMEM_LOADcS_v.shape, self.qk_acc_dtype) - tTMEM_LOADtS_v0 = cute.make_tensor( - tTMEM_LOADtS_v.iterator + self.tmem_skip_softmax0_offset, - tTMEM_LOADtS_v.layout, - ) - tTMEM_LOADtS_v1 = cute.make_tensor( - tTMEM_LOADtS_v.iterator + self.tmem_skip_softmax1_offset, - tTMEM_LOADtS_v.layout, - ) - - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - continue_cond = False - seqlen_q = mQ_qdl.shape[0] - seqlen_k = mK_kdl.shape[0] - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q, - ) - ) - if not continue_cond: - if cutlass.const_expr(cum_seqlen_k is not None): - cuseqlen_k = cum_seqlen_k[batch_coord] - seqlen_k = cum_seqlen_k[batch_coord + 1] - cuseqlen_k - continue_cond = seqlen_k <= 0 - if not continue_cond: - # Wait for Q0 - q0_handle = load_q_consumer.wait_and_advance() - tSrQ0 = tSrQ[None, None, None, q0_handle.index] - # Wait for K0 - k_handle = load_kv_consumer.wait_and_advance() - tSrK0 = tSrK[None, None, None, k_handle.index] - q0_sf_stage = (None, None, None, None, q0_handle.index) - k_sf_stage = (None, None, None, None, k_handle.index) - # GEMM_QK00 (Q0 * K0 -> S0) - mma_s0_producer, s0_corr_producer, qk_sf_inplace_0_consumer = self.mma_qk( - qk_tiled_mma, - (tSrQ0, tSrK0, tStS0), - ( - tiled_copy_s2t_qsf0, - tCsQSF_s2t[q0_sf_stage], - tCtQSF0_s2t, - tCtQSF0, - tiled_copy_s2t_ksf0, - tCsKSF_s2t[k_sf_stage], - tCtKSF0_s2t, - tCtKSF0, - ), - (mma_s0_producer, s0_corr_producer), - qk_sf_inplace_0_consumer, - ) - # Wait for Q1 - q1_handle = load_q_consumer.wait_and_advance() - tSrQ1 = tSrQ[None, None, None, q1_handle.index] - q1_sf_stage = (None, None, None, None, q1_handle.index) - # GEMM_QK10 (Q1 * K0 -> S1), K0 is ready in GEMM_QK00 - mma_s1_producer, s1_corr_producer, qk_sf_inplace_1_consumer = self.mma_qk( - qk_tiled_mma, - (tSrQ1, tSrK0, tStS1), - ( - tiled_copy_s2t_qsf1, - tCsQSF1_s2t[q1_sf_stage], - tCtQSF1_s2t, - tCtQSF1, - tiled_copy_s2t_ksf1, - tCsKSF1_s2t[k_sf_stage], - tCtKSF1_s2t, - tCtKSF1, - ), - (mma_s1_producer, s1_corr_producer), - qk_sf_inplace_1_consumer, - ) - # Release K0 - k_handle.release() - # Note: Q0 & Q1 are still needed in the seqlen_kv loop - # so we need to release them after the seqlen_kv loop - seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - # O1 hasn't been accumulated yet, its first MMA calculation doesn't need to accumulate - pv_whether_acc = False - for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): - # Wait for Ki - k_handle = load_kv_consumer.wait_and_advance() - tSrKi = tSrK[None, None, None, k_handle.index] - k_sf_stage = (None, None, None, None, k_handle.index) - # GEMM_QK0i (Q0 * Ki -> S0) - mma_s0_producer, s0_corr_producer, qk_sf_inplace_0_consumer = self.mma_qk( - qk_tiled_mma, - (tSrQ0, tSrKi, tStS0), - ( - tiled_copy_s2t_qsf0, - tCsQSF_s2t[q0_sf_stage], - tCtQSF0_s2t, - tCtQSF0, - tiled_copy_s2t_ksf0, - tCsKSF_s2t[k_sf_stage], - tCtKSF0_s2t, - tCtKSF0, - ), - (mma_s0_producer, s0_corr_producer), - qk_sf_inplace_0_consumer, - ) - # Wait for Vi-1 - v_handle = load_kv_consumer.wait_and_advance() - tOrVi = tOrV[None, None, None, v_handle.index] - # GEMM_PV0(i-1) (P0 * Vi-1 -> O0_partial) - mma_corr_producer, p0_mma_consumer = self.mma_pv( - pv_tiled_mma, - pv_whether_acc, - (tOrP0, tOrVi, tOtO0), - (mma_corr_producer, p0_mma_consumer), - ( - enable_skip_softmax, - tiled_tmem_load_v, - tTMEM_LOADtS_v0, - tTMEM_LOADrS_v0, - ), - ) - # GEMM_QK1i (Q1 * Ki -> S1) - mma_s1_producer, s1_corr_producer, qk_sf_inplace_1_consumer = self.mma_qk( - qk_tiled_mma, - (tSrQ1, tSrKi, tStS1), - ( - tiled_copy_s2t_qsf1, - tCsQSF1_s2t[q1_sf_stage], - tCtQSF1_s2t, - tCtQSF1, - tiled_copy_s2t_ksf1, - tCsKSF1_s2t[k_sf_stage], - tCtKSF1_s2t, - tCtKSF1, - ), - (mma_s1_producer, s1_corr_producer), - qk_sf_inplace_1_consumer, - ) - # Release Ki - k_handle.release() - # GEMM_PV1(i-1) (P1 * Vi-1 -> O1_partial) - mma_corr_producer, p1_mma_consumer = self.mma_pv( - pv_tiled_mma, - pv_whether_acc, - (tOrP1, tOrVi, tOtO1), - (mma_corr_producer, p1_mma_consumer), - ( - enable_skip_softmax, - tiled_tmem_load_v, - tTMEM_LOADtS_v1, - tTMEM_LOADrS_v1, - ), - ) - pv_whether_acc = True - # Release Vi-1 - v_handle.release() - # End of seqlen_kv loop - # release Q0 & Q1 - q0_handle.release() - q1_handle.release() - # Wait for Vi_end - v_handle = load_kv_consumer.wait_and_advance() - tOrVi = tOrV[None, None, None, v_handle.index] - # GEMM_PV0(i_end) (P0 * Vi_end -> O0) - mma_corr_producer, p0_mma_consumer = self.mma_pv( - pv_tiled_mma, - pv_whether_acc, - (tOrP0, tOrVi, tOtO0), - (mma_corr_producer, p0_mma_consumer), - ( - enable_skip_softmax, - tiled_tmem_load_v, - tTMEM_LOADtS_v0, - tTMEM_LOADrS_v0, - ), - ) - # GEMM_PV1(i_end) (P1 * Vi_end -> O1) - mma_corr_producer, p1_mma_consumer = self.mma_pv( - pv_tiled_mma, - pv_whether_acc, - (tOrP1, tOrVi, tOtO1), - (mma_corr_producer, p1_mma_consumer), - ( - enable_skip_softmax, - tiled_tmem_load_v, - tTMEM_LOADtS_v1, - tTMEM_LOADrS_v1, - ), - ) - # Release Vi_end - v_handle.release() - # Empty step for correction epilog - vec0_handle = s0_corr_producer.acquire_and_advance() - vec0_handle.commit() - vec1_handle = s1_corr_producer.acquire_and_advance() - vec1_handle.commit() - # End of if not continue_cond - # Advance to next tile - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - # End of persistent scheduler loop - # /////////////////////////////////////////////////////////////////////////////// - # Epilogue (TMA store path only) - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx == self.epilogue_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_other) - if cutlass.const_expr(self.use_tma_store): - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - continue_cond = False - cuseqlen_q = Int32(0) - seqlen_q = mQ_qdl.shape[0] - - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q, - ) - ) - if not continue_cond: - mO_qdl_ = mO_qdl - if cutlass.const_expr(cum_seqlen_q is not None): - mO_qdl_ = cute.domain_offset( - (cum_seqlen_q[batch_coord], 0, ((0, 0), 0)), mO_qdl - ) - - o0_coord = 2 * curr_block_coord[0] - o1_coord = o0_coord + 1 - gO_qdl = cute.flat_divide( - mO_qdl_, cute.select(self.pv_mma_tiler, mode=[0, 1]) - ) - gO = gO_qdl[None, None, None, 0, curr_block_coord[2]] - tOsO, tOgO = cute.nvgpu.cpasync.tma_partition( - tma_atom_o, - 0, - cute.make_layout(1), - cute.group_modes(sO, 0, 2), - cute.group_modes(gO, 0, 2), - ) - - # O0 O1 using the same pipeline - # wait from corr, issue tma store on smem - # O0 - # 1. Wait for O0 final - o0_handle = corr_epi_consumer.wait_and_advance() - # 2. Copy O0 to gmem - cute.copy(tma_atom_o, tOsO[None, 0], tOgO[None, o0_coord]) - cute.arch.cp_async_bulk_commit_group() - # O1 - # 1. Wait for O1 final - o1_handle = corr_epi_consumer.wait_and_advance() - # 2. Copy O1 to gmem - cute.copy(tma_atom_o, tOsO[None, 1], tOgO[None, o1_coord]) - cute.arch.cp_async_bulk_commit_group() - - # Ensure O0 buffer is ready to be released - cute.arch.cp_async_bulk_wait_group(1, read=True) - o0_handle.release() - # Ensure O1 buffer is ready to be released - cute.arch.cp_async_bulk_wait_group(0, read=True) - o1_handle.release() - - # Advance to next tile - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - cute.arch.griddepcontrol_launch_dependents() - # End of persistent scheduler loop - # /////////////////////////////////////////////////////////////////////////////// - # Softmax0 - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx < self.softmax1_warp_ids[0]: - cute.arch.setmaxregister_increase(self.num_regs_softmax) - tmem.wait_for_alloc() - tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) - tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( - self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr - ) - softmax_fn( - stage=0, - tensor_args=( - tStS, - tStS0, - cum_seqlen_k, - cum_seqlen_q, - s0_warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ), - pipeline_args=(mma_s0_consumer, s0_corr_producer, p0_mma_producer), - inplace_args=(s0_p1_inplace_producer, s1_p0_inplace_consumer), - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Softmax1 - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx < self.correction_warp_ids[0] and warp_idx >= self.softmax1_warp_ids[0]: - cute.arch.setmaxregister_increase(self.num_regs_softmax) - tmem.wait_for_alloc() - tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) - tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( - self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr - ) - softmax_fn( - stage=1, - tensor_args=( - tStS, - tStS1, - cum_seqlen_k, - cum_seqlen_q, - s1_warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ), - pipeline_args=(mma_s1_consumer, s1_corr_producer, p1_mma_producer), - inplace_args=(s1_p0_inplace_producer, s0_p1_inplace_consumer), - ) - - # /////////////////////////////////////////////////////////////////////////////// - # Correction - # /////////////////////////////////////////////////////////////////////////////// - if warp_idx >= self.correction_warp_ids[0] and warp_idx < self.mma_warp_id: - cute.arch.setmaxregister_decrease(self.num_regs_correction) - tmem.allocate(self.num_tmem_alloc_cols) - tmem.wait_for_alloc() - tmem_ptr = tmem.retrieve_ptr(self.qk_acc_dtype) - tStS, tStS0, tStS1, tOtO0, tOtO1, tOrP0, tOrP1 = make_tmem_tensors( - self, qk_thr_mma, pv_thr_mma, p_tmem_layout_staged, tmem_ptr - ) - cS = cute.make_identity_tensor((self.qk_mma_tiler[0], self.qk_mma_tiler[1])) - tScS = qk_thr_mma.partition_C(cS) - - tStS_vec_layout = cute.composition(tStS.layout, cute.make_layout((128, 2))) - - tStS_vec0 = cute.make_tensor(tStS.iterator + self.tmem_vec0_offset, tStS_vec_layout) - tStS_vec1 = cute.make_tensor(tStS.iterator + self.tmem_vec1_offset, tStS_vec_layout) - - tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) - tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) - tmem_load_v_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(2)), - self.qk_acc_dtype, - ) - tiled_tmem_load_vec = tcgen05.make_tmem_copy(tmem_load_v_atom, tStS_vec0) - thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) - thr_tmem_load_vec = tiled_tmem_load_vec.get_slice(thread_idx) - tTMEM_LOAD_VECtS0 = thr_tmem_load_vec.partition_S(tStS_vec0) - tTMEM_LOAD_VECtS1 = thr_tmem_load_vec.partition_S(tStS_vec1) - tTMEM_LOAD_VECcS = thr_tmem_load_vec.partition_D(tScS_vec) - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - seqlen_k = mK_kdl.shape[0] - row_idx = Int32(0) - continue_cond = False - cuseqlen_q = Int32(0) - seqlen_q = mQ_qdl.shape[0] - - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q, - ) - ) - if not continue_cond: - row_idx = curr_block_coord[0] * self.cta_tiler[0] + tTMEM_LOAD_VECcS[0][0] - if cutlass.const_expr(cum_seqlen_k is not None): - cuseqlen_k = cum_seqlen_k[batch_coord] - seqlen_k = cum_seqlen_k[batch_coord + 1] - cuseqlen_k - continue_cond = seqlen_k <= 0 - if not continue_cond: - # Ignore first signal from softmax as no correction is required - vec0_handle = s0_corr_consumer.wait_and_advance() - vec0_handle.release() - vec1_handle = s1_corr_consumer.wait_and_advance() - vec1_handle.release() - # O0/O1 share the same mma_corr consumer state, so the Oi - # peek token rolls from O0 -> O1 -> next O0. Seed with a - # blocking token; the rescale helper refreshes it near the - # end of each iteration. - oi_peek_status = cutlass.Boolean(False) - seqlen_kv_loop_steps = fmha_utils.FusedMask.get_trip_count( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - for i in cutlass.range(1, seqlen_kv_loop_steps, 1, unroll=1): - # Rescale O0 - ( - (s0_corr_consumer, mma_corr_consumer), - oi_peek_status, - ) = self.correction_rescale( - pv_thr_mma, - tiled_tmem_load_vec, - scale_softmax_log2, - (tOtO0, tTMEM_LOAD_VECtS0, tTMEM_LOAD_VECcS), - (s0_corr_consumer, mma_corr_consumer), - oi_peek_status, - ) - # Rescale O1 - ( - (s1_corr_consumer, mma_corr_consumer), - oi_peek_status, - ) = self.correction_rescale( - pv_thr_mma, - tiled_tmem_load_vec, - scale_softmax_log2, - (tOtO1, tTMEM_LOAD_VECtS1, tTMEM_LOAD_VECcS), - (s1_corr_consumer, mma_corr_consumer), - oi_peek_status, - ) - # End of seqlen_corr_loop_steps - value_args = ( - cuseqlen_q, - seqlen_q, - curr_block_coord, - scale_softmax, - scale_output, - ) - if cutlass.const_expr(self.use_tma_store): - # TMA store path: write to sO, signal epilogue warp - # Normalize O0 - s0_corr_consumer, mma_corr_consumer, corr_epi_producer = ( - self.correction_epilog( - pv_thr_mma, - tiled_tmem_load_vec, - ( - tOtO0, - tTMEM_LOAD_VECtS0, - tTMEM_LOAD_VECcS, - sO[None, None, 0], - mLSE, - mSink, - m_scale_v_channels, - ), - ( - s0_corr_consumer, - mma_corr_consumer, - corr_epi_producer, - ), - (row_idx, *value_args), - ) - ) - row_idx += self.qk_mma_tiler[0] - # Normalize O1 - s1_corr_consumer, mma_corr_consumer, corr_epi_producer = ( - self.correction_epilog( - pv_thr_mma, - tiled_tmem_load_vec, - ( - tOtO1, - tTMEM_LOAD_VECtS1, - tTMEM_LOAD_VECcS, - sO[None, None, 1], - mLSE, - mSink, - m_scale_v_channels, - ), - ( - s1_corr_consumer, - mma_corr_consumer, - corr_epi_producer, - ), - (row_idx, *value_args), - ) - ) - else: - # st.global path: store directly to global memory - block_offset_o = Int32(0) - if cutlass.const_expr(cum_seqlen_q is not None): - block_offset_o = cum_seqlen_q[batch_coord] - mO_ = cute.make_tensor( - mO.iterator + block_offset_o * mO.stride[0], - cute.make_layout( - (seqlen_q, mO.shape[1], mO.shape[2]), - stride=mO.stride, - ), - ) - o0_coord = 2 * curr_block_coord[0] - o1_coord = o0_coord + 1 - gO_stg = cute.local_tile( - mO_, - (self.pv_mma_tiler[0], self.pv_mma_tiler[1]), - (None, None, None), - ) - gO0 = gO_stg[None, None, o0_coord, 0, curr_block_coord[2]] - gO1 = gO_stg[None, None, o1_coord, 0, curr_block_coord[2]] - # Normalize O0 and store to global memory - s0_corr_consumer, mma_corr_consumer = self.correction_epilog( - pv_thr_mma, - tiled_tmem_load_vec, - ( - tOtO0, - tTMEM_LOAD_VECtS0, - tTMEM_LOAD_VECcS, - gO0, - mLSE, - mSink, - m_scale_v_channels, - ), - (s0_corr_consumer, mma_corr_consumer), - (row_idx, *value_args), - ) - row_idx += self.qk_mma_tiler[0] - # Normalize O1 and st.global to global memory - s1_corr_consumer, mma_corr_consumer = self.correction_epilog( - pv_thr_mma, - tiled_tmem_load_vec, - ( - tOtO1, - tTMEM_LOAD_VECtS1, - tTMEM_LOAD_VECcS, - gO1, - mLSE, - mSink, - m_scale_v_channels, - ), - (s1_corr_consumer, mma_corr_consumer), - (row_idx, *value_args), - ) - # End of if not continue_cond - # Advance to next tile - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - if cutlass.const_expr(not self.use_tma_store): - cute.arch.griddepcontrol_launch_dependents() - # End of persistent scheduler loop - tmem.relinquish_alloc_permit() - # Synchronize before TMEM dealloc (done by the caller) - self.tmem_dealloc_barrier.arrive_and_wait() - tmem.free(tmem_ptr) - return - - def mainloop_s2t_copy_and_partition( - self, - sSF: cute.Tensor, - tSF: cute.Tensor, - ) -> Tuple[cute.TiledCopy, cute.Tensor, cute.Tensor]: - """Partition one block-scaled QK scale-factor tensor for SMEM to TMEM copy.""" - tCsSF_compact = cute.filter_zeros(sSF) - tCtSF_compact = cute.filter_zeros(tSF) - - copy_atom_s2t = cute.make_copy_atom( - tcgen05.Cp4x32x128bOp(tcgen05.CtaGroup.ONE), - self.qk_sf_dtype, - ) - tiled_copy_s2t = tcgen05.make_s2t_copy(copy_atom_s2t, tCtSF_compact) - thr_copy_s2t = tiled_copy_s2t.get_slice(0) - tCsSF_compact_s2t_ = thr_copy_s2t.partition_S(tCsSF_compact) - tCsSF_compact_s2t = tcgen05.get_s2t_smem_desc_tensor(tiled_copy_s2t, tCsSF_compact_s2t_) - tCtSF_compact_s2t = thr_copy_s2t.partition_D(tCtSF_compact) - return tiled_copy_s2t, tCsSF_compact_s2t, tCtSF_compact_s2t - - @cute.jit - def kv_producer_update_tx_acquire_and_advance( - self, tma_producer, empty_mbar_ptr, full_mbar_ptr, tx_bytes - ): - # This utility function is a special version of tma_producer.acquire_and_advance(). - # This is used to customize the tx bytes which is different from - # the initialized tx bytes of tma_producer. - state = tma_producer._PipelineProducer__state.clone() - cute.arch.mbarrier_wait(empty_mbar_ptr + state.index, state.phase) - with cute.arch.elect_one(): - cute.arch.mbarrier_arrive_and_expect_tx( - full_mbar_ptr + state.index, - tx_bytes, - ) - tma_producer.advance() - return state, tma_producer - - @cute.jit - def get_skip_softmax_flag(self, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v): - cute.copy(tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v) - tTMEM_LOADrS_v_i32 = cute.recast_tensor(tTMEM_LOADrS_v, dtype=cutlass.Int32) - skip_softmax_flag = cute.arch.make_warp_uniform(tTMEM_LOADrS_v_i32[0]) - return skip_softmax_flag - - @cute.jit - def mma_qk( - self, - tiled_mma: cute.TiledMma, - tensor_args: Tuple, - scale_args: Tuple, - pipeline_args: Tuple, - qk_sf_inplace_consumer: pipeline.PipelineConsumer, - pipeline_tokens: Tuple = (None, None), - ) -> Tuple[ - pipeline.PipelineProducer, - pipeline.PipelineProducer, - pipeline.PipelineConsumer, - ]: - """Perform a single step of the QK GEMM computation on a block of attention scores. - - :param tiled_mma: Tiled MMA for QK GEMM - :type tiled_mma: cute.TiledMma - :param tensor_args: Tuple containing Qi, K, and Si - :type tensor_args: Tuple - :param pipeline_args: Tuple containing mma_si_producer and si_corr_producer - :type pipeline_args: Tuple - :param qk_sf_inplace_consumer: PipelineAsync consumer guarding the SFQ/SFK TMEM slot - (the tail of the OPPOSITE-stage S region). - :type qk_sf_inplace_consumer: pipeline.PipelineConsumer - :param pipeline_tokens: Optional non-blocking peek tokens for the Si and - vec_i producers, in the form ``(si_peek_status, veci_peek_status)``. - ``None`` for either token falls back to a blocking acquire. - :type pipeline_tokens: Tuple - :return: Tuple containing mma_si_producer, si_corr_producer, and the - (advanced) qk_sf_inplace_consumer. - :rtype: Tuple[pipeline.PipelineProducer, pipeline.PipelineProducer, - pipeline.PipelineConsumer] - """ - tSrQi, tSrK, tStSi = tensor_args - ( - tiled_copy_s2t_qsf, - tCsQSF_stage, - tCtQSF_s2t, - tCtQSF, - tiled_copy_s2t_ksf, - tCsKSF_stage, - tCtKSF_s2t, - tCtKSF, - ) = scale_args - mma_si_producer, si_corr_producer = pipeline_args - si_peek_status, veci_peek_status = pipeline_tokens - qk_tiled_mma = cutlass.new_from_mlir_values( - tiled_mma, cutlass.extract_mlir_values(tiled_mma) - ) - # 0. Make sure Qi & K are ready when calling mma_qk - # 1. acquire S0 - si_handle = mma_si_producer.acquire_and_advance(si_peek_status) - # 2. make sure vec is already released in corr - veci_handle = si_corr_producer.acquire_and_advance(veci_peek_status) - veci_handle.commit() - # 3. Wait until softmax_{1-j} has finished T2R-loading S_{1-j} to avoid SFQK_j race cond - qk_sf_inplace_consumer.wait_and_advance() - # 4. Copy MXFP8 scale factors and issue block-scaled gemm - cute.copy(tiled_copy_s2t_qsf, tCsQSF_stage, tCtQSF_s2t) - cute.copy(tiled_copy_s2t_ksf, tCsKSF_stage, tCtKSF_s2t) - qk_tiled_mma.set(tcgen05.Field.ACCUMULATE, False) - cute.gemm( - qk_tiled_mma, - tStSi, - [tSrQi, tCtQSF], - [tSrK, tCtKSF], - tStSi, - ) - # 4. release S0 - si_handle.commit() - return mma_si_producer, si_corr_producer, qk_sf_inplace_consumer - - @cute.jit - def mma_pv( - self, - tiled_mma: cute.TiledMma, - whether_acc: bool, - tensor_args: Tuple, - pipeline_args: Tuple, - skip_pv_args: Tuple, - ) -> Tuple[ - pipeline.PipelineProducer, - pipeline.PipelineConsumer, - ]: - """Perform a single step of the PV GEMM computation on accumulating O. - - :param tiled_mma: Tiled MMA for PV GEMM - :type tiled_mma: cute.TiledMma - :param whether_acc: Whether to accumulate O - :type whether_acc: bool - :param tensor_args: Tuple containing Pi, Vi, and Oi - :type tensor_args: Tuple - :param pipeline_args: Tuple containing mma_corr_producer and pi_mma_consumer - :type pipeline_args: Tuple - :param skip_pv_args: Tuple containing enable_skip_softmax, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v - :type skip_pv_args: Tuple - :return: Tuple containing mma_corr_producer and pi_mma_consumer - :rtype: Tuple[pipeline.PipelineProducer, pipeline.PipelineConsumer] - """ - tOrPi, tOrVi, tOtOi = tensor_args - mma_corr_producer, pi_mma_consumer = pipeline_args - enable_skip_softmax, tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v = skip_pv_args - # 0. Make sure Vi is ready when calling mma_pv - # 1. acquire Oi - oi_handle = mma_corr_producer.acquire_and_advance() - # 2. wait for Pi - pi_handle = pi_mma_consumer.wait_and_advance() - # 3. gemm - num_kphases = cute.size(tOrPi, mode=[2]) - if cutlass.const_expr(enable_skip_softmax): - skip_pv = self.get_skip_softmax_flag(tiled_tmem_load_v, tTMEM_LOADtS_v, tTMEM_LOADrS_v) - if not skip_pv: - for kphase_idx in cutlass.range(num_kphases, unroll_full=True): - kphase_coord = (None, None, kphase_idx) - tiled_mma.set(tcgen05.Field.ACCUMULATE, whether_acc or kphase_idx != 0) - cute.gemm( - tiled_mma, - tOtOi, - tOrPi[kphase_coord], - tOrVi[kphase_coord], - tOtOi, - ) - else: - for kphase_idx in cutlass.range(num_kphases, unroll_full=True): - kphase_coord = (None, None, kphase_idx) - tiled_mma.set(tcgen05.Field.ACCUMULATE, whether_acc or kphase_idx != 0) - cute.gemm( - tiled_mma, - tOtOi, - tOrPi[kphase_coord], - tOrVi[kphase_coord], - tOtOi, - ) - # 4. commit Pi - pi_handle.release() - # 5. commit Oi - oi_handle.commit() - return mma_corr_producer, pi_mma_consumer - - @cute.jit - def calculate_skip_softmax_flag( - self, - row_max, - tile_row_max, - scale_softmax_log2, - skip_softmax_threshold_log2, - seqlen_q, - thread_idx, - logical_offset, - warp_wants_skip_softmax_exchange, - stage, - skip_softmax_count, - total_softmax_count, - ) -> Tuple[bool, float]: - """Calculate the skip softmax flag and the row maximum. - - :param row_max: The row maximum. - :type row_max: float - :param tile_row_max: The tile row maximum. - :type tile_row_max: float - :param scale_softmax_log2: The scale softmax log2. - :type scale_softmax_log2: float - :param skip_softmax_threshold_log2: The skip softmax threshold log2. - :type skip_softmax_threshold_log2: float - :param seqlen_q: The sequence length q. - :type seqlen_q: int - :param thread_idx: The thread index. - :type thread_idx: int - :param logical_offset: The logical offset. - :type logical_offset: Tuple[int, int] - :param warp_wants_skip_softmax_exchange: The warp wants skip softmax exchange. - :type warp_wants_skip_softmax_exchange: cute.Tensor - :param stage: The stage. - :type stage: int - :param skip_softmax_count: The skip softmax count. - :type skip_softmax_count: cute.Tensor - :param total_softmax_count: The total softmax count. - :type total_softmax_count: cute.Tensor - :return: Tuple containing the skip softmax flag and the row maximum. - :rtype: Tuple[bool, float] - """ - thread_wants_skip = ( - tile_row_max * scale_softmax_log2 - row_max * scale_softmax_log2 - ) < skip_softmax_threshold_log2 - thread_wants_skip = thread_wants_skip or ((logical_offset[0] + thread_idx) >= seqlen_q) - warp_wants_skip = cute.arch.vote_all_sync(thread_wants_skip) - - with cute.arch.elect_one(): - warp_wants_skip_softmax_exchange[cute.arch.warp_idx() % 4] = warp_wants_skip - softmax_barrier = self.s0_warpgroup_barrier if stage == 0 else self.s1_warpgroup_barrier - softmax_barrier.arrive_and_wait() - warp_wants_skip_softmax_exchange_i32 = cute.make_tensor( - cute.recast_ptr(warp_wants_skip_softmax_exchange.iterator, dtype=cutlass.Int32), - cute.make_layout((1,)), - ) - skip_softmax = cute.arch.popc(warp_wants_skip_softmax_exchange_i32[0]) == 4 - - if not skip_softmax: - row_max = max(row_max, tile_row_max) - - if cutlass.const_expr(skip_softmax_count is not None): - if thread_idx == 0: - if skip_softmax: - cute.arch.atomic_add(skip_softmax_count.iterator.llvm_ptr, Int32(1)) - cute.arch.atomic_add(total_softmax_count.iterator.llvm_ptr, Int32(1)) - return skip_softmax, row_max - - @cute.jit - def apply_exp_and_cvt( - self, - tTMEM_LOADrS, - tTMEM_LOADrS_cvt, - tTMEM_STORErS_x4_e_cvt, - stage, - scale, - minus_row_max_scale, - local_row_sum, - inplace_consumer, - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_COUNT, - CVT_COUNT, - CVT_PER_STEP, - FMA_COUNT, - ARV_COUNT, - ): - """Apply the exp and conversion to the P data type on fragment. - - :param tTMEM_LOADrS: The tTMEM_LOADrS tensor. - :type tTMEM_LOADrS: cute.Tensor - :param tTMEM_LOADrS_cvt: The tTMEM_LOADrS_cvt tensor. - :type tTMEM_LOADrS_cvt: cute.Tensor - :param tTMEM_STORErS_x4_e_cvt: The tTMEM_STORErS_x4_e_cvt tensor. - :type tTMEM_STORErS_x4_e_cvt: cute.Tensor - :param stage: The stage. - :type stage: int - :param scale: The scale. - :type scale: float - :param minus_row_max_scale: The minus row maximum scale. - :type minus_row_max_scale: float - :param local_row_sum: The local row sum. - :type local_row_sum: float - :param inplace_consumer: The inplace consumer. - :type inplace_consumer: cute.Tensor - :param EXP2_EMULATION_OFFSET: The exp2 emulation offset. - :type EXP2_EMULATION_OFFSET: int - :param EXP2_EMULATION_COUNT: The exp2 emulation count. - :type EXP2_EMULATION_COUNT: int - :param CVT_COUNT: The cvt count. - :type CVT_COUNT: int - :param CVT_PER_STEP: The cvt per step. - :type CVT_PER_STEP: int - :param FMA_COUNT: The fma count. - :type FMA_COUNT: int - :param ARV_COUNT: The arv count. - :type ARV_COUNT: int - :return: The local row sum and the inplace consumer. - :rtype: Tuple[float, cute.Tensor] - """ - for i in cutlass.range_constexpr(0, EXP2_EMULATION_OFFSET, 2): - if cutlass.const_expr(i >= CVT_COUNT): - if cutlass.const_expr(i % CVT_PER_STEP == 0): - if cutlass.const_expr(self.v_dtype.width == 8): - fmha_utils.cvt_f32x4_to_f8x4( - tTMEM_LOADrS_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP], - tTMEM_STORErS_x4_e_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP], - ) - else: - s_vec = tTMEM_LOADrS_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP].load() - tTMEM_STORErS_x4_e_cvt[None, (i - CVT_COUNT) // CVT_PER_STEP].store( - s_vec.to(self.v_dtype) - ) - local_row_sum = cute.arch.add_packed_f32x2( - local_row_sum, - ( - tTMEM_LOADrS[i - CVT_COUNT], - tTMEM_LOADrS[i - CVT_COUNT + 1], - ), - ) - tTMEM_LOADrS[i] = cute.math.exp2(tTMEM_LOADrS[i], fastmath=True) - if cutlass.const_expr(i + FMA_COUNT < EXP2_EMULATION_OFFSET): - ( - tTMEM_LOADrS[i + FMA_COUNT], - tTMEM_LOADrS[i + FMA_COUNT + 1], - ) = cute.arch.fma_packed_f32x2( - ( - tTMEM_LOADrS[i + FMA_COUNT], - tTMEM_LOADrS[i + FMA_COUNT + 1], - ), - (scale, scale), - (minus_row_max_scale, minus_row_max_scale), - ) - tTMEM_LOADrS[i + 1] = cute.math.exp2(tTMEM_LOADrS[i + 1], fastmath=True) - if cutlass.const_expr(i == EXP2_EMULATION_OFFSET - ARV_COUNT): - if cutlass.const_expr(self.enable_sequence_barrier): - if cutlass.const_expr(stage == 0): - self.sequence_s1_s0_barrier.arrive() - else: - self.sequence_s0_s1_barrier.arrive() - - # The remaining conversion steps - for i in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET - CVT_COUNT, - EXP2_EMULATION_OFFSET, - 2, - ): - if cutlass.const_expr(i % CVT_PER_STEP == 0): - if cutlass.const_expr(self.v_dtype.width == 8): - fmha_utils.cvt_f32x4_to_f8x4( - tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], - ) - else: - s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) - local_row_sum = cute.arch.add_packed_f32x2( - local_row_sum, (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]) - ) - for i in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET, EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT // 2, 2 - ): - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( - (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), - (scale, scale), - (minus_row_max_scale, minus_row_max_scale), - ) - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = fmha_utils.ex2_emulation_packed_f32x2( - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] - ) - if cutlass.const_expr((i + 2) % CVT_PER_STEP == 0): - if cutlass.const_expr(self.v_dtype.width == 8): - fmha_utils.cvt_f32x4_to_f8x4( - tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], - ) - else: - s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) - - inplace_peek_status = inplace_consumer.try_wait() - for i in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT // 2, - EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, - 2, - ): - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( - (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), - (scale, scale), - (minus_row_max_scale, minus_row_max_scale), - ) - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = fmha_utils.ex2_emulation_packed_f32x2( - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] - ) - if cutlass.const_expr((i + 2) % CVT_PER_STEP == 0): - if cutlass.const_expr(self.v_dtype.width == 8): - fmha_utils.cvt_f32x4_to_f8x4( - tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP], - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP], - ) - else: - s_vec = tTMEM_LOADrS_cvt[None, i // CVT_PER_STEP].load() - tTMEM_STORErS_x4_e_cvt[None, i // CVT_PER_STEP].store(s_vec.to(self.v_dtype)) - inplace_consumer.wait_and_advance(inplace_peek_status) - return local_row_sum, inplace_consumer - - @cute.jit - def softmax_step( - self, - stage: int, - whether_apply_mask: bool, - iter_args: Tuple, - stats_args: Tuple, - pipeline_args: Tuple, - value_args: Tuple, - atom_args: Tuple, - tensor_args: Tuple, - ) -> Tuple[Tuple, Tuple]: - """Perform a single step of the softmax computation on a block of attention scores. - - This method processes one block of the attention matrix, computing numerically stable - softmax by first finding the row maximum, subtracting it from all elements, applying - exponential function, and then normalizing by the sum of exponentials. It also handles - optional masking of attention scores. - - The method involves several key operations: - 1. Loading attention scores from tensor memory - 2. Applying optional masking based on position - 3. Computing row-wise maximum values for numerical stability - 4. Transforming scores using exp2(x*scale - max*scale) - 5. Computing row sums for normalization - 6. Coordinating pipeline synchronization between different processing stages - - :param stage: Processing stage (0 for first half, 1 for second half) - :type stage: int - :param whether_apply_mask: Whether to apply attention masking - :type whether_apply_mask: bool - :param iter_args: Tuple containing the counting tensor, row_max, row_sum, and vector buffer's handle for current iteration - :type iter_args: Tuple - :param stats_args: Tuple containing row_sum and row_max - :type stats_args: Tuple - :param pipeline_args: Tuple containing pipeline related arguments for MMA, correction, and sequence synchronization - :type pipeline_args: Tuple - :param value_args: Tuple containing seqlen_k, seqlen_q, and scale_softmax_log2 - :type value_args: Tuple - :param atom_args: Tuple containing mma & copy atoms - :type atom_args: Tuple - :param tensor_args: Tuple containing softmax related tensors - :type tensor_args: Tuple - :param fused_mask: Compute trip counts and apply masking for attention blocks - :type fused_mask: fmha_utils.FusedMask - :return: Updated stats_args and pipeline_args - :rtype: Tuple[Tuple, Tuple] - """ - row_sum, row_max = stats_args - cS, is_last_iter = iter_args - ( - seqlen_k, - seqlen_q, - scale_softmax_log2, - window_size_left, - window_size_right, - skip_softmax_threshold_log2, - thread_idx, - logical_offset, - qk_sf_inplace_producer, - ) = value_args - ( - si_peek_status, - mma_si_consumer, - si_corr_producer, - pi_mma_producer, - inplace_producer, - inplace_consumer, - ) = pipeline_args - ( - qk_thr_mma, - tiled_tmem_load, - tiled_tmem_store, - tiled_tmem_store_vec, - thr_tmem_load, - thr_tmem_store, - thr_tmem_store_vec, - ) = atom_args - ( - tTMEM_LOADtS, - tTMEM_STORE_VECtS, - tTMEM_STORE_SKIP_SOFTMAX, - tTMEM_STOREtS_x4, - warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ) = tensor_args - tilePlikeFP32 = self.qk_mma_tiler[1] // Float32.width * self.o_dtype.width - tScS = qk_thr_mma.partition_C(cS) - enable_skip_softmax = skip_softmax_threshold_log2 is not None - tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) - tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) - tScS_P_layout = cute.composition(tScS.layout, cute.make_layout((128, tilePlikeFP32))) - tScS_P = cute.make_tensor(tScS.iterator, tScS_P_layout) - tTMEM_LOADcS = thr_tmem_load.partition_D(tScS) - tTMEM_STORE_VECcS = thr_tmem_store_vec.partition_S(tScS_vec) - tTMEM_STOREcS = thr_tmem_store.partition_S(tScS_P) - # Wait for Si - si_handle = mma_si_consumer.wait_and_advance(si_peek_status) - tTMEM_LOADrS = cute.make_rmem_tensor(tTMEM_LOADcS.shape, self.qk_acc_dtype) - old_row_max = row_max - skip_softmax = cutlass.Boolean(False) - if whether_apply_mask: - if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): - cute.copy(tiled_tmem_load, tTMEM_LOADtS, tTMEM_LOADrS) - else: - tTMEM_LOADrMax = cute.make_rmem_tensor( - cute.make_layout((1, cute.size(tTMEM_LOADrS, mode=[1]))), - self.qk_acc_dtype, - ) - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[1])): - cute.copy_atom_call( - tiled_tmem_load, - tTMEM_LOADtS[None, i, 0, 0], - (tTMEM_LOADrS[None, i, 0, 0], tTMEM_LOADrMax[None, i]), - ) - fmha_utils.FusedMask.apply_mask( - self.mask_type, - tTMEM_LOADrS, - tTMEM_LOADcS, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - tile_row_max = tTMEM_LOADrS.load().reduce(cute.ReductionOp.MAX, -cutlass.Float32.inf, 0) - if cutlass.const_expr(not enable_skip_softmax): - row_max = cute.arch.fmax(row_max, tile_row_max) - else: - skip_softmax, row_max = self.calculate_skip_softmax_flag( - row_max, - tile_row_max, - scale_softmax_log2, - skip_softmax_threshold_log2, - seqlen_q, - thread_idx, - logical_offset, - warp_wants_skip_softmax_exchange, - stage, - skip_softmax_count, - total_softmax_count, - ) - # Fence the T2R load above, then signal MMA that the opposite-stage S is loaded. - cute.arch.fence_view_async_tmem_load() - qk_sf_inplace_producer.commit() - qk_sf_inplace_producer.advance() - si_handle.release() - # S0 -> P1 / S1 -> P0 - inplace_producer.commit() - inplace_producer.advance() - else: - if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): - cute.copy( - tiled_tmem_load, - tTMEM_LOADtS[None, 0, None, None], - tTMEM_LOADrS[None, 0, None, None], - ) - cute.copy( - tiled_tmem_load, - tTMEM_LOADtS[None, 1, None, None], - tTMEM_LOADrS[None, 1, None, None], - ) - tile_row_max = -cutlass.Float32.inf - tile_row_max_ = tile_row_max - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 0, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 0, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 0, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 0, 0, 0]) - cute.copy( - tiled_tmem_load, - tTMEM_LOADtS[None, 2, None, None], - tTMEM_LOADrS[None, 2, None, None], - ) - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 1, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 1, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 1, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 1, 0, 0]) - cute.copy( - tiled_tmem_load, - tTMEM_LOADtS[None, 3, None, None], - tTMEM_LOADrS[None, 3, None, None], - ) - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 2, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 2, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 2, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 2, 0, 0]) - cute.arch.fence_view_async_tmem_store() - # Fence T2R loads then release the QK SF TMEM slot before we release S to MMA - cute.arch.fence_view_async_tmem_load() - qk_sf_inplace_producer.commit() - qk_sf_inplace_producer.advance() - si_handle.release() - # S0 -> P1 / S1 -> P0 - inplace_producer.commit() - inplace_producer.advance() - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[0]), 4): - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i, 3, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tTMEM_LOADrS[i + 1, 3, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 2, 3, 0, 0]) - tile_row_max_ = cute.arch.fmax(tile_row_max_, tTMEM_LOADrS[i + 3, 3, 0, 0]) - tile_row_max = cute.arch.fmax(tile_row_max, tile_row_max_) - if cutlass.const_expr(not enable_skip_softmax): - row_max = cute.arch.fmax(tile_row_max, row_max) - else: - skip_softmax, row_max = self.calculate_skip_softmax_flag( - row_max, - tile_row_max, - scale_softmax_log2, - skip_softmax_threshold_log2, - seqlen_q, - thread_idx, - logical_offset, - warp_wants_skip_softmax_exchange, - stage, - skip_softmax_count, - total_softmax_count, - ) - else: - tTMEM_LOADrMax = cute.make_rmem_tensor( - cute.make_layout((1, cute.size(tTMEM_LOADrS, mode=[1]))), - self.qk_acc_dtype, - ) - for i in cutlass.range_constexpr(0, cute.size(tTMEM_LOADrS, mode=[1])): - cute.copy_atom_call( - tiled_tmem_load, - tTMEM_LOADtS[None, i, 0, 0], - (tTMEM_LOADrS[None, i, 0, 0], tTMEM_LOADrMax[None, i]), - ) - cute.arch.fence_view_async_tmem_store() - tile_row_max = tTMEM_LOADrMax.load().reduce( - cute.ReductionOp.MAX, -cutlass.Float32.inf, 0 - ) - if cutlass.const_expr(not enable_skip_softmax): - row_max = cute.arch.fmax(tile_row_max, row_max) - else: - skip_softmax, row_max = self.calculate_skip_softmax_flag( - row_max, - tile_row_max, - scale_softmax_log2, - skip_softmax_threshold_log2, - seqlen_q, - thread_idx, - logical_offset, - warp_wants_skip_softmax_exchange, - stage, - skip_softmax_count, - total_softmax_count, - ) - # Fence the T2R loads then release the QK SF TMEM slot. - cute.arch.fence_view_async_tmem_load() - qk_sf_inplace_producer.commit() - qk_sf_inplace_producer.advance() - si_handle.release() - # S0 -> P1 / S1 -> P0 - inplace_producer.commit() - inplace_producer.advance() - - row_max_safe = row_max - if row_max == -cutlass.Float32.inf: - row_max_safe = 0.0 - if cutlass.const_expr(self.rescale_threshold > 0.0): - if (row_max_safe - old_row_max) * scale_softmax_log2 <= self.rescale_threshold: - row_max_safe = old_row_max - tTMEM_STORE_VECrS = cute.make_rmem_tensor(tTMEM_STORE_VECcS.shape, self.qk_acc_dtype) - tTMEM_STORE_VECrS[0] = old_row_max - tTMEM_STORE_VECrS[1] = row_max_safe - vec_i_peek_status = si_corr_producer.try_acquire() - tTMEM_STORErS_x4 = cute.make_rmem_tensor(tTMEM_STOREcS.shape, self.qk_acc_dtype) - tTMEM_STORErS_x4_e = cute.make_tensor( - cute.recast_ptr(tTMEM_STORErS_x4.iterator, dtype=self.v_dtype), - tTMEM_LOADrS.layout, - ) - scale = scale_softmax_log2 - minus_row_max_scale = (0.0 - row_max_safe) * scale - if cutlass.const_expr(self.v_dtype.width == 8 and self.p_fp8_prescale_log2 > 0): - minus_row_max_scale = minus_row_max_scale + self.p_fp8_prescale_log2 - - ARV_COUNT = 4 - FMA_COUNT = 8 - CVT_COUNT = 8 if self.v_dtype.width == 8 else 4 - CVT_PER_STEP = 4 if self.v_dtype.width == 8 else 2 - assert CVT_COUNT % CVT_PER_STEP == 0, ( - f"CVT_COUNT {CVT_COUNT} must be divisible by CVT_PER_STEP {CVT_PER_STEP}" - ) - tTMEM_LOADrS_cvt = cute.logical_divide(tTMEM_LOADrS, cute.make_layout(CVT_PER_STEP)) - tTMEM_STORErS_x4_e_cvt = cute.logical_divide( - tTMEM_STORErS_x4_e, cute.make_layout(CVT_PER_STEP) - ) - for i in cutlass.range_constexpr(0, FMA_COUNT, 2): - tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1] = cute.arch.fma_packed_f32x2( - (tTMEM_LOADrS[i], tTMEM_LOADrS[i + 1]), - (scale, scale), - (minus_row_max_scale, minus_row_max_scale), - ) - vec_i_handle = si_corr_producer.acquire_and_advance(vec_i_peek_status) - cute.copy(tiled_tmem_store_vec, tTMEM_STORE_VECrS, tTMEM_STORE_VECtS) - cute.arch.fence_view_async_tmem_store() - # Notify correction wg that row_max is ready - vec_i_handle.commit() - - EXP2_EMULATION_COUNT = 20 if self.enable_ex2_emulation and not whether_apply_mask else 0 - EXP2_EMULATION_OFFSET = cute.size(tTMEM_LOADrS) - EXP2_EMULATION_COUNT - acc_scale_ = scale * (old_row_max - row_max_safe) - acc_scale = cute.math.exp2(acc_scale_, fastmath=True) * 0.5 - if cutlass.const_expr(self.enable_sequence_barrier): - if cutlass.const_expr(stage == 0): - self.sequence_s0_s1_barrier.arrive_and_wait() - else: - self.sequence_s1_s0_barrier.arrive_and_wait() - - if cutlass.const_expr(enable_skip_softmax): - if not skip_softmax: - row_sum *= acc_scale - local_row_sum = (row_sum, row_sum) - local_row_sum, inplace_consumer = self.apply_exp_and_cvt( - tTMEM_LOADrS, - tTMEM_LOADrS_cvt, - tTMEM_STORErS_x4_e_cvt, - stage, - scale, - minus_row_max_scale, - local_row_sum, - inplace_consumer, - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_COUNT, - CVT_COUNT, - CVT_PER_STEP, - FMA_COUNT, - ARV_COUNT, - ) - tTMEM_STORE_VECrS_i32 = cute.recast_tensor(tTMEM_STORE_VECrS, dtype=cutlass.Int32) - tTMEM_STORE_VECrS_i32[0] = 0 - pi_handle = pi_mma_producer.acquire_and_advance() - # store skip softmax flag - cute.copy( - tiled_tmem_store_vec, - tTMEM_STORE_VECrS_i32, - tTMEM_STORE_SKIP_SOFTMAX, - ) - # store P - cute.copy(tiled_tmem_store, tTMEM_STORErS_x4, tTMEM_STOREtS_x4) - cute.arch.fence_view_async_tmem_store() - pi_handle.commit() - for j in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, - 2, - ): - local_row_sum = cute.arch.add_packed_f32x2( - (tTMEM_LOADrS[j], tTMEM_LOADrS[j + 1]), - local_row_sum, - ) - row_sum = local_row_sum[0] + local_row_sum[1] - cute.arch.fence_view_async_tmem_store() - else: - if cutlass.const_expr(self.enable_sequence_barrier): - if cutlass.const_expr(stage == 0): - self.sequence_s1_s0_barrier.arrive() - else: - self.sequence_s0_s1_barrier.arrive() - inplace_peek_status = inplace_consumer.try_wait() - inplace_consumer.wait_and_advance(inplace_peek_status) - tTMEM_STORE_VECrS_i32 = cute.recast_tensor(tTMEM_STORE_VECrS, dtype=cutlass.Int32) - tTMEM_STORE_VECrS_i32[0] = 1 - pi_handle = pi_mma_producer.acquire_and_advance() - # store skip softmax flag - cute.copy( - tiled_tmem_store_vec, - tTMEM_STORE_VECrS_i32, - tTMEM_STORE_SKIP_SOFTMAX, - ) - cute.arch.fence_view_async_tmem_store() - pi_handle.commit() - else: - row_sum *= acc_scale - local_row_sum = (row_sum, row_sum) - local_row_sum, inplace_consumer = self.apply_exp_and_cvt( - tTMEM_LOADrS, - tTMEM_LOADrS_cvt, - tTMEM_STORErS_x4_e_cvt, - stage, - scale, - minus_row_max_scale, - local_row_sum, - inplace_consumer, - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_COUNT, - CVT_COUNT, - CVT_PER_STEP, - FMA_COUNT, - ARV_COUNT, - ) - pi_handle = pi_mma_producer.acquire_and_advance() - # store P - cute.copy(tiled_tmem_store, tTMEM_STORErS_x4, tTMEM_STOREtS_x4) - cute.arch.fence_view_async_tmem_store() - for j in cutlass.range_constexpr( - EXP2_EMULATION_OFFSET, - EXP2_EMULATION_OFFSET + EXP2_EMULATION_COUNT, - 2, - ): - local_row_sum = cute.arch.add_packed_f32x2( - (tTMEM_LOADrS[j], tTMEM_LOADrS[j + 1]), - local_row_sum, - ) - row_sum = local_row_sum[0] + local_row_sum[1] - cute.arch.fence_view_async_tmem_store() - # Notify tensor core warp that softmax(S->P) is ready - pi_handle.commit() - if not is_last_iter: - si_peek_status = mma_si_consumer.try_wait() - - stats_args = (row_sum, row_max_safe) - pipeline_args = ( - si_peek_status, - mma_si_consumer, - si_corr_producer, - pi_mma_producer, - inplace_producer, - inplace_consumer, - ) - return stats_args, pipeline_args - - # For both softmax0 and softmax1 warp group - @cute.jit - def softmax( - self, - stage: int, - tensor_args: Tuple, - pipeline_args: Tuple, - inplace_args: Tuple, - qk_thr_mma: cute.ThrMma, - value_args: Tuple, - mask_args: Tuple, - sched_args: Tuple, - qk_sf_inplace_producers: Tuple[pipeline.PipelineProducer, pipeline.PipelineProducer], - ): - """Compute softmax on attention scores from QK matrix multiplication. - - This method handles the softmax computation for either the first or second half of the - attention matrix, depending on the 'stage' parameter. It calculates row-wise maximum - and sum values needed for stable softmax computation, applies optional masking, and - transforms raw attention scores into probability distributions. - - The implementation uses specialized memory access patterns and efficient math operations - for computing exp(x) using exp2 functions. It also coordinates pipeline - synchronization between MMA, correction, and sequence processing stages. - - :param stage: Processing stage (0 for first half, 1 for second half of attention matrix) - :type stage: int - :param seqlen_k: Length of the key sequence - :type seqlen_k: Int32 - :param seqlen_q: Length of the query sequence - :type seqlen_q: Int32 - :param cum_seqlen_q: Cumulative sequence lengths for queries - :type cum_seqlen_q: cute.Tensor | None - :param cum_seqlen_k: Cumulative sequence lengths for keys - :type cum_seqlen_k: cute.Tensor | None - :param scale_softmax_log2: Log2 scale factor for softmax operation - :type scale_softmax_log2: Float32 - :param qk_thr_mma: Thread MMA operation for QK matrix multiplication - :type qk_thr_mma: cute.ThrMma - :param tStS: Shared tensor for softmax input/output - :type tStS: cute.Tensor - :param tStSi: Input tensor containing attention scores - :type tStSi: cute.Tensor - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - :param mma_si_consumer: Pipeline for synchronizing with Si tensors - :type mma_si_consumer: pipeline.PipelineConsumer - :param si_corr_producer: Pipeline for synchronizing with correction operations - :type si_corr_producer: pipeline.PipelineProducer - :param pi_mma_producer: Pipeline for synchronizing with Pi tensors - :type pi_mma_producer: pipeline.PipelineProducer - :param tile_sched_params: Parameters for tile scheduling - :type tile_sched_params: fmha_utils.FmhaStaticTileSchedulerParams - :param fused_mask: Compute trip counts and apply masking for attention blocks - :type fused_mask: fmha_utils.FusedMask - """ - ( - tStS, - tStSi, - cum_seqlen_k, - cum_seqlen_q, - warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ) = tensor_args - mma_si_consumer, si_corr_producer, pi_mma_producer = pipeline_args - inplace_producer, inplace_consumer = inplace_args - ( - seqlen_k, - seqlen_q, - scale_softmax_log2, - skip_softmax_threshold_log2, - ) = value_args - window_size_left, window_size_right = mask_args - tile_sched, work_tile = sched_args - - tidx, _, _ = cute.arch.thread_idx() - thread_idx = tidx % (self.threads_per_warp * len(self.softmax0_warp_ids)) - - cS_base = cute.make_identity_tensor((self.qk_mma_tiler[0], self.qk_mma_tiler[1])) - tilePlikeFP32 = self.qk_mma_tiler[1] // 32 * self.o_dtype.width - tScS = qk_thr_mma.partition_C(cS_base) - tStS_vec_layout = cute.composition(tStS.layout, cute.make_layout((128, 2))) - tmem_vec_offset = self.tmem_vec0_offset if stage == 0 else self.tmem_vec1_offset - tStS_vec = cute.make_tensor(tStS.iterator + tmem_vec_offset, tStS_vec_layout) - tmem_skip_softmax_offset = ( - self.tmem_skip_softmax0_offset if stage == 0 else self.tmem_skip_softmax1_offset - ) - tStS_skip_softmax = cute.make_tensor( - tStS.iterator + tmem_skip_softmax_offset, tStS_vec_layout - ) - tScS_vec_layout = cute.composition(tScS.layout, cute.make_layout((128, 2))) - tScS_vec = cute.make_tensor(tScS.iterator, tScS_vec_layout) - tStS_P_layout = cute.composition(tStS.layout, cute.make_layout((128, tilePlikeFP32))) - tmem_p_offset = self.tmem_p0_offset if stage == 0 else self.tmem_p1_offset - tStS_P = cute.make_tensor(tStS.iterator + tmem_p_offset, tStS_P_layout) - if cutlass.const_expr(self.arch >= Arch.sm_100 and self.arch <= Arch.sm_100f): - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(32)), - self.qk_acc_dtype, - ) - else: - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.LdRed32x32bOp( - tcgen05.copy.Repetition(32), redOp=tcgen05.TmemLoadRedOp.MAX - ), - self.qk_acc_dtype, - ) - - tiled_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tStSi) - thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) - tTMEM_LOADtS = thr_tmem_load.partition_S(tStSi) - tmem_store_vec_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(2)), - self.qk_acc_dtype, - ) - tiled_tmem_store_vec = tcgen05.make_tmem_copy(tmem_store_vec_atom, tStS_vec) - thr_tmem_store_vec = tiled_tmem_store_vec.get_slice(thread_idx) - tTMEM_STORE_VECtS = thr_tmem_store_vec.partition_D(tStS_vec) - tTMEM_STORE_VECcS = thr_tmem_store_vec.partition_S(tScS_vec) - tTMEM_STORE_SKIP_SOFTMAX = thr_tmem_store_vec.partition_D(tStS_skip_softmax) - tmem_store_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(32)), - self.qk_acc_dtype, - ) - tiled_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tStS_P) - thr_tmem_store = tiled_tmem_store.get_slice(thread_idx) - tTMEM_STOREtS_x4 = thr_tmem_store.partition_D(tStS_P) - # Each softmax warpgroup releases the OPPOSITE-stage S region (where - # SFQ_{1-stage}/SFK_{1-stage} live in the tail) by committing to its qk_sf_inplace producer - # once per kv block: - # softmax0 (stage=0) drives qk_sf_inplace_1 - # softmax1 (stage=1) drives qk_sf_inplace_0 - qk_sf_inplace_producer = qk_sf_inplace_producers[stage] - # Bootstrap: MMA's first mma_qk(stage=0) waits on qk_sf_inplace_0 before - # softmax1 has produced any in-loop commit. Each thread of softmax1 - # pre-commits once so the first wait succeeds. qk_sf_inplace_1 needs no - # pre-commit because mma_qk(stage=1) runs after mma_qk(stage=0), - # giving softmax0 enough time to consume S0 and commit naturally. - if cutlass.const_expr(stage == 1): - qk_sf_inplace_producer.commit() - qk_sf_inplace_producer.advance() - if cutlass.const_expr(self.enable_sequence_barrier): - if cutlass.const_expr(stage == 1): - self.sequence_s0_s1_barrier.arrive() - while work_tile.is_valid_tile: - curr_block_coord = work_tile.tile_idx - batch_coord = curr_block_coord[2][1] - seqlen_k_ = seqlen_k - seqlen_q_ = seqlen_q - continue_cond = False - cuseqlen_q = Int32(0) - seqlen_q_ = seqlen_q - if cutlass.const_expr(cum_seqlen_q is not None): - cuseqlen_q = cum_seqlen_q[batch_coord] - seqlen_q_ = cum_seqlen_q[batch_coord + 1] - cuseqlen_q - continue_cond = ( - not fmha_utils.FmhaStaticTileScheduler.check_valid_work_for_seqlen_q( - self.cta_tiler[0], - curr_block_coord[0], - seqlen_q_, - ) - ) - if not continue_cond: - if cutlass.const_expr(cum_seqlen_k is not None): - cuseqlen_k = cum_seqlen_k[batch_coord] - seqlen_k_ = cum_seqlen_k[batch_coord + 1] - cuseqlen_k - continue_cond = seqlen_k_ <= 0 - if not continue_cond: - logical_offset = ( - curr_block_coord[0] * self.cta_tiler[0] + stage * self.qk_mma_tiler[0], - 0, - ) - cS = cute.domain_offset(logical_offset, cS_base) - value_args_ = ( - seqlen_k_, - seqlen_q_, - scale_softmax_log2, - window_size_left, - window_size_right, - skip_softmax_threshold_log2, - thread_idx, - logical_offset, - qk_sf_inplace_producer, - ) - atom_args = ( - qk_thr_mma, - tiled_tmem_load, - tiled_tmem_store, - tiled_tmem_store_vec, - thr_tmem_load, - thr_tmem_store, - thr_tmem_store_vec, - ) - tensor_args_ = ( - tTMEM_LOADtS, - tTMEM_STORE_VECtS, - tTMEM_STORE_SKIP_SOFTMAX, - tTMEM_STOREtS_x4, - warp_wants_skip_softmax_exchange, - skip_softmax_count, - total_softmax_count, - ) - st_cnt, end_cnt, ld_mask_cnt, unmask_cnt, tl_mask_cnt = ( - fmha_utils.FusedMask.get_masked_info( - self.mask_type, - curr_block_coord, - self.cta_tiler, - seqlen_q_, - seqlen_k_, - window_size_left, - window_size_right, - ) - ) - row_max = -Float32.inf - row_sum = 0.0 - stats_args = (row_sum, row_max) - - def softmax_loop( - whether_apply_mask: bool, - loop_args: Tuple, - stats_args: Tuple, - pipeline_args: Tuple, - inner_fn: Callable, - value_args: Tuple, - atom_args: Tuple, - tensor_args: Tuple, - cS: cute.Tensor, - ) -> Tuple[Tuple, Tuple]: - start_index, iter_num, upper_bound = loop_args - for i in cutlass.range(start_index, start_index + iter_num, 1, unroll=1): - cS_iter = cute.domain_offset((0, i * self.qk_mma_tiler[1]), cS) - iter_args = (cS_iter, i == upper_bound - 1) - stats_args, pipeline_args = inner_fn( - stage, - whether_apply_mask, - iter_args, - stats_args, - pipeline_args, - value_args, - atom_args, - tensor_args, - ) - return stats_args, pipeline_args - - softmax_step_fn = self.softmax_step - softmax_loop_fn = partial( - softmax_loop, - inner_fn=softmax_step_fn, - value_args=value_args_, - atom_args=atom_args, - tensor_args=tensor_args_, - cS=cS, - ) - si_peek_status = mma_si_consumer.try_wait() - if cutlass.const_expr(stage == 1): - inplace_consumer.wait_and_advance() - pipeline_args_ = ( - si_peek_status, - mma_si_consumer, - si_corr_producer, - pi_mma_producer, - inplace_producer, - inplace_consumer, - ) - # 1. Leading mask loop - loop_args = (st_cnt, ld_mask_cnt, end_cnt) - stats_args, pipeline_args_ = softmax_loop_fn( - True, loop_args, stats_args, pipeline_args_ - ) - # 2. Unmasked loop - loop_args = (st_cnt + ld_mask_cnt, unmask_cnt, end_cnt) - stats_args, pipeline_args_ = softmax_loop_fn( - False, loop_args, stats_args, pipeline_args_ - ) - # 3. Trailing mask loop - loop_args = (st_cnt + ld_mask_cnt + unmask_cnt, tl_mask_cnt, end_cnt) - stats_args, pipeline_args_ = softmax_loop_fn( - True, loop_args, stats_args, pipeline_args_ - ) - - # Unpack pipeline_args - ( - _, - mma_si_consumer, - si_corr_producer, - pi_mma_producer, - inplace_producer, - inplace_consumer, - ) = pipeline_args_ - if cutlass.const_expr(stage == 0): - inplace_producer.commit() - inplace_producer.advance() - # 4. Copy the final stats for correction epilog - tTMEM_STORE_VECrS = cute.make_rmem_tensor( - tTMEM_STORE_VECcS.shape, self.qk_acc_dtype - ) - tTMEM_STORE_VECrS[0] = stats_args[0] - tTMEM_STORE_VECrS[1] = stats_args[1] - vec_i_handle = si_corr_producer.acquire_and_advance() - cute.copy(tiled_tmem_store_vec, tTMEM_STORE_VECrS, tTMEM_STORE_VECtS) - cute.arch.fence_view_async_tmem_store() - vec_i_handle.commit() - # End of if not continue_cond - # Advance to next tile - tile_sched.advance_to_next_work() - work_tile = tile_sched.get_current_work() - # End of persistent scheduler loop - - @cute.jit - def correction_rescale( - self, - thr_mma: cute.ThrMma, - tiled_tmem_load_vec: cute.TiledCopy, - scale_softmax_log2: Float32, - tensor_args: Tuple, - pipeline_args: Tuple, - oi_peek_status=None, - ): - """Rescale intermediate attention results based on softmax normalization factor. - - This method performs a crucial correction step in the attention computation pipeline. - When processing attention in blocks, the softmax normalization factors may change - as new blocks are processed. This method rescales previously computed partial - output values to account for updated normalization factors. - - The implementation uses efficient tensor memory operations to: - 1. Load existing partial attention output from tensor memory - 2. Apply the scaling factor to all elements - 3. Store the rescaled results back to tensor memory - - When ``self.enable_correction_double_buffer`` is True, the rescale loop - runs as a 2-buffer tensor-memory load / multiply / store pipeline. - - :param thr_mma: Thread MMA operation for the computation - :type thr_mma: cute.ThrMma - :param tiled_tmem_load_vec: Tiled memory load operation for the vectorized row-wise max - :type tiled_tmem_load_vec: cute.TiledCopy - :param scale_softmax_log2: Log2 of the softmax factor - :type scale_softmax_log2: Float32 - :param tensor_args: Tuple containing the tensors for the correction - :type tensor_args: Tuple[cute.Tensor, cute.Tensor, cute.Tensor] - :param pipeline_args: Tuple containing the pipeline arguments for the correction - :type pipeline_args: Tuple[pipeline.PipelineConsumer, pipeline.PipelineConsumer] - :param oi_peek_status: Optional non-blocking token for the Oi consumer - wait. ``None`` or ``False`` falls back to a blocking wait. - :return: ``((si_corr_consumer, mma_corr_consumer), next_oi_peek_status)`` - where ``next_oi_peek_status`` is the peek for the next Oi (only - refreshed when a rescale actually ran; otherwise stays - ``cutlass.Boolean(False)``). - """ - tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS = tensor_args - si_corr_consumer, mma_corr_consumer = pipeline_args - - pv_tiled_mma_shape = ( - self.pv_mma_tiler[0], - self.pv_mma_tiler[1], - ) - cO = cute.make_identity_tensor(pv_tiled_mma_shape) - tOcO = thr_mma.partition_C(cO) - corr_tile_size = 16 # tuneable parameter - tmem_load_atom = cute.make_copy_atom( - tcgen05.copy.Ld32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), - self.pv_acc_dtype, - ) - tmem_store_atom = cute.make_copy_atom( - tcgen05.copy.St32x32bOp(tcgen05.copy.Repetition(corr_tile_size)), - self.pv_acc_dtype, - ) - tOtO_i_layout = cute.composition(tOtO.layout, cute.make_layout((128, corr_tile_size))) - tOcO_i_layout = cute.composition(tOcO.layout, cute.make_layout((128, corr_tile_size))) - tOtO_i = cute.make_tensor(tOtO.iterator, tOtO_i_layout) - tOcO_i = cute.make_tensor(tOcO.iterator, tOcO_i_layout) - tiled_tmem_load = tcgen05.make_tmem_copy(tmem_load_atom, tOtO_i) - tiled_tmem_store = tcgen05.make_tmem_copy(tmem_store_atom, tOtO_i) - tidx, _, _ = cute.arch.thread_idx() - thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) - thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) - thr_tmem_store = tiled_tmem_store.get_slice(thread_idx) - tTMEM_LOADtO = thr_tmem_load.partition_S(tOtO_i) - tTMEM_LOADcO = thr_tmem_load.partition_D(tOcO_i) - tTMEM_STOREtO = thr_tmem_store.partition_D(tOtO_i) - num_tiles = self.cta_tiler[2] // corr_tile_size - if cutlass.const_expr(self.enable_correction_double_buffer): - # Double buffer: 2 register buffers to pipeline tensor-memory load / multiply / store. - tTMrO = cute.make_rmem_tensor((tTMEM_LOADcO.shape, 2), self.pv_acc_dtype) - # Rank-matching views for cute.copy (TMEM partitions are rank-3, - # raw tTMrO[None, idx] is rank-1; composition restores the rank). - copy_layout = cute.make_layout(tTMrO.shape[0]) - view_0 = tTMrO[None, 0] - view_1 = tTMrO[None, 1] - tTMrO_copy = ( - cute.make_tensor( - view_0.iterator, - cute.composition(view_0.layout, copy_layout), - ), - cute.make_tensor( - view_1.iterator, - cute.composition(view_1.layout, copy_layout), - ), - ) - else: - tTMrO = cute.make_rmem_tensor((tTMEM_LOADcO.shape, num_tiles), self.pv_acc_dtype) - tTMEM_LOAD_VECrS = cute.make_rmem_tensor(tTMEM_LOAD_VECcS.shape, self.qk_acc_dtype) - # Wait for vec_i (row_wise current max & previous max) - vec_i_handle = si_corr_consumer.wait_and_advance() - cute.copy(tiled_tmem_load_vec, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECrS) - cute.arch.fence_view_async_tmem_load() - vec_i_handle.release() - # Wait for Oi (peek-token aware: None/False falls back to blocking). - oi_handle = mma_corr_consumer.wait_and_advance(oi_peek_status) - next_oi_peek_status = cutlass.Boolean(False) - vote_ballot_cnt = cute.arch.vote_ballot_sync(tTMEM_LOAD_VECrS[0] != tTMEM_LOAD_VECrS[1]) - should_rescale = vote_ballot_cnt != 0 - if should_rescale: - scale_ = scale_softmax_log2 * (tTMEM_LOAD_VECrS[0] - tTMEM_LOAD_VECrS[1]) - scale = cute.math.exp2(scale_, fastmath=True) - if cutlass.const_expr(self.enable_correction_double_buffer): - num_elems = cute.size(tTMrO, mode=[0]) - # Prologue: load first tile into buffer 0 - cute.copy(tiled_tmem_load, tTMEM_LOADtO, tTMrO_copy[0]) - # Steady state. Refresh the next Oi probe near the end of the - # current rescale pipeline. - for i in cutlass.range_constexpr(1, num_tiles): - cute.copy( - tiled_tmem_load, - cute.make_tensor( - tTMEM_LOADtO.iterator + i * corr_tile_size, - tTMEM_LOADtO.layout, - ), - tTMrO_copy[i % 2], - ) - for j in range(0, num_elems, 2): - tTMrO[j, (i - 1) % 2], tTMrO[j + 1, (i - 1) % 2] = ( - cute.arch.mul_packed_f32x2( - ( - tTMrO[j, (i - 1) % 2], - tTMrO[j + 1, (i - 1) % 2], - ), - (scale, scale), - ) - ) - cute.copy( - tiled_tmem_store, - tTMrO_copy[(i - 1) % 2], - cute.make_tensor( - tTMEM_STOREtO.iterator + (i - 1) * corr_tile_size, - tTMEM_STOREtO.layout, - ), - ) - next_oi_peek_status = mma_corr_consumer.try_wait() - # Epilogue: compute and store last tile - last = (num_tiles - 1) % 2 - for j in range(0, num_elems, 2): - tTMrO[j, last], tTMrO[j + 1, last] = cute.arch.mul_packed_f32x2( - (tTMrO[j, last], tTMrO[j + 1, last]), - (scale, scale), - ) - cute.copy( - tiled_tmem_store, - tTMrO_copy[last], - cute.make_tensor( - tTMEM_STOREtO.iterator + (num_tiles - 1) * corr_tile_size, - tTMEM_STOREtO.layout, - ), - ) - else: - for i in cutlass.range_constexpr(0, num_tiles): - tTMrO_i_ = tTMrO[None, i] - tTMrO_i = cute.make_tensor( - tTMrO_i_.iterator, - cute.composition(tTMrO_i_.layout, cute.make_layout(tTMrO.shape[0])), - ) - cute.copy( - tiled_tmem_load, - cute.make_tensor( - tTMEM_LOADtO.iterator + i * corr_tile_size, - tTMEM_LOADtO.layout, - ), - tTMrO_i, - ) - for j in range(0, cute.size(tTMrO_i), 2): - tTMrO_i[j], tTMrO_i[j + 1] = cute.arch.mul_packed_f32x2( - (tTMrO_i[j], tTMrO_i[j + 1]), - (scale, scale), - ) - cute.copy( - tiled_tmem_store, - tTMrO_i, - cute.make_tensor( - tTMEM_STOREtO.iterator + i * corr_tile_size, - tTMEM_STOREtO.layout, - ), - ) - next_oi_peek_status = mma_corr_consumer.try_wait() - # Release Oi - cute.arch.fence_view_async_tmem_store() - oi_handle.release() - return (si_corr_consumer, mma_corr_consumer), next_oi_peek_status - - @cute.jit - def correction_epilog( - self, - thr_mma: cute.ThrMma, - tiled_tmem_load_vec: cute.TiledCopy, - tensor_args: Tuple, - pipeline_args: Tuple, - value_args: Tuple, - ): - """Apply final scaling and transformation to attention output. - - When use_tma_store=True: writes to shared memory and signals epilogue warp for TMA store. - When use_tma_store=False: writes directly to global memory via st.global. - - :param thr_mma: Thread MMA operation for the computation - :type thr_mma: cute.ThrMma - :param tiled_tmem_load_vec: Tiled memory load operation for the vectorized row-wise max - :type tiled_tmem_load_vec: cute.TiledCopy - :param tensor_args: Tuple containing (tOtO, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECcS, sO_or_gO, mLSE, mSink) - :type tensor_args: Tuple - :param pipeline_args: When use_tma_store: (si_corr_consumer, mma_corr_consumer, corr_epi_producer). - When not use_tma_store: (si_corr_consumer, mma_corr_consumer). - :type pipeline_args: Tuple - :param value_args: Tuple containing (row_idx, cuseqlen_q, seqlen_q, blk_coord, scale_softmax, scale_output) - :type value_args: Tuple - """ - ( - tOtO, - tTMEM_LOAD_VECtSi, - tTMEM_LOAD_VECcS, - dest_O, - mLSE, - mSink, - m_scale_v_channels, - ) = tensor_args - row_idx, cuseqlen_q, seqlen_q, blk_coord, scale_softmax, scale_output = value_args - - pv_tiled_mma_shape = ( - self.pv_mma_tiler[0], - self.pv_mma_tiler[1], - ) - cO = cute.make_identity_tensor(pv_tiled_mma_shape) - - corr_tile_size = 32 * 8 // self.o_dtype.width - tOdO = thr_mma.partition_C(dest_O) - tOcO = thr_mma.partition_C(cO) - tOtO_i = cute.logical_divide(tOtO, cute.make_layout((128, corr_tile_size))) - tOcO_i = cute.logical_divide(tOcO, cute.make_layout((128, corr_tile_size))) - tOdO_i = cute.logical_divide(tOdO, cute.make_layout((128, corr_tile_size))) - - tidx, _, _ = cute.arch.thread_idx() - thread_idx = tidx % (self.threads_per_warp * len(self.correction_warp_ids)) - epi_subtile = (self.epi_tile[0], corr_tile_size) - tmem_copy_atom = sm100_utils.get_tmem_load_op( - self.pv_mma_tiler, - self.o_layout, - self.o_dtype, - self.pv_acc_dtype, - epi_subtile, - use_2cta_instrs=False, - ) - tiled_tmem_load = tcgen05.make_tmem_copy(tmem_copy_atom, tOtO_i[(None, None), 0]) - thr_tmem_load = tiled_tmem_load.get_slice(thread_idx) - tTMEM_LOADtO = thr_tmem_load.partition_S(tOtO_i[(None, None), None]) - tTMEM_LOADdO = thr_tmem_load.partition_D(tOdO_i[(None, None), None]) - tTMEM_LOADoO = thr_tmem_load.partition_D(tOcO_i[(None, None), None]) - - if cutlass.const_expr(self.use_tma_store): - si_corr_consumer, mma_corr_consumer, corr_epi_producer = pipeline_args - smem_copy_atom = sm100_utils.get_smem_store_op( - self.o_layout, self.o_dtype, self.pv_acc_dtype, tiled_tmem_load - ) - tiled_smem_store = cute.make_tiled_copy_D(smem_copy_atom, tiled_tmem_load) - else: - si_corr_consumer, mma_corr_consumer = pipeline_args - - # Wait for vec_i (row_wise global sum) - vec_i_handle = si_corr_consumer.wait_and_advance() - tTMEM_LOAD_VECrS = cute.make_rmem_tensor(tTMEM_LOAD_VECcS.shape, self.qk_acc_dtype) - cute.copy(tiled_tmem_load_vec, tTMEM_LOAD_VECtSi, tTMEM_LOAD_VECrS) - cute.arch.fence_view_async_tmem_load() - vec_i_handle.release() - - # Wait for Oi - oi_handle = mma_corr_consumer.wait_and_advance() - if cutlass.const_expr(self.use_tma_store): - oi_final_handle = corr_epi_producer.acquire_and_advance() - row_sum = tTMEM_LOAD_VECrS[0] - if cutlass.const_expr(mSink is not None): - sink_val = mSink[blk_coord[2]] - row_max_raw = tTMEM_LOAD_VECrS[1] - # sink is already in scaled logit space, row_max_raw is unscaled - # exp2((sink - max_scaled) * log2(e)) = exp(sink - max_scaled) - log2_e = Float32(1.4426950408889634) - sink_exp = cute.math.exp2( - (sink_val - row_max_raw * scale_softmax) * log2_e, fastmath=True - ) - row_sum = row_sum + sink_exp - scale = scale_output / row_sum - - if cutlass.const_expr(m_scale_v_channels is not None): - scale_v_ch_h = m_scale_v_channels[None, blk_coord[2]] - for i in range(self.cta_tiler[2] // corr_tile_size): - tTMEM_LOADtO_i = tTMEM_LOADtO[None, 0, 0, i] - tTMEM_LOADdO_i = tTMEM_LOADdO[None, 0, 0, i] - tTMEM_LOADoO_i = tTMEM_LOADoO[None, 0, 0, i] - tTMrO = cute.make_rmem_tensor(tTMEM_LOADoO_i.shape, self.pv_acc_dtype) - cute.copy(tiled_tmem_load, tTMEM_LOADtO_i, tTMrO) - for j in range(0, cute.size(tTMrO), 2): - tTMrO[j], tTMrO[j + 1] = cute.arch.mul_packed_f32x2( - (tTMrO[j], tTMrO[j + 1]), - (scale, scale), - ) - if cutlass.const_expr(m_scale_v_channels is not None): - for j in range(0, cute.size(tTMrO), 2): - _, n0 = tTMEM_LOADoO_i[j] - _, n1 = tTMEM_LOADoO_i[j + 1] - tTMrO[j], tTMrO[j + 1] = cute.arch.mul_packed_f32x2( - (tTMrO[j], tTMrO[j + 1]), - (scale_v_ch_h[n0], scale_v_ch_h[n1]), - ) - tDMrO = cute.make_rmem_tensor(tTMrO.shape, self.o_dtype) - o_vec = tTMrO.load() - tDMrO.store(o_vec.to(self.o_dtype)) - if cutlass.const_expr(self.use_tma_store): - # TMA store path: write to shared memory - cute.copy(tiled_smem_store, tDMrO, tTMEM_LOADdO_i) - else: - # st.global path: write directly to global memory with bounds check - if row_idx < seqlen_q: - cute.autovec_copy(tDMrO, tTMEM_LOADdO_i) - - if cutlass.const_expr(mLSE is not None): - scaled_tmp = scale_softmax * tTMEM_LOAD_VECrS[1] - # Convert LSE from natural log to log2 space, consistent with flashinfer trtllm-gen backend - lse = (cute.math.log(row_sum, fastmath=True) + scaled_tmp) * Float32(1.4426950408889634) - # Pre-scale correction: row_sum was inflated by 2^offset, so the - # log2-space LSE is too large by exactly `p_fp8_prescale_log2`. - if cutlass.const_expr(self.v_dtype.width == 8 and self.p_fp8_prescale_log2 > 0): - lse = lse - self.p_fp8_prescale_log2 - if row_idx < seqlen_q: - mLSE[row_idx + cuseqlen_q, blk_coord[2]] = lse - if cutlass.const_expr(self.use_tma_store): - # fence view async shared - cute.arch.fence_view_async_shared() - oi_handle.release() - oi_final_handle.commit() - return (si_corr_consumer, mma_corr_consumer, corr_epi_producer) - else: - oi_handle.release() - return (si_corr_consumer, mma_corr_consumer) - - def check_supported_dtypes( - self, - qk_dtype: Type[cutlass.Numeric], - pv_dtype: Type[cutlass.Numeric], - out_dtype: Type[cutlass.Numeric], - qk_sf_dtype: Type[cutlass.Numeric], - qk_sf_vec_size: int, - qk_acc_dtype: Type[cutlass.Numeric], - pv_acc_dtype: Type[cutlass.Numeric], - ): - if qk_dtype in {cutlass.Float8E4M3FN, cutlass.Float8E5M2}: - if qk_sf_dtype is not cutlass.Float8E8M0FNU: - raise NotImplementedError("MXFP8 QK requires qk_sf_dtype Float8E8M0FNU") - if qk_sf_vec_size != 32: - raise NotImplementedError("MXFP8 QK requires qk_sf_vec_size 32") - elif qk_dtype is cutlass.Float4E2M1FN: - if qk_sf_dtype is not cutlass.Float8E4M3FN: - raise NotImplementedError("NVFP4 QK requires qk_sf_dtype Float8E4M3FN") - if qk_sf_vec_size != 16: - raise NotImplementedError("NVFP4 QK requires qk_sf_vec_size 16") - else: - raise NotImplementedError( - "qk_dtype must be Float8E4M3FN/Float8E5M2 for MXFP8 or Float4E2M1FN for NVFP4" - ) - - if pv_dtype not in {cutlass.Float8E4M3FN, cutlass.BFloat16}: - raise NotImplementedError("pv_dtype must be Float8E4M3FN or BFloat16") - if out_dtype not in {cutlass.Float8E4M3FN, cutlass.Float16, cutlass.BFloat16}: - raise NotImplementedError("Unsupported out_dtype") - if qk_acc_dtype not in {cutlass.Float32}: - raise NotImplementedError("Unsupported qk_acc_dtype") - if pv_acc_dtype not in {cutlass.Float32}: - raise NotImplementedError("Unsupported pv_acc_dtype") - - def check_invalid_shape( - self, - qk_dtype: Type[cutlass.Numeric], - qk_sf_vec_size: int, - q_shape: Tuple[int, int, int, int], - k_shape: Tuple[int, int, int, int], - ): - # Shapes are passed around this example as (batch, seq_len, num_heads, head_dim). - b, s_q, h_q, d = q_shape - b_, s_k, h_k, d_ = k_shape - - if b != b_: - raise NotImplementedError("q & k must have the same batch size") - if d != d_: - raise NotImplementedError("q & k must have the same head dimension") - if qk_dtype is cutlass.Float4E2M1FN: - if d not in {64, 128}: - raise NotImplementedError("NVFP4 QK supports headdim 64 or 128") - elif d not in {32, 64, 128}: - raise NotImplementedError("MXFP8 QK supports headdim 32, 64, or 128") - if d % qk_sf_vec_size != 0: - raise NotImplementedError("head dimension must be divisible by qk_sf_vec_size") - if h_q % h_k != 0: - raise NotImplementedError("h_q must be divisible by h_k") - if isinstance(s_q, tuple) and len(s_q) != b: - raise NotImplementedError("variable_seqlen s_q must have the length of batch size") - if isinstance(s_k, tuple) and len(s_k) != b: - raise NotImplementedError("variable_seqlen s_k must have the length of batch size") - - def can_implement( - self, - q_shape: Tuple[int, int, int, int], - k_shape: Tuple[int, int, int, int], - qk_dtype: Type[cutlass.Numeric], - pv_dtype: Type[cutlass.Numeric], - out_dtype: Type[cutlass.Numeric], - qk_sf_dtype: Type[cutlass.Numeric], - qk_sf_vec_size: int, - qk_acc_dtype: Type[cutlass.Numeric], - pv_acc_dtype: Type[cutlass.Numeric], - ) -> bool: - """ - :param q_shape: Shape of the query tensor. - :type q_shape: Tuple[int, int, int, int] - :param k_shape: Shape of the key tensor. - :type k_shape: Tuple[int, int, int, int] - :param qk_dtype: Data type for Q and K (Bmm1 inputs). - :type qk_dtype: Type[cutlass.Numeric] - :param pv_dtype: Data type for P and V (Bmm2 inputs). - :type pv_dtype: Type[cutlass.Numeric] - :param out_dtype: Data type of the output tensor. - :type out_dtype: Type[cutlass.Numeric] - :param qk_acc_dtype: Data type of the qk accumulator tensor. - :type qk_acc_dtype: Type[cutlass.Numeric] - :param pv_acc_dtype: Data type of the pv accumulator tensor. - :type pv_acc_dtype: Type[cutlass.Numeric] - :return: True if the kernel can be implemented, False otherwise. - :rtype: bool - """ - try: - # Skip unsupported types - self.check_supported_dtypes( - qk_dtype, - pv_dtype, - out_dtype, - qk_sf_dtype, - qk_sf_vec_size, - qk_acc_dtype, - pv_acc_dtype, - ) - # Skip invalid shape - self.check_invalid_shape( - qk_dtype, - qk_sf_vec_size, - q_shape, - k_shape, - ) - except NotImplementedError: - return False - return True diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/__init__.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/__init__.py deleted file mode 100644 index 467079831e16..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/fmha_helpers.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/fmha_helpers.py deleted file mode 100644 index f2034556e1d8..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/fmha_helpers.py +++ /dev/null @@ -1,1176 +0,0 @@ -# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: BSD-3-Clause - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# ruff: noqa: I001, E501, F841 - -import enum -from typing import Tuple, Optional -import cutlass -from cutlass.cute.typing import Boolean -from cutlass._mlir.dialects import llvm, vector - -from cutlass.cutlass_dsl import ( - Int32, - Float32, - T, - min, - extract_mlir_values, - new_from_mlir_values, - dsl_user_op, -) -from cutlass.utils.hardware_info import HardwareInfo -from .static_persistent_tile_scheduler import WorkTileInfo -import cutlass.cute as cute - - -############################################################################## -# Fmha static tile scheduler -############################################################################## - - -class FmhaStaticTileSchedulerParams: - """A class to represent parameters for the FMHA (Fused Multi-Head Attention) static tile scheduler. - - This class holds the configuration parameters needed to initialize and configure - the tile scheduler for FMHA operations. - - :ivar is_persistent: Whether to use persistent kernel mode. - :type is_persistent: bool - :ivar problem_shape_mhb: Problem shape in (M, H, B) format. - :type problem_shape_mhb: cute.Shape - """ - - def __init__( - self, - is_persistent: bool, - problem_shape_mhb: cute.Shape, - *, - loc=None, - ip=None, - ): - """ - Initializes the FmhaStaticTileSchedulerParams with the given parameters. - - :param is_persistent: Whether to use persistent kernel mode. - :type is_persistent: bool - :param problem_shape_mhb: Problem shape in (M, H, B) format. - :type problem_shape_mhb: cute.Shape - """ - self.is_persistent = is_persistent - self.problem_shape_mhb = problem_shape_mhb - self._loc = loc - self._ip = ip - - def __extract_mlir_values__(self): - values, self._values_pos = [], [] - for obj in [self.problem_shape_mhb]: - obj_values = extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - return values - - def __new_from_mlir_values__(self, values): - obj_list = [] - for obj, n_items in zip([self.problem_shape_mhb], self._values_pos): - obj_list.append(new_from_mlir_values(obj, values[:n_items])) - values = values[n_items:] - return FmhaStaticTileSchedulerParams(self.is_persistent, *(tuple(obj_list)), loc=self._loc) - - -class FmhaStaticTileScheduler: - """A static tile scheduler for FMHA (Fused Multi-Head Attention) operations. - - This class manages the scheduling of work tiles for FMHA kernels, supporting - both persistent and non-persistent kernel modes. It tracks the current work - position and advances through the problem space efficiently. - - :ivar _params: Scheduler parameters. - :type _params: FmhaStaticTileSchedulerParams - :ivar _blk_coord: Block coordinates. - :type _blk_coord: cute.Coord - :ivar _grid_shape: Grid shape for the kernel. - :type _grid_shape: cute.Shape - :ivar _is_persistent: Whether to use persistent kernel mode. - :type _is_persistent: bool - :ivar _current_work_linear_idx: Current linear work index. - :type _current_work_linear_idx: Int32 - :ivar _problem_shape_mhb: Problem shape in (M, H, B) format. - :type _problem_shape_mhb: cute.Layout - :ivar _num_blocks: Number of blocks in the problem. - :type _num_blocks: Int32 - :ivar _is_first_block: Whether this is the first block. - :type _is_first_block: bool - :ivar num_persistent_sm: Number of persistent SMs. - :type num_persistent_sm: Int32 - """ - - def __init__( - self, - params: FmhaStaticTileSchedulerParams, - current_work_linear_idx: Int32, - blk_coord: cute.Coord, - grid_shape: cute.Shape, - *, - loc=None, - ip=None, - ): - """ - Initializes the FmhaStaticTileScheduler with the given parameters. - - :param params: Scheduler parameters. - :type params: FmhaStaticTileSchedulerParams - :param current_work_linear_idx: Current linear work index. - :type current_work_linear_idx: Int32 - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param grid_shape: Grid shape for the kernel. - :type grid_shape: cute.Shape - """ - self._params = params - self._blk_coord = blk_coord - self._grid_shape = grid_shape - self._is_persistent = params.is_persistent - self._current_work_linear_idx = current_work_linear_idx - self._problem_shape_mhb = cute.make_layout(params.problem_shape_mhb, loc=loc, ip=ip) - self._num_blocks = cute.size(self._problem_shape_mhb, loc=loc, ip=ip) - self._is_first_block = True - self.num_persistent_sm = cute.size(grid_shape, loc=loc, ip=ip) - self._loc = loc - self._ip = ip - - # called by host - @staticmethod - def get_grid_shape( - params: FmhaStaticTileSchedulerParams, - *, - loc=None, - ip=None, - ) -> cute.Shape: - """ - Determine the grid shape for the FMHA kernel. - - For persistent kernels, the grid shape is limited by the number of SMs - (Streaming Multiprocessors) available on the device. For non-persistent - kernels, the grid shape matches the problem shape. - - :param params: Scheduler parameters. - :type params: FmhaStaticTileSchedulerParams - - :return: Grid shape as (M, H, B) tuple. - :rtype: cute.Shape - """ - if params.is_persistent: - hardware_info = HardwareInfo() - sm_count = hardware_info.get_device_multiprocessor_count() - return ( - min(sm_count, cute.size(params.problem_shape_mhb, loc=loc, ip=ip)), - 1, - 1, - ) - else: - return params.problem_shape_mhb - - @staticmethod - def check_valid_work_for_seqlen_q( - q_tiler: int, - current_idx: Int32, - seqlen_q: Int32, - ) -> Boolean: - """ - Check if the current work index is valid for the given query sequence length. - - This method verifies that the current work tile index multiplied by the - query tiler size is within the bounds of the query sequence length. - - :param q_tiler: Query tiler size. - :type q_tiler: int - :param current_idx: Current work index. - :type current_idx: Int32 - :param seqlen_q: Query sequence length. - :type seqlen_q: Int32 - - :return: True if the work is valid, False otherwise. - :rtype: Boolean - """ - return current_idx * q_tiler < seqlen_q - - def get_current_work(self, *, loc=None, ip=None) -> WorkTileInfo: - """ - Get information about the current work tile. - - Determines if the current work is valid and computes the tile coordinates - based on whether the kernel is persistent or non-persistent. - - :return: WorkTileInfo containing tile coordinates and validity flag. - :rtype: WorkTileInfo - """ - is_valid = ( - self._current_work_linear_idx < self._num_blocks - if self._is_persistent - else self._is_first_block - ) - - blk_coord = (0, 0, 0) - if self._is_persistent: - blk_coord = self._problem_shape_mhb.get_hier_coord( - self._current_work_linear_idx, loc=loc, ip=ip - ) - else: - blk_coord = self._blk_coord - - # cur_tile_coord is (mid, 0, (bid, hid)) - cur_tile_coord = ( - blk_coord[0], - 0, - (blk_coord[1], blk_coord[2]), - ) - - return WorkTileInfo(cur_tile_coord, is_valid) - - def initial_work_tile_info(self, *, loc=None, ip=None): - """ - Get the initial work tile information. - - :return: Initial WorkTileInfo. - :rtype: WorkTileInfo - """ - return self.get_current_work(loc=loc, ip=ip) - - def advance_to_next_work(self, *, advance_count=1, loc=None, ip=None): - """ - Advance to the next work tile. - - For persistent kernels, advances by the number of persistent SMs. - For non-persistent kernels, marks that the first block has been processed. - - :param advance_count: Number of steps to advance (default: 1). - :type advance_count: int - """ - if self._is_persistent: - self._current_work_linear_idx += advance_count * self.num_persistent_sm - self._is_first_block = False - - def __extract_mlir_values__(self): - # Only pass mutable per-iteration state as scf.while block arguments. - # _params and _grid_shape are loop-invariant and captured from outer - # scope, keeping block arg count low (4 instead of 10). - values = extract_mlir_values(self._current_work_linear_idx) - values.extend(extract_mlir_values(self._blk_coord)) - return values - - def __new_from_mlir_values__(self, values): - assert len(values) == 4 - new_current_work_linear_idx = new_from_mlir_values( - self._current_work_linear_idx, [values[0]] - ) - new_blk_coord = new_from_mlir_values(self._blk_coord, values[1:4]) - return FmhaStaticTileScheduler( - self._params, new_current_work_linear_idx, new_blk_coord, self._grid_shape - ) - - -def create_fmha_static_tile_scheduler( - params: FmhaStaticTileSchedulerParams, - blk_coord: cute.Coord, - grid_shape: cute.Shape, -) -> FmhaStaticTileScheduler: - """ - Create a new FMHA static tile scheduler. - - :param params: Scheduler parameters. - :type params: FmhaStaticTileSchedulerParams - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param grid_shape: Grid shape. - :type grid_shape: cute.Shape - - :return: New FmhaStaticTileScheduler instance. - :rtype: FmhaStaticTileScheduler - """ - return FmhaStaticTileScheduler(params, blk_coord[0], blk_coord, grid_shape) - - -def create_fmha_static_tile_scheduler_params( - is_persistent: bool, - problem_shape_mhb: cute.Shape, -) -> FmhaStaticTileSchedulerParams: - """ - Create FMHA static tile scheduler parameters. - - :param is_persistent: Whether to use persistent kernel mode. - :type is_persistent: bool - :param problem_shape_mhb: Problem shape in (M, H, B) format. - :type problem_shape_mhb: cute.Shape - - :return: New FmhaStaticTileSchedulerParams instance. - :rtype: FmhaStaticTileSchedulerParams - """ - return FmhaStaticTileSchedulerParams(is_persistent, problem_shape_mhb) - - -def compute_grid( - o_shape: cute.Shape, - cta_tiler: Tuple[int, int, int], - is_persistent: bool, - is_2cta: bool = False, -) -> Tuple[FmhaStaticTileSchedulerParams, Tuple[int, int, int]]: - """ - Compute grid parameters for FMHA operation. - - This function calculates the appropriate grid shape and scheduler parameters - based on the output tensor shape, CTA (Cooperative Thread Array) tiler, - and whether to use persistent kernel mode. - - The output tensor o has shape (s, d, ((h_r, h_k), b)) where: - - s: sequence length - - d: head dimension - - h_r: number of heads for query - - h_k: number of heads for key - - b: batch size - - :param o_shape: Output tensor shape for grid computation. - :type o_shape: cute.Shape - :param cta_tiler: CTA tiler dimensions (M, N, K). - :type cta_tiler: Tuple[int, int, int] - :param is_persistent: Whether to use persistent kernel mode. - :type is_persistent: bool - :param is_2cta: Whether to use 2CTA mode. - :type is_2cta: bool - - :return: Tuple of (scheduler_params, grid_shape). - :rtype: Tuple[FmhaStaticTileSchedulerParams, Tuple[int, int, int]] - """ - tile_sched_params = create_fmha_static_tile_scheduler_params( - is_persistent, - ( - cute.round_up(cute.ceil_div(cute.size(o_shape[0]), cta_tiler[0]), 2 if is_2cta else 1), - cute.size(o_shape[2][0]), - cute.size(o_shape[2][1]), - ), - ) - grid = FmhaStaticTileScheduler.get_grid_shape(tile_sched_params) - - return tile_sched_params, grid - - -############################################################################## -# Fused Mask -############################################################################## - - -class MaskEnum(enum.Enum): - """Enumeration of mask types for FMHA operations. - - - RESIDUAL_MASK: Residual mask for handling variable sequence lengths - - WINDOW_MASK: Window mask for attention which also includes causal and no mask - - WINDOW_MASK_INFERENCE: Same as the window mask, but has the limitation that the end of q is aligned with the end of k - - WINDOW_MASK_BWD: Window mask for backward pass - - WINDOW_MASK_BWD_INFERENCE: Same as the window mask for backward pass, but has the limitation that the end of q is aligned with the end of k - """ - - RESIDUAL_MASK = enum.auto() - RESIDUAL_MASK_BWD = enum.auto() - WINDOW_MASK = enum.auto() - WINDOW_MASK_INFERENCE = enum.auto() - WINDOW_MASK_BWD = enum.auto() - WINDOW_MASK_BWD_INFERENCE = enum.auto() - - -class FusedMask: - """A fused mask implementation for FMHA operations. - - This class handles different types of attention masks including no mask, - residual mask for variable sequence lengths, and causal mask for - autoregressive attention patterns. - - The class provides methods to: - - Calculate trip counts for different mask types - - Apply masks to attention scores - - Handle masked and unmasked trip calculations - """ - - def get_trip_count( - mask_type: MaskEnum, - blk_coord: cute.Coord, - tile_shape: cute.Shape, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - ) -> Int32: - """ - Calculate the number of trips needed for the current block. - - The trip count depends on the mask type and the block coordinates. - For causal masks, it considers the autoregressive constraint. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param tile_shape: Shape of the tile. - :type tile_shape: cute.Shape - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Int32 - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - - :return: Number of trips needed. - :rtype: Int32 - """ - result = 0 - offset = 0 - if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_INFERENCE): - offset = seqlen_k - seqlen_q - if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE): - offset = seqlen_q - seqlen_k - if cutlass.const_expr(mask_type == MaskEnum.RESIDUAL_MASK): - result = cute.ceil_div(seqlen_k, tile_shape[1]) - if cutlass.const_expr(mask_type is MaskEnum.RESIDUAL_MASK_BWD): - result = cute.ceil_div(seqlen_q, tile_shape[0]) - if cutlass.const_expr( - mask_type == MaskEnum.WINDOW_MASK or mask_type == MaskEnum.WINDOW_MASK_INFERENCE - ): - if cutlass.const_expr(window_size_right is None): - result = cute.ceil_div(seqlen_k, tile_shape[1]) - else: - max_idx_q = (blk_coord[0] + 1) * tile_shape[0] - idx_k = max_idx_q + offset + window_size_right - tmp_blocks_k = cute.ceil_div(idx_k, tile_shape[1]) - max_blocks_k = cute.ceil_div(seqlen_k, tile_shape[1]) - result = min(max_blocks_k, tmp_blocks_k) - if cutlass.const_expr( - mask_type == MaskEnum.WINDOW_MASK_BWD or mask_type == MaskEnum.WINDOW_MASK_BWD_INFERENCE - ): - if cutlass.const_expr(window_size_left is None): - result = cute.ceil_div(seqlen_q, tile_shape[0]) - else: - max_idx_k = (blk_coord[1] + 1) * tile_shape[1] - idx_k = max_idx_k + offset + window_size_left - tmp_blocks_q = cute.ceil_div(idx_k, tile_shape[0]) - max_blocks_q = cute.ceil_div(seqlen_q, tile_shape[0]) - result = min(max_blocks_q, tmp_blocks_q) - start_block = FusedMask.get_trip_start( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - result = result - start_block - return result - - @cute.jit - def get_trip_start( - mask_type: MaskEnum, - blk_coord: cute.Coord, - tile_shape: cute.Shape, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - ) -> Int32: - """ - Get the start of the trip for the current block. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param tile_shape: Shape of the tile. - :type tile_shape: cute.Shape - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Int32 - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - """ - result = 0 - offset = 0 - if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_INFERENCE): - offset = seqlen_k - seqlen_q - if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE): - offset = seqlen_q - seqlen_k - if cutlass.const_expr( - mask_type is MaskEnum.WINDOW_MASK or mask_type is MaskEnum.WINDOW_MASK_INFERENCE - ): - if cutlass.const_expr(window_size_left is not None): - min_idx_q = blk_coord[0] * tile_shape[0] - idx_k = min_idx_q + offset - window_size_left - tmp_blocks_k = idx_k // tile_shape[1] - result = max(tmp_blocks_k, result) - if cutlass.const_expr( - mask_type is MaskEnum.WINDOW_MASK_BWD or mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE - ): - if cutlass.const_expr(window_size_right is not None): - min_idx_k = blk_coord[1] * tile_shape[1] - idx_q = min_idx_k + offset - window_size_right - tmp_blocks_q = idx_q // tile_shape[0] - result = max(tmp_blocks_q, result) - return result - - @cute.jit - def get_leading_mask_id( - mask_type: MaskEnum, - blk_coord: cute.Coord, - tile_shape: cute.Shape, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - ) -> Tuple[Int32, Int32]: - """ - Get the begin and end tile idx for the leading mask. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param tile_shape: Shape of the tile. - :type tile_shape: cute.Shape - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Int32 - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - - :return: Tuple of (begin, end) tile idx for the leading mask. - :rtype: Tuple[Int32, Int32] - """ - offset = 0 - if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_INFERENCE): - offset = seqlen_k - seqlen_q - if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE): - offset = seqlen_q - seqlen_k - leading_mask_begin = FusedMask.get_trip_start( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - trip_count = FusedMask.get_trip_count( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - - leading_mask_end = leading_mask_begin - if cutlass.const_expr( - mask_type is MaskEnum.WINDOW_MASK or mask_type is MaskEnum.WINDOW_MASK_INFERENCE - ): - if cutlass.const_expr(window_size_left is not None): - min_idx_q = (blk_coord[0] + 1) * tile_shape[0] + offset - window_size_left - leading_mask_end = min( - cute.ceil_div(min_idx_q, tile_shape[1]) - 1, - trip_count + leading_mask_begin - 1, - ) - else: - leading_mask_end = leading_mask_begin - 1 - elif cutlass.const_expr( - mask_type is MaskEnum.WINDOW_MASK_BWD or mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE - ): - if cutlass.const_expr(window_size_right is not None): - min_idx_k = (blk_coord[1] + 1) * tile_shape[1] + offset - window_size_right - leading_mask_end = cute.ceil_div(min_idx_k, tile_shape[0]) - 1 - else: - leading_mask_end = leading_mask_begin - 1 - return leading_mask_begin, leading_mask_end - - @cute.jit - def get_trailing_mask_id( - mask_type: MaskEnum, - blk_coord: cute.Coord, - tile_shape: cute.Shape, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - ) -> Tuple[Optional[Int32], Optional[Int32]]: - """ - Get the begin and end tile idx for the trailing mask. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param tile_shape: Shape of the tile. - :type tile_shape: cute.Shape - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Int32 - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - - :return: Tuple of (begin, end) tile idx for the trailing mask. - :rtype: Tuple[Int32, Int32] - """ - offset = 0 - if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_INFERENCE): - offset = seqlen_k - seqlen_q - if cutlass.const_expr(mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE): - offset = seqlen_q - seqlen_k - trip_start = FusedMask.get_trip_start( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - trip_count = FusedMask.get_trip_count( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - - trailing_mask_begin, trailing_mask_end = None, None - if cutlass.const_expr( - mask_type is MaskEnum.WINDOW_MASK or mask_type is MaskEnum.WINDOW_MASK_INFERENCE - ): - if cutlass.const_expr(window_size_right is not None): - min_idx_q = blk_coord[0] * tile_shape[0] + offset + window_size_right - trailing_mask_begin = min(min_idx_q // tile_shape[1], trip_count + trip_start - 1) - trailing_mask_end = trip_count + trip_start - 1 - else: - # last tile, we always apply mask on it regardless whether it's a residual tile - trailing_mask_begin = trip_count + trip_start - 1 - trailing_mask_end = trip_count + trip_start - 1 - else: - if cutlass.const_expr(window_size_left is not None): - min_idx_k = blk_coord[1] * tile_shape[1] + offset + window_size_left + 1 - max_idx_k = (blk_coord[1] + 1) * tile_shape[1] + offset + window_size_left - trailing_mask_begin = min( - cute.ceil_div(min_idx_k, tile_shape[0]) - 1, - trip_count + trip_start - 1, - ) - trailing_mask_end = min( - cute.ceil_div(max_idx_k, tile_shape[0]) - 1, - trip_count + trip_start - 1, - ) - else: - # last tile, we always apply mask on it regardless whether it's a residual tile - trailing_mask_begin = trip_count + trip_start - 1 - trailing_mask_end = trip_count + trip_start - 1 - - return trailing_mask_begin, trailing_mask_end - - @cute.jit - def get_masked_leading_count( - mask_type: MaskEnum, - blk_coord: cute.Coord, - tile_shape: cute.Shape, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - ) -> Int32: - """ - Calculate the number of masked trips for the leading mask. - - This is used for blocks that need special handling due to masking. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param tile_shape: Shape of the tile. - :type tile_shape: cute.Shape - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Int32 - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - - :return: Number of masked trips. - :rtype: Int32 - """ - result = 0 - if cutlass.const_expr( - mask_type is not MaskEnum.RESIDUAL_MASK and mask_type is not MaskEnum.RESIDUAL_MASK_BWD - ): - if cutlass.const_expr(window_size_left is not None or window_size_right is not None): - leading_mask_begin, leading_mask_end = FusedMask.get_leading_mask_id( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - result = max(leading_mask_end - leading_mask_begin + 1, 0) - - return result - - @cute.jit - def get_masked_trailing_count( - mask_type: MaskEnum, - blk_coord: cute.Coord, - tile_shape: cute.Shape, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - rem_count: Optional[Int32] = 0, - ) -> Int32: - """ - Calculate the number of masked trips for the trailing mask. - - This is used for blocks that need special handling due to masking. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param tile_shape: Shape of the tile. - :type tile_shape: cute.Shape - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Int32 - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - :param rem_count: Remaining count from previous calculations. - :type rem_count: Int32 - - :return: Number of masked trips. - :rtype: Int32 - """ - result = 0 - - if cutlass.const_expr( - mask_type is not MaskEnum.RESIDUAL_MASK and mask_type is not MaskEnum.RESIDUAL_MASK_BWD - ): - if cutlass.const_expr(window_size_left is not None or window_size_right is not None): - trailing_mask_begin, trailing_mask_end = FusedMask.get_trailing_mask_id( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - leading_mask_begin, leading_mask_end = FusedMask.get_leading_mask_id( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - if cutlass.const_expr( - trailing_mask_begin is not None and trailing_mask_end is not None - ): - if trailing_mask_begin <= leading_mask_end: - result = max(trailing_mask_end - leading_mask_end, 0) - else: - result = max(trailing_mask_end - trailing_mask_begin + 1, 0) - else: - if seqlen_k % tile_shape[1] != 0: - result = 1 - else: - result = 0 - - return result + rem_count - - @cute.jit - def get_unmasked_trip_count( - mask_type: MaskEnum, - blk_coord: cute.Coord, - tile_shape: cute.Shape, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - ) -> Int32: - """ - Calculate the number of unmasked trips for the current block. - - This represents the number of trips that don't require special - masking treatment. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param tile_shape: Shape of the tile. - :type tile_shape: cute.Shape - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Int32 - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - - :return: Number of unmasked trips. - :rtype: Int32 - """ - result = ( - FusedMask.get_trip_count( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - - FusedMask.get_masked_leading_count( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - - FusedMask.get_masked_trailing_count( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - 0, - ) - ) - return result - - @cute.jit - def get_masked_info( - mask_type: MaskEnum, - blk_coord: cute.Coord, - tile_shape: cute.Shape, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[Int32] = None, - window_size_right: Optional[Int32] = None, - rem_count: Optional[Int32] = 0, - ) -> Tuple[Int32, Int32, Int32, Int32, Int32]: - """ - Calculate the number of masked trips for the trailing mask. - - This is used for blocks that need special handling due to masking. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param blk_coord: Block coordinates. - :type blk_coord: cute.Coord - :param tile_shape: Shape of the tile. - :type tile_shape: cute.Shape - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Int32 - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[Int32] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[Int32] - :param rem_count: Remaining count from previous calculations. - :type rem_count: Int32 - - :return: Number of masked info. - :rtype: Tuple[Int32, Int32, Int32, Int32, Int32] - """ - start_count = FusedMask.get_trip_start( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - leading_mask_count = FusedMask.get_masked_leading_count( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - unmask_count = FusedMask.get_unmasked_trip_count( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - ) - trailing_mask_count = FusedMask.get_masked_trailing_count( - mask_type, - blk_coord, - tile_shape, - seqlen_q, - seqlen_k, - window_size_left, - window_size_right, - rem_count, - ) - end_count = start_count + leading_mask_count + unmask_count + trailing_mask_count - return ( - start_count, - end_count, - leading_mask_count, - unmask_count, - trailing_mask_count, - ) - - @cute.jit - def apply_mask( - mask_type: MaskEnum, - acc_qk: cute.Tensor, - index_qk: cute.Tensor, - seqlen_q: Int32, - seqlen_k: Int32, - window_size_left: Optional[int] = None, - window_size_right: Optional[int] = None, - index_transform: cutlass.Constexpr = lambda index_q, index_k: ( - index_q, - index_k, - ), - ): - """ - Apply the appropriate mask to the attention scores. - - This method modifies the attention scores (acc_qk) based on the mask type - and the positions in the index tensor. - - :param mask_type: Type of mask to use - :type mask_type: utils.MaskEnum - :param acc_qk: Accumulated QK attention scores tensor. - :type acc_qk: cute.Tensor - :param index_qk: Index tensor containing position information. - :type index_qk: cute.Tensor - :param seqlen_k: Key sequence length for attention computation. - :type seqlen_k: Int32 - :param seqlen_q: Query sequence length for attention computation. - :type seqlen_q: Optional[int] - :param window_size_left: Left-side sliding window size for attention masking. - :type window_size_left: Optional[int] - :param window_size_right: Right-side sliding window size for attention masking. - :type window_size_right: Optional[int] - """ - - tidx, tidy, tidx = cute.arch.thread_idx() - offset = 0 - offset = ( - seqlen_k - seqlen_q - if cutlass.const_expr( - mask_type is MaskEnum.WINDOW_MASK_INFERENCE - or mask_type is MaskEnum.WINDOW_MASK_BWD_INFERENCE - ) - else 0 - ) - for i in cutlass.range(cute.size(acc_qk), unroll_full=True): - index_q, index_k = index_transform(*index_qk[i]) - if cutlass.const_expr(window_size_left is not None or window_size_right is not None): - if cutlass.const_expr(window_size_left is None): - if index_q + offset + window_size_right < index_k: - acc_qk[i] = -Float32.inf - if index_k >= seqlen_k or index_q >= seqlen_q: # residual mask - acc_qk[i] = -Float32.inf - elif cutlass.const_expr(window_size_right is None): - if index_q + offset - window_size_left > index_k: - acc_qk[i] = -Float32.inf - if index_k >= seqlen_k or index_q >= seqlen_q: # residual mask - acc_qk[i] = -Float32.inf - else: - max_K_index = min(index_q + offset + window_size_right, seqlen_k) - min_K_index = max(0, index_q + offset - window_size_left) - if index_k > max_K_index or index_k < min_K_index: - acc_qk[i] = -Float32.inf - if index_k >= seqlen_k or index_q >= seqlen_q: # residual mask - acc_qk[i] = -Float32.inf - - if cutlass.const_expr( - mask_type == MaskEnum.RESIDUAL_MASK or mask_type == MaskEnum.RESIDUAL_MASK_BWD - ): - if index_k >= seqlen_k or index_q >= seqlen_q: - acc_qk[i] = -Float32.inf - - -@dsl_user_op -def ex2_emulation_packed_f32x2( - x: Float32, y: Float32, *, loc=None, ip=None -) -> Tuple[Float32, Float32]: - # Clamp the xy so they fit within the FP32 exponent range - # The upper side is ensured by the (s - row_max) - xy_clamped = (cute.arch.fmax(x, -127.0), cute.arch.fmax(y, -127.0)) - - fp32_round_int = float(2**23 + 2**22) - # | 0 | 10010110 | 10000000000000000000000 | - # | sign | exponent | mantissa | - # ^ - # | - # digit of ones place - # During FP32 addition, any number that smaller than this will be - # aligned the exponent to 2^23, so that the digit of ones - # is at the rightest place - # We want to round down here, so that the fractional part is in [0, 1) - xy_rounded = cute.arch.add_packed_f32x2(xy_clamped, (fp32_round_int, fp32_round_int), rnd="rm") - # The integer floor of x & y are now in the last 8 bits of xy_rounded - # We want the next 2 ops to round to nearest even. The rounding mode is important. - xy_rounded_back = cute.arch.sub_packed_f32x2(xy_rounded, (fp32_round_int, fp32_round_int)) - xy_frac = cute.arch.sub_packed_f32x2(xy_clamped, xy_rounded_back) - - @dsl_user_op - @cute.jit - def polynomial_deg3_packed_f32x2( - x: Float32, y: Float32, *, loc=None, ip=None - ) -> Tuple[Float32, Float32]: - # 2^x ~= (0.077 * x + 0.228) * x + 0.695) * x + 1, for x in [0, 1) - coeff = ( - 1.0, # coeff of deg0 - 0.695146143436431884765625, # coeff of deg1 - 0.227564394474029541015625, # coeff of deg2 - 0.077119089663028717041015625, # coeff of deg3 - ) - deg = len(coeff) - 1 # started with highest degree - out = (coeff[deg], coeff[deg]) - for i in cutlass.range_constexpr(deg - 1, -1, -1): - out = cute.arch.fma_packed_f32x2(out, (x, y), (coeff[i], coeff[i]), loc=loc, ip=ip) - return out - - xy_frac_ex2 = polynomial_deg3_packed_f32x2(*xy_frac, loc=loc, ip=ip) - - @dsl_user_op - def combine_int_frac_ex2( - x_rounded: Float32, frac_ex2: Float32, *, loc=None, ip=None - ) -> Float32: - return cutlass.Float32( - llvm.inline_asm( - T.f32(), - [ - Float32(x_rounded).ir_value(loc=loc, ip=ip), - Float32(frac_ex2).ir_value(loc=loc, ip=ip), - ], - "{\n\t" - ".reg .s32 x_rounded_i, frac_ex_i, x_rounded_e, out_i;\n\t" - "mov.b32 x_rounded_i, $1;\n\t" - "mov.b32 frac_ex_i, $2;\n\t" - "shl.b32 x_rounded_e, x_rounded_i, 23;\n\t" - "add.s32 out_i, x_rounded_e, frac_ex_i;\n\t" - "mov.b32 $0, out_i;\n\t" - "}\n", - "=f,f,f", - has_side_effects=False, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - ) - - x_out = combine_int_frac_ex2(xy_rounded[0], xy_frac_ex2[0], loc=loc, ip=ip) - y_out = combine_int_frac_ex2(xy_rounded[1], xy_frac_ex2[1], loc=loc, ip=ip) - - return x_out, y_out - - -@cute.jit -def cvt_f32x4_to_f8x4_pack_i32(fp32x4, fp8_type, *, loc=None, ip=None): - fp32x4 = fp32x4.load() - src_vec4 = fp32x4.ir_value(loc=loc, ip=ip) if hasattr(fp32x4, "ir_value") else fp32x4 - - src0 = Float32(vector.extract(src_vec4, [], [0])).ir_value(loc=loc, ip=ip) - src1 = Float32(vector.extract(src_vec4, [], [1])).ir_value(loc=loc, ip=ip) - src2 = Float32(vector.extract(src_vec4, [], [2])).ir_value(loc=loc, ip=ip) - src3 = Float32(vector.extract(src_vec4, [], [3])).ir_value(loc=loc, ip=ip) - - cvt_instruction = "" - if cutlass.const_expr(fp8_type == cutlass.Float8E4M3FN): - cvt_instruction = "cvt.rn.satfinite.e4m3x2.f32" - else: - assert False, "Unsupported fp8 element type" - - asm_tmpl = ( - "{\n" - " .reg .b16 lo;\n" - " .reg .b16 hi;\n" - f" {cvt_instruction} lo, $2, $1;\n" - f" {cvt_instruction} hi, $4, $3;\n" - " mov.b32 $0, {lo, hi};\n" - "}" - ) - packed_i32 = llvm.inline_asm( - T.i32(), - [src0, src1, src2, src3], - asm_tmpl, - "=r,f,f,f,f", - has_side_effects=True, - is_align_stack=False, - asm_dialect=llvm.AsmDialect.AD_ATT, - ) - - return packed_i32 - - -@cute.jit -def cvt_f32x4_to_f8x4(fp32x4, fp8x4, *, loc=None, ip=None): - packed_i32 = cvt_f32x4_to_f8x4_pack_i32(fp32x4, fp8x4.element_type) - fp8x4_i32 = cute.recast_tensor(fp8x4, cutlass.Int32) - fp8x4_i32[0] = cutlass.Int32(packed_i32) - return diff --git a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/static_persistent_tile_scheduler.py b/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/static_persistent_tile_scheduler.py deleted file mode 100644 index 21e80c07b5cd..000000000000 --- a/tensorrt_llm/_torch/visual_gen/cute_dsl_kernels/helpers/static_persistent_tile_scheduler.py +++ /dev/null @@ -1,814 +0,0 @@ -# Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: BSD-3-Clause - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# ruff: noqa: I001, E501 - -import inspect -from typing import Optional, Tuple - -from cutlass.cutlass_dsl import ( - Boolean, - Integer, - Int32, - min, - extract_mlir_values, - new_from_mlir_values, - dsl_user_op, - const_expr, -) -from cutlass._mlir import ir -import cutlass.cute as cute - -############################################################################## -# Static persistent tile scheduler -############################################################################## - - -class WorkTileInfo: - """A class to represent information about a work tile. - - :ivar tile_idx: The index of the tile. - :type tile_idx: cute.Coord - :ivar is_valid_tile: Whether the tile is valid. - :type is_valid_tile: Boolean - """ - - def __init__(self, tile_idx: cute.Coord, is_valid_tile: Boolean): - self._tile_idx = tile_idx - self._is_valid_tile = Boolean(is_valid_tile) - - def __extract_mlir_values__(self) -> list[ir.Value]: - values = extract_mlir_values(self.tile_idx) - values.extend(extract_mlir_values(self.is_valid_tile)) - return values - - def __new_from_mlir_values__(self, values: list[ir.Value]) -> "WorkTileInfo": - assert len(values) == 4 - new_tile_idx = new_from_mlir_values(self._tile_idx, values[:-1]) - new_is_valid_tile = new_from_mlir_values(self._is_valid_tile, [values[-1]]) - return WorkTileInfo(new_tile_idx, new_is_valid_tile) - - @property - @cute.jit - def is_valid_tile(self) -> Boolean: - """Check latest tile returned by the scheduler is valid or not. Any scheduling - requests after all tasks completed will return an invalid tile. - - :return: The validity of the tile. - :rtype: Boolean - """ - return self._is_valid_tile - - @property - @cute.jit - def tile_idx(self) -> cute.Coord: - """ - Get the index of the tile. - - :return: The index of the tile. - :rtype: cute.Coord - """ - return self._tile_idx - - -class PersistentTileSchedulerParams: - """A class to represent parameters for a persistent tile scheduler. - - This class is designed to manage and compute the layout of clusters and tiles - in a batched gemm problem. - - :ivar cluster_shape_mn: Shape of the cluster in (m, n) dimensions (K dimension cta count must be 1). - :type cluster_shape_mn: tuple - :ivar problem_layout_ncluster_mnl: Layout of the problem in terms of - number of clusters in (m, n, l) dimensions. - :type problem_layout_ncluster_mnl: cute.Layout - """ - - @dsl_user_op - def __init__( - self, - problem_shape_ntile_mnl: cute.Shape, - cluster_shape_mnk: cute.Shape, - swizzle_size: int = 1, - raster_along_m: bool = True, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> None: - """ - Initializes the PersistentTileSchedulerParams with the given parameters. - - :param problem_shape_ntile_mnl: The shape of the problem in terms of - number of CTA (Cooperative Thread Array) in (m, n, l) dimensions. - :type problem_shape_ntile_mnl: cute.Shape - :param cluster_shape_mnk: The shape of the cluster in (m, n) dimensions. - :type cluster_shape_mnk: cute.Shape - :param swizzle_size: Swizzling size in the unit of cluster. 1 means no swizzle - :type swizzle_size: int - :param raster_along_m: Rasterization order of clusters. Only used when swizzle_size > 1. - True means along M, false means along N. - :type raster_along_m: bool - - :raises ValueError: If cluster_shape_k is not 1. - """ - - if cluster_shape_mnk[2] != 1: # type: ignore[index] - raise ValueError(f"unsupported cluster_shape_k {cluster_shape_mnk[2]}") # type: ignore[index] - if swizzle_size < 1: - raise ValueError(f"expect swizzle_size >= 1, but get {swizzle_size}") - - self.problem_shape_ntile_mnl = problem_shape_ntile_mnl - # cluster_shape_mnk is kept for reconstruction - self._cluster_shape_mnk = cluster_shape_mnk - self.cluster_shape_mn = cluster_shape_mnk[:2] # type: ignore[index] - self.swizzle_size = swizzle_size - self.raster_along_m = raster_along_m - self._loc = loc - - # By default, we follow m major (col-major) raster order, so make a col-major layout - self.problem_layout_ncluster_mnl = cute.make_layout( - cute.ceil_div( - self.problem_shape_ntile_mnl, - cluster_shape_mnk[:2], # type: ignore[index] - loc=loc, - ip=ip, - ), - loc=loc, - ip=ip, - ) - - # Apply swizzle if swizzle_size > 1 - if swizzle_size > 1: - problem_shape_ncluster_mnl = cute.round_up( - self.problem_layout_ncluster_mnl.shape, - (1, swizzle_size, 1) if raster_along_m else (swizzle_size, 1, 1), - ) - - if raster_along_m: - self.problem_layout_ncluster_mnl = cute.make_layout( - ( - problem_shape_ncluster_mnl[0], # type: ignore[index] - (swizzle_size, problem_shape_ncluster_mnl[1] // swizzle_size), # type: ignore[index, operator] - problem_shape_ncluster_mnl[2], # type: ignore[index] - ), - stride=( - swizzle_size, - (1, swizzle_size * problem_shape_ncluster_mnl[0]), # type: ignore[index] - problem_shape_ncluster_mnl[0] * problem_shape_ncluster_mnl[1], # type: ignore[index, operator] - ), - loc=loc, - ip=ip, - ) - else: - self.problem_layout_ncluster_mnl = cute.make_layout( - ( - (swizzle_size, problem_shape_ncluster_mnl[0] // swizzle_size), # type: ignore[index, operator] - problem_shape_ncluster_mnl[1], # type: ignore[index] - problem_shape_ncluster_mnl[2], # type: ignore[index] - ), - stride=( - (1, swizzle_size * problem_shape_ncluster_mnl[1]), # type: ignore[index] - swizzle_size, - problem_shape_ncluster_mnl[0] * problem_shape_ncluster_mnl[1], # type: ignore[index, operator] - ), - loc=loc, - ip=ip, - ) - - # Create FastDivmod divisors (only when swizzle_size == 1 for correctness) - # FastDivmod assumes simple col-major layout, incompatible with swizzled layouts - if swizzle_size == 1: - _problem_layout_size = cute.size(self.problem_layout_ncluster_mnl, loc=loc, ip=ip) - cluster_count_m = self.problem_layout_ncluster_mnl.shape[0] - cluster_count_n = self.problem_layout_ncluster_mnl.shape[1] - - if raster_along_m: - cluster_count_major = cluster_count_m - cluster_count_minor = cluster_count_n - else: - cluster_count_major = cluster_count_n - cluster_count_minor = cluster_count_m - - # cluster_shape_major_fdd: Used to decode work_unit_id to cluster coordinates - self.cluster_shape_major_fdd = cute.fast_divmod_create_divisor( - cluster_count_major, loc=loc, ip=ip - ) - - # cluster_shape_minor_fdd: Used for the second level decomposition - self.cluster_shape_minor_fdd = cute.fast_divmod_create_divisor( - cluster_count_minor, loc=loc, ip=ip - ) - else: - # FastDivmod not applicable with swizzling, set to None - self.cluster_shape_major_fdd = None - self.cluster_shape_minor_fdd = None - - def __extract_mlir_values__(self) -> list[ir.Value]: - values, self._values_pos = [], [] - for obj in [ - self.problem_shape_ntile_mnl, - self._cluster_shape_mnk, - self.swizzle_size, - self.raster_along_m, - ]: - obj_values = extract_mlir_values(obj) - values += obj_values - self._values_pos.append(len(obj_values)) - - # Add FastDivmod divisors to MLIR values for Host->Device transfer - # Only add non-None values to avoid MLIR type errors - fastdivmod_values = [] - fastdivmod_indices = [] # Track which FastDivmod objects are present - fastdivmod_lengths = [] # Track serialized MLIR value count per FDD - - for i, (fdd_name, fdd_obj) in enumerate( - [ - ("cluster_shape_major_fdd", self.cluster_shape_major_fdd), - ("cluster_shape_minor_fdd", self.cluster_shape_minor_fdd), - ] - ): - if fdd_obj is not None: - # Extract MLIR values from FastDivmodDivisor objects - fdd_values = extract_mlir_values(fdd_obj) - fastdivmod_values.extend(fdd_values) - fastdivmod_indices.append(i) - fastdivmod_lengths.append(len(fdd_values)) - - values += fastdivmod_values - self._values_pos.append( - len(fastdivmod_indices) - ) # Store count of FastDivmod objects, not values - self._fastdivmod_indices = fastdivmod_indices # Store for reconstruction - self._fastdivmod_lengths = fastdivmod_lengths # Per-FDD value count - - return values - - def __new_from_mlir_values__(self, values: list[ir.Value]) -> "PersistentTileSchedulerParams": - obj_list = [] - values_copy = list(values) # Make a copy to avoid modifying original - - # Reconstruct original objects from MLIR values - for obj, n_items in zip( - [ - self.problem_shape_ntile_mnl, - self._cluster_shape_mnk, - self.swizzle_size, - self.raster_along_m, - ], - self._values_pos[:-1], # Exclude FastDivmod count - ): - obj_list.append(new_from_mlir_values(obj, values_copy[:n_items])) - values_copy = values_copy[n_items:] - - # Create new params object by calling __init__ with reconstructed values - # This properly recreates layouts and other derived attributes in the device context - new_params = PersistentTileSchedulerParams(*(tuple(obj_list)), loc=self._loc) - - # Restore FastDivmod divisors from remaining values - fdd_names = ["cluster_shape_major_fdd", "cluster_shape_minor_fdd"] - - if hasattr(self, "_fastdivmod_indices") and len(self._fastdivmod_indices) > 0: - # Override the FastDivmod divisors created by __init__ with reconstructed ones. - # FastDivmodDivisor now emits multiple MLIR values per object (see issue #3243); - # use the per-FDD length recorded during __extract_mlir_values__ to slice the - # tail of values_copy without re-emitting IR. - fdd_tail_offset = 0 - for original_index, n_fdd in zip(self._fastdivmod_indices, self._fastdivmod_lengths): - fdd_name = fdd_names[original_index] - original_fdd = getattr(self, fdd_name) - if original_fdd is None: - continue - end = fdd_tail_offset + n_fdd - if end > len(values_copy): - raise ValueError( - f"FastDivmod values out of range for {fdd_name}: " - f"need {end}, have {len(values_copy)}" - ) - reconstructed_fdd = new_from_mlir_values( - original_fdd, values_copy[fdd_tail_offset:end] - ) - setattr(new_params, fdd_name, reconstructed_fdd) - fdd_tail_offset = end - if fdd_tail_offset != len(values_copy): - raise ValueError( - "Unexpected trailing FastDivmod MLIR values: " - f"consumed {fdd_tail_offset}, have {len(values_copy)}" - ) - - return new_params - - @dsl_user_op - def get_grid_shape( - self, - max_active_clusters: Int32, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> Tuple[Integer, Integer, Integer]: - """ - Computes the grid shape based on the maximum active clusters allowed. - - :param max_active_clusters: The maximum number of active clusters that - can run in one wave. - :type max_active_clusters: Int32 - - :return: A tuple containing the grid shape in (m, n, persistent_clusters). - - m: self.cluster_shape_m. - - n: self.cluster_shape_n. - - persistent_clusters: Number of persistent clusters that can run. - """ - - # Total ctas in problem size - num_ctas_mnl = tuple( - cute.size(x) * y - for x, y in zip(self.problem_layout_ncluster_mnl.shape, self.cluster_shape_mn) - ) + (self.problem_layout_ncluster_mnl.shape[2],) - - num_ctas_in_problem = cute.size(num_ctas_mnl, loc=loc, ip=ip) - - num_ctas_per_cluster = cute.size(self.cluster_shape_mn, loc=loc, ip=ip) - # Total ctas that can run in one wave - num_ctas_per_wave = max_active_clusters * num_ctas_per_cluster - - num_persistent_ctas = min(num_ctas_in_problem, num_ctas_per_wave) - num_persistent_clusters = num_persistent_ctas // num_ctas_per_cluster - - return (*self.cluster_shape_mn, num_persistent_clusters) - - -# Set explicit signature for Sphinx documentation to avoid issues with @dsl_user_op decorator -PersistentTileSchedulerParams.__init__.__signature__ = inspect.Signature( # type: ignore[attr-defined] - [ - inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD), - ] -) - - -class StaticPersistentTileScheduler: - """A scheduler for static persistent tile execution in CUTLASS/CuTe kernels. - - :ivar params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl - :type params: PersistentTileSchedulerParams - :ivar num_persistent_clusters: Number of persistent clusters that can be launched - :type num_persistent_clusters: Int32 - :ivar cta_id_in_cluster: ID of the CTA within its cluster - :type cta_id_in_cluster: cute.Coord - :ivar _num_tiles_executed: Counter for executed tiles - :type _num_tiles_executed: Int32 - :ivar _current_work_linear_idx: Current cluster index - :type _current_work_linear_idx: Int32 - """ - - def __init__( - self, - params: PersistentTileSchedulerParams, - num_persistent_clusters: Int32, - current_work_linear_idx: Int32, - cta_id_in_cluster: cute.Coord, - num_tiles_executed: Int32, - ): - """ - Initializes the StaticPersistentTileScheduler with the given parameters. - - :param params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl. - :type params: PersistentTileSchedulerParams - :param num_persistent_clusters: Number of persistent clusters that can be launched. - :type num_persistent_clusters: Int32 - :param current_work_linear_idx: Current cluster index. - :type current_work_linear_idx: Int32 - :param cta_id_in_cluster: ID of the CTA within its cluster. - :type cta_id_in_cluster: cute.Coord - :param num_tiles_executed: Counter for executed tiles. - :type num_tiles_executed: Int32 - """ - self.params = params - self.num_persistent_clusters = num_persistent_clusters - self._current_work_linear_idx = current_work_linear_idx - self.cta_id_in_cluster = cta_id_in_cluster - self._num_tiles_executed = num_tiles_executed - - def __extract_mlir_values__(self) -> list[ir.Value]: - values = extract_mlir_values(self.num_persistent_clusters) - values.extend(extract_mlir_values(self._current_work_linear_idx)) - values.extend(extract_mlir_values(self.cta_id_in_cluster)) - values.extend(extract_mlir_values(self._num_tiles_executed)) - - # CRITICAL: Also extract FastDivmod divisors from params - values.extend(extract_mlir_values(self.params)) - - return values - - def __new_from_mlir_values__(self, values: list[ir.Value]) -> "StaticPersistentTileScheduler": - assert len(values) >= 6 - new_num_persistent_clusters = new_from_mlir_values( - self.num_persistent_clusters, [values[0]] - ) - new_current_work_linear_idx = new_from_mlir_values( - self._current_work_linear_idx, [values[1]] - ) - new_cta_id_in_cluster = new_from_mlir_values(self.cta_id_in_cluster, values[2:5]) - new_num_tiles_executed = new_from_mlir_values(self._num_tiles_executed, [values[5]]) - - # Reconstruct params with FastDivmod divisors - params_values = values[6:] # Remaining values are from params - new_params = new_from_mlir_values(self.params, params_values) - - return StaticPersistentTileScheduler( - new_params, # Use reconstructed params with FastDivmod divisors - new_num_persistent_clusters, - new_current_work_linear_idx, - new_cta_id_in_cluster, - new_num_tiles_executed, - ) - - @staticmethod - @dsl_user_op - def create( - params: PersistentTileSchedulerParams, - block_idx: Tuple[Integer, Integer, Integer], - grid_dim: Tuple[Integer, Integer, Integer], - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> "StaticPersistentTileScheduler": - """Initialize the static persistent tile scheduler. - - :param params: Parameters for the persistent - tile scheduler. - :type params: PersistentTileSchedulerParams - :param block_idx: The 3d block index in the format (bidx, bidy, bidz). - :type block_idx: Tuple[Integer, Integer, Integer] - :param grid_dim: The 3d grid dimensions for kernel launch. - :type grid_dim: Tuple[Integer, Integer, Integer] - - :return: A StaticPersistentTileScheduler object. - :rtype: StaticPersistentTileScheduler - """ - - # Calculate the number of persistent clusters by dividing the total grid size - # by the number of CTAs per cluster - num_persistent_clusters = cute.size(grid_dim, loc=loc, ip=ip) // cute.size( - params.cluster_shape_mn, loc=loc, ip=ip - ) - - bidx, bidy, bidz = block_idx - - # Initialize workload index equals to the cluster index in the grid - current_work_linear_idx = Int32(bidz) - - # CTA id in the cluster - cta_id_in_cluster = ( - Int32(bidx % params.cluster_shape_mn[0]), - Int32(bidy % params.cluster_shape_mn[1]), - Int32(0), - ) - # Initialize number of tiles executed to zero - num_tiles_executed = Int32(0) - return StaticPersistentTileScheduler( - params, - num_persistent_clusters, - current_work_linear_idx, - cta_id_in_cluster, - num_tiles_executed, - ) - - # called by host - @staticmethod - def get_grid_shape( - params: PersistentTileSchedulerParams, - max_active_clusters: Int32, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> Tuple[Integer, Integer, Integer]: - """Calculates the grid shape to be launched on GPU using problem shape, - threadblock shape, and active cluster size. - - :param params: Parameters for grid shape calculation. - :type params: PersistentTileSchedulerParams - :param max_active_clusters: Maximum active clusters allowed. - :type max_active_clusters: Int32 - - :return: The calculated 3d grid shape. - :rtype: Tuple[Integer, Integer, Integer] - """ - - return params.get_grid_shape(max_active_clusters, loc=loc, ip=ip) - - # private method - def _get_current_work_for_linear_idx( - self, - current_work_linear_idx: Int32, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> WorkTileInfo: - """Compute current tile coord given current_work_linear_idx and cta_id_in_cluster. - - :param current_work_linear_idx: The linear index of the current work. - :type current_work_linear_idx: Int32 - - :return: An object containing information about the current tile coordinates - and validity status. - :rtype: WorkTileInfo - """ - - is_valid = current_work_linear_idx < cute.size( - self.params.problem_layout_ncluster_mnl, loc=loc, ip=ip - ) - - # Choose coordinate calculation method based on swizzle configuration - if self.params.swizzle_size == 1: - # Use FastDivmod optimization for non-swizzled layouts - cur_cluster_coord = self._get_cluster_work_idx_with_fastdivmod( - current_work_linear_idx, loc=loc, ip=ip - ) - else: - # Use get_flat_coord for swizzled layouts (FastDivmod doesn't support them) - cur_cluster_coord = self.params.problem_layout_ncluster_mnl.get_flat_coord( - current_work_linear_idx, loc=loc, ip=ip - ) - - cur_tile_coord = tuple( - cute.arch.make_warp_uniform(Int32(x) * Int32(z) + Int32(y)) - for x, y, z in zip( - cur_cluster_coord, - self.cta_id_in_cluster, # type: ignore[arg-type] - (*self.params.cluster_shape_mn, Int32(1)), - ) - ) - - return WorkTileInfo(cur_tile_coord, cute.arch.make_warp_uniform(is_valid)) - - def _get_cluster_work_idx_with_fastdivmod( - self, - current_work_linear_idx: Int32, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> Tuple[Int32, Int32, Int32]: - """ - FastDivmod optimized CLUSTER coordinate calculation. - - CRITICAL: This should mimic problem_layout_ncluster_mnl.get_hier_coord() - which returns CLUSTER coordinates, not tile coordinates! - - :param current_work_linear_idx: Linear index in the work space - :type current_work_linear_idx: Int32 - :return: Cluster coordinates (m, n, l) or None if FastDivmod not available - :rtype: Tuple[Int32, Int32, Int32] or None - """ - - # Step 1: Decode current_work_linear_idx using FastDivmod objects - # The layout structure is: problem_layout_ncluster_mnl has shape (cluster_count_m, cluster_count_n, batch_count) - # current_work_linear_idx needs to be decomposed into (batch_l, cluster_minor, cluster_major) in little-endian order - - # First, get cluster_major using cluster_shape_major_fdd - cluster_minor_batch, cluster_major = divmod( - current_work_linear_idx, self.params.cluster_shape_major_fdd - ) - - # Then decode cluster_minor_batch to get cluster_minor and batch_l using FastDivmod - batch_l, cluster_minor = divmod(cluster_minor_batch, self.params.cluster_shape_minor_fdd) - - if self.params.raster_along_m: - cluster_m = cluster_major - cluster_n = cluster_minor - else: - cluster_m = cluster_minor - cluster_n = cluster_major - - return (cluster_m, cluster_n, batch_l) - - @dsl_user_op - @cute.jit - def get_current_work( - self, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> WorkTileInfo: - return self._get_current_work_for_linear_idx(self._current_work_linear_idx, loc=loc, ip=ip) - - @dsl_user_op - @cute.jit - def initial_work_tile_info( - self, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> WorkTileInfo: - return self.get_current_work(loc=loc, ip=ip) - - @dsl_user_op - @cute.jit - def advance_to_next_work( - self, - *, - advance_count: int = 1, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> None: - self._current_work_linear_idx += Int32(advance_count) * Int32(self.num_persistent_clusters) - self._num_tiles_executed += Int32(1) - - @property - @cute.jit - def num_tiles_executed(self) -> Int32: - return self._num_tiles_executed - - -class StaticPersistentRuntimeTileScheduler(StaticPersistentTileScheduler): - """A scheduler for static persistent runtime tile execution in CUTLASS/CuTe kernels. - This scheduler will always launch all the SMs and the scheduler will generate the real tile info for each SM. - - :ivar params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl - :type params: PersistentTileSchedulerParams - :ivar num_persistent_clusters: Number of persistent clusters that can be launched - :type num_persistent_clusters: Int32 - :ivar cta_id_in_cluster: ID of the CTA within its cluster - :type cta_id_in_cluster: cute.Coord - :ivar _num_tiles_executed: Counter for executed tiles - :type _num_tiles_executed: Int32 - :ivar _current_work_linear_idx: Current cluster index - :type _current_work_linear_idx: Int32 - """ - - def __init__( - self, - params: PersistentTileSchedulerParams, - num_persistent_clusters: Int32, - current_work_linear_idx: Int32, - cta_id_in_cluster: cute.Coord, - num_tiles_executed: Int32, - inner_mode: int = 1, - ): - """ - Initializes the StaticPersistentRuntimeTileScheduler with the given parameters. - - :param params: Tile schedule related params, including cluster shape and problem_layout_ncluster_mnl. - :type params: PersistentTileSchedulerParams - :param num_persistent_clusters: Number of persistent clusters that can be launched. - :type num_persistent_clusters: Int32 - :param current_work_linear_idx: Current cluster index. - :type current_work_linear_idx: Int32 - :param cta_id_in_cluster: ID of the CTA within its cluster. - :type cta_id_in_cluster: cute.Coord - :param num_tiles_executed: Counter for executed tiles. - :type num_tiles_executed: Int32 - :param inner_mode: The inner mode along which the linear index will be decomposed first. - :type inner_mode: int - """ - super().__init__( - params, - num_persistent_clusters, - current_work_linear_idx, - cta_id_in_cluster, - num_tiles_executed, - ) - if inner_mode not in [0, 1]: - raise ValueError( - f"inner_mode must be 0(for M mode) or 1(for N mode), but got {inner_mode}" - ) - self.inner_mode = inner_mode - - def __new_from_mlir_values__( - self, values: list[ir.Value] - ) -> "StaticPersistentRuntimeTileScheduler": - assert len(values) >= 6 - new_num_persistent_clusters = new_from_mlir_values( - self.num_persistent_clusters, [values[0]] - ) - new_current_work_linear_idx = new_from_mlir_values( - self._current_work_linear_idx, [values[1]] - ) - new_cta_id_in_cluster = new_from_mlir_values(self.cta_id_in_cluster, values[2:5]) - new_num_tiles_executed = new_from_mlir_values(self._num_tiles_executed, [values[5]]) - - # Reconstruct params with FastDivmod divisors (same as parent class) - params_values = values[6:] # Remaining values are from params - new_params = new_from_mlir_values(self.params, params_values) - - return StaticPersistentRuntimeTileScheduler( - new_params, # Use reconstructed params with FastDivmod divisors - new_num_persistent_clusters, - new_current_work_linear_idx, - new_cta_id_in_cluster, - new_num_tiles_executed, - self.inner_mode, - ) - - @staticmethod - @dsl_user_op - def create( - params: PersistentTileSchedulerParams, - block_idx: Tuple[Integer, Integer, Integer], - grid_dim: Tuple[Integer, Integer, Integer], - inner_mode: int = 1, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> "StaticPersistentRuntimeTileScheduler": - """Initialize the static persistent tile scheduler. - - :param params: Parameters for the persistent - tile scheduler. - :type params: PersistentTileSchedulerParams - :param block_idx: The 3d block index in the format (bidx, bidy, bidz). - :type block_idx: Tuple[Integer, Integer, Integer] - :param grid_dim: The 3d grid dimensions for kernel launch. - :type grid_dim: Tuple[Integer, Integer, Integer] - :param inner_mode: The inner mode along which the linear index will be decomposed first. - :type inner_mode: int - - :return: A StaticPersistentRuntimeTileScheduler object. - :rtype: StaticPersistentRuntimeTileScheduler - """ - - # Calculate the number of persistent clusters by dividing the total grid size - # by the number of CTAs per cluster - num_persistent_clusters = cute.size(grid_dim, loc=loc, ip=ip) // cute.size( - params.cluster_shape_mn, loc=loc, ip=ip - ) - - bidx, bidy, bidz = block_idx - - # Initialize workload index equals to the cluster index in the grid - current_work_linear_idx = Int32(bidz) - - # CTA id in the cluster - cta_id_in_cluster = ( - Int32(bidx % params.cluster_shape_mn[0]), - Int32(bidy % params.cluster_shape_mn[1]), - Int32(0), - ) - # Initialize number of tiles executed to zero - num_tiles_executed = Int32(0) - return StaticPersistentRuntimeTileScheduler( - params, - num_persistent_clusters, - current_work_linear_idx, - cta_id_in_cluster, - num_tiles_executed, - inner_mode, - ) - - # private method - def _get_current_work_for_linear_idx( - self, - current_work_linear_idx: Int32, - *, - loc: Optional[ir.Location] = None, - ip: Optional[ir.InsertionPoint] = None, - ) -> WorkTileInfo: - """Compute current tile coord given current_work_linear_idx and cta_id_in_cluster. - - :param current_work_linear_idx: The linear index of the current work. - :type current_work_linear_idx: Int32 - - :return: An object containing information about the current tile coordinates - and validity status. - :rtype: WorkTileInfo - """ - ntile_shape = self.params.problem_layout_ncluster_mnl.shape - int_max = 2147483647 - if const_expr(self.inner_mode == 1): - ntile_layout = cute.make_layout((int_max, ntile_shape[1]), stride=(ntile_shape[1], 1)) - else: - ntile_layout = cute.make_layout((ntile_shape[0], int_max), stride=(1, ntile_shape[0])) - cluster_tile_coord_mn = ntile_layout.get_hier_coord(current_work_linear_idx) - cur_tile_coord = ( - cluster_tile_coord_mn[0], - cluster_tile_coord_mn[1], - Int32(0), - ) - - # it is determined by kernel implementation - is_valid = Boolean(True) - - return WorkTileInfo(cur_tile_coord, is_valid) diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/attention.py b/tensorrt_llm/_torch/visual_gen/models/flux/attention.py index 136695d9d2e9..c308c1979b23 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/attention.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/attention.py @@ -109,11 +109,6 @@ def __init__( mapping=config.mapping, tensor_parallel_mode=TensorParallelMode.COLUMN, reduce_output=False, - override_tp_sharding={ - "q": (self.local_q_dim_start, self.local_q_dim_end), - "k": (self.local_kv_dim_start, self.local_kv_dim_end), - "v": (self.local_kv_dim_start, self.local_kv_dim_end), - }, ) # Need not pass any mapping info since this is intra-head normalization @@ -143,7 +138,6 @@ def __init__( allreduce_strategy=config.allreduce_strategy, tensor_parallel_mode=TensorParallelMode.ROW, reduce_output=True, - override_tp_sharding=(self.local_kv_dim_start, self.local_kv_dim_end), ) def apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: @@ -364,7 +358,6 @@ def __init__( skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, config=config, - attn_shard=(self.local_q_dim_start, self.local_q_dim_end), ) def _init_qkv_proj(self): @@ -381,11 +374,6 @@ def _init_qkv_proj(self): skip_create_weights_in_init=self.skip_create_weights_in_init, force_dynamic_quantization=self.force_dynamic_quantization, mapping=self.mapping, - override_qkv_sharding={ - "q": (self.local_q_dim_start, self.local_q_dim_end), - "k": (self.local_kv_dim_start, self.local_kv_dim_end), - "v": (self.local_kv_dim_start, self.local_kv_dim_end), - }, ) def _apply_norm_rope_unfused( diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py b/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py index db6d2e336818..edb908c2d80a 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/joint_proj.py @@ -53,7 +53,6 @@ def __init__( skip_create_weights_in_init: bool = False, force_dynamic_quantization: bool = False, config: Optional[DiffusionModelConfig] = None, - attn_shard: Optional[tuple[int, int]] = None, ): super().__init__() mapping = config.mapping if config else None @@ -61,11 +60,6 @@ def __init__( self.tp_rank = getattr(mapping, "tp_rank", 0) self.attn_dim = attn_dim self.has_bias = bias - self.attn_shard = attn_shard - - assert attn_dim % self.tp_size == 0 or self.attn_shard is not None, ( - "Explicit attention sharding required for uneven TP" - ) if self.tp_size == 1: self.proj = Linear( @@ -90,7 +84,6 @@ def __init__( mapping=config.mapping, tensor_parallel_mode=TensorParallelMode.ROW, reduce_output=False, - override_tp_sharding=self.attn_shard, ) self.mlp_proj = Linear( mlp_dim, @@ -169,12 +162,10 @@ def __init__( skip_create_weights_in_init: bool = False, force_dynamic_quantization: bool = False, mapping: Optional[Mapping] = None, - override_qkv_sharding=None, ): super().__init__() self.tp_size = mapping.tp_size if mapping else 1 - self.tp_rank = mapping.tp_rank if mapping else 0 # Store full (pre-TP) dims for weight loading (splitting checkpoint weight) self.full_q_dim = q_dim @@ -197,15 +188,9 @@ def __init__( self.local_qkv_dim = q_dim + 2 * kv_dim self.local_mlp_dim = mlp_dim else: - assert override_qkv_sharding is not None, ( - "override_qkv_sharding required when tp_size > 1" - ) - - def range_size(r): - return r[1] - r[0] - - local_q_dim = range_size(override_qkv_sharding["q"]) - local_kv_dim = range_size(override_qkv_sharding["k"]) + local_q_dim = q_dim // self.tp_size + local_kv_dim = kv_dim // self.tp_size + shard_mlp_hidden_dim = self.mlp_hidden_dim // self.tp_size # QKV: column-parallel with fused Q/K/V sharding self.qkv_proj = Linear( in_dim, @@ -226,17 +211,8 @@ def range_size(r): mapping=mapping, tensor_parallel_mode=TensorParallelMode.COLUMN, reduce_output=False, - override_tp_sharding=override_qkv_sharding, ) - - local_mlp_hidden_start = Linear._calc_shard( - self.mlp_hidden_dim, self.tp_size, self.tp_rank - ) - local_mlp_hidden_end = Linear._calc_shard( - self.mlp_hidden_dim, self.tp_size, self.tp_rank + 1 - ) - local_mlp_hidden_size = local_mlp_hidden_end - local_mlp_hidden_start - + # MLP gate+up: column-parallel with fused gate/up sharding self.mlp_proj = Linear( in_dim, mlp_dim, @@ -249,19 +225,15 @@ def range_size(r): weight_mode=WeightMode.FUSED_GATE_UP_LINEAR, ), fused_weight_shard_indices_mapping={ - "gate": (0, local_mlp_hidden_size), - "up": (local_mlp_hidden_size, local_mlp_hidden_size), + "gate": (0, shard_mlp_hidden_dim), + "up": (shard_mlp_hidden_dim, shard_mlp_hidden_dim), }, mapping=mapping, tensor_parallel_mode=TensorParallelMode.COLUMN, reduce_output=False, - override_tp_sharding={ - "gate": (local_mlp_hidden_start, local_mlp_hidden_end), - "up": (local_mlp_hidden_start, local_mlp_hidden_end), - }, ) - self.local_qkv_dim = local_q_dim + 2 * local_kv_dim - self.local_mlp_dim = local_mlp_hidden_size + self.local_qkv_dim = (q_dim + 2 * kv_dim) // self.tp_size + self.local_mlp_dim = mlp_dim // self.tp_size def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """Returns (qkv, mlp_gate_up) with local (post-TP) sizes.""" diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux.py b/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux.py index e08d07047878..0a35ce659122 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux.py @@ -242,9 +242,9 @@ def post_load_weights(self) -> None: ) ) - # TeaCache or Cache-DiT: resolve coefficients here; the loader enables - # cache acceleration after torch.compile (see PipelineLoader.load). + # TeaCache or Cache-DiT self._apply_teacache_coefficients(FLUX_TEACACHE_COEFFICIENTS) + self._setup_cache_acceleration() @property def default_generation_params(self): diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux2.py b/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux2.py index 1b2116f7f27f..4f0fa7391895 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux2.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/pipeline_flux2.py @@ -336,9 +336,8 @@ def post_load_weights(self) -> None: ) ) - # Cache acceleration itself is enabled by the loader after - # torch.compile (see PipelineLoader.load). self._apply_teacache_coefficients(FLUX2_TEACACHE_COEFFICIENTS) + self._setup_cache_acceleration() @property def default_generation_params(self): diff --git a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py index 527dce31df23..d6551918141d 100644 --- a/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py +++ b/tensorrt_llm/_torch/visual_gen/models/flux/transformer_flux.py @@ -467,22 +467,11 @@ def __init__( ) self.act_mlp = _gelu_tanh_eager - # Attention (no added_kv_proj_dim since tokens are already concatenated) - self.attn = FluxJointAttention( - hidden_size=dim, - num_attention_heads=num_attention_heads, - head_dim=attention_head_dim, - bias=True, - eps=1e-6, - pre_only=True, # No output projection in attention - config=config, - layer_idx=layer_idx, - module_name=f"single_transformer_blocks.{layer_idx}.attn", - ) + kv_dim = num_attention_heads * attention_head_dim # MLP + Attn Output projection, requires special handling for TP self.proj_out = FluxJointAttnMLPProj( - attn_dim=self.attn.q_dim, + attn_dim=kv_dim, mlp_dim=self.mlp_hidden_dim, out_dim=dim, bias=True, @@ -491,8 +480,19 @@ def __init__( skip_create_weights_in_init=skip_create_weights, force_dynamic_quantization=force_dynamic_quant, config=config, - # need explicit shard because we are aligned on head boundaries - attn_shard=(self.attn.local_q_dim_start, self.attn.local_q_dim_end), + ) + + # Attention (no added_kv_proj_dim since tokens are already concatenated) + self.attn = FluxJointAttention( + hidden_size=dim, + num_attention_heads=num_attention_heads, + head_dim=attention_head_dim, + bias=True, + eps=1e-6, + pre_only=True, # No output projection in attention + config=config, + layer_idx=layer_idx, + module_name=f"single_transformer_blocks.{layer_idx}.attn", ) def forward( diff --git a/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2.py b/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2.py index 45a46ea3b015..1336bb7249bc 100644 --- a/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2.py +++ b/tensorrt_llm/_torch/visual_gen/models/ltx2/pipeline_ltx2.py @@ -1026,9 +1026,7 @@ def post_load_weights(self) -> None: """Finalize after weight loading: TeaCache, Cache-DiT, derived attributes.""" super().post_load_weights() - # LTX-2: single transformer (one DiT for video+audio); TeaCache only with - # explicit coefficients. Cache acceleration itself is enabled by the - # loader after torch.compile (see PipelineLoader.load). + # LTX-2: single transformer (one DiT for video+audio); TeaCache only with explicit coefficients. if self.transformer is not None and self.pipeline_config.cache_backend == "teacache": if self.pipeline_config.teacache.coefficients is None: raise ValueError( @@ -1039,6 +1037,11 @@ def post_load_weights(self) -> None: "LTXModel", LTX2TeaCacheExtractor(self._compute_ltx2_timestep_embedding), ) + self._setup_cache_acceleration() + + # Cache-DiT + if self.transformer is not None and self.pipeline_config.cache_backend == "cache_dit": + self._setup_cache_acceleration() # Compression ratios from native scale factors self.vae_spatial_compression_ratio = VIDEO_SCALE_FACTORS.width diff --git a/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image.py b/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image.py index 54c2310cd56d..1290453c57a0 100644 --- a/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image.py +++ b/tensorrt_llm/_torch/visual_gen/models/qwen_image/pipeline_qwen_image.py @@ -486,8 +486,6 @@ def forward( # Denoise loop. timer.mark_denoise_start() logger.info("Denoising (%d steps)...", len(timesteps)) - pipeline_config = getattr(self, "pipeline_config", None) - cuda_graph_enabled = getattr(getattr(pipeline_config, "cuda_graph", None), "enable", False) for i, t in enumerate(timesteps): timestep = t.expand(latents.shape[0]).to(latents.dtype) noise_pred = self.transformer( @@ -499,11 +497,6 @@ def forward( return_dict=False, )[0] - if do_true_cfg and cuda_graph_enabled: - # CUDA graph outputs are graph-owned buffers; the negative CFG - # replay may reuse the same pool before guidance consumes this one. - noise_pred = noise_pred.clone() - if do_true_cfg: neg_noise_pred = self.transformer( hidden_states=latents, diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/parallel_vae.py b/tensorrt_llm/_torch/visual_gen/models/wan/parallel_vae.py index 68f9a4e68a5d..6c1e3e1c2c2d 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/parallel_vae.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/parallel_vae.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from typing import Literal import torch @@ -21,7 +6,6 @@ from diffusers.models.autoencoders.autoencoder_kl_wan import WanAttentionBlock, WanCausalConv3d from diffusers.models.autoencoders.vae import DecoderOutput, DiagonalGaussianDistribution -from tensorrt_llm._torch.visual_gen.models.wan import wan_vae from tensorrt_llm._torch.visual_gen.modules.vae import ( HaloExchangeConv, HaloExchangeConv2dStride2, @@ -51,12 +35,6 @@ def forward(self, x, cache_x=None, *args, **kwargs): class ParallelVAE_Wan(ParallelVAEBase): """Parallel VAE wrapper for ``AutoencoderKLWan``.""" - # Module classes replaced with parallel variants. Subclasses that wrap a - # different VAE implementation (e.g. the native ``WanVAE``) override these - # to target their own module classes; everything else is inherited. - _conv3d_cls: type = WanCausalConv3d - _attn_cls: type = WanAttentionBlock - @staticmethod def make_spec(split_dim: Literal["height", "width"]) -> SplitSpec: # WAN tensor shapes: @@ -110,7 +88,7 @@ def _replace_conv3d(self, model: nn.Module) -> None: targets = [ (name, module) for name, module in model.named_modules() - if isinstance(module, self._conv3d_cls) and max(module.kernel_size) > 1 + if isinstance(module, WanCausalConv3d) and max(module.kernel_size) > 1 ] for name, module in targets: self._replace_module( @@ -130,7 +108,7 @@ def _replace_attention(self, model: nn.Module) -> None: targets = [ (name, module) for name, module in model.named_modules() - if isinstance(module, self._attn_cls) + if isinstance(module, WanAttentionBlock) ] for name, module in targets: self._replace_module( @@ -194,24 +172,3 @@ def _replace_resample_conv2d_stride2(self, model: nn.Module) -> None: pad_before_conv=pad_module.padding, ), ) - - -# Two parallel-VAE wrappers, one per VAE *class* (not a temporary transition): -# ParallelVAE_Wan wraps the diffusers AutoencoderKLWan -- used by Cosmos3 -# (models/cosmos3) and the Wan TRTLLM_USE_DIFFUSER_VAE -# debug fallback. -# ParallelVAE_TrtllmWan wraps the native WanVAE -- the default for Wan2.1/2.2. -# They share all splitting logic via the base class; only the conv3d/attention -# module classes differ. ParallelVAE_Wan stays as long as any model uses the -# diffusers AutoencoderKLWan. -class ParallelVAE_TrtllmWan(ParallelVAE_Wan): - """Parallel VAE wrapper for the native ``WanVAE``. - - Identical parallelisation to ``ParallelVAE_Wan``; only the conv3d/attention - module classes differ (the native ``WanCausalConv3d`` / ``WanAttentionBlock`` - in ``wan_vae.py``). Resample Conv2d replacement is inherited unchanged because - the native ``WanConv2d`` subclasses ``nn.Conv2d``. - """ - - _conv3d_cls = wan_vae.WanCausalConv3d - _attn_cls = wan_vae.WanAttentionBlock diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py index 8230f10823e5..41a284e040c3 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import os import time from typing import List, Optional, Union @@ -266,6 +251,7 @@ def load_standard_components( self.vae = load_wan_vae( checkpoint_dir, device, + self.pipeline_config.visual_gen_mapping, dtype=self.pipeline_config.torch_dtype, ) @@ -345,14 +331,16 @@ def post_load_weights(self) -> None: if not self.is_wan22_14b: self._apply_teacache_coefficients(WAN_TEACACHE_COEFFICIENTS) + self._setup_cache_acceleration() + else: + if self.pipeline_config.cache_backend == "cache_dit": + self._setup_cache_acceleration() if self.transformer_2 is not None: if hasattr(self.transformer_2, "post_load_weights"): self.transformer_2.post_load_weights() - # Wan 2.2 TeaCache validation after both transformers' post_load_weights - # (FP8 scales, etc.). Cache acceleration itself is enabled by the loader - # after torch.compile (see PipelineLoader.load). + # Wan 2.2 TeaCache after both transformers' post_load_weights (FP8 scales, etc.) if ( self.transformer is not None and self.transformer_2 is not None @@ -365,6 +353,7 @@ def post_load_weights(self) -> None: "teacache.coefficients_2 (high-noise and low-noise stage polynomials). " "There is no built-in coefficient table for Wan 2.2." ) + self._setup_cache_acceleration() def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: with torch.no_grad(): diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py index f83d5be29f1b..f472c4c1fe45 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import json import os import time @@ -251,6 +236,7 @@ def load_standard_components( self.vae = load_wan_vae( checkpoint_dir, device, + self.pipeline_config.visual_gen_mapping, dtype=self.pipeline_config.torch_dtype, ) @@ -356,13 +342,15 @@ def post_load_weights(self) -> None: if not self.is_wan22_14b: self._apply_teacache_coefficients(WAN_I2V_TEACACHE_COEFFICIENTS) + self._setup_cache_acceleration() + else: + if self.pipeline_config.cache_backend == "cache_dit": + self._setup_cache_acceleration() if self.transformer_2 is not None: if hasattr(self.transformer_2, "post_load_weights"): self.transformer_2.post_load_weights() - # Wan 2.2 TeaCache validation; cache acceleration itself is enabled by - # the loader after torch.compile (see PipelineLoader.load). if ( self.transformer is not None and self.transformer_2 is not None @@ -375,6 +363,7 @@ def post_load_weights(self) -> None: "teacache.coefficients_2 (high-noise and low-noise stage polynomials). " "There is no built-in coefficient table for Wan 2.2." ) + self._setup_cache_acceleration() def _run_warmup(self, height: int, width: int, num_frames: int, steps: int) -> None: dummy_image = PIL.Image.new("RGB", (width, height)) diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py index 5a4077652cdd..71852014121c 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/transformer_wan.py @@ -425,7 +425,6 @@ def __init__( force_dynamic_quantization=force_dynamic_quant, tensor_parallel_mode=tp_mode, reduce_output=False, - override_tp_sharding=(self.attn2.local_kv_dim_start, self.attn2.local_kv_dim_end), ) self.add_v_proj = Linear( added_kv_proj_dim, @@ -437,7 +436,6 @@ def __init__( force_dynamic_quantization=force_dynamic_quant, tensor_parallel_mode=tp_mode, reduce_output=False, - override_tp_sharding=(self.attn2.local_kv_dim_start, self.attn2.local_kv_dim_end), ) self.norm_added_k = RMSNormTPAware( hidden_size=hidden_size, @@ -446,7 +444,6 @@ def __init__( has_weights=True, enable_tp=(tp_size > 1), mapping=model_config.mapping, - override_tp_sharding=(self.attn2.local_kv_dim_start, self.attn2.local_kv_dim_end), ) # Use torch.empty().normal_(std=...) instead of torch.randn()/scale for MetaInitMode compatibility diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/vae_loader.py b/tensorrt_llm/_torch/visual_gen/models/wan/vae_loader.py index ff2b73f02862..eaf2e6afe64a 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/vae_loader.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/vae_loader.py @@ -15,6 +15,7 @@ import os from pathlib import Path +from typing import Any import torch import torch.nn as nn @@ -49,24 +50,36 @@ def _use_diffuser_vae_env() -> bool: ) from None -def _use_native_wan_vae() -> bool: +def _vae_is_parallel(visual_gen_mapping: Any | None) -> bool: + if visual_gen_mapping is None: + return False + return getattr(visual_gen_mapping, "parallel_vae_size", 1) > 1 + + +def _use_native_wan_vae(visual_gen_mapping: Any | None) -> bool: """Select the Wan VAE backend and log the reason. - The native ``WanVAE`` is the default; the diffusers ``AutoencoderKLWan`` - is used only when forced via ``TRTLLM_USE_DIFFUSER_VAE``. + The native ``WanVAE`` is the default. Diffusers ``AutoencoderKLWan`` is used + only when the VAE is tensor-parallel (``parallel_vae_size > 1``), which the + native path does not support yet, or when forced via + ``TRTLLM_USE_DIFFUSER_VAE``. """ if _use_diffuser_vae_env(): logger.info(f"Loading Diffusers Wan VAE because {TRTLLM_USE_DIFFUSER_VAE_ENV} is non-zero.") return False + if _vae_is_parallel(visual_gen_mapping): + logger.info("Loading Diffusers Wan VAE because parallel VAE is not supported natively yet.") + return False return True def load_wan_vae( checkpoint_dir: str, device: torch.device, + visual_gen_mapping: Any | None = None, dtype: torch.dtype = torch.bfloat16, ) -> nn.Module: - if not _use_native_wan_vae(): + if not _use_native_wan_vae(visual_gen_mapping): return AutoencoderKLWan.from_pretrained( checkpoint_dir, subfolder="vae", diff --git a/tensorrt_llm/_torch/visual_gen/modules/attention.py b/tensorrt_llm/_torch/visual_gen/modules/attention.py index 7677401cf41b..58b61a280b02 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/attention.py +++ b/tensorrt_llm/_torch/visual_gen/modules/attention.py @@ -76,7 +76,10 @@ def __init__( self.bias = bias self.tp_size = self.mapping.tp_size if self.mapping else 1 - self.tp_rank = self.mapping.tp_rank if self.mapping else 0 + assert ( + self.num_attention_heads % self.tp_size == 0 + and self.num_key_value_heads % self.tp_size == 0 + ), "TP size must divide the number of Query and KV Heads" # Fused QK Norm + RoPE: each model class opts in via fuse_qk_norm_rope. # Backed by torch.ops.trtllm.fused_dit_qk_norm_rope which auto-dispatches: @@ -124,7 +127,11 @@ def __init__( self.q_dim = self.num_attention_heads * self.head_dim self.kv_dim = self.num_key_value_heads * self.head_dim - self._calculate_tp_parameters(ulysses_size if enable_sequence_parallel else None) + self.local_num_attention_heads = self.num_attention_heads // self.tp_size + self.local_num_key_value_heads = self.num_key_value_heads // self.tp_size + self.local_q_dim = self.local_num_attention_heads * self.head_dim + self.local_kv_dim = self.local_num_key_value_heads * self.head_dim + self._init_qkv_proj() # Structural eligibility for SEPARATE_QKV self-attn quantize dedup. @@ -150,12 +157,6 @@ def __init__( q_norm_dim = self.head_dim if qk_norm_mode == "per_head" else self.q_dim k_norm_dim = self.head_dim if qk_norm_mode == "per_head" else self.kv_dim enable_tp_rms = self.tp_size > 1 and qk_norm_mode == "full" - - q_start = self.local_q_dim_start - q_end = self.local_q_dim_end - k_start = self.local_kv_dim_start - k_end = self.local_kv_dim_end - self.norm_q = RMSNormTPAware( hidden_size=q_norm_dim, eps=self.eps, @@ -163,7 +164,6 @@ def __init__( has_weights=True, enable_tp=enable_tp_rms, mapping=self.mapping, - override_tp_sharding=(q_start, q_end) if qk_norm_mode == "full" else None, ) self.norm_k = RMSNormTPAware( hidden_size=k_norm_dim, @@ -172,7 +172,6 @@ def __init__( has_weights=True, enable_tp=enable_tp_rms, mapping=self.mapping, - override_tp_sharding=(k_start, k_end) if qk_norm_mode == "full" else None, ) # TODO: Use weight mapper to create just a Linear module @@ -190,7 +189,6 @@ def __init__( tensor_parallel_mode=TensorParallelMode.ROW if self.tp_size > 1 else None, reduce_output=(self.tp_size > 1), allreduce_strategy=self.allreduce_strategy, - override_tp_sharding=(self.local_q_dim_start, self.local_q_dim_end), ) ] ) @@ -278,41 +276,6 @@ def _qualified_module_name( prefix = f"{component_name}." return module_name if module_name.startswith(prefix) else f"{prefix}{module_name}" - def _calculate_tp_parameters(self, ulysses_size: Optional[int]): - assert self.num_attention_heads % self.num_key_value_heads == 0 - gqa_ratio = self.num_attention_heads // self.num_key_value_heads - - if not ulysses_size: - ulysses_size = 1 - - assert self.num_key_value_heads % ulysses_size == 0 - assert self.num_key_value_heads // ulysses_size >= self.tp_size - - kv_heads_per_ulysses = self.num_key_value_heads // ulysses_size - self.local_key_value_head_start = ( - Linear._calc_shard(kv_heads_per_ulysses, self.tp_size, self.tp_rank) * ulysses_size - ) - self.local_key_value_head_end = ( - Linear._calc_shard(kv_heads_per_ulysses, self.tp_size, self.tp_rank + 1) * ulysses_size - ) - self.local_num_key_value_heads = ( - self.local_key_value_head_end - self.local_key_value_head_start - ) - - self.local_attention_head_start = gqa_ratio * self.local_key_value_head_start - self.local_attention_head_end = gqa_ratio * self.local_key_value_head_end - self.local_num_attention_heads = ( - self.local_attention_head_end - self.local_attention_head_start - ) - - self.local_q_dim_start = self.local_attention_head_start * self.head_dim - self.local_q_dim_end = self.local_attention_head_end * self.head_dim - self.local_q_dim = self.local_q_dim_end - self.local_q_dim_start - - self.local_kv_dim_start = self.local_key_value_head_start * self.head_dim - self.local_kv_dim_end = self.local_key_value_head_end * self.head_dim - self.local_kv_dim = self.local_kv_dim_end - self.local_kv_dim_start - def _init_qkv_proj(self) -> None: tp_mode = TensorParallelMode.COLUMN if self.tp_size > 1 else None @@ -340,11 +303,6 @@ def _init_qkv_proj(self) -> None: }, tensor_parallel_mode=tp_mode, reduce_output=False, - override_tp_sharding={ - "q": (self.local_q_dim_start, self.local_q_dim_end), - "k": (self.local_kv_dim_start, self.local_kv_dim_end), - "v": (self.local_kv_dim_start, self.local_kv_dim_end), - }, ) else: self.to_q = Linear( @@ -358,7 +316,6 @@ def _init_qkv_proj(self) -> None: force_dynamic_quantization=self.force_dynamic_quantization, tensor_parallel_mode=tp_mode, reduce_output=False, - override_tp_sharding=(self.local_q_dim_start, self.local_q_dim_end), ) self.to_k = Linear( self.hidden_size, @@ -371,7 +328,6 @@ def _init_qkv_proj(self) -> None: force_dynamic_quantization=self.force_dynamic_quantization, tensor_parallel_mode=tp_mode, reduce_output=False, - override_tp_sharding=(self.local_kv_dim_start, self.local_kv_dim_end), ) self.to_v = Linear( self.hidden_size, @@ -384,7 +340,6 @@ def _init_qkv_proj(self) -> None: force_dynamic_quantization=self.force_dynamic_quantization, tensor_parallel_mode=tp_mode, reduce_output=False, - override_tp_sharding=(self.local_kv_dim_start, self.local_kv_dim_end), ) def get_qkv( diff --git a/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py b/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py index 76da6b2ad18e..b58239ca8b9b 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py +++ b/tensorrt_llm/_torch/visual_gen/modules/rms_norm.py @@ -19,7 +19,6 @@ from torch import nn from tensorrt_llm._torch.distributed import AllReduce -from tensorrt_llm._torch.modules.linear import Linear # for Linear._calc_shard from tensorrt_llm.functional import AllReduceStrategy from tensorrt_llm.mapping import Mapping @@ -37,7 +36,6 @@ def __init__( enable_tp: bool = False, mapping: Optional[Mapping] = None, allreduce_strategy: AllReduceStrategy = AllReduceStrategy.NCCL, - override_tp_sharding: Optional[tuple] = None, ): super().__init__() @@ -47,43 +45,30 @@ def __init__( self.mapping = mapping self.enable_tp = enable_tp - self.hidden_size = hidden_size - if enable_tp: assert mapping is not None + self.full_size = hidden_size + shard = hidden_size // mapping.tp_size + start = shard * mapping.tp_rank + end = min(shard * (mapping.tp_rank + 1), hidden_size) + hidden_size = end - start - if override_tp_sharding: - self.tp_sharding = override_tp_sharding - else: - start = Linear._calc_shard(self.hidden_size, mapping.tp_size, mapping.tp_rank) - end = Linear._calc_shard(self.hidden_size, mapping.tp_size, mapping.tp_rank + 1) - self.tp_sharding = (start, end) - - start, end = self.tp_sharding - self.local_hidden_size = end - start self.allreduce = AllReduce( mapping=mapping, strategy=allreduce_strategy, dtype=torch.float32 ) else: - self.local_hidden_size = self.hidden_size self.allreduce = None if use_gemma and not has_weights: raise ValueError("has_weights must be True if use_gemma is True") if has_weights: if not use_gemma: - self.weight = nn.Parameter( - torch.ones(self.local_hidden_size, dtype=dtype, device=device) - ) + self.weight = nn.Parameter(torch.ones(hidden_size, dtype=dtype, device=device)) else: - self.weight = nn.Parameter( - torch.zeros(self.local_hidden_size, dtype=dtype, device=device) - ) + self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype, device=device)) else: self.register_buffer( - "weight", - torch.ones(self.local_hidden_size, dtype=dtype, device=device), - persistent=False, + "weight", torch.ones(hidden_size, dtype=dtype, device=device), persistent=False ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -93,7 +78,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: x2 = hidden_states.pow(2) if self.allreduce: x2_sum = x2.sum(-1, keepdim=True) - variance = self.allreduce(x2_sum) / self.hidden_size + variance = self.allreduce(x2_sum) / self.full_size else: variance = x2.mean(-1, keepdim=True) @@ -110,7 +95,9 @@ def load_weights(self, weights: torch.Tensor): if param is None or param_name not in weights: continue if param_name == "weight" and self.enable_tp: - start, end = self.tp_sharding + shard = self.full_size // self.mapping.tp_size + start = shard * self.mapping.tp_rank + end = min(shard * (self.mapping.tp_rank + 1), self.full_size) data = weights[param_name][..., start:end] else: data = weights[param_name] diff --git a/tensorrt_llm/_torch/visual_gen/modules/vae/conv.py b/tensorrt_llm/_torch/visual_gen/modules/vae/conv.py index 74684998352b..bdf56dc6ee71 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/vae/conv.py +++ b/tensorrt_llm/_torch/visual_gen/modules/vae/conv.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from typing import List, Optional import torch @@ -20,25 +5,6 @@ import torch.nn as nn -def _spatial_channels_last_format(x: torch.Tensor) -> Optional[torch.memory_format]: - """Return ``x``'s channels-last memory format (2D or 3D), or ``None``. - - The halo exchange builds row-major send/recv buffers and concatenates them - with ``x`` along the split dimension. When ``x`` is channels-last (as the - native Wan VAE convs require and produce), a mixed-format ``torch.cat`` falls - back to row-major, so every downstream conv re-converts via a full-tensor - ``channels_last`` copy. Materialising the halo slices in ``x``'s format - before the ``cat`` lets it preserve channels-last, making the conv's - ``_channels_last_*_if_needed`` a no-op. Returns ``None`` when ``x`` is not - unambiguously channels-last, in which case the caller leaves layout untouched. - """ - if x.dim() == 5 and x.is_contiguous(memory_format=torch.channels_last_3d): - return torch.channels_last_3d - if x.dim() == 4 and x.is_contiguous(memory_format=torch.channels_last): - return torch.channels_last - return None - - class HaloExchangeConv(nn.Module): """Wraps a stride-1 convolution with halo exchange for spatial-parallel decoding. @@ -156,13 +122,6 @@ def _exchange_halos(self, x: torch.Tensor) -> torch.Tensor: if self.halo_right < exchange_size: recv_from_right = torch.narrow(recv_from_right, dim, 0, self.halo_right) - # Match the halo slices' layout to ``x`` so the cat preserves - # channels-last and downstream convs skip a full-tensor re-conversion. - mf = _spatial_channels_last_format(x) - if mf is not None: - recv_from_left = recv_from_left.contiguous(memory_format=mf) - recv_from_right = recv_from_right.contiguous(memory_format=mf) - return torch.cat([recv_from_left, x, recv_from_right], dim=dim) def _strip_halo(self, x: torch.Tensor) -> torch.Tensor: @@ -274,11 +233,6 @@ def _recv_from_right(self, x: torch.Tensor) -> torch.Tensor: dist.send(send_left, dst=self.rank - 1) if self.rank != self.world_size - 1: - # Match the halo slice's layout to ``x`` so the cat preserves - # channels-last (see ``_spatial_channels_last_format``). - mf = _spatial_channels_last_format(x) - if mf is not None: - right_context = right_context.contiguous(memory_format=mf) x = torch.cat([x, right_context], dim=dim) if self.rank == self.world_size - 1: diff --git a/tensorrt_llm/_torch/visual_gen/modules/vae/parallel_vae_interface.py b/tensorrt_llm/_torch/visual_gen/modules/vae/parallel_vae_interface.py index 85d57d911bd0..8d0d23c72673 100644 --- a/tensorrt_llm/_torch/visual_gen/modules/vae/parallel_vae_interface.py +++ b/tensorrt_llm/_torch/visual_gen/modules/vae/parallel_vae_interface.py @@ -1,18 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - from dataclasses import dataclass from typing import Any, Dict, List, Literal, Optional, Tuple, Type @@ -151,10 +136,6 @@ class ParallelVAEFactory: "tensorrt_llm._torch.visual_gen.models.wan.parallel_vae", "ParallelVAE_Wan", ), - "tensorrt_llm._torch.visual_gen.models.wan.wan_vae.WanVAE": ( - "tensorrt_llm._torch.visual_gen.models.wan.parallel_vae", - "ParallelVAE_TrtllmWan", - ), } @classmethod diff --git a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py index 884f90f97d45..3551371fa173 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py @@ -303,15 +303,6 @@ def load( else: logger.info("torch.compile disabled by config") - # Cache acceleration (TeaCache / Cache-DiT) is enabled AFTER torch.compile - # on purpose: Cache-DiT captures references to the transformer block - # modules at enable time, while torch_compile() replaces the block lists - # with compiled copies. If Cache-DiT were enabled first, it would keep - # running the stale eager blocks and torch.compile would contribute - # nothing. - if getattr(pipeline, "transformer", None) is not None: - pipeline._setup_cache_acceleration() - if not skip_warmup: if config.torch_compile.enable_autotune: with autotune( diff --git a/tensorrt_llm/_torch/weight_sharing/__init__.py b/tensorrt_llm/_torch/weight_sharing/__init__.py index 38ec2471c41c..31b89d648298 100644 --- a/tensorrt_llm/_torch/weight_sharing/__init__.py +++ b/tensorrt_llm/_torch/weight_sharing/__init__.py @@ -14,18 +14,6 @@ # limitations under the License. """Backend-agnostic weight-sharing utilities (MX, GMS, ...).""" -from tensorrt_llm._torch.weight_sharing.artifact_identity import ( - ARTIFACT_IDENTITY_FORMAT_VERSION, - ArtifactIdentity, -) -from tensorrt_llm._torch.weight_sharing.post_transform_profiles import ( - PostTransformFeature, - PostTransformProfile, - PostTransformProfileRegistry, - PostTransformQualificationDecision, - PostTransformQualificationReason, - PostTransformTransferScope, -) from tensorrt_llm._torch.weight_sharing.source_identity import ( SOURCE_IDENTITY_FORMAT_VERSION, IdentityCheckDecision, @@ -37,15 +25,7 @@ ) __all__ = [ - "ARTIFACT_IDENTITY_FORMAT_VERSION", - "ArtifactIdentity", "SOURCE_IDENTITY_FORMAT_VERSION", - "PostTransformFeature", - "PostTransformProfile", - "PostTransformProfileRegistry", - "PostTransformQualificationDecision", - "PostTransformQualificationReason", - "PostTransformTransferScope", "SourceIdentity", "IdentityMatchResult", "IdentityCheckPolicy", diff --git a/tensorrt_llm/_torch/weight_sharing/artifact_identity.py b/tensorrt_llm/_torch/weight_sharing/artifact_identity.py deleted file mode 100644 index d1b8908e241a..000000000000 --- a/tensorrt_llm/_torch/weight_sharing/artifact_identity.py +++ /dev/null @@ -1,221 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Immutable checkpoint identity for shared-weight compatibility checks.""" - -from __future__ import annotations - -import hashlib -import json -import os -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -ARTIFACT_IDENTITY_FORMAT_VERSION = 1 - -_HF_SNAPSHOT_SCHEME = "hf_snapshot_revision" -_CHECKPOINT_MANIFEST_SCHEME = "checkpoint_manifest_sha256" -_SUPPORTED_SCHEMES = frozenset({_HF_SNAPSHOT_SCHEME, _CHECKPOINT_MANIFEST_SCHEME}) -_IGNORED_DIRECTORY_NAMES = frozenset({".cache", ".git", "__pycache__"}) -_IGNORED_FILE_NAMES = frozenset({".DS_Store"}) -_HASH_CHUNK_SIZE = 1024 * 1024 - - -def _canonical_hash(value: Any) -> str: - payload = json.dumps(value, sort_keys=True, separators=(",", ":")) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def _is_hex(value: str, lengths: tuple[int, ...]) -> bool: - return len(value) in lengths and all(char in "0123456789abcdef" for char in value) - - -def _hf_snapshot_descriptor(path: Path) -> tuple[str, str] | None: - """Return an immutable HF revision and repository-relative subpath.""" - parts = path.resolve().parts - for index, part in enumerate(parts[:-1]): - if part != "snapshots" or index == 0: - continue - if not parts[index - 1].startswith("models--"): - continue - - revision = parts[index + 1].lower() - if not _is_hex(revision, (40, 64)): - continue - subpath = "/".join(parts[index + 2 :]) - return revision, subpath - return None - - -def _raise_walk_error(error: OSError) -> None: - raise error - - -def _checkpoint_files(path: Path) -> tuple[Path, list[Path]]: - if path.is_file(): - return path.parent, [path] - - files = [] - for directory, directory_names, file_names in os.walk(path, onerror=_raise_walk_error): - retained_directories = [] - for directory_name in directory_names: - if directory_name in _IGNORED_DIRECTORY_NAMES: - continue - nested_directory = Path(directory) / directory_name - if nested_directory.is_symlink(): - raise ValueError( - "Checkpoint manifests do not support nested symlinked directories: " - f"{nested_directory}" - ) - retained_directories.append(directory_name) - directory_names[:] = retained_directories - for file_name in file_names: - if file_name in _IGNORED_FILE_NAMES: - continue - candidate = Path(directory) / file_name - if candidate.is_file(): - files.append(candidate) - files.sort(key=lambda candidate: candidate.relative_to(path).as_posix()) - if not files: - raise ValueError(f"Checkpoint path contains no files: {path}") - return path, files - - -def _sha256_file(path: Path) -> tuple[int, str]: - before = path.stat() - digest = hashlib.sha256() - with path.open("rb") as checkpoint_file: - for chunk in iter(lambda: checkpoint_file.read(_HASH_CHUNK_SIZE), b""): - digest.update(chunk) - after = path.stat() - if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns): - raise RuntimeError(f"Checkpoint file changed while being fingerprinted: {path}") - return after.st_size, digest.hexdigest() - - -def _checkpoint_manifest_digest(path: Path) -> str: - root, files = _checkpoint_files(path) - manifest = [] - for checkpoint_file in files: - size, digest = _sha256_file(checkpoint_file) - manifest.append( - { - "path": checkpoint_file.relative_to(root).as_posix(), - "size": size, - "sha256": digest, - } - ) - return _canonical_hash( - { - "format_version": ARTIFACT_IDENTITY_FORMAT_VERSION, - "files": manifest, - } - ) - - -@dataclass(frozen=True) -class ArtifactIdentity: - """Versioned identity of the immutable checkpoint artifact being loaded. - - `SourceIdentity` embeds this value as a global compatibility component. - Hugging Face cache snapshots use their immutable commit revision; local - checkpoints use a canonical manifest of relative paths, sizes, and file - content digests. Absolute paths are intentionally excluded. - """ - - format_version: int - scheme: str - digest: str - - def __post_init__(self) -> None: - if not isinstance(self.format_version, int) or isinstance(self.format_version, bool): - raise ValueError("ArtifactIdentity format version must be an integer") - if self.format_version != ARTIFACT_IDENTITY_FORMAT_VERSION: - raise ValueError(f"Unsupported ArtifactIdentity format version: {self.format_version}") - if not isinstance(self.scheme, str): - raise ValueError("ArtifactIdentity scheme must be a string") - if self.scheme not in _SUPPORTED_SCHEMES: - raise ValueError(f"Unsupported ArtifactIdentity scheme: {self.scheme}") - if not isinstance(self.digest, str): - raise ValueError("ArtifactIdentity digest must be a string") - - normalized_digest = self.digest.lower() - if not _is_hex(normalized_digest, (64,)): - raise ValueError("ArtifactIdentity digest must be a 64-character hex value") - object.__setattr__(self, "digest", normalized_digest) - - @classmethod - def from_checkpoint(cls, checkpoint_path: str | os.PathLike[str]) -> "ArtifactIdentity": - """Build an identity from an immutable snapshot or local checkpoint. - - Args: - checkpoint_path: A model checkpoint file or directory. - - Returns: - The path-independent checkpoint identity. - - Raises: - FileNotFoundError: If `checkpoint_path` does not exist. - ValueError: If a local checkpoint directory contains no files. - RuntimeError: If a local checkpoint changes while it is hashed. - - Note: - Local checkpoints have no authoritative immutable revision, so - their regular files are read in full to derive a content-bound - manifest. Hugging Face cache snapshots use the resolved immutable - revision without rereading model shards. - """ - path = Path(checkpoint_path).expanduser() - if not path.exists(): - raise FileNotFoundError(f"Checkpoint path does not exist: {path}") - - snapshot_descriptor = _hf_snapshot_descriptor(path) - if snapshot_descriptor is not None: - revision, subpath = snapshot_descriptor - digest = _canonical_hash( - { - "scheme": _HF_SNAPSHOT_SCHEME, - "revision": revision, - "subpath": subpath, - } - ) - return cls( - format_version=ARTIFACT_IDENTITY_FORMAT_VERSION, - scheme=_HF_SNAPSHOT_SCHEME, - digest=digest, - ) - - return cls( - format_version=ARTIFACT_IDENTITY_FORMAT_VERSION, - scheme=_CHECKPOINT_MANIFEST_SCHEME, - digest=_checkpoint_manifest_digest(path), - ) - - def to_dict(self) -> dict[str, Any]: - """Return a JSON-serializable representation.""" - return { - "format_version": self.format_version, - "scheme": self.scheme, - "digest": self.digest, - } - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "ArtifactIdentity": - """Reconstruct and validate a serialized artifact identity.""" - return cls( - format_version=data["format_version"], - scheme=data["scheme"], - digest=data["digest"], - ) diff --git a/tensorrt_llm/_torch/weight_sharing/post_transform_profiles.py b/tensorrt_llm/_torch/weight_sharing/post_transform_profiles.py deleted file mode 100644 index e6b4c82b3d2c..000000000000 --- a/tensorrt_llm/_torch/weight_sharing/post_transform_profiles.py +++ /dev/null @@ -1,246 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Qualified model profiles for sharing post-transform weights. - -Staged post-load hooks make it possible to receive weights that already use -their final runtime layout. They do not prove that every root model and feature -combination is safe to skip one-shot transforms. This module records the exact -profiles that have completed that qualification. - -The registry deliberately matches root classes by identity rather than -``isinstance``. A subclass must have its own profile unless it was explicitly -qualified under the same architecture and lifecycle contract. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from enum import Enum -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from torch import nn - - -class PostTransformTransferScope(str, Enum): - """The model component represented by a post-transform transfer.""" - - TARGET_MODEL = "target_model" - LANGUAGE_MODEL = "language_model" - COMPLETE_MODEL = "complete_model" - - -class PostTransformFeature(str, Enum): - """Optional lifecycle features that require explicit qualification.""" - - SEPARATE_DRAFT_MODEL = "separate_draft_model" - - -class PostTransformQualificationReason(str, Enum): - """Structured outcome of matching a request to a qualified profile.""" - - QUALIFIED = "qualified" - ROOT_MODEL_CLASS_NOT_REGISTERED = "root_model_class_not_registered" - ARCHITECTURE_NOT_REGISTERED = "architecture_not_registered" - MODEL_TYPE_NOT_REGISTERED = "model_type_not_registered" - SPECULATIVE_MODE_NOT_REGISTERED = "speculative_mode_not_registered" - PROTOCOL_NOT_REGISTERED = "protocol_not_registered" - TRANSFER_SCOPE_NOT_REGISTERED = "transfer_scope_not_registered" - FEATURE_NOT_SUPPORTED = "feature_not_supported" - - -@dataclass(frozen=True) -class PostTransformProfile: - """One exact root-model profile qualified for post-transform sharing.""" - - profile_id: str - root_model_class: type[nn.Module] - architecture: str - model_type: str - speculative_mode: str | None - protocol_version: int - transfer_scope: PostTransformTransferScope - supported_features: frozenset[PostTransformFeature] = field(default_factory=frozenset) - - def __post_init__(self) -> None: - if not self.profile_id: - raise ValueError("Post-transform profile_id must not be empty") - if not self.architecture: - raise ValueError("Post-transform architecture must not be empty") - if not self.model_type: - raise ValueError("Post-transform model_type must not be empty") - if self.speculative_mode == "": - raise ValueError("Post-transform speculative_mode must not be empty") - if self.protocol_version < 1: - raise ValueError("Post-transform protocol_version must be positive") - object.__setattr__(self, "supported_features", frozenset(self.supported_features)) - - -@dataclass(frozen=True) -class PostTransformQualificationDecision: - """Result of looking up a requested post-transform receiver profile.""" - - reason: PostTransformQualificationReason - profile: PostTransformProfile | None = None - unsupported_features: frozenset[PostTransformFeature] = field(default_factory=frozenset) - - def __post_init__(self) -> None: - object.__setattr__(self, "unsupported_features", frozenset(self.unsupported_features)) - if self.reason is PostTransformQualificationReason.QUALIFIED: - if self.profile is None: - raise ValueError("A qualified decision must include its profile") - if self.unsupported_features: - raise ValueError("A qualified decision cannot include unsupported features") - elif self.unsupported_features and ( - self.reason is not PostTransformQualificationReason.FEATURE_NOT_SUPPORTED - ): - raise ValueError("Unsupported features require a feature_not_supported decision") - - @property - def qualified(self) -> bool: - """Whether the request exactly matched a qualified profile.""" - - return ( - self.reason is PostTransformQualificationReason.QUALIFIED and self.profile is not None - ) - - -@dataclass(frozen=True) -class PostTransformProfileRegistry: - """Immutable collection of audited post-transform profiles.""" - - profiles: tuple[PostTransformProfile, ...] - - def __post_init__(self) -> None: - object.__setattr__(self, "profiles", tuple(self.profiles)) - - profile_ids: set[str] = set() - profile_keys: set[ - tuple[type[nn.Module], str, str, str | None, int, PostTransformTransferScope] - ] = set() - for profile in self.profiles: - if profile.profile_id in profile_ids: - raise ValueError(f"Duplicate post-transform profile_id: {profile.profile_id!r}") - profile_ids.add(profile.profile_id) - - key = ( - profile.root_model_class, - profile.architecture, - profile.model_type, - profile.speculative_mode, - profile.protocol_version, - profile.transfer_scope, - ) - if key in profile_keys: - raise ValueError( - "Duplicate post-transform profile for " - f"{profile.root_model_class.__name__}/{profile.architecture}/" - f"{profile.model_type}/{profile.speculative_mode or 'target-only'}/" - f"v{profile.protocol_version}/{profile.transfer_scope.value}" - ) - profile_keys.add(key) - - def qualify( - self, - *, - root_model_class: type[nn.Module], - architecture: str | None, - model_type: str | None, - speculative_mode: str | None, - protocol_version: int, - transfer_scope: PostTransformTransferScope, - enabled_features: frozenset[PostTransformFeature] = frozenset(), - ) -> PostTransformQualificationDecision: - """Match a receiver request against the exact qualified profile set. - - Args: - root_model_class: Exact constructed root model class. - architecture: Canonical architecture from the resolved model config. - model_type: Canonical Hugging Face model type. - speculative_mode: Canonical speculative decoding mode, or `None` - for target-only loading. - protocol_version: Post-transform transfer protocol version. - transfer_scope: Component represented by the transfer. - enabled_features: Optional lifecycle features active for this load. - - Returns: - A structured qualification decision and the matching profile, when - one exists. - """ - - model_profiles = tuple( - profile for profile in self.profiles if profile.root_model_class is root_model_class - ) - if not model_profiles: - return PostTransformQualificationDecision( - PostTransformQualificationReason.ROOT_MODEL_CLASS_NOT_REGISTERED - ) - - architecture_profiles = tuple( - profile for profile in model_profiles if profile.architecture == architecture - ) - if not architecture_profiles: - return PostTransformQualificationDecision( - PostTransformQualificationReason.ARCHITECTURE_NOT_REGISTERED - ) - - model_type_profiles = tuple( - profile for profile in architecture_profiles if profile.model_type == model_type - ) - if not model_type_profiles: - return PostTransformQualificationDecision( - PostTransformQualificationReason.MODEL_TYPE_NOT_REGISTERED - ) - - speculative_mode_profiles = tuple( - profile - for profile in model_type_profiles - if profile.speculative_mode == speculative_mode - ) - if not speculative_mode_profiles: - return PostTransformQualificationDecision( - PostTransformQualificationReason.SPECULATIVE_MODE_NOT_REGISTERED - ) - - protocol_profiles = tuple( - profile - for profile in speculative_mode_profiles - if profile.protocol_version == protocol_version - ) - if not protocol_profiles: - return PostTransformQualificationDecision( - PostTransformQualificationReason.PROTOCOL_NOT_REGISTERED - ) - - scope_profiles = tuple( - profile for profile in protocol_profiles if profile.transfer_scope is transfer_scope - ) - if not scope_profiles: - return PostTransformQualificationDecision( - PostTransformQualificationReason.TRANSFER_SCOPE_NOT_REGISTERED - ) - - profile = scope_profiles[0] - unsupported_features = frozenset(enabled_features) - profile.supported_features - if unsupported_features: - return PostTransformQualificationDecision( - PostTransformQualificationReason.FEATURE_NOT_SUPPORTED, - profile=profile, - unsupported_features=unsupported_features, - ) - - return PostTransformQualificationDecision( - PostTransformQualificationReason.QUALIFIED, profile=profile - ) diff --git a/tensorrt_llm/_torch/weight_sharing/source_identity.py b/tensorrt_llm/_torch/weight_sharing/source_identity.py index c6b010634760..18168703c868 100644 --- a/tensorrt_llm/_torch/weight_sharing/source_identity.py +++ b/tensorrt_llm/_torch/weight_sharing/source_identity.py @@ -14,19 +14,17 @@ # limitations under the License. """Backend-agnostic source identity for weight-sharing receivers. -A :class:`SourceIdentity` is a serializable fingerprint of an immutable -checkpoint artifact and the configuration choices that affect how its weights -are laid out in memory. It exists so that a *receiver* of pre-laid-out weights -(e.g. MX peer-to-peer transfer, or a GMS read-only materialize) can verify that -both the producer ("source") and the consumer built identities and agree on the -artifact and every layout-affecting choice before consuming shared weights. +A :class:`SourceIdentity` is a serializable fingerprint of configuration choices +that affect how a model's weights are laid out in memory. It exists so that a +*receiver* of pre-laid-out weights (e.g. MX peer-to-peer transfer, or a GMS +read-only materialize) can verify that both the producer ("source") and the +consumer built identities and agree on every layout-affecting choice before the +receiver consumes shared weights. The identity is intentionally decoupled from any specific weight-sharing technology (neither MX nor GMS appears here). Both consume it identically: - local = SourceIdentity.from_model_config( - model_config, checkpoint_dir="/path/to/checkpoint" - ) + local = SourceIdentity.from_model_config(model_config) decision = check_weight_sharing_compatibility(local, source_identity, policy) if decision.should_share: ... # pull / materialize shared weights @@ -42,9 +40,8 @@ -------------- The fingerprint is split so comparison can be selective: -* **global fingerprint** -- immutable checkpoint artifact, rank-invariant - model identity, quantization, backend selection, fusion flags, and parallel - *sizes* (TP/PP/EP/CP). +* **global fingerprint** -- rank-invariant model identity, quantization, + backend selection, fusion flags, and parallel *sizes* (TP/PP/EP/CP). * **shard fingerprint** -- this rank's TP/PP/EP/CP *rank* slice plus the realized local parameter/buffer `(shape, dtype)` layout. Receiver rank `N` must align with the source rank that produced shard `N`. @@ -70,7 +67,6 @@ from enum import Enum from typing import TYPE_CHECKING, Any, List, Optional -from tensorrt_llm._torch.weight_sharing.artifact_identity import ArtifactIdentity from tensorrt_llm.logger import logger if TYPE_CHECKING: @@ -82,7 +78,7 @@ # Bump when the fingerprint projection changes in a way that makes previously # stored identities incomparable. Two identities with different format versions # never match. -SOURCE_IDENTITY_FORMAT_VERSION = 2 +SOURCE_IDENTITY_FORMAT_VERSION = 1 _PRETRAINED_METADATA_FIELDS = frozenset( { @@ -228,7 +224,6 @@ class SourceIdentity: format_version: int # --- global parts (must match across all ranks) --- - artifact_identity: ArtifactIdentity model_fingerprint: str quant_fingerprint: str backend_fingerprint: str @@ -251,8 +246,6 @@ def from_model_config( model_config: "ModelConfig", model: Optional["nn.Module"] = None, *, - checkpoint_dir: Optional[str] = None, - artifact_identity: Optional[ArtifactIdentity] = None, model_name: Optional[str] = None, ) -> "SourceIdentity": """Build an identity from a torch-backend :class:`ModelConfig`. @@ -265,12 +258,6 @@ def from_model_config( Producer and consumer must build the identity at the same lifecycle point (model construction, before weight load). When `None`, the shard fingerprint contains no tensor-layout data. - checkpoint_dir: Checkpoint file or directory used to derive the - nested artifact identity. Required unless `artifact_identity` - is supplied explicitly. - artifact_identity: Precomputed immutable checkpoint identity, - primarily for callers that resolve provenance outside this - method. Mutually exclusive with `checkpoint_dir`. model_name: Human-readable model identity used by discovery layers (e.g. the MX server's source catalog). Does not affect the compatibility fingerprints. @@ -278,17 +265,7 @@ def from_model_config( Returns: A fully populated :class:`SourceIdentity` for `model_config.mapping.rank`. - - Raises: - ValueError: If neither or both artifact identity inputs are given. """ - if checkpoint_dir is None and artifact_identity is None: - raise ValueError("Exactly one of checkpoint_dir or artifact_identity must be provided") - if checkpoint_dir is not None and artifact_identity is not None: - raise ValueError("Exactly one of checkpoint_dir or artifact_identity must be provided") - if artifact_identity is None: - artifact_identity = ArtifactIdentity.from_checkpoint(checkpoint_dir) - mapping = model_config.mapping rank = getattr(mapping, "rank", 0) @@ -298,7 +275,6 @@ def from_model_config( return cls( format_version=SOURCE_IDENTITY_FORMAT_VERSION, - artifact_identity=artifact_identity, model_fingerprint=cls._build_model_fingerprint(model_config), quant_fingerprint=cls._build_quant_fingerprint(model_config), backend_fingerprint=cls._build_backend_fingerprint(model_config), @@ -442,7 +418,6 @@ def global_fingerprint(self) -> str: return _canonical_hash( { "format_version": self.format_version, - "artifact": self.artifact_identity.to_dict(), "model": self.model_fingerprint, "quant": self.quant_fingerprint, "backend": self.backend_fingerprint, @@ -472,8 +447,6 @@ def matches( mismatched.append("format_version") if compare_global: - if self.artifact_identity != other.artifact_identity: - mismatched.append("artifact_identity") for name in ( "model_fingerprint", "quant_fingerprint", @@ -500,7 +473,6 @@ def to_dict(self) -> dict: """ return { "format_version": self.format_version, - "artifact_identity": self.artifact_identity.to_dict(), "model_fingerprint": self.model_fingerprint, "quant_fingerprint": self.quant_fingerprint, "backend_fingerprint": self.backend_fingerprint, @@ -524,13 +496,8 @@ def from_dict(cls, data: dict) -> "SourceIdentity": Returns: The reconstructed :class:`SourceIdentity`. """ - format_version = data["format_version"] - if format_version != SOURCE_IDENTITY_FORMAT_VERSION: - raise ValueError(f"Unsupported SourceIdentity format version: {format_version}") - return cls( - format_version=format_version, - artifact_identity=ArtifactIdentity.from_dict(data["artifact_identity"]), + format_version=data["format_version"], model_fingerprint=data["model_fingerprint"], quant_fingerprint=data["quant_fingerprint"], backend_fingerprint=data["backend_fingerprint"], @@ -581,8 +548,7 @@ def check_weight_sharing_compatibility( result = IdentityMatchResult(matched=False, mismatched_fields=missing_fields) message = ( "SourceIdentity unavailable for fields " - f"{missing_fields}; receiver cannot verify the source checkpoint " - "artifact and weight layout." + f"{missing_fields}; receiver cannot verify source weight layout." ) if policy is IdentityCheckPolicy.STRICT: raise SourceIdentityMismatchError(message) @@ -608,7 +574,7 @@ def check_weight_sharing_compatibility( message = ( "SourceIdentity mismatch on fields " f"{result.mismatched_fields}; receiver and source disagree on " - "the checkpoint artifact or weight layout." + "weight layout." ) if policy is IdentityCheckPolicy.STRICT: raise SourceIdentityMismatchError(message) diff --git a/tensorrt_llm/_utils.py b/tensorrt_llm/_utils.py index 71574f3e1aa5..67f1c9d3e950 100644 --- a/tensorrt_llm/_utils.py +++ b/tensorrt_llm/_utils.py @@ -31,6 +31,7 @@ from ctypes import byref from enum import EnumMeta from functools import lru_cache, partial, wraps +from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Sequence, TypeVar, Union import numpy as np @@ -63,7 +64,7 @@ has_nvml = False # isort: on -from tensorrt_llm.bindings import DataType, LayerType +from tensorrt_llm.bindings import DataType, GptJsonConfig, LayerType from tensorrt_llm.bindings.BuildInfo import ENABLE_MULTI_DEVICE from tensorrt_llm.logger import logger @@ -774,6 +775,13 @@ def __contains__(cls, item): return True +def supports_inflight_batching(engine_dir): + config_path = Path(engine_dir) / "config.json" + json_config = GptJsonConfig.parse_file(config_path) + model_config = json_config.model_config + return model_config.supports_inflight_batching + + class QuantModeWrapper: def __init__(self, objs): diff --git a/tensorrt_llm/bench/benchmark/__init__.py b/tensorrt_llm/bench/benchmark/__init__.py index 7dcf3cc9bbed..8386e115f505 100644 --- a/tensorrt_llm/bench/benchmark/__init__.py +++ b/tensorrt_llm/bench/benchmark/__init__.py @@ -6,9 +6,9 @@ from tensorrt_llm import LLM as PyTorchLLM from tensorrt_llm.bench.benchmark.utils.processes import IterationWriter +from tensorrt_llm.bench.build.build import get_model_config from tensorrt_llm.bench.dataclasses.configuration import RuntimeConfig from tensorrt_llm.bench.dataclasses.general import BenchmarkEnvironment -from tensorrt_llm.bench.tuning.settings import get_model_config from tensorrt_llm.commands.utils import \ collect_explicit_cli_keys as _collect_explicit_cli_keys from tensorrt_llm.logger import logger diff --git a/tensorrt_llm/bench/benchmark/throughput.py b/tensorrt_llm/bench/benchmark/throughput.py index b31a4608a5c3..844e63fb18ee 100755 --- a/tensorrt_llm/bench/benchmark/throughput.py +++ b/tensorrt_llm/bench/benchmark/throughput.py @@ -449,16 +449,6 @@ def throughput_command( kwargs = kwargs | runtime_config.get_llm_args() kwargs['skip_tokenizer_init'] = not no_skip_tokenizer_init kwargs['backend'] = options.backend - if (options.modality is None and options.backend == "pytorch" - and "disable_mm_encoder" not in kwargs - and not kwargs.get("mm_encoder_only", False)): - # Text-only benchmark: skip a multimodal checkpoint's encoder so - # its GPU memory goes to the KV cache pool instead. Text-only - # models ignore this flag. Overridable via extra_llm_api_options. - kwargs["disable_mm_encoder"] = True - logger.info( - "Text-only benchmark (--modality not set): the multimodal " - "encoder, if the model has one, will not be loaded.") if bench_env.telemetry_config is not None: kwargs["telemetry_config"] = bench_env.telemetry_config diff --git a/tensorrt_llm/bench/benchmark/utils/general.py b/tensorrt_llm/bench/benchmark/utils/general.py index ba3c312bd45a..e28121b18aba 100755 --- a/tensorrt_llm/bench/benchmark/utils/general.py +++ b/tensorrt_llm/bench/benchmark/utils/general.py @@ -9,12 +9,12 @@ from tensorrt_llm._torch.pyexecutor.model_loader import \ validate_and_set_kv_cache_quant +from tensorrt_llm.bench.build.build import (get_benchmark_engine_settings, + get_model_config) +from tensorrt_llm.bench.build.dataclasses import (NemotronHybridConfig, + Qwen3HybridConfig) from tensorrt_llm.bench.dataclasses.general import (DatasetMetadata, InferenceRequest) -from tensorrt_llm.bench.tuning.dataclasses import (NemotronHybridConfig, - Qwen3HybridConfig) -from tensorrt_llm.bench.tuning.settings import (get_benchmark_engine_settings, - get_model_config) from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo diff --git a/tensorrt_llm/bench/build/__init__.py b/tensorrt_llm/bench/build/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tensorrt_llm/bench/tuning/settings.py b/tensorrt_llm/bench/build/build.py similarity index 77% rename from tensorrt_llm/bench/tuning/settings.py rename to tensorrt_llm/bench/build/build.py index 9f47292975ac..16bf6d23d9aa 100644 --- a/tensorrt_llm/bench/tuning/settings.py +++ b/tensorrt_llm/bench/build/build.py @@ -3,28 +3,21 @@ from pathlib import Path from typing import Tuple -from tensorrt_llm._torch.pyexecutor.config_utils import ( - is_nemotron_hybrid, - is_qwen3_hybrid, - load_pretrained_config, -) -from tensorrt_llm.bench.tuning.dataclasses import ( - ModelConfig, - NemotronHybridConfig, - Qwen3HybridConfig, -) -from tensorrt_llm.bench.tuning.heuristics import calc_engine_setting +from tensorrt_llm._torch.pyexecutor.config_utils import (is_nemotron_hybrid, + is_qwen3_hybrid, + load_pretrained_config) +from tensorrt_llm.bench.build.dataclasses import (ModelConfig, + NemotronHybridConfig, + Qwen3HybridConfig) +from tensorrt_llm.bench.build.tuning import calc_engine_setting from tensorrt_llm.llmapi.llm_args import TorchLlmArgs from tensorrt_llm.llmapi.llm_utils import QuantConfig from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo TUNED_QUANTS = { - QuantAlgo.NVFP4, - QuantAlgo.FP8, - QuantAlgo.FP8_BLOCK_SCALES, - QuantAlgo.NO_QUANT, - None, + QuantAlgo.NVFP4, QuantAlgo.FP8, QuantAlgo.FP8_BLOCK_SCALES, + QuantAlgo.NO_QUANT, None } # Sourced from TorchLlmArgs so the bench defaults track the args-class field # defaults and can't drift. @@ -42,7 +35,7 @@ def get_benchmark_engine_settings( kv_cache_gpu_mem_fraction: float = 0.95, enable_attention_dp: bool = False, ) -> Tuple[int, int]: - """Retrieve benchmark settings for a specific model + configuration. + """ Retrieve benchmark settings for a specific model + configuration. Args: model_config (ModelConfig): Model specific configurations. @@ -84,14 +77,13 @@ def get_benchmark_engine_settings( ) if max_batch_size <= 0 or max_num_tokens <= 0: - raise RuntimeError("Unable to obtain correct settings for benchmark.") + raise RuntimeError(f"Unable to obtain correct settings for benchmark.") return max_batch_size, max_num_tokens def get_model_config(model_name: str, model_path: Path = None) -> ModelConfig: - """Obtain the model-related parameters from Hugging Face. - + """ Obtain the model-related parameters from Hugging Face. Args: model_name (str): Huggingface model name. model_path (Path): Path to a local Huggingface checkpoint. @@ -99,7 +91,8 @@ def get_model_config(model_name: str, model_path: Path = None) -> ModelConfig: Raises: ValueError: When model is not supported. """ - pretrained_config = load_pretrained_config(model_path or model_name, trust_remote_code=True) + pretrained_config = load_pretrained_config(model_path or model_name, + trust_remote_code=True) if is_nemotron_hybrid(pretrained_config): return NemotronHybridConfig.from_hf(model_name, model_path) if is_qwen3_hybrid(pretrained_config): diff --git a/tensorrt_llm/bench/tuning/dataclasses.py b/tensorrt_llm/bench/build/dataclasses.py similarity index 65% rename from tensorrt_llm/bench/tuning/dataclasses.py rename to tensorrt_llm/bench/build/dataclasses.py index ca7e58a4fb51..37d63062f23a 100755 --- a/tensorrt_llm/bench/tuning/dataclasses.py +++ b/tensorrt_llm/bench/build/dataclasses.py @@ -1,42 +1,40 @@ -import json -import os -import struct -from typing import Literal, Optional - +from typing import Optional, Literal +from pydantic import AliasPath, BaseModel, Field, AliasChoices, model_validator import huggingface_hub from huggingface_hub.constants import ( SAFETENSORS_INDEX_FILE, SAFETENSORS_MAX_HEADER_LENGTH, SAFETENSORS_SINGLE_FILE, ) -from huggingface_hub.utils import SafetensorsFileMetadata, SafetensorsRepoMetadata, TensorInfo +from huggingface_hub.utils import SafetensorsRepoMetadata, SafetensorsFileMetadata, TensorInfo from huggingface_hub.utils import tqdm as hf_tqdm -from pydantic import AliasChoices, AliasPath, BaseModel, Field, model_validator from tqdm.contrib.concurrent import thread_map +import os +import json +import struct from tensorrt_llm._torch.pyexecutor.config_utils import ( - get_qwen3_hybrid_layer_types, - load_pretrained_config, -) + load_pretrained_config, get_qwen3_hybrid_layer_types) # Mapping from safetensors dtype strings to bytes per element. # Used to compute checkpoint size from per-dtype element counts. SAFETENSORS_DTYPE_BYTES = { - "F64": 8, - "F32": 4, - "F16": 2, - "BF16": 2, - "I64": 8, - "I32": 4, - "I16": 2, - "I8": 1, - "U8": 1, - "BOOL": 1, - "F8_E4M3": 1, + 'F64': 8, + 'F32': 4, + 'F16': 2, + 'BF16': 2, + 'I64': 8, + 'I32': 4, + 'I16': 2, + 'I8': 1, + 'U8': 1, + 'BOOL': 1, + 'F8_E4M3': 1, } def parse_safetensors_file_metadata(model_path, filename): + with open(os.path.join(model_path, filename), "rb") as f: metadata_size = f.read(8) metadata_size = struct.unpack(" None: files_metadata[filename] = parse_safetensors_file_metadata( - model_path=model_name_or_path, filename=filename - ) + model_path=model_name_or_path, filename=filename) thread_map( _parse, @@ -124,8 +123,7 @@ def _parse(filename: str) -> None: else: # Not a safetensors repo raise RuntimeError( - f"'{model_name_or_path}' is not a safetensors repo. Couldn't find " - f"'{SAFETENSORS_INDEX_FILE}' or '{SAFETENSORS_SINGLE_FILE}' files." + f"'{model_name_or_path}' is not a safetensors repo. Couldn't find '{SAFETENSORS_INDEX_FILE}' or '{SAFETENSORS_SINGLE_FILE}' files." ) else: return huggingface_hub.get_safetensors_metadata(model_name_or_path) @@ -136,28 +134,23 @@ class ModelConfig(BaseModel): The parameters are needed in engine setting calculation. """ - name: str model_type: str param_count: int checkpoint_size_in_gb: float = Field(default=0.0) - num_hidden_layers: int = Field( - validation_alias=AliasChoices( - "num_hidden_layers", - "n_layer", - AliasPath("text_config", "num_hidden_layers"), - AliasPath("language_config", "num_hidden_layers"), - ) - ) + num_hidden_layers: int = Field(validation_alias=AliasChoices( + "num_hidden_layers", + "n_layer", + AliasPath("text_config", "num_hidden_layers"), + AliasPath("language_config", "num_hidden_layers"), + )) num_attention_layers: Optional[int] = Field(default=None) - num_attention_heads: int = Field( - validation_alias=AliasChoices( - "num_attention_heads", - "n_head", - AliasPath("text_config", "num_attention_heads"), - AliasPath("language_config", "num_attention_heads"), - ) - ) + num_attention_heads: int = Field(validation_alias=AliasChoices( + "num_attention_heads", + "n_head", + AliasPath("text_config", "num_attention_heads"), + AliasPath("language_config", "num_attention_heads"), + )) num_key_value_heads: Optional[int] = Field( default=None, validation_alias=AliasChoices( @@ -167,37 +160,33 @@ class ModelConfig(BaseModel): AliasPath("language_config", "num_key_value_heads"), ), ) - hidden_size: int = Field( - validation_alias=AliasChoices( - "hidden_size", - "n_embd", - AliasPath("text_config", "hidden_size"), - ) - ) - head_size: Optional[int] = Field( - default=None, - validation_alias=AliasChoices( - "head_size", - "head_dim", - "attention_head_dim", - AliasPath("text_config", "head_dim"), - ), - ) + hidden_size: int = Field(validation_alias=AliasChoices( + "hidden_size", + "n_embd", + AliasPath("text_config", "hidden_size"), + )) + head_size: Optional[int] = Field(default=None, + validation_alias=AliasChoices( + "head_size", + "head_dim", + "attention_head_dim", + AliasPath("text_config", "head_dim"), + )) max_position_embeddings: Optional[int] = Field( default=None, validation_alias=AliasChoices( "max_position_embeddings", "n_positions", AliasPath("text_config", "max_position_embeddings"), - ), - ) - dtype: Literal["float16", "bfloat16", "float32", None] = Field( - default="float16", validation_alias=AliasChoices("dtype", "torch_dtype") - ) + )) + dtype: Literal["float16", "bfloat16", "float32", + None] = Field(default="float16", + validation_alias=AliasChoices( + "dtype", "torch_dtype")) @model_validator(mode="after") def set_values_if_none(self): - """Set the values if cannot get values from HF config.json.""" + """ Set the values if cannot get values from HF config.json. """ if not self.dtype: # for GPT-J self.dtype = "float16" if self.num_key_value_heads is None: @@ -233,32 +222,28 @@ def get_param_count_and_checkpoint_size(cls, model_hf_name, hf_model_path): # BF16 for non-quantized layers, F8_E4M3 for scales, etc.). checkpoint_size_in_bytes = sum( count * SAFETENSORS_DTYPE_BYTES.get(dtype, 1) - for dtype, count in metadata.parameter_count.items() - ) + for dtype, count in metadata.parameter_count.items()) checkpoint_size_in_gb = checkpoint_size_in_bytes / (1024**3) if not param_count: - raise ValueError( - f"Can't get valid parameter count for model: {hf_model_path or model_hf_name}." - ) + raise ValueError(f"Can't get valid parameter count for model: " + f"{hf_model_path or model_hf_name}.") return param_count, checkpoint_size_in_gb @classmethod def from_hf(cls, model_hf_name, hf_model_path): - pretrained_config = load_pretrained_config( - hf_model_path or model_hf_name, trust_remote_code=True - ) + pretrained_config = load_pretrained_config(hf_model_path + or model_hf_name, + trust_remote_code=True) hf_config = pretrained_config.to_dict() - param_count, checkpoint_size_in_gb = cls.get_param_count_and_checkpoint_size( - model_hf_name, hf_model_path - ) + param_count, checkpoint_size_in_gb = ( + cls.get_param_count_and_checkpoint_size(model_hf_name, + hf_model_path)) - return cls( - name=model_hf_name, - param_count=param_count, - checkpoint_size_in_gb=checkpoint_size_in_gb, - **hf_config, - ) + return cls(name=model_hf_name, + param_count=param_count, + checkpoint_size_in_gb=checkpoint_size_in_gb, + **hf_config) def extra_model_cache_in_gb(self, bytes_per_elem, target_seq_len=None): return 0 @@ -268,35 +253,27 @@ def cache_memory_fraction(self, cache_memory_fraction): class NemotronHybridConfig(ModelConfig): - hybrid_override_pattern: str = Field( - validation_alias=AliasChoices( - "hybrid_override_pattern", - AliasPath("text_config", "hybrid_override_pattern"), - AliasPath("language_config", "hybrid_override_pattern"), - ) - ) - num_hidden_layers: int = Field( - validation_alias=AliasChoices( - "num_hidden_layers", - "n_layer", - AliasPath("text_config", "num_hidden_layers"), - AliasPath("language_config", "num_hidden_layers"), - ) - ) - d_state: int = Field( - validation_alias=AliasChoices( - "d_state", - "mamba_d_state", - "ssm_state_size", - ) - ) - d_conv: int = Field( - validation_alias=AliasChoices( - "d_conv", - "mamba_d_conv", - "conv_kernel", - ) - ) + hybrid_override_pattern: str = Field(validation_alias=AliasChoices( + "hybrid_override_pattern", + AliasPath("text_config", "hybrid_override_pattern"), + AliasPath("language_config", "hybrid_override_pattern"), + )) + num_hidden_layers: int = Field(validation_alias=AliasChoices( + "num_hidden_layers", + "n_layer", + AliasPath("text_config", "num_hidden_layers"), + AliasPath("language_config", "num_hidden_layers"), + )) + d_state: int = Field(validation_alias=AliasChoices( + "d_state", + "mamba_d_state", + "ssm_state_size", + )) + d_conv: int = Field(validation_alias=AliasChoices( + "d_conv", + "mamba_d_conv", + "conv_kernel", + )) mamba_num_heads: int n_groups: int mamba_head_dim: int @@ -306,7 +283,7 @@ class NemotronHybridConfig(ModelConfig): @model_validator(mode="after") def set_values_if_none(self): - """Set the values if cannot get values from HF config.json.""" + """ Set the values if cannot get values from HF config.json. """ if not self.d_inner: self.d_inner = self.mamba_num_heads * self.mamba_head_dim if self.num_mamba_layers is None: @@ -321,17 +298,12 @@ def extra_model_cache_in_gb(self, bytes_per_elem, target_seq_len=None): conv_dim = self.d_inner + 2 * self.n_groups * self.d_state conv_state_elems = conv_dim * (self.d_conv - 1) ssm_state_elems = self.mamba_num_heads * self.mamba_head_dim * self.d_state - gb_per_mamba_cache = ( - bytes_per_elem - * self.num_mamba_layers - * (conv_state_elems + ssm_state_elems) - / (1024**3) - ) + gb_per_mamba_cache = bytes_per_elem * self.num_mamba_layers * ( + conv_state_elems + ssm_state_elems) / (1024**3) return gb_per_mamba_cache def cache_memory_fraction(self, cache_memory_fraction): - # Each mamba cache entry is pretty large (~50MB for 8B model), so we are - # more conservative when estimating the max batch size + # Each mamba cache entry is pretty large (~50MB for 8B model), so we are more conservative when estimating the max batch size return cache_memory_fraction**2 def set_mamba_ssm_cache_dtype(self, mamba_ssm_cache_dtype: str): @@ -339,13 +311,13 @@ def set_mamba_ssm_cache_dtype(self, mamba_ssm_cache_dtype: str): @classmethod def from_hf(cls, model_hf_name, hf_model_path): - pretrained_config = load_pretrained_config( - hf_model_path or model_hf_name, trust_remote_code=True - ) + pretrained_config = load_pretrained_config(hf_model_path + or model_hf_name, + trust_remote_code=True) hf_config = pretrained_config.to_dict() - param_count, checkpoint_size_in_gb = cls.get_param_count_and_checkpoint_size( - model_hf_name, hf_model_path - ) + param_count, checkpoint_size_in_gb = ( + cls.get_param_count_and_checkpoint_size(model_hf_name, + hf_model_path)) # HuggingFace PretrainedConfig.to_dict() only serializes attributes known to # the base class; custom configs (e.g. NemotronHConfig) have num_hidden_layers @@ -354,8 +326,8 @@ def from_hf(cls, model_hf_name, hf_model_path): text_config = getattr(pretrained_config, "text_config", None) language_config = getattr(pretrained_config, "language_config", None) for key in ( - "num_hidden_layers", - "hybrid_override_pattern", + "num_hidden_layers", + "hybrid_override_pattern", ): if hf_config.get(key) is None: value = None @@ -370,12 +342,10 @@ def from_hf(cls, model_hf_name, hf_model_path): if value is not None: hf_config[key] = value - return cls( - name=model_hf_name, - param_count=param_count, - checkpoint_size_in_gb=checkpoint_size_in_gb, - **hf_config, - ) + return cls(name=model_hf_name, + param_count=param_count, + checkpoint_size_in_gb=checkpoint_size_in_gb, + **hf_config) class Qwen3HybridConfig(ModelConfig): @@ -384,7 +354,6 @@ class Qwen3HybridConfig(ModelConfig): Maps Qwen3.5 linear-attention parameters to the same cache estimation formulas used by NemotronHybridConfig. """ - linear_key_head_dim: int # d_state linear_conv_kernel_dim: int # d_conv linear_num_value_heads: int # num_heads (mamba_num_heads) @@ -395,38 +364,34 @@ class Qwen3HybridConfig(ModelConfig): @classmethod def from_hf(cls, model_hf_name, hf_model_path): - pretrained_config = load_pretrained_config( - hf_model_path or model_hf_name, trust_remote_code=True - ) + pretrained_config = load_pretrained_config(hf_model_path + or model_hf_name, + trust_remote_code=True) hf_config = pretrained_config.to_dict() - param_count, checkpoint_size_in_gb = cls.get_param_count_and_checkpoint_size( - model_hf_name, hf_model_path - ) + param_count, checkpoint_size_in_gb = ( + cls.get_param_count_and_checkpoint_size(model_hf_name, + hf_model_path)) layer_types = get_qwen3_hybrid_layer_types(pretrained_config) - hf_config.setdefault("num_attention_layers", layer_types.count("full_attention")) - hf_config.setdefault("num_linear_attention_layers", layer_types.count("linear_attention")) - - return cls( - name=model_hf_name, - param_count=param_count, - checkpoint_size_in_gb=checkpoint_size_in_gb, - **hf_config, - ) + hf_config.setdefault("num_attention_layers", + layer_types.count("full_attention")) + hf_config.setdefault("num_linear_attention_layers", + layer_types.count("linear_attention")) + + return cls(name=model_hf_name, + param_count=param_count, + checkpoint_size_in_gb=checkpoint_size_in_gb, + **hf_config) def extra_model_cache_in_gb(self, bytes_per_elem, target_seq_len=None): d_inner = self.linear_value_head_dim * self.linear_num_value_heads conv_dim = d_inner + 2 * self.linear_num_key_heads * self.linear_key_head_dim conv_state_elems = conv_dim * (self.linear_conv_kernel_dim - 1) - ssm_state_elems = ( - self.linear_num_value_heads * self.linear_value_head_dim * self.linear_key_head_dim - ) - gb_per_cache = ( - bytes_per_elem - * self.num_linear_attention_layers - * (conv_state_elems + ssm_state_elems) - / (1024**3) - ) + ssm_state_elems = (self.linear_num_value_heads * + self.linear_value_head_dim * + self.linear_key_head_dim) + gb_per_cache = bytes_per_elem * self.num_linear_attention_layers * ( + conv_state_elems + ssm_state_elems) / (1024**3) return gb_per_cache def cache_memory_fraction(self, cache_memory_fraction): diff --git a/tensorrt_llm/bench/tuning/heuristics.py b/tensorrt_llm/bench/build/tuning.py similarity index 79% rename from tensorrt_llm/bench/tuning/heuristics.py rename to tensorrt_llm/bench/build/tuning.py index 0b41fee9767f..d77cf6591dbb 100755 --- a/tensorrt_llm/bench/tuning/heuristics.py +++ b/tensorrt_llm/bench/build/tuning.py @@ -1,25 +1,20 @@ -import math from typing import Tuple import torch from tensorrt_llm._utils import str_dtype_to_torch -from tensorrt_llm.bench.tuning.dataclasses import ( - ModelConfig, - NemotronHybridConfig, - Qwen3HybridConfig, -) from tensorrt_llm.llmapi.llm_utils import QuantConfig from tensorrt_llm.logger import logger from tensorrt_llm.quantization.mode import QuantAlgo - +from tensorrt_llm.bench.build.dataclasses import ModelConfig, NemotronHybridConfig, Qwen3HybridConfig from .utils import get_device_memory +import math BYTES_PER_ELEM = { QuantAlgo.NO_QUANT: 2.0, QuantAlgo.FP8: 1.0, QuantAlgo.FP8_BLOCK_SCALES: 1.0, - QuantAlgo.NVFP4: 0.5, + QuantAlgo.NVFP4: .5, } @@ -33,14 +28,13 @@ def calc_engine_setting( kv_cache_gpu_mem_fraction: float = 0.95, enable_attention_dp: bool = False, ) -> Tuple[int, int]: - """Calculate the engine build settings (max batch size and max num tokens). - - For a specific model + parallelism mapping + dataset configuration, - trtllm-bench sets a slightly optimistic upper bound for max batch size - and max num tokens to avoid over-allocation of memory in activation, - runtime, and decoder buffers. In runtime, TRT-LLM relies on its runtime - tuning features to adjust the runtime max batch size according to - incoming traffic. + """ Calculate the engine build settings (max batch size and max num tokens) + for a specific model + parallelism mapping + dataset configuration. + trtllm-bench sets a slightly optimistic upper bound for max batch size + and max num tokens to avoid over-allocation of memory in activation, + runtime, and decoder buffers. In runtime, TRT-LLM relies on its runtime + tuning features to adjust the runtime max batch size according to + incoming traffic. Args: model_config (ModelConfig): Model specific configurations. @@ -69,16 +63,11 @@ def calc_engine_setting( # Each GPU in TP group has at least 1 kv head adjusted_num_kv_heads = max(tp_size, model_config.num_key_value_heads) - logger.info(f"Number of attention layers: {model_config.num_attention_layers}") + logger.info( + f"Number of attention layers: {model_config.num_attention_layers}") - gb_per_token = ( - 2 - * model_config.num_attention_layers - * adjusted_num_kv_heads - * model_config.head_size - * byte_per_kv_elem - / (1024**3) - ) + gb_per_token = 2 * model_config.num_attention_layers * adjusted_num_kv_heads \ + * model_config.head_size * byte_per_kv_elem / (1024 ** 3) # Number of GPU used for this run. n_gpus = tp_size * pp_size @@ -103,11 +92,13 @@ def calc_engine_setting( # Available memory to allocate KV cache. available_memory = total_gpu_memory - engine_size logger.info(f"Estimated engine size: {engine_size:.2f} GB") - logger.info(f"Estimated total available memory for KV cache: {available_memory:.2f} GB") + logger.info("Estimated total available memory for KV cache: " + f"{available_memory:.2f} GB") # Calculate max requests in KV cache based on target ISL and OSL. target_seq_len = target_input_len + target_output_len - cache_memory = available_memory * model_config.cache_memory_fraction(kv_cache_gpu_mem_fraction) + cache_memory = available_memory * model_config.cache_memory_fraction( + kv_cache_gpu_mem_fraction) bytes_per_elem = BYTES_PER_ELEM.get(QuantAlgo.NO_QUANT) if isinstance(model_config, (NemotronHybridConfig, Qwen3HybridConfig)): @@ -116,31 +107,30 @@ def calc_engine_setting( if str_dtype_to_torch(mamba_ssm_cache_dtype) == torch.float32: bytes_per_elem = 4.0 - gb_per_extra_cache = model_config.extra_model_cache_in_gb(bytes_per_elem, target_seq_len) - kv_cache_max_requests = cache_memory / (gb_per_token * target_seq_len + gb_per_extra_cache) + gb_per_extra_cache = model_config.extra_model_cache_in_gb( + bytes_per_elem, target_seq_len) + kv_cache_max_requests = cache_memory / (gb_per_token * target_seq_len + + gb_per_extra_cache) extra_cache_memory = gb_per_extra_cache * kv_cache_max_requests kv_cache_memory = cache_memory - extra_cache_memory kv_cache_max_tokens = kv_cache_memory / gb_per_token logger.info( - f"Estimated total cache memory: {cache_memory:.2f} GB. " - f"KV cache: {kv_cache_memory:.2f} GB, Extra cache: {extra_cache_memory:.2f} GB" + f"Estimated total cache memory: {cache_memory:.2f} GB. KV cache: {kv_cache_memory:.2f} GB, Extra cache: {extra_cache_memory:.2f} GB" ) logger.info(f"Estimated kv cache max tokens: {kv_cache_max_tokens:.2f}") - logger.info(f"Estimated max number of requests in KV cache memory: {kv_cache_max_requests:.2f}") + logger.info("Estimated max number of requests in KV cache memory: " + f"{kv_cache_max_requests:.2f}") # Fine-tune the max batch size and num token setting for performance. - # For mamba-attn hybrid models, we disable optimistic tuning because the - # mamba cache leaves less memory for the KV cache + # For mamba-attn hybrid models, we disable optimistic tuning because the mamba cache leaves less memory for the KV cache max_batch_size, max_num_tokens = finetune_setting( kv_cache_max_requests, target_input_len, target_output_len, pp_size, disable_optimistic_tuning=isinstance( - model_config, (NemotronHybridConfig, Qwen3HybridConfig) - ), - ) + model_config, (NemotronHybridConfig, Qwen3HybridConfig))) # Functional and performance if total_gpu_memory < engine_size: @@ -161,26 +151,21 @@ def calc_engine_setting( f"Number of Tensor Parallel Shards: {tp_size}\n" f"Number of Pipeline Parallel Stages: {pp_size}\n" f"KV Cache GPU Memory Fraction: {kv_cache_gpu_mem_fraction}\n" - "----------------------------------------------------------\n" - ) + "----------------------------------------------------------\n") if kv_cache_max_requests < 1: - raise RuntimeError( - "The amount of KV cache memory is insufficient to " - "run this model. Please try with more GPUs." - ) + raise RuntimeError("The amount of KV cache memory is insufficient to " + "run this model. Please try with more GPUs.") warning_gpu_count = pp_size if enable_attention_dp else n_gpus if cache_memory / warning_gpu_count < 10.0: logger.warning( - "The KV cache memory per GPU is less than 10 GB. " + f"The KV cache memory per GPU is less than 10 GB. " "Performance may be undesirable. Please consider using a different " - "mapping or more GPUs." - ) + "mapping or more GPUs.") if kv_cache_max_requests < 32: logger.warning( - "The maximum number of requests in the KV cache is too " + f"The maximum number of requests in the KV cache is too " "small. Performance may be undesirable. Please consider using more " - "GPUs or a different mapping to process more concurrent requests." - ) + "GPUs or a different mapping to process more concurrent requests.") return max_batch_size, max_num_tokens @@ -192,10 +177,9 @@ def finetune_setting( pp_size: int, disable_optimistic_tuning: bool = False, ) -> Tuple[int, int]: - """Calculate and fine-tune the engine build settings. - - Both max batch size and max num tokens are fine-tuned to be slightly - optimistic. + """ Calculate and fine-tune the engine build settings (max batch size and + max num tokens). Both max batch size and max num tokens are fine-tuned + to be slightly optimistic. Args: kv_cache_max_requests (float): Max number of requests that can fits in @@ -235,12 +219,11 @@ def finetune_setting( else: max_token = 1024 * math.ceil(raw_token / 1024) - logger.debug( - f"Estimated max batch size (before fine-tune): {kv_cache_max_requests / pp_size:.2f}" - ) + logger.debug(f"Estimated max batch size (before fine-tune): " + f"{kv_cache_max_requests / pp_size:.2f}") logger.debug( f"Estimated max num tokens (before fine-tune): " - f"{kv_cache_max_requests / pp_size * (1 + input_len / output_len):.2f}" + f"{kv_cache_max_requests / pp_size * (1 + input_len / output_len) :.2f}" ) logger.info(f"Estimated max batch size (after fine-tune): {max_bs}") logger.info(f"Estimated max num tokens (after fine-tune): {max_token}") diff --git a/tensorrt_llm/bench/build/utils.py b/tensorrt_llm/bench/build/utils.py new file mode 100644 index 000000000000..18d0bac8f022 --- /dev/null +++ b/tensorrt_llm/bench/build/utils.py @@ -0,0 +1,34 @@ +import pynvml + +DEFAULT_HF_MODEL_DIRS = { + 'BaichuanForCausalLM': 'baichuan-inc/Baichuan-13B-Chat', + 'BloomForCausalLM': 'bigscience/bloom-560m', + 'GLMModel': 'THUDM/glm-10b', + 'ChatGLMModel': 'THUDM/chatglm3-6b', + 'ChatGLMForCausalLM': 'THUDM/chatglm3-6b', + 'FalconForCausalLM': 'tiiuae/falcon-rw-1b', + 'GPTForCausalLM': 'gpt2-medium', + 'GPTJForCausalLM': 'EleutherAI/gpt-j-6b', + 'GPTNeoXForCausalLM': 'EleutherAI/gpt-neox-20b', + 'InternLMForCausalLM': 'internlm/internlm-chat-7b', + 'InternLM2ForCausalLM': 'internlm/internlm2-chat-7b', + 'LlamaForCausalLM': 'meta-llama/Llama-2-7b-hf', + 'MPTForCausalLM': 'mosaicml/mpt-7b', + 'PhiForCausalLM': 'microsoft/phi-2', + 'OPTForCausalLM': 'facebook/opt-350m', + 'QWenLMHeadModel': 'Qwen/Qwen-7B', + 'QWenForCausalLM': 'Qwen/Qwen-7B', + 'Qwen2ForCausalLM': 'Qwen/Qwen1.5-7B', + 'Qwen2MoeForCausalLM': 'Qwen/Qwen1.5-MoE-A2.7B', + 'RecurrentGemmaForCausalLM': 'google/recurrentgemma-2b', +} + + +def get_device_memory(): + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) + total_memory = mem_info.total / (1024**3) + pynvml.nvmlShutdown() + + return total_memory diff --git a/tensorrt_llm/bench/dataset/prepare_dataset.py b/tensorrt_llm/bench/dataset/prepare_dataset.py index 3f286a998553..aa7f4eb722e9 100644 --- a/tensorrt_llm/bench/dataset/prepare_dataset.py +++ b/tensorrt_llm/bench/dataset/prepare_dataset.py @@ -25,7 +25,7 @@ class RootArgs(BaseModel): tokenizer: str - output: Optional[str] + output: str random_seed: int task_id: int trust_remote_code: bool = False @@ -54,12 +54,6 @@ def validate_tokenizer(self): @click.option( "--output", type=str, help="Output json filename.", default="preprocessed_dataset.json" ) -@click.option( - "--stdout", - is_flag=True, - default=False, - help="Print the dataset to stdout with a JSON entry on each line instead of writing a file.", -) @click.option( "--random-seed", required=False, type=int, help="random seed for token_ids", default=420 ) @@ -80,15 +74,12 @@ def validate_tokenizer(self): def prepare_dataset(ctx, **kwargs): """Prepare dataset for benchmarking with trtllm-bench.""" model = ctx.obj.model or ctx.obj.checkpoint_path - # --stdout is encoded as a null output path (mutually exclusive with a file). - output = None if kwargs["stdout"] else kwargs["output"] - if output is not None: - output_path = Path(output) - output_path.parent.mkdir(parents=True, exist_ok=True) + output_path = Path(kwargs["output"]) + output_path.parent.mkdir(parents=True, exist_ok=True) ctx.obj = RootArgs( tokenizer=model, - output=output, + output=kwargs["output"], random_seed=kwargs["random_seed"], task_id=kwargs["task_id"], rand_task_id=kwargs["rand_task_id"], diff --git a/tensorrt_llm/bench/dataset/prepare_real_data.py b/tensorrt_llm/bench/dataset/prepare_real_data.py index 0a140482ee92..641d95a948f5 100644 --- a/tensorrt_llm/bench/dataset/prepare_real_data.py +++ b/tensorrt_llm/bench/dataset/prepare_real_data.py @@ -70,12 +70,11 @@ def query(self): def get_prompt(self, req): """Get the prompt sentence from the given request.""" if self.prompt_key: - if self.prompt_key not in req: - raise ValueError( - f"Dataset {self.name} does not have key '{self.prompt_key}'. " - "Please set --prompt-key to one of the available keys: " - f"{req.keys()}" - ) + assert self.prompt_key in req, ( + f"Dataset {self.name} does not have key '{self.prompt_key}'. " + "Please set --prompt-key to one of the available keys: " + f"{req.keys()}" + ) return req[self.prompt_key] elif self.prompt: return self.prompt @@ -84,23 +83,21 @@ def get_prompt(self, req): def get_input(self, req): """Get the input sentence from the given request.""" - if self.input_key not in req: - raise ValueError( - f"Dataset {self.name} does not have key '{self.input_key}'. " - "Please set --input-key to one of the available keys: " - f"{req.keys()}" - ) + assert self.input_key in req, ( + f"Dataset {self.name} does not have key '{self.input_key}'. " + "Please set --input-key to one of the available keys: " + f"{req.keys()}" + ) return req[self.input_key] def get_images(self, req): """Get the images from the given request.""" image_keys = [self.image_key] + [f"{self.image_key}_{i}" for i in range(1, 8)] - if not any(key in req for key in image_keys): - raise ValueError( - f"Dataset {self.name} does not have key '{self.image_key}'. " - "Please set --dataset-image-key to one of the available keys: " - f"{req.keys()}" - ) + assert any(key in req for key in image_keys), ( + f"Dataset {self.name} does not have key '{self.image_key}'. " + "Please set --dataset-image-key to one of the available keys: " + f"{req.keys()}" + ) images = [] for key in image_keys: if key in req and req[key] is not None: @@ -117,12 +114,11 @@ def get_output(self, req): "you wish to set output length to the length of the golden " "output, set --output-key." ) - if self.output_key not in req: - raise ValueError( - f"Dataset {self.name} does not have key '{self.output_key}'. " - "Please set --output-key to one of the available keys: " - f"{req.keys()}" - ) + assert self.output_key in req, ( + f"Dataset {self.name} does not have key '{self.output_key}'. " + "Please set --output-key to one of the available keys: " + f"{req.keys()}" + ) return req[self.output_key] @@ -244,11 +240,7 @@ def real_dataset(root_args, **kwargs): if any(key in req for key in ["image", "image_1", "video"]): # multimodal input if "video" in req and req["video"] is not None: - # Video inputs are not supported yet; warn and fall through to - # image handling. Raising here would reject datasets that carry - # a video column alongside images, which previously worked (the - # original `assert "Not supported yet"` never fired). - logging.warning("Video modality is not supported yet; ignoring 'video' field.") + assert "Not supported yet" assert kwargs["output_len_dist"] is not None, ( "Output length distribution must be set for multimodal requests." ) diff --git a/tensorrt_llm/bench/dataset/utils.py b/tensorrt_llm/bench/dataset/utils.py index 39d507419207..2ecea0320a5b 100644 --- a/tensorrt_llm/bench/dataset/utils.py +++ b/tensorrt_llm/bench/dataset/utils.py @@ -103,10 +103,6 @@ def get_sample_from_population(population_range, sample_size): def write_dataset_to_file(dataset_generator, output_file): - if output_file is None: - for item in dataset_generator: - print(item) - return output_file = Path(output_file) os.makedirs(output_file.parent, exist_ok=True) with open(output_file, "w") as f: diff --git a/tensorrt_llm/bench/tuning/utils.py b/tensorrt_llm/bench/tuning/utils.py deleted file mode 100644 index 37d92e197f9b..000000000000 --- a/tensorrt_llm/bench/tuning/utils.py +++ /dev/null @@ -1,34 +0,0 @@ -import pynvml - -DEFAULT_HF_MODEL_DIRS = { - "BaichuanForCausalLM": "baichuan-inc/Baichuan-13B-Chat", - "BloomForCausalLM": "bigscience/bloom-560m", - "GLMModel": "THUDM/glm-10b", - "ChatGLMModel": "THUDM/chatglm3-6b", - "ChatGLMForCausalLM": "THUDM/chatglm3-6b", - "FalconForCausalLM": "tiiuae/falcon-rw-1b", - "GPTForCausalLM": "gpt2-medium", - "GPTJForCausalLM": "EleutherAI/gpt-j-6b", - "GPTNeoXForCausalLM": "EleutherAI/gpt-neox-20b", - "InternLMForCausalLM": "internlm/internlm-chat-7b", - "InternLM2ForCausalLM": "internlm/internlm2-chat-7b", - "LlamaForCausalLM": "meta-llama/Llama-2-7b-hf", - "MPTForCausalLM": "mosaicml/mpt-7b", - "PhiForCausalLM": "microsoft/phi-2", - "OPTForCausalLM": "facebook/opt-350m", - "QWenLMHeadModel": "Qwen/Qwen-7B", - "QWenForCausalLM": "Qwen/Qwen-7B", - "Qwen2ForCausalLM": "Qwen/Qwen1.5-7B", - "Qwen2MoeForCausalLM": "Qwen/Qwen1.5-MoE-A2.7B", - "RecurrentGemmaForCausalLM": "google/recurrentgemma-2b", -} - - -def get_device_memory(): - pynvml.nvmlInit() - handle = pynvml.nvmlDeviceGetHandleByIndex(0) - mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) - total_memory = mem_info.total / (1024**3) - pynvml.nvmlShutdown() - - return total_memory diff --git a/tensorrt_llm/bench/utils/__init__.py b/tensorrt_llm/bench/utils/__init__.py index 6d818578b9be..ea8e5a10996a 100644 --- a/tensorrt_llm/bench/utils/__init__.py +++ b/tensorrt_llm/bench/utils/__init__.py @@ -20,7 +20,7 @@ VALID_QUANT_ALGOS = Literal[f"{QuantAlgo.W8A16}", f"{QuantAlgo.W4A16}", f"{QuantAlgo.W4A16_AWQ}", f"{QuantAlgo.W4A8_AWQ}", f"{QuantAlgo.W4A16_GPTQ}", f"{QuantAlgo.FP8}", - f"{QuantAlgo.NVFP4}"] + f"{QuantAlgo.INT8}", f"{QuantAlgo.NVFP4}"] VALID_SCHEDULING_POLICIES = \ Literal["max_utilization", "guaranteed_no_evict", "static"] diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index ffc560b2d6b6..2f39353c521c 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -1,20 +1,17 @@ import asyncio -import atexit import gc import importlib import inspect import json import os import secrets -import select import signal import socket import subprocess # nosec B404 import sys -import time import uuid from pathlib import Path -from typing import Any, Dict, NamedTuple, Optional, Sequence, Set +from typing import Any, Dict, Optional, Sequence, Set import click import torch @@ -25,11 +22,11 @@ from tensorrt_llm import LLM as PyTorchLLM from tensorrt_llm import MultimodalEncoder -from tensorrt_llm._utils import mpi_rank, set_prometheus_multiproc_dir +from tensorrt_llm._utils import mpi_rank from tensorrt_llm.commands._serve_stability import stability_option from tensorrt_llm.commands.utils import (collect_explicit_cli_keys, get_is_diffusion_only_model) -from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS, LlmLauncherEnvs +from tensorrt_llm.executor.utils import LlmLauncherEnvs from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import KvCacheConfig from tensorrt_llm.llmapi.disagg_utils import (DisaggClusterConfig, @@ -40,7 +37,7 @@ validate_config_bool) from tensorrt_llm.llmapi.llm_args import MultimodalConfig, TorchLlmArgs from tensorrt_llm.llmapi.llm_utils import update_llm_args_with_extra_dict -from tensorrt_llm.llmapi.mpi_session import find_free_ipc_addr, split_mpi_env +from tensorrt_llm.llmapi.mpi_session import find_free_ipc_addr from tensorrt_llm.llmapi.reasoning_parser import (ReasoningParserFactory, resolve_auto_reasoning_parser) from tensorrt_llm.logger import logger, severity_map @@ -200,7 +197,6 @@ def get_llm_args( free_gpu_memory_fraction: float = 0.9, kv_cache_dtype: str = "auto", num_postprocess_workers: int = 0, - num_serve_frontends: int = 1, trust_remote_code: bool = False, revision: Optional[str] = None, reasoning_parser: Optional[str] = None, @@ -274,8 +270,6 @@ def get_llm_args( max_seq_len, "num_postprocess_workers": num_postprocess_workers, - "num_serve_frontends": - num_serve_frontends, "enable_chunked_prefill": enable_chunked_prefill, "enable_attention_dp": @@ -364,156 +358,6 @@ def _diagnose_port_in_use(port: int) -> str: return "; ".join(details) -class MultiFrontendMode(NamedTuple): - """This process's role under multi-frontend serving (prototype).""" - num_frontends: int - is_attached_frontend: bool - - @property - def is_launcher(self) -> bool: - """Owns the engine and spawns/cleans the attached frontends.""" - return self.num_frontends > 1 and not self.is_attached_frontend - - -def _init_multi_frontend_mode(llm_args: dict, - enabled: bool) -> MultiFrontendMode: - """Resolve this process's multi-frontend serving role. - - num_serve_frontends=K runs K HTTP frontend processes against ONE - executor: the launcher (frontend 0) owns the engine and spawns K-1 - attached frontends (classic IPC executor path only). enabled=False - entry points (e.g. disaggregated MPI workers) never honor the knob. - """ - if not enabled: - if llm_args.pop("num_serve_frontends", 1) > 1: - logger.warning("num_serve_frontends is only supported on plain " - "trtllm-serve; ignored on this entry point.") - return MultiFrontendMode(1, False) - - mode = MultiFrontendMode(llm_args.get("num_serve_frontends", 1), - os.getenv("TLLM_EXECUTOR_ATTACH_INFO") is not None) - if mode.is_launcher and llm_args.get("orchestrator_type") is not None: - raise ValueError( - "num_serve_frontends > 1 requires the default (classic IPC) " - "executor path, not orchestrator_type=" - f"{llm_args.get('orchestrator_type')!r}") - return mode - - -def _spawn_attached_frontends(llm, num_frontends: int) -> list: - """Spawn num_frontends - 1 attached serving frontend processes. - - Each child re-execs this trtllm-serve command line with env vars - carrying the launcher executor's attach endpoints; its executor - attaches to the already-running worker instead of launching one (see - GenerationExecutor.create / GenerationExecutorFrontendProxy). - - Blocks until every child signals READY over its inherited pipe: a - successful Popen only proves the process exists, while the frontend - can still fail during executor attach or server setup. Any child - failure (or a missed deadline) fails the whole group, terminating - the children already started, so num_serve_frontends=K never - silently degrades to fewer frontends. - """ - from tensorrt_llm.executor.proxy import GenerationExecutorProxy - - executor = getattr(llm, "_executor", None) - if not isinstance(executor, GenerationExecutorProxy) or ( - attach_info := executor.multi_frontend_attach_info()) is None: - raise ValueError( - "num_serve_frontends > 1 requires the classic IPC executor " - f"proxy in multi-frontend mode, got {type(executor).__name__}") - # Carries the executor HMAC keys; the child deletes it from its env - # once consumed (GenerationExecutor.create). - attach_env = json.dumps(attach_info) - - children, ready_fds = [], [] - try: - for frontend_id in range(1, num_frontends): - # Strip MPI/SLURM identity vars: an inherited rank identity would - # make the child's mpi4py try to (re-)join the launcher's job. - env, _ = split_mpi_env() - env["TLLM_EXECUTOR_ATTACH_INFO"] = attach_env - env["TLLM_EXECUTOR_FRONTEND_ID"] = str(frontend_id) - env["TLLM_DISABLE_MPI"] = "1" - read_fd, write_fd = os.pipe() - ready_fds.append(read_fd) - env["TLLM_FRONTEND_READY_FD"] = str(write_fd) - try: - child = subprocess.Popen([sys.executable] + sys.argv, - env=env, - pass_fds=(write_fd, )) # nosec B603 - finally: - # The child now holds the only write end; its exit before - # READY surfaces as EOF on read_fd. - os.close(write_fd) - children.append(child) - logger.info( - f"Launched attached serving frontend {frontend_id} (pid {child.pid})" - ) - _wait_attached_frontends_ready(children, ready_fds) - except BaseException: - _terminate_attached_frontends(children) - raise - finally: - for fd in ready_fds: - os.close(fd) - return children - - -def _wait_attached_frontends_ready(children: list, ready_fds: list) -> None: - """Block until every attached frontend writes its READY byte.""" - timeout = float(os.getenv("TLLM_FRONTEND_READY_TIMEOUT", "300")) - deadline = time.monotonic() + timeout - pending = dict(zip(ready_fds, children)) - while pending: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise RuntimeError( - f"{len(pending)} attached frontend(s) not ready within " - f"{timeout:.0f}s (TLLM_FRONTEND_READY_TIMEOUT)") - readable, _, _ = select.select(list(pending), [], [], - min(remaining, 1.0)) - for fd in readable: - child = pending.pop(fd) - if os.read(fd, 1) != b"R": # EOF: pipe closed without READY - raise RuntimeError( - f"Attached frontend (pid {child.pid}) exited before " - "signaling READY") - logger.info(f"Attached frontend (pid {child.pid}) is ready") - for fd, child in list(pending.items()): - if child.poll() is not None: - raise RuntimeError( - f"Attached frontend (pid {child.pid}) exited with code " - f"{child.returncode} before signaling READY") - - -def _signal_frontend_ready(multi_frontend: MultiFrontendMode) -> None: - """Report READY to the launcher over the inherited pipe. - - Called once everything fallible in an attached frontend's startup - (port bind, executor attach, LLM and OpenAIServer construction, - middleware registration) has succeeded; the launcher blocks group - startup on this byte (see _wait_attached_frontends_ready). - """ - ready_fd = os.environ.pop("TLLM_FRONTEND_READY_FD", None) - if not (multi_frontend.is_attached_frontend and ready_fd): - return - fd = int(ready_fd) - os.write(fd, b"R") - os.close(fd) - - -def _terminate_attached_frontends(children: list) -> None: - for child in children: - child.terminate() - for child in children: - try: - child.wait(timeout=10) - except subprocess.TimeoutExpired: - child.kill() - - def launch_server( host: str, port: int, @@ -528,25 +372,10 @@ def launch_server( served_model_name: Optional[str] = None, allow_request_chat_template: bool = False, num_input_processor_workers: int = 8, - num_media_load_workers: int = 8, - multi_frontend_enabled: bool = True): + num_media_load_workers: int = 8): backend = llm_args["backend"] model = served_model_name or llm_args["model"] - - multi_frontend = _init_multi_frontend_mode(llm_args, multi_frontend_enabled) - if multi_frontend.is_launcher or multi_frontend.is_attached_frontend: - # The Responses API store is per-process in-memory: with several - # frontends behind one SO_REUSEPORT port, a follow-up request may - # land on a sibling that has no record of the previous response. - # Disable storage group-wide (OpenAIServer.enable_store). - if not os.getenv("TRTLLM_RESPONSES_API_DISABLE_STORE"): - logger.warning( - "num_serve_frontends > 1: stateful Responses API storage " - "(store/previous_response_id) is disabled; the per-frontend " - "in-memory store cannot be shared across frontends.") - os.environ["TRTLLM_RESPONSES_API_DISABLE_STORE"] = "1" - addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) address_family = socket.AF_INET6 if all( @@ -554,10 +383,6 @@ def launch_server( with socket.socket(address_family, socket.SOCK_STREAM) as s: # If disagg cluster config is provided and port is not specified, try to find a free port, otherwise try to bind to the specified port assert port > 0 or disagg_cluster_config is not None, "Port must be specified if disagg cluster config is not provided" - if multi_frontend.is_launcher or multi_frontend.is_attached_frontend: - # Every frontend process binds its own listening socket on the - # same port; the kernel load-balances accepts across them. - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) try: s.bind((host, port)) if port == 0: @@ -584,39 +409,25 @@ def launch_server( f"{backend} is not a known backend, check help for available options.", param_hint="backend") - # The finally below is the cleanup boundary for the attached - # frontends: it must cover everything from their spawn through - # server construction, middleware registration, and runtime, or a - # failure in between leaks the child processes. - frontend_children = [] - try: - if multi_frontend.is_launcher: - frontend_children = _spawn_attached_frontends( - llm, multi_frontend.num_frontends) - - server = OpenAIServer( - generator=llm, - model=model, - tool_parser=tool_parser, - server_role=server_role, - metadata_server_cfg=metadata_server_cfg, - disagg_cluster_config=disagg_cluster_config, - multimodal_server_config=multimodal_server_config, - chat_template=chat_template, - allow_request_chat_template=allow_request_chat_template, - input_processor_workers=num_input_processor_workers, - media_load_workers=num_media_load_workers) - _apply_fastapi_middlewares(server.app, middleware) + server = OpenAIServer( + generator=llm, + model=model, + tool_parser=tool_parser, + server_role=server_role, + metadata_server_cfg=metadata_server_cfg, + disagg_cluster_config=disagg_cluster_config, + multimodal_server_config=multimodal_server_config, + chat_template=chat_template, + allow_request_chat_template=allow_request_chat_template, + input_processor_workers=num_input_processor_workers, + media_load_workers=num_media_load_workers) + _apply_fastapi_middlewares(server.app, middleware) - # Optionally disable GC (default: not disabled) - if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": - gc.disable() + # Optionally disable GC (default: not disabled) + if os.getenv("TRTLLM_SERVER_DISABLE_GC", "0") == "1": + gc.disable() - _signal_frontend_ready(multi_frontend) - uvloop.run(server(host, port, sockets=[s])) - finally: - if frontend_children: - _terminate_attached_frontends(frontend_children) + uvloop.run(server(host, port, sockets=[s])) def launch_grpc_server(host: str, @@ -1075,13 +886,6 @@ def launch_visual_gen_server( help="Number of workers to postprocess raw responses " "to comply with OpenAI protocol.", status="prototype") -@stability_option("--num_serve_frontends", - type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS), - default=1, - help="Number of HTTP frontend processes serving one " - "executor; values > 1 share the serving port via " - "SO_REUSEPORT (classic IPC executor path only).", - status="prototype") @stability_option("--num_input_processor_workers", type=click.IntRange(min=1), default=8, @@ -1263,30 +1067,29 @@ def launch_visual_gen_server( help= "Types of agents to schedule. Now Only Support Open Deep Research agent.", status="prototype") -def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], - post_processor_hook: Optional[str], host: str, port: int, - log_level: str, backend: str, max_beam_width: int, - max_batch_size: int, max_num_tokens: int, max_seq_len: int, - tensor_parallel_size: int, pipeline_parallel_size: int, - context_parallel_size: int, moe_expert_parallel_size: Optional[int], - moe_cluster_parallel_size: Optional[int], - gpus_per_node: Optional[int], free_gpu_memory_fraction: float, - kv_cache_dtype: str, num_postprocess_workers: int, - num_serve_frontends: int, num_input_processor_workers: int, - num_media_load_workers: int, trust_remote_code: bool, - revision: Optional[str], extra_llm_api_options: Optional[str], - reasoning_parser: Optional[str], tool_parser: Optional[str], - metadata_server_config_file: Optional[str], - server_role: Optional[str], - fail_fast_on_attention_window_too_large: bool, - otlp_traces_endpoint: Optional[str], enable_chunked_prefill: bool, - enable_attention_dp: bool, disagg_cluster_uri: Optional[str], - media_io_kwargs: Optional[str], agent_percentage: float, - agent_types: Optional[str], video_pruning_rate: Optional[float], - telemetry: bool, custom_module_dirs: list[Path], - chat_template: Optional[str], allow_request_chat_template: bool, - middleware: tuple[str, ...], grpc: bool, enable_visual_gen: bool, - served_model_name: Optional[str], visual_gen_args: Optional[str]): +def serve( + model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], + post_processor_hook: Optional[str], host: str, port: int, + log_level: str, backend: str, max_beam_width: int, max_batch_size: int, + max_num_tokens: int, max_seq_len: int, tensor_parallel_size: int, + pipeline_parallel_size: int, context_parallel_size: int, + moe_expert_parallel_size: Optional[int], + moe_cluster_parallel_size: Optional[int], gpus_per_node: Optional[int], + free_gpu_memory_fraction: float, kv_cache_dtype: str, + num_postprocess_workers: int, num_input_processor_workers: int, + num_media_load_workers: int, trust_remote_code: bool, + revision: Optional[str], extra_llm_api_options: Optional[str], + reasoning_parser: Optional[str], tool_parser: Optional[str], + metadata_server_config_file: Optional[str], server_role: Optional[str], + fail_fast_on_attention_window_too_large: bool, + otlp_traces_endpoint: Optional[str], enable_chunked_prefill: bool, + enable_attention_dp: bool, disagg_cluster_uri: Optional[str], + media_io_kwargs: Optional[str], agent_percentage: float, + agent_types: Optional[str], video_pruning_rate: Optional[float], + telemetry: bool, custom_module_dirs: list[Path], + chat_template: Optional[str], allow_request_chat_template: bool, + middleware: tuple[str, ...], grpc: bool, enable_visual_gen: bool, + served_model_name: Optional[str], visual_gen_args: Optional[str]): """Running an OpenAI API compatible server MODEL: model name | HF checkpoint path | TensorRT engine path @@ -1368,7 +1171,6 @@ def _serve_llm(): free_gpu_memory_fraction=free_gpu_memory_fraction, kv_cache_dtype=kv_cache_dtype, num_postprocess_workers=num_postprocess_workers, - num_serve_frontends=num_serve_frontends, trust_remote_code=trust_remote_code, revision=revision, reasoning_parser=reasoning_parser, @@ -1822,7 +1624,6 @@ def disaggregated( """Running server in disaggregated mode""" logger.set_level(log_level) - set_prometheus_multiproc_dir() if metrics_log_interval != 0: logger.warning( @@ -1842,35 +1643,6 @@ def disaggregated( logger.info(f"Reserving disaggregated server address " f"{disagg_cfg.hostname}:{disagg_cfg.port} (pid={os.getpid()})") - metadata_server_cfg = parse_metadata_server_config_file( - metadata_server_config_file) - - # Topology from explicit config (num_workers + disagg_coordinator_url): - # (a) url set -> delegate to that external coordinator; (b) url absent, - # num_workers>1 -> implicit in-process coordinator + delegating fleet; - # (c) url absent, num_workers==1 -> single self-contained server. - num_workers = disagg_cfg.num_workers - coordinator_url = disagg_cfg.disagg_coordinator_url - - if coordinator_url: - # (a) External coordinator: fork a fleet of delegating servers (or a - # single one) pointed at it; never start a coordinator in this process. - _serve_disagg_fleet(disagg_cfg, config_file, - metadata_server_config_file, request_timeout, - server_start_timeout, num_workers, coordinator_url) - return - - if num_workers > 1: - # (b) Implicit coordinator in this process (on port-1) + a delegating - # uvicorn fleet (workers=N) on the public port. See below. - _serve_coordinator_and_fleet(disagg_cfg, config_file, - metadata_server_config_file, - metadata_server_cfg, request_timeout, - server_start_timeout, num_workers) - return - - # (c) num_workers==1, no external coordinator: a single disagg server with an - # in-process (local) coordinator. Pre-bind the socket (validates port), serve. with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: try: s.bind((disagg_cfg.hostname, disagg_cfg.port)) @@ -1884,6 +1656,9 @@ def disaggregated( f"Failed to bind socket to {disagg_cfg.hostname}:{disagg_cfg.port}: {e}. " f"Port holder(s): {holder}") + metadata_server_cfg = parse_metadata_server_config_file( + metadata_server_config_file) + server = OpenAIDisaggServer( config=disagg_cfg, req_timeout_secs=request_timeout, @@ -1907,302 +1682,6 @@ def disaggregated( uvloop.run(server(disagg_cfg.hostname, disagg_cfg.port, sockets=[s])) -def _launch_disagg_fleet(disagg_cfg, config_file, metadata_server_config_file, - request_timeout, server_start_timeout, num_workers, - coordinator_url): - """Launch delegating disaggregated-server workers. - - Each worker has its own ``Popen`` running ``_run_fleet_worker`` on the shared - public port and pointing at ``coordinator_url``. Separate processes allow an - explicit per-worker ``TLLM_DISAGG_WORKER_PROCESS_ID`` and independent - ``SO_REUSEPORT`` sockets. MPI/PMIX/SLURM environment variables are stripped. - - Returns the list of ``Popen`` handles. - """ - from tensorrt_llm.llmapi.disagg_utils import disagg_process_id_space - public_host, public_port = disagg_cfg.hostname, disagg_cfg.port - base_env = { - k: v - for k, v in os.environ.items() - if not k.startswith(("SLURM_", "PMIX_", "PMI_", "OMPI_", "UCX_", - "I_MPI_", "HYDRA_", "MPI_")) - } - # num_workers is explicit config now; ensure no stale WEB_CONCURRENCY leaks in - # and re-forks each plain-HTTP worker into a nested fleet. - base_env.pop("WEB_CONCURRENCY", None) - base_env[DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_URL] = coordinator_url - base_env[DisaggWorkerEnvs.TLLM_DISAGG_CONFIG_FILE] = os.path.abspath( - config_file) - if metadata_server_config_file: - base_env[DisaggWorkerEnvs.TLLM_DISAGG_METADATA_CONFIG_FILE] = \ - os.path.abspath(metadata_server_config_file) - base_env[DisaggWorkerEnvs.TLLM_DISAGG_REQUEST_TIMEOUT] = str( - request_timeout) - base_env[DisaggWorkerEnvs.TLLM_DISAGG_SERVER_START_TIMEOUT] = str( - server_start_timeout) - base_env[DisaggWorkerEnvs.TLLM_DISAGG_SCHEDULE_STYLE] = \ - disagg_cfg.schedule_style - # Propagate the parent's log level so fleet workers' INFO logs (e.g. the - # per-request [ttft_split] / [coord_api] breakdowns) are not dropped. - base_env[DisaggWorkerEnvs.TLLM_DISAGG_LOG_LEVEL] = logger.level - - cmd = [ - sys.executable, "-c", - "from tensorrt_llm.commands.serve import _run_fleet_worker; " - "_run_fleet_worker()" - ] - logger.info( - f"Launching disagg fleet: {num_workers} SO_REUSEPORT workers on " - f"{public_host}:{public_port}, coordinator={coordinator_url}") - - procid_space = disagg_process_id_space() - if not 1 <= num_workers <= procid_space: - raise ValueError( - f"num_workers must be between 1 and {procid_space}, got {num_workers}" - ) - fleet = [] - for i in range(num_workers): - worker_env = dict(base_env) - # Explicit per-worker process index (no shared counter file). - worker_env[DisaggWorkerEnvs.TLLM_DISAGG_WORKER_PROCESS_ID] = str(i) - p = subprocess.Popen(cmd, - env=worker_env, - stdout=sys.stdout, - stderr=sys.stderr) - logger.info(f"Disagg fleet worker {i} launched (pid={p.pid})") - fleet.append(p) - - def _cleanup(): - for p in fleet: - if p.poll() is None: - p.terminate() - deadline = time.monotonic() + 10 - for p in fleet: - try: - p.wait(timeout=max(0, deadline - time.monotonic())) - except Exception: - p.kill() - p.wait() - - def _handle_signal(signum, _frame): - _cleanup() - raise SystemExit(128 + signum) - - atexit.register(_cleanup) - signal.signal(signal.SIGTERM, _handle_signal) - signal.signal(signal.SIGINT, _handle_signal) - return fleet - - -def _serve_disagg_fleet(disagg_cfg, config_file, metadata_server_config_file, - request_timeout, server_start_timeout, num_workers, - coordinator_url): - """External coordinator: fork the delegating fleet and block on it. - - No coordinator is started in this process -- the fleet delegates to the - already-running coordinator at ``coordinator_url``. - """ - fleet = _launch_disagg_fleet(disagg_cfg, config_file, - metadata_server_config_file, request_timeout, - server_start_timeout, num_workers, - coordinator_url) - # Block until any worker exits; a nonzero exit from any worker is a failure. - try: - while True: - for i, p in enumerate(fleet): - rc = p.poll() - if rc is not None: - if rc != 0: - raise RuntimeError( - f"Disagg fleet worker {i} (pid={p.pid}) exited with " - f"code {rc}") - # A clean exit of one worker ends the fleet. - return - time.sleep(1) - finally: - for process in fleet: - if process.poll() is None: - process.terminate() - - -def _serve_coordinator_and_fleet(disagg_cfg, config_file, - metadata_server_config_file, - metadata_server_cfg, request_timeout, - server_start_timeout, num_workers): - """workers>1, no external URL: coordinator server here + a delegating fleet. - - This process runs the coordinator server (owns the ctx/gen routers, cluster - state, and centralized ZMQ ingest) on ``port-1``; a uvicorn fleet on the - public port delegates to it. - """ - from tensorrt_llm.serve.coordinator_server import CoordinatorServer - from tensorrt_llm.serve.disagg_coordinator import DisaggCoordinatorService - - public_host, public_port = disagg_cfg.hostname, disagg_cfg.port - coord_port = int( - os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_PORT, - public_port - 1)) - # The fleet is co-located with the implicit coordinator, so route its hot - # /select,/finish over a Unix domain socket (avoids the TCP loopback stack - # that dominated per-request latency). Opt-out via the UDS env = "0". - use_uds = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_UDS, - "1") == "1" - coord_uds = ( - f"/tmp/trtllm_disagg_coord_{public_port}.sock" # nosec B108 - if use_uds else None) - # Fleet points at the UDS when enabled; the TCP port stays up for health. - coord_url = f"unix:{coord_uds}" if coord_uds else \ - f"http://{public_host}:{coord_port}" - - # 1. Launch the delegating fleet pointed at the implicit coordinator we start - # below (port-1 for TCP; UDS for the hot path). Workers hold - # CoordinatorClients (no core), so they can't race the ZMQ ingest bind. - fleet = _launch_disagg_fleet(disagg_cfg, config_file, - metadata_server_config_file, request_timeout, - server_start_timeout, num_workers, coord_url) - - # 2. Build + serve the coordinator in this process. It OWNS routing state and - # builds the owner routers itself (single shared namespace-aware core + ONE - # ZMQ ingest server, started once here). - def _client_factory(router, role, max_retries=1): - from tensorrt_llm.serve.openai_client import OpenAIHttpClient - return OpenAIHttpClient(router, role, request_timeout, max_retries) - - coordinator = DisaggCoordinatorService( - disagg_cfg, - _client_factory, - metadata_config=metadata_server_cfg, - server_start_timeout_secs=server_start_timeout) - logger.info(f"Coordinator serving on {public_host}:{coord_port} " - f"(uds={coord_uds}) (fleet on public port {public_port})") - - async def _serve_and_monitor(): - server_task = asyncio.create_task( - CoordinatorServer(coordinator)(public_host, - coord_port, - uds=coord_uds)) - - async def _monitor_fleet(): - while True: - for i, process in enumerate(fleet): - return_code = process.poll() - if return_code is not None: - if return_code != 0: - raise RuntimeError( - f"Disagg fleet worker {i} (pid={process.pid}) " - f"exited with code {return_code}") - return - await asyncio.sleep(1) - - monitor_task = asyncio.create_task(_monitor_fleet()) - done, pending = await asyncio.wait((server_task, monitor_task), - return_when=asyncio.FIRST_COMPLETED) - for task in pending: - task.cancel() - await asyncio.gather(*pending, return_exceptions=True) - for task in done: - task.result() - - try: - asyncio.run(_serve_and_monitor()) - finally: - for process in fleet: - if process.poll() is None: - process.terminate() - - -def _init_fleet_worker_process(): - """Per-process setup shared by every fleet worker (one OS process each). - - Restores logging/GC state for the fresh ``python -c`` interpreter and tags - every log line with this worker's PID so the shared-stdout fleet output stays - attributable. - """ - # A worker is a plain HTTP process, never an MPI rank; drop WEB_CONCURRENCY so - # it is never itself re-forked into multiple uvicorn workers. - os.environ.pop("WEB_CONCURRENCY", None) - if os.getenv("TRTLLM_DISAGG_SERVER_DISABLE_GC", "1") == "1": - gc.disable() - - # This is a fresh Python process, so the TRT-LLM logger defaults to WARNING - # and would drop the workers' INFO logs (per-request [ttft_split] / - # [coord_api]). Restore the parent's level. - _worker_log_level = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_LOG_LEVEL) - if _worker_log_level: - logger.set_level(_worker_log_level) - - # All N fleet workers share one stdout; tag every trtllm log line from this - # worker with its PID so interleaved fleet output is attributable. - import logging as _logging - for _h in _logging.getLogger("TRT-LLM").handlers: - _h.setFormatter( - _logging.Formatter( - fmt= - f"[%(asctime)s] [fleet-worker pid={os.getpid()}] %(message)s", - datefmt="%m/%d/%Y-%H:%M:%S")) - - -def _build_disagg_server_from_env() -> "OpenAIDisaggServer": - """Build one delegating disagg server from the env the launcher exported. - - Fully stateless -- reads config + coordinator URL from ``DisaggWorkerEnvs``; - the server holds a remote ``CoordinatorClient`` so routing/readiness are - delegated to the coordinator. The worker's process index (for the snowflake - disagg-id) is read from ``TLLM_DISAGG_WORKER_PROCESS_ID``, which the launcher - set explicitly per worker (see ``worker_local_process_id``). - """ - config_file = os.environ[DisaggWorkerEnvs.TLLM_DISAGG_CONFIG_FILE] - coordinator_url = os.environ[DisaggWorkerEnvs.TLLM_DISAGG_COORDINATOR_URL] - metadata_config_file = os.environ.get( - DisaggWorkerEnvs.TLLM_DISAGG_METADATA_CONFIG_FILE) - request_timeout = int( - os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_REQUEST_TIMEOUT, "180")) - server_start_timeout = int( - os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_SERVER_START_TIMEOUT, - "180")) - - disagg_cfg = parse_disagg_config_file(config_file) - schedule_style = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_SCHEDULE_STYLE) - if schedule_style: - disagg_cfg.schedule_style = schedule_style - metadata_server_cfg = parse_metadata_server_config_file( - metadata_config_file) - - server = OpenAIDisaggServer(config=disagg_cfg, - req_timeout_secs=request_timeout, - server_start_timeout_secs=server_start_timeout, - metadata_server_cfg=metadata_server_cfg, - coordinator_url=coordinator_url) - logger.info(f"Disagg server built, coordinator={coordinator_url}") - return server - - -def _run_fleet_worker(): - """Run one fleet worker process. - - Each ``Popen`` receives an explicit ``TLLM_DISAGG_WORKER_PROCESS_ID`` and - binds its own ``SO_REUSEPORT`` socket on the shared public port. - """ - _init_fleet_worker_process() - server = _build_disagg_server_from_env() - host, port = server._config.hostname, server._config.port - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - # SO_REUSEPORT: every worker binds the same (host, port); the kernel spreads - # connections across the workers' accept queues by 4-tuple hash. - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - try: - s.bind((host, port)) - except OSError as e: - raise RuntimeError( - f"Fleet worker failed to SO_REUSEPORT-bind {host}:{port}: {e}") - pidx = os.environ.get(DisaggWorkerEnvs.TLLM_DISAGG_WORKER_PROCESS_ID, "0") - logger.info(f"Fleet worker process_id={pidx} bound {host}:{port} " - f"(SO_REUSEPORT)") - asyncio.run(server(host, port, sockets=[s])) - - def set_cuda_device(): if (os.getenv("OMPI_COMM_WORLD_RANK")): env_global_rank = int(os.environ["OMPI_COMM_WORLD_RANK"]) @@ -2300,29 +1779,6 @@ class DisaggLauncherEnvs(StrEnum): TLLM_DISAGG_ROLE = "TRTLLM_DISAGG_ROLE" -class DisaggWorkerEnvs(StrEnum): - # Passed from the `disaggregated` coordinator to the fleet of worker processes - # (one Popen per worker) via env, then read by _run_fleet_worker in each worker. - TLLM_DISAGG_COORDINATOR_URL = "TRTLLM_DISAGG_COORDINATOR_URL" - TLLM_DISAGG_CONFIG_FILE = "TRTLLM_DISAGG_CONFIG_FILE" - TLLM_DISAGG_METADATA_CONFIG_FILE = "TRTLLM_DISAGG_METADATA_CONFIG_FILE" - TLLM_DISAGG_REQUEST_TIMEOUT = "TRTLLM_DISAGG_REQUEST_TIMEOUT" - TLLM_DISAGG_SERVER_START_TIMEOUT = "TRTLLM_DISAGG_SERVER_START_TIMEOUT" - TLLM_DISAGG_SCHEDULE_STYLE = "TRTLLM_DISAGG_SCHEDULE_STYLE" - # Parent's logger level; the forked uvicorn fleet is a fresh process that - # otherwise defaults to WARNING and drops the workers' INFO logs. - TLLM_DISAGG_LOG_LEVEL = "TRTLLM_DISAGG_LOG_LEVEL" - # Coordinator transport (read in _serve_coordinator_and_fleet): PORT overrides - # the coordinator TCP port (default public_port-1); UDS="1" (default) routes the - # co-located fleet's hot /select,/finish over a Unix domain socket, "0" for TCP. - TLLM_DISAGG_COORDINATOR_PORT = "TRTLLM_DISAGG_COORDINATOR_PORT" - TLLM_DISAGG_COORDINATOR_UDS = "TRTLLM_DISAGG_COORDINATOR_UDS" - # Per-fleet-worker process index (0..N-1) → the snowflake disagg-id process_id - # field so co-located workers never collide; the launcher sets one distinct - # value per worker Popen (see worker_local_process_id). Defaults to 0 if unset. - TLLM_DISAGG_WORKER_PROCESS_ID = "TRTLLM_DISAGG_WORKER_PROCESS_ID" - - def _launch_disaggregated_server(disagg_config_file: str, llm_args: dict): # Launching the server instance_idx = os.environ.get(DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX) @@ -2342,9 +1798,7 @@ def _launch_disaggregated_server(disagg_config_file: str, llm_args: dict): host=server_cfg.hostname, port=server_cfg.port, llm_args=llm_args, - allow_request_chat_template=disagg_config.allow_request_chat_template, - # Disagg ctx/gen MPI workers must not enter multi-frontend mode. - multi_frontend_enabled=False) + allow_request_chat_template=disagg_config.allow_request_chat_template) def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, diff --git a/tensorrt_llm/executor/base_worker.py b/tensorrt_llm/executor/base_worker.py index 5e1413a4f0da..670d6f30508f 100644 --- a/tensorrt_llm/executor/base_worker.py +++ b/tensorrt_llm/executor/base_worker.py @@ -49,11 +49,9 @@ from .result import (GenerationResult, LogProbsResult, ResponseWrapper, compute_logprobs, get_metrics_dict) from .utils import (ErrorResponse, IntraProcessQueue, RequestError, - bucket_responses_by_frontend, frontend_lane_index, is_llm_response) if TYPE_CHECKING: - from .._torch.pyexecutor.kv_cache_transceiver import KvCacheTransceiver from ..disaggregated_params import DisaggregatedParams __all__ = [ @@ -120,9 +118,6 @@ def __init__( self.engine = None self.result_queue: Optional[IpcQueue] = None self.postproc_queues: Optional[List[IpcQueue]] = None - # Multi-frontend serving: one result lane per frontend process, - # selected by the frontend id in client_id's top bits. - self.frontend_result_queues: Optional[List[IpcQueue]] = None self.rank = mpi_rank() self.global_rank = global_mpi_rank() # mapping: client_id -> GenerationResult @@ -212,8 +207,24 @@ def _create_py_executor(): self.max_seq_len = _executor.max_seq_len return _executor - assert self.llm_args is not None, "llm_args is required to set up the worker engine" - self.engine = _create_py_executor() + def _create_engine(executor_config): + engine = self._engine + if executor_config is None: + executor_config = tllm.ExecutorConfig(1) + executor_config.logits_post_processor_config = tllm.LogitsPostProcessorConfig( + processor_batched=self._batched_logits_processor, + replicate=False) + comm_ranks, device_ids = self._get_comm_ranks_device_id() + executor_config.parallel_config = tllm.ParallelConfig( + participant_ids=comm_ranks, device_ids=device_ids) + + assert not hasattr(executor_config, "backend") + return tllm.Executor(engine, tllm.ModelType.DECODER_ONLY, + executor_config) + + self.engine = _create_py_executor( + ) if self.llm_args is not None else _create_engine( + self._executor_config) self._lora_manager: Optional[LoraManager] = None self._prompt_adapter_manager: Optional[PromptAdapterManager] = None @@ -234,10 +245,16 @@ def await_responses(self, timeout: Optional[float] = None) -> list: seconds=timeout) if timeout is not None else None) def fetch_stats(self) -> list: - return self.engine.get_latest_iteration_stats() + if isinstance(self.engine, tllm.Executor): + iter_stats = self.engine.get_latest_iteration_stats() + #TODO: Support req stats with TRT engine + # This would require ensuring iter and req stats have same size + return [(iter_stat, None, None) for iter_stat in iter_stats] + else: + return self.engine.get_latest_iteration_stats() def fetch_kv_cache_capacity(self) -> dict: - if self.engine is None: + if self.engine is None or isinstance(self.engine, tllm.Executor): return {} from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor @@ -247,26 +264,21 @@ def fetch_kv_cache_capacity(self) -> dict: return {} def fetch_kv_cache_events(self) -> list: - return self.engine.get_latest_kv_cache_events() + if isinstance(self.engine, tllm.Executor): + return self.engine.get_latest_kv_cache_events() + else: + return self.engine.get_latest_kv_cache_events() def set_result_queue(self, queue): """In multi-gpu mode, result_queue will be set here to communicate between the proxy and the worker 0 process.""" assert self.postproc_queues is None - assert self.frontend_result_queues is None self.result_queue = queue def set_postproc_queues(self, queues: List["IpcQueue"]): """ Set the IPC queues for feeding post-processing processes. """ assert self.result_queue is None - assert self.frontend_result_queues is None self.postproc_queues = queues - def set_frontend_result_queues(self, queues: List["IpcQueue"]): - """Multi-frontend serving: one result lane per frontend process.""" - assert self.result_queue is None - assert self.postproc_queues is None - self.frontend_result_queues = queues - def _set_iteration_result_queue(self, it_result_queue: IterationResultQueue, queue: Union[Queue, FusedIpcQueue, IntraProcessQueue]): @@ -425,10 +437,6 @@ def _deduce_max_tokens(request: GenerationRequest, llm_args: Optional[BaseLlmArgs] = None) -> int: # deduce max_tokens when it's not set by user max_tokens = request.sampling_params.max_tokens - output_prefix_len = len( - request.sampling_params._decoder_output_token_prefix) - if max_tokens is not None: - max_tokens -= output_prefix_len query_token_len = len( request.query_token_ids) if request.query_token_ids else 0 @@ -546,8 +554,6 @@ def _deduce_max_tokens(request: GenerationRequest, # E/P handoff embedding handles parked under "multimodal_embedding". request.multimodal_params.to_tensor("multimodal_data") executor_request.py_multimodal_data = request.multimodal_params.multimodal_data - if request.multimodal_params.mm_item_order: - executor_request.py_mm_item_order = request.multimodal_params.mm_item_order if self._is_pytorch_backend and request.sampling_params.logits_processor: # For PyTorch backend, we attach logits processors as a dynamic Python attribute @@ -973,16 +979,6 @@ def get_disaggregated_params(self) -> dict: return {} return self.engine.kv_cache_transceiver.get_disaggregated_params() - def get_cache_transceiver(self) -> Optional["KvCacheTransceiver"]: - if self.engine is None: - return None - return self.engine.kv_cache_transceiver - - def get_data_transceiver_state(self) -> bytes: - if self.engine is None or self.engine.kv_cache_transceiver is None: - return b"" - return self.engine.kv_cache_transceiver.get_data_transceiver_state() - @staticmethod def _stats_serializer(stats) -> str: # Per-rank path: stats is ("per_rank_dict", {..., "rank": N}). @@ -1091,20 +1087,20 @@ def responses_handler(self, responses: List[tllm.Response]): HandlerKind = AwaitResponseHelper.HandlerKind if self.handler_kind is HandlerKind.unknown: - has_ipc_queues = (self.worker.result_queue is not None - or self.worker.postproc_queues is not None - or self.worker.frontend_result_queues is not None) - if not has_ipc_queues: + if not (self.worker.result_queue is not None + or self.worker.postproc_queues is not None): logger_debug(f"creating await_response helper for Worker\n", color="yellow") # When ExecutorBindingWorker is used in the main process # aka the single process mode self.handler_kind = HandlerKind.single_process_worker - else: + elif self.worker.result_queue is not None or self.worker.postproc_queues is not None: # The ExecutorBindingProxy is used logger_debug(f"creating await_response helper for IPC\n", color="yellow") self.handler_kind = HandlerKind.ipc_batched + else: + raise NotImplementedError match self.handler_kind: case HandlerKind.single_process_worker: @@ -1283,13 +1279,7 @@ def handle_for_ipc_batched(self, responses: List[tllm.Response]) -> None: self.worker.postproc_queues[wid].put(batch) if rsp_batch: - if (lanes := self.worker.frontend_result_queues) is not None: - for frontend_id, sub_batch in enumerate( - bucket_responses_by_frontend(rsp_batch, len(lanes))): - if sub_batch: - lanes[frontend_id].put(sub_batch) - else: - self.worker.result_queue.put(rsp_batch) + self.worker.result_queue.put(rsp_batch) def _get_params_for_first_rsp( @@ -1403,16 +1393,7 @@ def _send_rsp( rsp_batch: Optional[List[tllm.Response]] = None): # if postproc_batches is set, append to batch instead of putting to IpcQueue - if worker.frontend_result_queues is not None: - # Route to the origin frontend's result lane; None/out-of-range ids - # fall back to lane 0 (see frontend_lane_index). - if rsp_batch is not None: - rsp_batch.append(response) - else: - lanes = worker.frontend_result_queues - lanes[frontend_lane_index(response.client_id, - len(lanes))].put(response) - elif worker.result_queue is not None: + if worker.result_queue is not None: if rsp_batch is not None: rsp_batch.append(response) else: diff --git a/tensorrt_llm/executor/executor.py b/tensorrt_llm/executor/executor.py index 98afe8daa44f..ba5b843ff347 100644 --- a/tensorrt_llm/executor/executor.py +++ b/tensorrt_llm/executor/executor.py @@ -1,8 +1,6 @@ import atexit import faulthandler -import json import multiprocessing -import os import platform import signal import traceback @@ -43,7 +41,6 @@ from .utils import IntraProcessQueue, ProcessPoolExecutorSession, RequestError if TYPE_CHECKING: - from .._torch.pyexecutor.kv_cache_transceiver import KvCacheTransceiver from .proxy import GenerationExecutorProxy from .worker import GenerationExecutorWorker @@ -465,12 +462,6 @@ def aget_kv_events(self, timeout=None) -> IterationResult: def get_disaggregated_params(self) -> dict: return {} - def get_cache_transceiver(self) -> Optional["KvCacheTransceiver"]: - return None - - def get_data_transceiver_state(self) -> bytes: - return b"" - @staticmethod def _create_ray_executor( worker_kwargs: Dict, @@ -569,35 +560,6 @@ def create( f"Using {postproc_worker_config.num_postprocess_workers} postprocess parallel processes.\n", "green") - # Multi-frontend serving: attach to an already-running executor - # instead of launching one (set by trtllm-serve for attached - # frontend processes). - attach_env = os.getenv("TLLM_EXECUTOR_ATTACH_INFO") - if attach_env: - attach_info = json.loads(attach_env) - # Consumed: it carries the HMAC keys and must not leak into - # descendant processes. - os.environ.pop("TLLM_EXECUTOR_ATTACH_INFO", None) - if attach_info.get("mode") != "classic": - raise ValueError( - "TLLM_EXECUTOR_ATTACH_INFO only supports the classic IPC " - f"executor path, got mode={attach_info.get('mode')!r}") - from .proxy import GenerationExecutorFrontendProxy - frontend_id_env = os.getenv("TLLM_EXECUTOR_FRONTEND_ID") - if frontend_id_env is None: - raise ValueError( - "TLLM_EXECUTOR_ATTACH_INFO is set but " - "TLLM_EXECUTOR_FRONTEND_ID is not; both are set together " - "by trtllm-serve when spawning attached frontends.") - frontend_id = int(frontend_id_env) - logger.info(f"Attaching executor frontend {frontend_id} to the " - "running classic IPC worker") - return GenerationExecutorFrontendProxy( - attach_info, - frontend_id=frontend_id, - postproc_worker_config=postproc_worker_config, - is_llm_executor=is_llm_executor) - worker_kwargs = { "engine": engine, "executor_config": executor_config, diff --git a/tensorrt_llm/executor/ipc.py b/tensorrt_llm/executor/ipc.py index d450ff276dec..a096f9c0433d 100644 --- a/tensorrt_llm/executor/ipc.py +++ b/tensorrt_llm/executor/ipc.py @@ -48,10 +48,7 @@ def __init__(self, use_hmac_encryption (bool): Whether to use HMAC encryption for pickled data. Defaults to True. ''' - if not use_hmac_encryption: - raise RuntimeError( - "use_hmac_encryption HMAC encryption is always required. Turning off HMAC encryption risks security vulnerability of unauthorized data serialization and deserialization. " - ) + assert use_hmac_encryption, "HMAC encryption is always required. Turning off HMAC encryption risks security vulnerability of unauthorized data serialization and deserialization. " self.socket_type = socket_type self.address_endpoint = address[ diff --git a/tensorrt_llm/executor/postproc_worker.py b/tensorrt_llm/executor/postproc_worker.py index 552c5c8f2bd3..ece60b2374b1 100644 --- a/tensorrt_llm/executor/postproc_worker.py +++ b/tensorrt_llm/executor/postproc_worker.py @@ -15,7 +15,7 @@ from ..sampling_params import SamplingParams from .ipc import ZeroMqQueue from .postprocessor_hook import load_post_processor_hook -from .utils import ErrorResponse, bucket_responses_by_frontend, is_llm_response +from .utils import ErrorResponse, is_llm_response if TYPE_CHECKING: from ..disaggregated_params import DisaggregatedParams @@ -86,7 +86,7 @@ class Output(NamedTuple): def __init__( self, pull_pipe_addr: tuple[str, Optional[bytes]], - push_pipe_addrs: List[tuple[str, Optional[bytes]]], + push_pipe_addr: tuple[str, Optional[bytes]], tokenizer_dir: str, record_creator: Callable[ ["PostprocWorker.Input", TransformersTokenizer], Any], @@ -95,9 +95,7 @@ def __init__( ''' Args: pull_pipe_addr (tuple[str, Optional[bytes]]): The address and HMAC key of the input IPC. - push_pipe_addrs: The addresses and HMAC keys of the output IPC - lanes, one per frontend (a single-element list in - single-frontend mode). + push_pipe_addr (tuple[str, Optional[bytes]]): The address and HMAC key of the output IPC. tokenizer_dir (str): The directory to load tokenizer. record_creator (Callable[["ResponsePostprocessWorker.Input"], Any]): A creator for creating a record for a request. result_handler (Optional[Callable[[GenerationResultBase], Any]]): A callback handles the final result. @@ -110,14 +108,11 @@ def __init__( is_async=True, is_server=False, name="postprocess_pull_pipe") - self._push_pipes = [ - ZeroMqQueue(address=addr, - is_async=True, - is_server=False, - socket_type=zmq.PUSH, - name=f"postprocess_push_pipe_{i}") - for i, addr in enumerate(push_pipe_addrs) - ] + self._push_pipe = ZeroMqQueue(address=push_pipe_addr, + is_async=True, + is_server=False, + socket_type=zmq.PUSH, + name="postprocess_push_pipe") self._to_stop = asyncio.Event() self._q = deque() @@ -197,19 +192,11 @@ async def _batched_put(self): ''' Batched IPC send. ''' async for batch in self._mainloop(): if batch is None: - # notify the dispatch_result coroutine in every frontend to - # quit - for pipe in self._push_pipes: - await pipe.put_async(None) + # notify dispatch_result corountine to quit + await self._push_pipe.put_async(None) break assert isinstance(batch, list) - if len(self._push_pipes) == 1: - await self._push_pipes[0].put_async(batch) - continue - for frontend_id, sub_batch in enumerate( - bucket_responses_by_frontend(batch, len(self._push_pipes))): - if sub_batch: - await self._push_pipes[frontend_id].put_async(sub_batch) + await self._push_pipe.put_async(batch) async def _mainloop(self): ''' The loop for handle_response and keep producing outputs. ''' @@ -302,13 +289,13 @@ async def main(): @print_traceback_on_error def postproc_worker_main(feedin_ipc_addr: tuple[str, Optional[bytes]], - feedout_ipc_addrs: List[tuple[str, Optional[bytes]]], + feedout_ipc_addr: tuple[str, Optional[bytes]], tokenizer_dir: str, record_creator: Callable, post_processor_hook: Optional[str] = None): # Pass the hook import path; PostprocWorker builds it once. worker = PostprocWorker(feedin_ipc_addr, - feedout_ipc_addrs, + feedout_ipc_addr, tokenizer_dir=tokenizer_dir, record_creator=record_creator, post_processor_hook=post_processor_hook) diff --git a/tensorrt_llm/executor/proxy.py b/tensorrt_llm/executor/proxy.py index d1baa0792785..5599cc8df2a8 100644 --- a/tensorrt_llm/executor/proxy.py +++ b/tensorrt_llm/executor/proxy.py @@ -16,8 +16,6 @@ import concurrent.futures import json import os -import shutil -import tempfile import threading import weakref from queue import Empty @@ -46,14 +44,11 @@ from .utils import (EngineDeadError, ErrorResponse, RequestError, WorkerCommIpcAddrs, create_mpi_comm_session, get_spawn_proxy_process_env, is_llm_response, - multi_frontend_request_addr, multi_frontend_result_addr, - namespace_client_id, print_alive_threads) + print_alive_threads) from .worker import GenerationExecutorWorker, worker_main -from .worker_process_monitor import WorkerProcessIdentity, WorkerProcessMonitor __all__ = [ "GenerationExecutorProxy", - "GenerationExecutorFrontendProxy", ] # Methods that are explicitly implemented for multi-rank MPI/IPC executor @@ -165,24 +160,6 @@ def __init__( self._enable_resource_governor = bool( getattr(_llm_args, "enable_resource_governor", False)) - # Multi-frontend serving: this launcher proxy owns the shared ipc - # dir + HMAC key for the per-frontend endpoints (_setup_queues); - # trtllm-serve hands them to the attached frontends via - # multi_frontend_attach_info(). - self._num_frontends = (_llm_args.num_serve_frontends - if _llm_args is not None else 1) - self._multi_frontend_ipc_dir: Optional[str] = None - self._multi_frontend_hmac: Optional[bytes] = None - if self._num_frontends > 1: - if self._enable_resource_governor: - raise ValueError( - "Multi-frontend serving does not support " - "enable_resource_governor: the resource-governor signal " - "only reaches the launcher frontend.") - self._multi_frontend_ipc_dir = tempfile.mkdtemp( - prefix="trtllm_frontends_") - self._multi_frontend_hmac = os.urandom(32) - # Generate RPC address and key for stats RPC self.rpc_addr = get_unique_ipc_addr() self.hmac_key = os.urandom(32) @@ -199,7 +176,6 @@ def __init__( self.dispatch_result_thread: Optional[ManagedThread] = None self.rpc_client: Optional[RPCClient] = None - self._worker_process_monitor = WorkerProcessMonitor() self._start_executor_workers(worker_kwargs) # Create RPC client after workers are started (worker starts RPC server) @@ -248,19 +224,6 @@ def _check_mpi_futures(self) -> bool: return True return False - def _check_mpi_workers(self) -> bool: - """Check OS process handles and MPI futures for worker death.""" - dead_worker = self._worker_process_monitor.find_dead_worker() - if dead_worker is not None: - self._set_fatal_error( - RuntimeError("MPI worker rank " - f"{dead_worker.rank} (pid {dead_worker.pid}) " - "exited unexpectedly")) - if not self.doing_shutdown: - self.pre_shutdown() - return True - return self._check_mpi_futures() - def _drain_error_queue(self) -> bool: """Drain all queued errors, skipping per-request errors. @@ -303,7 +266,7 @@ def check_health(self) -> bool: if self._drain_error_queue(): return self._fatal_error is None and not self.doing_shutdown - if self._check_mpi_workers(): + if self._check_mpi_futures(): return False return True @@ -333,18 +296,6 @@ def _mark_engine_dead(self, error: Optional[BaseException] = None) -> None: result.queue.put(dead_error) except Exception: # noqa: BLE001 - a full/closed queue must not stop the sweep pass - # Release the session's exit joins here, not at teardown: interpreter - # exit joins non-daemon threads before any teardown code runs, so a - # wedged pool manager thread must be deregistered while user code is - # still alive. Non-destructive, hence safe for unowned sessions. - release = getattr(getattr(self, 'mpi_session', None), - 'release_exit_joins', None) - if release is not None: - try: - release() - except Exception as e: # noqa: BLE001 - best-effort cleanup - logger.debug( - f"MPI session exit-join release failed (ignored): {e!r}") def _handle_worker_death(self, error: BaseException) -> None: """Event-driven worker-death handler. @@ -392,10 +343,9 @@ def _check_remote_worker_death(self) -> bool: def _error_monitor_loop(self) -> None: """Background thread that reaps a dead engine and drives pre_shutdown. - Checks local MPI worker process handles and futures, remote-session - worker-death notifications, and the error queue using the shared - ``_check_mpi_workers()``, ``_check_remote_worker_death()`` and - ``_drain_error_queue()`` helpers. + Checks MPI worker futures, remote-session worker-death notifications, + and the error queue using the shared ``_check_mpi_futures()``, + ``_check_remote_worker_death()`` and ``_drain_error_queue()`` helpers. Propagation to pending requests is event-driven via ``_handle_worker_death`` (the MPI future done-callback) where futures @@ -405,7 +355,7 @@ def _error_monitor_loop(self) -> None: """ while not self.doing_shutdown and self._fatal_error is None: try: - if self._check_mpi_workers(): + if self._check_mpi_futures(): logger.error("Error monitor: MPI worker crash detected, " "shutting down") return @@ -429,91 +379,34 @@ def _error_monitor_loop(self) -> None: self._shutdown_event.wait(timeout=5.0) def _setup_queues(self) -> WorkerCommIpcAddrs: - frontend_result_addrs = None - if self._num_frontends > 1: - # The rank0 worker BINDS the request ingress (PULL) so every - # frontend can PUSH-connect; each frontend (incl. this launcher, - # frontend 0) binds its own result lane (PULL). - ipc_dir = self._multi_frontend_ipc_dir - hmac_key = self._multi_frontend_hmac - request_addr = (multi_frontend_request_addr(ipc_dir), hmac_key) - frontend_result_addrs = [(multi_frontend_result_addr(ipc_dir, - i), hmac_key) - for i in range(self._num_frontends)] - self.request_queue = IpcQueue(request_addr, - is_server=False, - socket_type=zmq.PUSH, - name="proxy_request_queue") - self.result_queue = FusedIpcQueue(frontend_result_addrs[0], - is_server=True, - fuse_message=False, - socket_type=zmq.PULL, - name="proxy_result_queue") - else: - request_addr = None - self.request_queue = IpcQueue(is_server=True, - name="proxy_request_queue") - # TODO[chunweiy]: Unify IpcQueue and FusedIpcQueue - # Use PULL mode when enable_postprocess_parallel as there are - # multiple senders from multiple processes. - self.result_queue = FusedIpcQueue( - is_server=True, - fuse_message=False, - socket_type=zmq.PULL - if self.enable_postprocess_parallel else zmq.PAIR, - name="proxy_result_queue") + + self.request_queue = IpcQueue(is_server=True, + name="proxy_request_queue") self.worker_init_status_queue = IpcQueue( is_server=True, socket_type=zmq.ROUTER, name="worker_init_status_queue") + # TODO[chunweiy]: Unify IpcQueue and FusedIpcQueue + # Use PULL mode when enable_postprocess_parallel as there are + # multiple senders from multiple processes. + self.result_queue = FusedIpcQueue( + is_server=True, + fuse_message=False, + socket_type=zmq.PULL + if self.enable_postprocess_parallel else zmq.PAIR, + name="proxy_result_queue") self._resource_governor_queue = IpcQueue( is_server=True, name="proxy_resource_governor_queue" ) if self._enable_resource_governor else None # Stats and KV events are now fetched via RPC, not IPC queues. return WorkerCommIpcAddrs( - # A connect-mode queue has no bound .address; use the preset one. - request_queue_addr=request_addr - if request_addr is not None else self.request_queue.address, + request_queue_addr=self.request_queue.address, worker_init_status_queue_addr=self.worker_init_status_queue.address, result_queue_addr=self.result_queue.address, resource_governor_queue_addr=self._resource_governor_queue.address if self._resource_governor_queue is not None else None, - frontend_result_queue_addrs=frontend_result_addrs, ) - def multi_frontend_attach_info(self) -> Optional[dict]: - """The attach payload consumed by attached serving frontends. - - See GenerationExecutorFrontendProxy and commands/serve.py. Returns - None unless multi-frontend mode is active. - """ - if self._num_frontends <= 1: - return None - ipc_dir = self._multi_frontend_ipc_dir - hmac_key = self._multi_frontend_hmac - return { - "mode": - "classic", - "request_addr": - multi_frontend_request_addr(ipc_dir), - "result_addrs": [ - multi_frontend_result_addr(ipc_dir, i) - for i in range(self._num_frontends) - ], - "hmac_key": - hmac_key.hex(), - # Attached frontends must apply the same collective_rpc guard - # as the launcher (see _check_collective_rpc_guard). - "model_world_size": - self.model_world_size, - # Stats / KV events / disagg params RPC endpoint on the rank0 - # worker (ROUTER socket, natively multi-client). - "rpc_addr": - self.rpc_addr, - "rpc_hmac_key": - self.hmac_key.hex(), - } - @property def resource_governor_queue(self): return self._resource_governor_queue @@ -644,7 +537,7 @@ def mpi_done_callback(future: concurrent.futures.Future): while True: if self.worker_init_status_queue.poll(1): - status = self.worker_init_status_queue.get() + ready_signal, error_trace = self.worker_init_status_queue.get() # Send ACK to the worker self.worker_init_status_queue.put("ACK") logger.info("get signal from executor worker") @@ -654,7 +547,6 @@ def mpi_done_callback(future: concurrent.futures.Future): raise RuntimeError("Executor worker died during initialization") self._handle_background_error() - ready_signal, error_trace = status[:2] if ready_signal != GenerationExecutorProxy.READY_SIGNAL: logger.error(f"Executor worker initialization error: {error_trace}") # Only abort a session this proxy created; an externally owned @@ -664,21 +556,6 @@ def mpi_done_callback(future: concurrent.futures.Future): raise RuntimeError( "Executor worker returned error") from ready_signal - self._register_worker_processes(status) - - def _register_worker_processes(self, status: tuple) -> None: - """Register identities returned by locally spawned MPI workers. - - Test session reuse replaces this module's ``MpiPoolSession`` class - reference with a factory, so identify pool-backed sessions by excluding - the external communication session types. - """ - if not isinstance( - self.mpi_session, - (MpiCommSession, RemoteMpiCommSessionClient)) and len(status) == 3: - worker_process_identities: List[WorkerProcessIdentity] = status[2] - self._worker_process_monitor.register(worker_process_identities) - def _abort_all_requests(self): # The results can be finished during this loop, so self._results may be changed. for result in list(self._results.values()): @@ -694,8 +571,6 @@ def pre_shutdown(self): else: self.doing_shutdown = True - self._worker_process_monitor.close() - # Wake the error monitor thread immediately so it exits cleanly if hasattr(self, '_shutdown_event'): self._shutdown_event.set() @@ -721,28 +596,8 @@ def pre_shutdown(self): if not self.mpi_futures or any(not f.done() for f in self.mpi_futures): self.request_queue.put_noblock(None, retry=4) - def _get_next_client_id(self) -> int: - client_id = super()._get_next_client_id() - if self._num_frontends > 1: - # Lane 0 follows the same namespace rule as attached frontends - # so a long-lived counter can never bleed into the frontend-id - # bits (a no-op re-encode until the counter wraps). - client_id = namespace_client_id(0, client_id) - return client_id - - def _cleanup_multi_frontend_ipc_dir(self): - """Remove the launcher-owned multi-frontend ipc directory. - - Unlinking ipc socket paths does not disturb established zmq - connections; it only prevents new connects. - """ - if self._multi_frontend_ipc_dir is not None: - shutil.rmtree(self._multi_frontend_ipc_dir, ignore_errors=True) - self._multi_frontend_ipc_dir = None - def shutdown(self): if not self.workers_started: - self._cleanup_multi_frontend_ipc_dir() return if not self.doing_shutdown: @@ -750,15 +605,7 @@ def shutdown(self): logger_debug('Proxy.shutdown...\n', "yellow") - # An abruptly-killed worker world (MPI_Abort, SIGKILL, OOM) never - # completes its mpi4py futures: give them one short collective grace - # instead of blocking on each, and skip the ones still pending. - if self._engine_dead: - concurrent.futures.wait(self.mpi_futures, timeout=5.0) - for f in self.mpi_futures: - if self._engine_dead and not f.done(): - continue try: f.result() except: @@ -775,11 +622,7 @@ def shutdown(self): if self.dispatch_result_thread is not None and self.dispatch_result_thread.is_alive( ): self.dispatch_result_thread.stop() - # With the engine dead, the shutdown sentinel will never arrive - # and the dispatcher may be blocked in a ZMQ recv forever: bound - # the join and leak the daemon thread. - self.dispatch_result_thread.join( - timeout=5.0 if self._engine_dead else None) + self.dispatch_result_thread.join() # step3: finish all remaining work @@ -794,15 +637,10 @@ def shutdown(self): self.result_queue.close() if self._resource_governor_queue is not None: self._resource_governor_queue.close() - self._cleanup_multi_frontend_ipc_dir() self.workers_started = False if self._owns_mpi_session: - if self._engine_dead: - # Anything joining a dead worker world blocks forever. - self.mpi_session.abandon() - else: - self.mpi_session.shutdown() + self.mpi_session.shutdown() # Process the errors in-case error during shutting down the threads self._handle_background_error() @@ -948,16 +786,6 @@ def get_disaggregated_params(self) -> dict: logger.warning(f"Error fetching disaggregated params via RPC: {e}") return {} - def get_data_transceiver_state(self) -> bytes: - """Get serialized DataTransceiverState from worker runtime via RPC.""" - if self.rpc_client is None: - return b"" - try: - return self.rpc_client.get_data_transceiver_state().remote() - except RPCError as e: - logger.error(f"Error fetching data transceiver state via RPC: {e}") - raise - def aget_stats(self, timeout: float) -> IterationResult: """Get iteration statistics from the runtime via RPC (async). @@ -1049,126 +877,3 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self.shutdown() return False # propagate the exception - - -class GenerationExecutorFrontendProxy(GenerationExecutorProxy): - """An attached serving frontend for the classic IPC executor path. - - Used for multi-frontend serving (num_serve_frontends > 1). - PUSH-connects to the request ingress bound by the rank0 worker and binds - its own per-frontend result lane (PULL); the worker routes responses to - this lane by the frontend id embedded in the top bits of client_id (see - utils.namespace_client_id). It never owns the engine: no MPI - session, no worker launch, and shutdown never emits the worker's None - shutdown sentinel -- that right is the launcher frontend's alone. - """ - - def __init__( - self, - attach_info: dict, - *, - frontend_id: int, - postproc_worker_config: Optional[PostprocWorkerConfig] = None, - is_llm_executor: Optional[bool] = None, - ) -> None: - num_lanes = len(attach_info["result_addrs"]) - if not 0 < frontend_id < num_lanes: - raise ValueError( - f"frontend_id {frontend_id} out of range: attached frontends " - f"use ids 1..{num_lanes - 1} (id 0 is the launcher)") - postproc_worker_config = postproc_worker_config or PostprocWorkerConfig( - ) - # Deliberately skip GenerationExecutorProxy.__init__: it creates an - # MPI session, launches workers, and registers the pre_shutdown - # atexit hook that emits the engine shutdown sentinel. - GenerationExecutor.__init__( - self, - num_postprocess_workers=postproc_worker_config. - num_postprocess_workers, - postprocess_tokenizer_dir=postproc_worker_config. - postprocess_tokenizer_dir, - is_llm_executor=is_llm_executor) - - # State consumed by methods inherited from GenerationExecutorProxy - # (submit / check_health / collective_rpc). The engine lives with - # the launcher: there are no local MPI workers, so the monitor - # stays empty and worker death reaches this frontend through its - # result lane / error queue instead. - self._engine_dead = False - self.model_world_size = attach_info.get("model_world_size", 1) - self._worker_process_monitor = WorkerProcessMonitor() - - self._frontend_id = frontend_id - self._num_frontends = num_lanes - self._results: Dict[int, GenerationResult] = {} - self.garbage_collection_gen0_threshold = None - self.workers_started = False - self.dispatch_result_thread: Optional[ManagedThread] = None - # Must be None: OpenAIServer reads the resource_governor_queue - # property at init; the governor lives with the launcher only. - self._resource_governor_queue = None - - hmac_key = bytes.fromhex(attach_info["hmac_key"]) - self.request_queue = IpcQueue( - (attach_info["request_addr"], hmac_key), - is_server=False, - socket_type=zmq.PUSH, - name=f"frontend_{frontend_id}_request_queue") - self.result_queue = FusedIpcQueue( - (attach_info["result_addrs"][frontend_id], hmac_key), - is_server=True, - fuse_message=False, - socket_type=zmq.PULL, - name=f"frontend_{frontend_id}_result_queue") - - # Stats / KV events / disagg params share the rank0 worker's stats - # RPC server with the launcher (ROUTER socket, multi-client). - self.rpc_client: Optional[RPCClient] = None - if attach_info.get("rpc_addr"): - self.rpc_client = RPCClient(attach_info["rpc_addr"], - hmac_key=bytes.fromhex( - attach_info["rpc_hmac_key"])) - - def _get_next_client_id(self) -> int: - # Embed the frontend id in the top bits so the worker routes the - # responses back to this frontend's result lane. - return namespace_client_id(self._frontend_id, - super()._get_next_client_id()) - - def check_health(self) -> bool: - """Health contract of an attached frontend. - - An attached frontend owns no workers, so there is no process or - MPI-future liveness to poll: it is healthy while no fatal error - has been recorded and shutdown has not begun. Engine death - reaches it through the per-lane result socket / dispatch-thread - error path, which records the fatal error checked here. - """ - if self.doing_shutdown or self._fatal_error is not None: - return False - - if self._drain_error_queue(): - return self._fatal_error is None and not self.doing_shutdown - - return True - - def pre_shutdown(self): - if self.doing_shutdown: - return - self.doing_shutdown = True - # Abort this frontend's in-flight requests so the engine frees their - # slots. Never send the None engine-shutdown sentinel: the launcher - # frontend owns the engine lifecycle (see the class docstring). - self._abort_all_requests() - - def shutdown(self): - self.pre_shutdown() - if self.rpc_client is not None: - self.rpc_client.close() - self.rpc_client = None - self._worker_process_monitor.close() - # The dispatch thread blocks on result_queue.get(); it is a daemon - # ManagedThread that exits with the process or on the worker's - # per-lane None sentinel at engine teardown. Closing its socket from - # another thread is not ZMQ-safe, so leave the queues to process - # teardown. diff --git a/tensorrt_llm/executor/ray_executor.py b/tensorrt_llm/executor/ray_executor.py index accd9efc1632..0eb62678d85e 100644 --- a/tensorrt_llm/executor/ray_executor.py +++ b/tensorrt_llm/executor/ray_executor.py @@ -1,6 +1,5 @@ import asyncio import os -import time from typing import Any, Dict, List, Optional, Tuple try: @@ -356,17 +355,6 @@ def shutdown(self): except Exception as e: logger.warning(f"Error shutting down: {e}") - # The engines are already stopped by the shutdown RPC above, so - # kill the actor processes explicitly instead of relying on - # handle garbage collection. ray.kill() only *initiates* an - # asynchronous kill; _wait_for_cluster_resource_release() below - # blocks until Ray has reclaimed the workers' resources. - for worker in self.workers: - try: - ray.kill(worker, no_restart=True) - except Exception as e: - logger.warning(f"Error killing worker: {e}") - if hasattr(self, 'rpc_client') and self.rpc_client is not None: try: self.rpc_client.close() @@ -383,53 +371,10 @@ def shutdown(self): self.placement_group = None self.bundle_indices = None - # ray.kill() and remove_placement_group() above are asynchronous. - # Block until Ray has reclaimed the workers' resources so their GPU - # cleanup has completed before shutdown() returns. - self._wait_for_cluster_resource_release(timeout=30.0) - if self.has_start_local_cluser and ray.is_initialized(): logger.debug("Shutting down Ray cluster") ray.shutdown() - def _wait_for_cluster_resource_release(self, timeout: float = 30.0) -> None: - """Block until Ray returns the workers' resources to the cluster. - - ray.kill() and remove_placement_group() only initiate an - asynchronous teardown; Ray reclaims an actor's logical resources - after the raylet has reaped the worker process, by which point the - CUDA driver has already destroyed its context (GPU memory and IPC - mappings). Waiting here therefore guarantees that a subsequent LLM - instance will not race against the dying workers, which can - otherwise fail spuriously (e.g. cudaErrorMapBufferObjectFailed when - opening CUDA IPC handles). Full availability can only be expected on - a cluster dedicated to this executor, so the wait is skipped when - attached to an external cluster. Best-effort: logs a warning on - timeout instead of raising, since shutdown must not fail. - """ - if not self.has_start_local_cluser or not ray.is_initialized(): - return - deadline = time.monotonic() + timeout - busy = {} - while time.monotonic() < deadline: - try: - cluster = ray.cluster_resources() - available = ray.available_resources() - except Exception as e: - logger.debug(f"Could not query Ray resources: {e}") - return - busy = { - key: cluster[key] - available.get(key, 0.0) - for key in ("GPU", "CPU") if key in cluster and cluster[key] - - available.get(key, 0.0) > 1e-6 - } - if not busy: - return - time.sleep(0.1) - logger.warning( - f"Timed out after {timeout}s waiting for Ray to reclaim cluster " - f"resources; still in use: {busy}.") - def _get_worker_ready_futures(self): return [worker.__ray_ready__.remote() for worker in self.workers] diff --git a/tensorrt_llm/executor/result.py b/tensorrt_llm/executor/result.py index a6bbb60e8d20..a3014da7072f 100644 --- a/tensorrt_llm/executor/result.py +++ b/tensorrt_llm/executor/result.py @@ -295,21 +295,10 @@ def _handle_sequence(self, output = self._outputs[seq_idx] output.disaggregated_params = self.disaggregated_params output._last_token_ids_len = len(output.token_ids) - output._last_logprobs_len = len(output.logprobs) - decoder_output_prefix = () - if (self.sampling_params.exclude_input_from_output - or getattr(self, "_streaming", False)): - decoder_output_prefix = \ - self.sampling_params._decoder_output_token_prefix if self.sampling_params.use_beam_search: # Beam search enforces returning all generated tokens - output.token_ids = [ - *decoder_output_prefix, - *response_tensors.output_token_ids[src_idx], - ] + output.token_ids = response_tensors.output_token_ids[src_idx] else: - if decoder_output_prefix and not output.token_ids: - output.token_ids.extend(decoder_output_prefix) output.token_ids.extend(response_tensors.output_token_ids[src_idx]) if response_tensors.cum_log_probs is not None: @@ -321,15 +310,16 @@ def _handle_sequence(self, # generation logprobs handling (provenance varies by backend) if logprobs_result and logprobs_result.generation is not None: # TRT backend # update logprobs from ResponseWrapper (TRT top logprobs WAR) + output._last_logprobs_len = len(output.logprobs) output.logprobs += logprobs_result.generation elif response_tensors.log_probs is not None: # PyTorch backend # handle logprobs directly from response tensors given by sampler - if decoder_output_prefix and self.sampling_params.use_beam_search: - output.logprobs = [ - *self._get_decoder_output_prefix_logprobs(), - *response_tensors.log_probs[src_idx], - ] - elif self.use_trtllm_sampler: + output._last_logprobs_len = len(output.logprobs) + # In streaming mode, since out-of-order responses are not possible, + # each streamed response_tensors.log_probs[src_idx] + # contains a streamwise monotonically growing list of logprobs. + # so we need to accumulate only the new ones unique to that particular streamed response + if self.use_trtllm_sampler: assert output._last_logprobs_len <= len( response_tensors.log_probs[src_idx] ), (f"_last_logprobs_len ({output._last_logprobs_len}) > log_probs length (" @@ -337,9 +327,6 @@ def _handle_sequence(self, output.logprobs += response_tensors.log_probs[src_idx][ output._last_logprobs_len:] else: - if decoder_output_prefix and not output.logprobs: - output.logprobs.extend( - self._get_decoder_output_prefix_logprobs()) output.logprobs += response_tensors.log_probs[src_idx] # overcome some WAR in the cpp executor @@ -440,13 +427,6 @@ def _handle_sequence(self, if self._done: self.do_tracing(output, req_perf_metrics_dict) - def _get_decoder_output_prefix_logprobs( - self) -> TokenLogprobs | SimpleTokenLogprobs: - prefix = self.sampling_params._decoder_output_token_prefix - if self.sampling_params.logprobs_simple_format: - return [0.0] * len(prefix) - return [{token_id: Logprob(logprob=0.0, rank=1)} for token_id in prefix] - @print_traceback_on_error @nvtx_range_debug("handle_response", color="red", @@ -1015,11 +995,7 @@ def _handle_ray_response(self, response: Any): return response def _result_step(self, timeout: Optional[float] = None): - # Honor `timeout`: a bounded `queue.get()` lets the caller regain control if the executor - # worker dies silently without pushing a terminal response, instead of blocking potentially - # indefinitely. - # Raises `queue.Empty` on timeout; `result()` turns that into a `TimeoutError`. - response = self.queue.get(timeout=timeout) + response = self.queue.get() # Fast-fail: when a worker dies, the proxy enqueues EngineDeadError onto # every pending result so this get() unblocks instead of hanging forever # on a queue whose producer is gone. Record it as the sticky terminal @@ -1046,29 +1022,15 @@ def result(self, timeout: Optional[float] = None) -> "GenerationResult": """Wait for the completion of the request, and return the result. Args: - timeout (float, optional): The maximum number of seconds to wait for the request to - complete. `None` (default) waits indefinitely. - The timeout is a total budget across all streaming steps, not per-step. + timeout (float, optional): Timeout. Defaults to None. Returns: tensorrt_llm.executor.result.GenerationResult: generation result. - - Raises: - TimeoutError: If the request does not complete within `timeout` seconds. Bounding the - wait prevents a silently-dead executor worker from hanging the caller forever. """ if self._terminal_error is not None: raise self._terminal_error - deadline = None if timeout is None else time.monotonic() + timeout while not self._done: - remaining = (None if deadline is None else max( - 0.0, deadline - time.monotonic())) - try: - self._result_step(remaining) - except Empty: - raise TimeoutError( - f"Request {self.request_id} did not complete within " - f"{timeout} seconds.") from None + self._result_step(timeout) return self async def aresult(self) -> "GenerationResult": diff --git a/tensorrt_llm/executor/rpc_proxy.py b/tensorrt_llm/executor/rpc_proxy.py index 0763e09ad032..680b8d5c84e4 100644 --- a/tensorrt_llm/executor/rpc_proxy.py +++ b/tensorrt_llm/executor/rpc_proxy.py @@ -254,10 +254,6 @@ def collective_rpc( return [remote_call.remote_future()] return [remote_call.remote()] - def get_data_transceiver_state(self) -> bytes: - """Get serialized DataTransceiverState from worker runtime via RPC.""" - return self.rpc_client.get_data_transceiver_state().remote() - def setup_engine_remote(self): return self.rpc_client.setup_engine().remote(need_response=True) diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index e4a9f3333f4b..3bd0106241f8 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -229,66 +229,6 @@ class WorkerCommIpcAddrs(NamedTuple): worker_init_status_queue_addr: tuple[str, Optional[bytes]] result_queue_addr: tuple[str, Optional[bytes]] resource_governor_queue_addr: Optional[tuple[str, Optional[bytes]]] = None - # Multi-frontend serving: one result lane per frontend. When set, the - # rank0 worker BINDS the request queue (PULL) and routes responses to - # these lanes; result_queue_addr then aliases lane 0 (the launcher). - frontend_result_queue_addrs: Optional[list[tuple[str, - Optional[bytes]]]] = None - - -# Multi-frontend client_id namespacing: a FRONTEND_ID_BITS-wide frontend id -# sits just below the sign bit -- bit 63 stays clear so ids remain positive -# in signed-int64 contexts -- and the low bits carry the per-frontend -# request counter. A stray un-namespaced (small) id reads as frontend 0, -# the launcher. -FRONTEND_ID_BITS = 6 -# Keep llm_args.num_serve_frontends le= in sync (it cannot import this -# module; test_multi_frontend_routing pins the two together). -MAX_NUM_FRONTENDS = 1 << FRONTEND_ID_BITS -FRONTEND_ID_SHIFT = 63 - FRONTEND_ID_BITS -FRONTEND_COUNTER_MASK = (1 << FRONTEND_ID_SHIFT) - 1 - - -def get_frontend_id(client_id: Optional[int]) -> int: - """Extract the originating frontend id from a namespaced client id.""" - if not isinstance(client_id, int): - return 0 - return client_id >> FRONTEND_ID_SHIFT - - -def namespace_client_id(frontend_id: int, client_id: int) -> int: - """Embed frontend_id in the top bits of a per-frontend client id.""" - return (frontend_id << FRONTEND_ID_SHIFT) | (client_id - & FRONTEND_COUNTER_MASK) - - -def frontend_lane_index(client_id: Optional[int], num_lanes: int) -> int: - """The result-lane index for a response's originating frontend. - - None (e.g. ADP dummy requests) and out-of-range frontend ids go to - lane 0, the launcher, which silently discards them like today. - """ - frontend_id = get_frontend_id(client_id) - return frontend_id if frontend_id < num_lanes else 0 - - -def bucket_responses_by_frontend(responses: list, - num_frontends: int) -> list[list]: - """Bucket responses by frontend_lane_index of their client_id.""" - buckets = [[] for _ in range(num_frontends)] - for rsp in responses: - buckets[frontend_lane_index(rsp.client_id, num_frontends)].append(rsp) - return buckets - - -def multi_frontend_request_addr(ipc_dir: str) -> str: - """The request ingress bound by the rank0 worker; frontends PUSH-connect.""" - return f"ipc://{os.path.join(ipc_dir, 'request.sock')}" - - -def multi_frontend_result_addr(ipc_dir: str, frontend_id: int) -> str: - """The result lane bound by a frontend; worker/postproc PUSH-connect.""" - return f"ipc://{os.path.join(ipc_dir, f'result_{frontend_id}.sock')}" def is_llm_response(instance): diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index f173aabc224a..87841c1edd8e 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -27,7 +27,6 @@ from .rpc_worker_mixin import RpcWorkerMixin from .utils import (ErrorResponse, IntraProcessQueue, RequestError, WorkerCommIpcAddrs) -from .worker_process_monitor import capture_worker_process_identity __all__ = [ "GenerationExecutorWorker", @@ -153,6 +152,11 @@ def shutdown(self): def block_subordinates(self): if self.rank != 0: + if isinstance(self.engine, tllm.Executor): + self.shutdown() + raise self.WorkerExit( + "block_subordinates() should be used in a `with GenerationExecutorWorker() as ...:` block" + ) from tensorrt_llm._torch.pyexecutor.py_executor import PyExecutor if isinstance(self.engine, PyExecutor): self.engine.wait_shutdown() @@ -216,10 +220,6 @@ def _print_stacks(): postproc_worker_config = postproc_worker_config or PostprocWorkerConfig() is_leader: bool = mpi_rank() == 0 - # Multi-frontend serving: the worker binds the request ingress (PULL) - # and pushes responses to per-frontend result lanes. - multi_frontend_addrs = worker_queues.frontend_result_queue_addrs - frontend_result_queues: Optional[List[FusedIpcQueue]] = None if tracer_init_kwargs is not None and is_leader: tracer = VizTracer(**tracer_init_kwargs) tracer.register_exit() @@ -237,9 +237,7 @@ def _print_stacks(): # inherit the log level from "TLLM_LOG_LEVEL" environment variable logger.set_level(log_level) request_queue = IpcQueue(worker_queues.request_queue_addr, - is_server=multi_frontend_addrs is not None, - socket_type=zmq.PULL if multi_frontend_addrs - is not None else zmq.PAIR, + is_server=False, name="worker_request_queue") worker_init_status_queue = IpcQueue( worker_queues.worker_init_status_queue_addr, @@ -261,16 +259,6 @@ def _print_stacks(): name=f"postprocess_{i}_feedin_queue") for i in range(postproc_worker_config.num_postprocess_workers) ] - elif multi_frontend_addrs is not None: - # One PUSH lane per frontend (see base_worker._send_rsp). - frontend_result_queues = [ - FusedIpcQueue(addr, - is_server=False, - fuse_message=False, - socket_type=zmq.PUSH, - name=f"worker_result_queue_{i}") - for i, addr in enumerate(multi_frontend_addrs) - ] else: # IPC queue for sending results back to the proxy, and let the # Proxy process to handle the postprocess @@ -280,12 +268,9 @@ def _print_stacks(): name="worker_result_queue") def notify_proxy_threads_to_quit(): - # Signal the dispatcher thread in every frontend proxy to quit + # Signal the dispatcher thread in the proxy to quit if result_queue is not None: result_queue.put(None) - elif frontend_result_queues is not None: - for q in frontend_result_queues: - q.put(None) else: assert result_queues is not None for q in result_queues: @@ -295,20 +280,18 @@ def notify_proxy_threads_to_quit(): if is_leader and postproc_worker_config.enabled: logger_debug(f"initiate postprocess workers...", "yellow") - # Each postproc worker pushes to every frontend result lane (a - # single lane in single-frontend mode). - proxy_result_addrs = (multi_frontend_addrs - if multi_frontend_addrs is not None else - [worker_queues.result_queue_addr]) + proxy_result_queue: tuple[ + str, Optional[bytes]] = worker_queues.result_queue_addr assert result_queues is not None postproc_worker_pool = ProcessPoolExecutor( max_workers=postproc_worker_config.num_postprocess_workers) + assert isinstance(proxy_result_queue, tuple) for i in range(postproc_worker_config.num_postprocess_workers): fut = postproc_worker_pool.submit( postproc_worker_main, result_queues[i].address, - proxy_result_addrs, + proxy_result_queue, postproc_worker_config.postprocess_tokenizer_dir, PostprocWorker.default_record_creator, postproc_worker_config.post_processor_hook, @@ -327,8 +310,6 @@ def notify_proxy_threads_to_quit(): # error to the error_queue in the main thread. mpi_comm().barrier() - worker_process_identities = mpi_comm().allgather( - capture_worker_process_identity(mpi_rank())) logger_debug(f"Worker {mpi_rank()} ready to setup backend...\n", "green") try: @@ -365,13 +346,11 @@ def notify_proxy_threads_to_quit(): if is_leader: if postproc_worker_config.enabled: worker.set_postproc_queues(result_queues) - elif frontend_result_queues is not None: - worker.set_frontend_result_queues(frontend_result_queues) else: worker.set_result_queue(result_queue) # Send ready signal with confirmation - ready_msg = (ready_signal, None, worker_process_identities) + ready_msg = (ready_signal, None) if not worker_init_status_queue.notify_with_retry(ready_msg): logger.warning( "Failed to deliver ready signal to proxy, continuing anyway" diff --git a/tensorrt_llm/executor/worker_process_monitor.py b/tensorrt_llm/executor/worker_process_monitor.py deleted file mode 100644 index 4d00980c44a9..000000000000 --- a/tensorrt_llm/executor/worker_process_monitor.py +++ /dev/null @@ -1,200 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Linux process-liveness monitoring for locally spawned executor workers.""" - -from __future__ import annotations - -import logging -import os -import select -import socket -import threading -from typing import Dict, List, NamedTuple, Optional - -logger = logging.getLogger(__name__) - - -class WorkerProcessIdentity(NamedTuple): - """Stable identity for one executor worker process.""" - - rank: int - pid: int - start_time: Optional[int] - hostname: str - pid_namespace: Optional[int] - - -def _read_process_state(pid: int) -> Optional[tuple[str, int]]: - """Read the Linux process state and start time from ``/proc``. - - Returns: - A ``(state, start_time)`` tuple, or ``None`` when the process does not - exist or procfs is unavailable. - """ - try: - with open(f"/proc/{pid}/stat", encoding="utf-8") as stat_file: - stat = stat_file.read() - except (FileNotFoundError, ProcessLookupError): - return None - - # The second field is parenthesized and may contain spaces or parentheses. - command_end = stat.rfind(")") - if command_end < 0: - return None - fields = stat[command_end + 2 :].split() - if len(fields) <= 19: - return None - return fields[0], int(fields[19]) - - -def _read_pid_namespace(pid: int) -> Optional[int]: - """Return the inode identifying a Linux PID namespace.""" - try: - return os.stat(f"/proc/{pid}/ns/pid").st_ino - except OSError: - return None - - -def capture_worker_process_identity(rank: int, pid: Optional[int] = None) -> WorkerProcessIdentity: - """Capture the calling worker's rank, PID, and Linux start time.""" - pid = pid if pid is not None else os.getpid() - try: - process_state = _read_process_state(pid) - except (OSError, ValueError): - process_state = None - start_time = process_state[1] if process_state is not None else None - return WorkerProcessIdentity( - rank=rank, - pid=pid, - start_time=start_time, - hostname=socket.gethostname(), - pid_namespace=_read_pid_namespace(pid), - ) - - -class WorkerProcessMonitor: - """Monitor locally spawned workers without relying on MPI futures. - - Linux pidfds are used when Python exposes ``os.pidfd_open``. A procfs - identity check is retained as a fallback for older Python versions. If - neither mechanism is available, the worker is left to the existing MPI - future/error-queue checks. - """ - - def __init__(self) -> None: - self._pidfd_to_identity: Dict[int, WorkerProcessIdentity] = {} - self._procfs_identities: List[WorkerProcessIdentity] = [] - self._dead_identity: Optional[WorkerProcessIdentity] = None - self._poller = select.poll() if hasattr(select, "poll") else None - self._lock = threading.Lock() - self._local_hostname = socket.gethostname() - self._local_pid_namespace = _read_pid_namespace(os.getpid()) - - def register(self, identities: List[WorkerProcessIdentity]) -> None: - """Register locally spawned worker identities for liveness checks.""" - with self._lock: - self._close_unlocked() - pidfd_open = getattr(os, "pidfd_open", None) - for identity in identities: - if identity.hostname != self._local_hostname: - continue - if identity.pid_namespace != self._local_pid_namespace: - continue - if pidfd_open is not None and self._poller is not None: - try: - pidfd = pidfd_open(identity.pid) - except ProcessLookupError: - self._dead_identity = identity - continue - except OSError as error: - logger.debug("pidfd_open(%d) failed: %s", identity.pid, error) - else: - if identity.start_time is not None: - try: - process_state = _read_process_state(identity.pid) - except (OSError, ValueError) as error: - logger.debug( - "Failed to validate process identity for pid %d: %s", - identity.pid, - error, - ) - else: - if ( - process_state is None - or process_state[0] == "Z" - or process_state[1] != identity.start_time - ): - os.close(pidfd) - self._dead_identity = identity - continue - self._pidfd_to_identity[pidfd] = identity - self._poller.register( - pidfd, select.POLLIN | select.POLLERR | select.POLLHUP - ) - continue - - if identity.start_time is not None: - self._procfs_identities.append(identity) - - def find_dead_worker(self) -> Optional[WorkerProcessIdentity]: - """Return the first worker known to have exited, if any.""" - with self._lock: - if self._dead_identity is not None: - return self._dead_identity - - if self._poller is not None: - for pidfd, _ in self._poller.poll(0): - identity = self._pidfd_to_identity.get(pidfd) - if identity is not None: - self._dead_identity = identity - return identity - - for identity in self._procfs_identities: - try: - process_state = _read_process_state(identity.pid) - except (OSError, ValueError) as error: - logger.debug( - "Failed to check process state for pid %d: %s", identity.pid, error - ) - continue - if ( - process_state is None - or process_state[0] == "Z" - or process_state[1] != identity.start_time - ): - self._dead_identity = identity - return identity - return None - - def close(self) -> None: - """Close all process handles and reset the monitor.""" - with self._lock: - self._close_unlocked() - - def _close_unlocked(self) -> None: - for pidfd in self._pidfd_to_identity: - if self._poller is not None: - try: - self._poller.unregister(pidfd) - except (KeyError, OSError): - pass - try: - os.close(pidfd) - except OSError: - pass - self._pidfd_to_identity.clear() - self._procfs_identities.clear() - self._dead_identity = None diff --git a/tensorrt_llm/functional.py b/tensorrt_llm/functional.py index f39b6eac7a03..f4b97746f615 100644 --- a/tensorrt_llm/functional.py +++ b/tensorrt_llm/functional.py @@ -64,8 +64,7 @@ class PositionEmbeddingType(IntEnum): def is_rope(self) -> bool: return self in [ - self.rope_gptj, self.rope_gpt_neox, self.long_rope, self.mrope, - self.yarn + self.rope_gptj, self.rope_gpt_neox, self.long_rope, self.mrope ] def is_mrope(self) -> bool: diff --git a/tensorrt_llm/inputs/media_io.py b/tensorrt_llm/inputs/media_io.py index 0624978bcb94..9d456b23e4f1 100644 --- a/tensorrt_llm/inputs/media_io.py +++ b/tensorrt_llm/inputs/media_io.py @@ -36,7 +36,6 @@ import requests import soundfile import torch -from blake3 import blake3 from packaging.version import Version from PIL import Image @@ -410,7 +409,6 @@ def _load_video_by_cv2( device: str = "cpu", extract_audio: bool = False, cv2_backend: Optional[int] = None, - raw_bytes_hash: Optional[str] = None, ) -> VideoData: """Decode a video and return sampled frames as a list. @@ -421,13 +419,12 @@ def _load_video_by_cv2( tempfile required. Callers that have no stream-buffered backend available should spill to a tempfile themselves and pass a path. - `format` controls the return type: - `"pt"` - list[torch.Tensor], dtype=float32, shape=(C, H, W), range - [0, 1]; rescaled and permuted to CHW here. - `"np"` - np.ndarray of shape (N, H, W, 3), dtype=uint8; a single - contiguous 4D buffer that HF video processors pass through - without an extra frame-by-frame copy. - `"pil"` - list[PIL.Image], one per sampled frame. + `format` controls the per-frame return type: + `"pt"` - list[torch.Tensor], dtype=float32, shape=(C, H, W), range + [0, 1]; rescaled and permuted to CHW here. + `"np"` - list[np.ndarray], dtype=uint8, shape=(H, W, 3); returned as + decoded, leaving rescale/permute to the HF processor. + `"pil"` - list[PIL.Image], one per sampled frame. """ assert format in ("pt", "np", "pil"), "format must be one of 'pt', 'np', 'pil'" @@ -478,66 +475,42 @@ def _load_video_by_cv2( indices = np.linspace(0, frame_count - 1, num_frames_to_sample, dtype=int).tolist() - # Defer allocating the stacked buffer until the first frame is actually - # decoded: container metadata (CAP_PROP_FRAME_WIDTH/HEIGHT) can be 0 - # or stale before the first decode for some codecs. - stacked_rgb: Optional[np.ndarray] = None - H = W = None - + # Sequential forward scan — grab() without per-frame seek target_set = set(indices) max_idx = indices[-1] - bgr_scratch: Optional[np.ndarray] = None - valid_indices: list[int] = [] - # Log at most once per decode to avoid spamming when a whole stream is - # affected (e.g. every frame retrieves with a drifted shape). - skip_warned = False + raw_frames: dict[int, np.ndarray] = {} frame_idx = 0 - while frame_idx <= max_idx and vidcap.grab(): + while frame_idx <= max_idx: + grab_succeeded = vidcap.grab() + if not grab_succeeded: + break if frame_idx in target_set: - # Reuse a single BGR buffer across retrieves; cv2 replaces its - # contents in place when the argument is shape-compatible. - ok, bgr_scratch = vidcap.retrieve(bgr_scratch) - if ok: - fh, fw = bgr_scratch.shape[:2] - if stacked_rgb is None: - H, W = fh, fw - stacked_rgb = np.empty((num_frames_to_sample, H, W, 3), dtype=np.uint8) - if (fh, fw) == (H, W): - cv2.cvtColor( - bgr_scratch, - cv2.COLOR_BGR2RGB, - dst=stacked_rgb[len(valid_indices)], - ) - valid_indices.append(frame_idx) - elif not skip_warned: - logger.warning( - f"Skipping frame {frame_idx} and subsequent size-drifted frames: " - f"shape={(fh, fw)} differs from first decoded shape=({H}, {W})." - ) - skip_warned = True - elif not skip_warned: - logger.warning( - f"Skipping frame {frame_idx} and subsequent retrieve failures: retrieve returned ok=False." - ) - skip_warned = True + # cv2 decodes frames in BGR order; convert to RGB for downstream use + retrieve_succeeded, bgr_frame = vidcap.retrieve() + if retrieve_succeeded: + raw_frames[frame_idx] = cv2.cvtColor(bgr_frame, cv2.COLOR_BGR2RGB) frame_idx += 1 vidcap.release() - if stacked_rgb is None or not valid_indices: + if not raw_frames: raise ValueError("Video has no readable frames.") - stacked_rgb = stacked_rgb[: len(valid_indices)] + valid_indices = [i for i in indices if i in raw_frames] if format == "pt": - stacked_f32 = stacked_rgb.astype(np.float32) - stacked_f32 *= 1.0 / 255.0 + # uint8 -> float32 + /255 rescale done once on the stacked buffer + # so the dtype conversion is a single memory pass and there's one + # Python torch call instead of one per frame. + stacked_uint8 = np.stack([raw_frames[i] for i in valid_indices]) + stacked_f32 = stacked_uint8.astype(np.float32) * (1.0 / 255.0) tensor_nchw = torch.from_numpy(stacked_f32).permute(0, 3, 1, 2).contiguous() if device != "cpu": tensor_nchw = tensor_nchw.to(device) loaded_frames = list(torch.unbind(tensor_nchw, dim=0)) elif format == "np": - loaded_frames = stacked_rgb + # uint8 HWC frames as-is; the HF processor rescales/permutes. + loaded_frames = [raw_frames[i] for i in valid_indices] else: # "pil" - loaded_frames = [Image.fromarray(frame) for frame in stacked_rgb] + loaded_frames = [Image.fromarray(raw_frames[i]) for i in valid_indices] metadata = { "total_num_frames": frame_count, @@ -569,12 +542,7 @@ def _load_video_by_cv2( else: raise - return VideoData( - frames=loaded_frames, - metadata=metadata, - audio=audio, - raw_bytes_hash=raw_bytes_hash, - ) + return VideoData(frames=loaded_frames, metadata=metadata, audio=audio) def _normalize_file_uri(uri: str) -> str: @@ -792,7 +760,6 @@ def load_bytes(self, data: bytes) -> VideoData: # is unlinked on context exit; the inode survives until cv2 closes # its own fd (Linux semantics), so the decode inside the `with` # block reads safely. - raw_bytes_hash = blake3(data).hexdigest() cv2_backend = _select_cv2_stream_buffered_backend() if cv2_backend is not None: return _load_video_by_cv2( @@ -803,7 +770,6 @@ def load_bytes(self, data: bytes) -> VideoData: self._device, extract_audio=self._extract_audio, cv2_backend=cv2_backend, - raw_bytes_hash=raw_bytes_hash, ) with tempfile.NamedTemporaryFile(suffix=".mp4", dir=_VIDEO_TEMPFILE_DIR) as tmp: tmp.write(data) @@ -815,14 +781,20 @@ def load_bytes(self, data: bytes) -> VideoData: self._format, self._device, extract_audio=self._extract_audio, - raw_bytes_hash=raw_bytes_hash, ) def load_base64(self, media_type: str, data: str) -> VideoData: return self.load_bytes(base64.b64decode(data)) def load_file(self, url: str) -> VideoData: - return self.load_bytes(Path(_normalize_file_uri(url)).read_bytes()) + return _load_video_by_cv2( + _normalize_file_uri(url), + self._num_frames, + self._fps, + self._format, + self._device, + extract_audio=self._extract_audio, + ) MEDIA_IO_REGISTRY: Mapping[MediaModality, Type[BaseMediaIO]] = MappingProxyType( diff --git a/tensorrt_llm/inputs/multimodal.py b/tensorrt_llm/inputs/multimodal.py index fb4b33779d32..597011ed3fee 100644 --- a/tensorrt_llm/inputs/multimodal.py +++ b/tensorrt_llm/inputs/multimodal.py @@ -3,7 +3,7 @@ """Multimodal utilities for handling images and other media types in TensorRT-LLM.""" from dataclasses import dataclass, field -from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np import torch @@ -503,12 +503,6 @@ class MultimodalParams: multimodal_data: Optional[Dict[str, Any]] = field(default_factory=dict) multimodal_runtime: Optional[MultimodalRuntimeData] = None input_ids_start_offset: int = 0 - # Prompt-order manifest of data-backed items. Each entry is - # ``{"modality": m, "index": i, "placeholder": p}`` where ``i`` indexes - # ``multi_modal_data[m]``. Encoders that interleave items across modalities - # in prompt order read ``modality``/``index``; ``placeholder`` is carried - # for parity with the tracker manifest. - mm_item_order: Optional[List[Dict[str, Union[str, int]]]] = None # CUDA event recorded on a side stream by the MM encoder prefetch path. # When set, the consume site in `get_multimodal_embeddings` issues a # `wait_event` on the current stream before reading cached embeddings. @@ -820,26 +814,11 @@ def _update_hash(hasher, item: object) -> None: hasher.update(serialize_item(item)) -class MMHashResult(NamedTuple): - """Parallel per-modality dicts of content hashes and their UUIDs. - - Both dicts are indexed first by modality name (`"image"`, `"video"`, - ...) then by within-modality item position, so `hashes[m][i]` and - `uuids[m][i]` describe the same item. - """ - - hashes: Dict[str, List[str]] - """Modality -> list of BLAKE3 hex digests (64 chars each).""" - - uuids: Optional[Dict[str, List[Optional[str]]]] - """Modality -> list of original UUID strings. `None` entries mark - items that fell back to content-only hashing. The whole dict is - `None` when the caller supplied no UUIDs at all.""" - - -def apply_mm_hashes(mm_data: Dict[str, Any], - mm_uuids: Optional[Dict[str, List[Optional[str]]]] = None, - hash_lib=default_hasher) -> "MMHashResult": +def apply_mm_hashes( + mm_data: Dict[str, Any], + mm_uuids: Optional[Dict[str, List[Optional[str]]]] = None, + hash_lib=default_hasher +) -> Tuple[Dict[str, List[str]], Optional[List[Optional[str]]]]: """Apply hashing to multimodal data, one hash per multimodal item. When a UUID is provided for an item, the hash is computed from both the UUID @@ -855,8 +834,9 @@ def apply_mm_hashes(mm_data: Dict[str, Any], hash_lib: Hash function to use (default: blake3) Returns: - `MMHashResult` with per-modality `hashes` and `uuids` dicts. See - the type's own docstring for the field-level contract. + Tuple of: + - Dictionary of modality -> list of hash hex strings (64 chars each) + - Flattened list of original UUID strings (or None for content-hashed items) """ def _hash_item(item): @@ -888,8 +868,9 @@ def _hash_item_with_uuid(item, uuid: str): for modality, items in mm_data.items() } + # Collect UUIDs in the same order as items + all_uuids: List[Optional[str]] = [] mm_hashes: Dict[str, List[str]] = {} - mm_uuids_by_key: Dict[str, List[Optional[str]]] = {} for modality, items in mm_items.items(): modality_uuids = None @@ -903,25 +884,22 @@ def _hash_item_with_uuid(item, uuid: str): f"data items length ({len(items)}) for modality '{modality}'" ) - hashes: List[str] = [] - uuids: List[Optional[str]] = [] + hashes = [] for i, item in enumerate(items): uuid = modality_uuids[i] if modality_uuids else None if uuid is not None: # Hash UUID + content together for cache correctness hashes.append(_hash_item_with_uuid(item, uuid)) + all_uuids.append(uuid) # Store original UUID else: # Fall back to content-only hashing hashes.append(_hash_item(item)) - uuids.append(uuid) # `None` when the caller didn't supply one + all_uuids.append(None) mm_hashes[modality] = hashes - mm_uuids_by_key[modality] = uuids - return MMHashResult( - hashes=mm_hashes, - uuids=mm_uuids_by_key if mm_uuids is not None else None, - ) + # Return None for uuids if no UUIDs were provided at all + return mm_hashes, all_uuids if mm_uuids is not None else None def hexdigest_to_int32(hex_digest: str) -> List[int]: diff --git a/tensorrt_llm/inputs/multimodal_data.py b/tensorrt_llm/inputs/multimodal_data.py index 65895f9fa435..4c8e468e1f00 100644 --- a/tensorrt_llm/inputs/multimodal_data.py +++ b/tensorrt_llm/inputs/multimodal_data.py @@ -147,52 +147,32 @@ class VideoData(BaseModalityData): """Data class for video loading results. Attributes: - frames: Video frames as a list of PIL Images, a list of PyTorch - tensors, or a single 4D numpy array of shape (N, H, W, 3). + frames: List of video frames, either as PIL Images or PyTorch tensors. metadata: Dictionary containing video metadata including: - total_num_frames: Total number of frames in the video - fps: Original frames per second of the video - duration: Duration of the video in seconds - frames_indices: List of indices of the sampled frames audio: Structured audio payload from the video, when extracted. - raw_bytes_hash: BLAKE3 hex digest of the source video bytes when - available. Populated by media loaders that hold the source - (e.g. `VideoMediaIO`); `None` when the `VideoData` is - constructed from a frame list directly. When set, it acts as - the source anchor in `update_hash` — combined with the - sampling metadata, it uniquely identifies the decoded frame - content without walking pixels. """ - frames: list[Image.Image] | list[torch.Tensor] | np.ndarray + frames: list[Image.Image] | list[torch.Tensor] metadata: dict[str, Any] audio: AudioData | None = None - raw_bytes_hash: str | None = None def __post_init__(self) -> None: - if len(self.frames) == 0: - raise ValueError("frames cannot be empty") + if not self.frames: + raise ValueError("frames list cannot be empty") if not isinstance(self.metadata, dict): raise TypeError("metadata must be a dictionary") def update_hash(self, hasher: ContentHasher) -> None: hasher.update(b"